Add allocateGraphicsMemoryInDevicePool

- do not always expect failures in tests with failure injections
there is retry mechanism for some cases

Change-Id: If7589d2dacc41216d2f3b08f861209bbab179615

runtime/indirect_heap/indirect_heap.cpp

@@ -21,7 +21,6 @@
  */
 
 #include "indirect_heap.h"
-#include "runtime/memory_manager/graphics_allocation.h"
 
 namespace OCLRT {
 

runtime/memory_manager/CMakeLists.txt

@@ -36,6 +36,7 @@ set(RUNTIME_SRCS_MEMORY_MANAGER
   ${CMAKE_CURRENT_SOURCE_DIR}/memory_pool.h
   ${CMAKE_CURRENT_SOURCE_DIR}/os_agnostic_memory_manager.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/os_agnostic_memory_manager.h
+  ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/os_agnostic_memory_manager_allocate_in_device_pool.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/page_table.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/page_table.h
   ${CMAKE_CURRENT_SOURCE_DIR}/residency_container.h

runtime/memory_manager/memory_manager.cpp

@@ -411,10 +411,17 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, bool mustB
 
 GraphicsAllocation *MemoryManager::allocateGraphicsMemoryInPreferredPool(bool mustBeZeroCopy, bool allocateMemory, bool forcePin, bool uncacheable, const void *hostPtr, size_t size, GraphicsAllocation::AllocationType type) {
     AllocationData allocationData;
+    AllocationStatus status = AllocationStatus::Error;
+
     getAllocationData(allocationData, mustBeZeroCopy, allocateMemory, forcePin, uncacheable, hostPtr, size, type);
     UNRECOVERABLE_IF(allocationData.type == GraphicsAllocation::AllocationType::IMAGE || allocationData.type == GraphicsAllocation::AllocationType::SHARED_RESOURCE);
+    GraphicsAllocation *allocation = nullptr;
 
-    return allocateGraphicsMemory(allocationData);
+    allocation = allocateGraphicsMemoryInDevicePool(allocationData, status);
+    if (!allocation && status == AllocationStatus::RetryInNonDevicePool) {
+        allocation = allocateGraphicsMemory(allocationData);
+    }
+    return allocation;
 }
 
 GraphicsAllocation *MemoryManager::allocateGraphicsMemory(const AllocationData &allocationData) {

runtime/memory_manager/memory_manager.h

@@ -104,6 +104,7 @@ class MemoryManager {
         Success = 0,
         Error,
         InvalidHostPointer,
+        RetryInNonDevicePool
     };
 
     MemoryManager(bool enable64kbpages);
@@ -139,6 +140,19 @@ class MemoryManager {
 
     virtual GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) = 0;
 
+    virtual GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) {
+        status = AllocationStatus::Error;
+        if (!allocationData.flags.useSystemMemory && !(allocationData.flags.allow32Bit && this->force32bitAllocations)) {
+            auto allocation = allocateGraphicsMemory(allocationData);
+            if (allocation) {
+                status = AllocationStatus::Success;
+            }
+            return allocation;
+        }
+        status = AllocationStatus::RetryInNonDevicePool;
+        return nullptr;
+    }
+
     virtual bool mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) { return false; };
 
     virtual void *lockResource(GraphicsAllocation *graphicsAllocation) = 0;

runtime/memory_manager/os_agnostic_memory_manager.h

@@ -68,6 +68,8 @@ class OsAgnosticMemoryManager : public MemoryManager {
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness, bool reuseBO) override;
     GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) override { return nullptr; }
     GraphicsAllocation *allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) override;
+    GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
+
     void addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) override;
     void removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) override;
     void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override;

runtime/memory_manager/os_agnostic_memory_manager_allocate_in_device_pool.cpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "runtime/memory_manager/os_agnostic_memory_manager.h"
+
+namespace OCLRT {
+GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) {
+    return MemoryManager::allocateGraphicsMemoryInDevicePool(allocationData, status);
+}
+} // namespace OCLRT

runtime/os_interface/linux/drm_memory_manager.cpp

@@ -170,7 +170,6 @@ DrmAllocation *DrmMemoryManager::createGraphicsAllocation(OsHandleStorage &handl
 DrmAllocation *DrmMemoryManager::allocateGraphicsMemory(size_t size, size_t alignment, bool forcePin, bool uncacheable) {
     const size_t minAlignment = MemoryConstants::allocationAlignment;
     size_t cAlignment = alignUp(std::max(alignment, minAlignment), minAlignment);
-    DrmAllocation *allocation = nullptr;
     // When size == 0 allocate allocationAlignment
     // It's needed to prevent overlapping pages with user pointers
     size_t cSize = std::max(alignUp(size, minAlignment), minAlignment);
@@ -191,8 +190,7 @@ DrmAllocation *DrmMemoryManager::allocateGraphicsMemory(size_t size, size_t alig
     if (forcePinEnabled && pinBB != nullptr && forcePin && size >= this->pinThreshold) {
         pinBB->pin(&bo, 1);
     }
-    allocation = new DrmAllocation(bo, res, cSize, MemoryPool::System4KBPages);
-    return allocation;
+    return new DrmAllocation(bo, res, cSize, MemoryPool::System4KBPages);
 }
 
 DrmAllocation *DrmMemoryManager::allocateGraphicsMemory(size_t size, const void *ptr, bool forcePin) {
@@ -394,7 +392,6 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromSharedHandle(o
 }
 
 GraphicsAllocation *DrmMemoryManager::createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding) {
-    DrmAllocation *drmAllocation = nullptr;
     void *gpuRange = mmapFunction(nullptr, sizeWithPadding, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
 
     auto srcPtr = inputGraphicsAllocation->getUnderlyingBuffer();
@@ -411,8 +408,7 @@ GraphicsAllocation *DrmMemoryManager::createPaddedAllocation(GraphicsAllocation
     bo->softPin(reinterpret_cast<uint64_t>(gpuRange));
     bo->setUnmapSize(sizeWithPadding);
     bo->setAllocationType(MMAP_ALLOCATOR);
-    drmAllocation = new DrmAllocation(bo, (void *)srcPtr, (uint64_t)ptrOffset(gpuRange, offset), sizeWithPadding, inputGraphicsAllocation->getMemoryPool());
-    return drmAllocation;
+    return new DrmAllocation(bo, (void *)srcPtr, (uint64_t)ptrOffset(gpuRange, offset), sizeWithPadding, inputGraphicsAllocation->getMemoryPool());
 }
 
 void DrmMemoryManager::addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) {

runtime/os_interface/windows/CMakeLists.txt

@@ -73,6 +73,7 @@ set(RUNTIME_SRCS_OS_INTERFACE_WINDOWS
   ${CMAKE_CURRENT_SOURCE_DIR}/wddm_engine_mapper.inl
   ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.h
+  ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/windows_defs.h
   ${CMAKE_CURRENT_SOURCE_DIR}/windows_inc.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/windows_wrapper.h

runtime/os_interface/windows/wddm_memory_manager.h

@@ -53,6 +53,8 @@ class WddmMemoryManager : public MemoryManager {
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness, bool reuseBO) override;
     GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) override;
     GraphicsAllocation *allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) override;
+    GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
+
     void addAllocationToHostPtrManager(GraphicsAllocation *memory) override;
     void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) override;
     void *lockResource(GraphicsAllocation *graphicsAllocation) override;

runtime/os_interface/windows/wddm_memory_manager_allocate_in_device_pool.cpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "runtime/os_interface/windows/wddm_memory_manager.h"
+
+namespace OCLRT {
+GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) {
+    return MemoryManager::allocateGraphicsMemoryInDevicePool(allocationData, status);
+}
+} // namespace OCLRT