Extend DRM memory manager

Signed-off-by: Daniel Chabrowski daniel.chabrowski@intel.com
Related-To: NEO-6591

opencl/test/unit_test/test_files/igdrcl.config

@@ -378,6 +378,7 @@ ForceSimdMessageSizeInWalker = -1
 UseNewQueryTopoIoctl = 1
 EnableRecoverablePageFaults = -1
 EnableImplicitMigrationOnFaultableHardware = -1
+SetVmAdviseAtomicAttribute = -1
 UseDrmVirtualEnginesForCcs = -1
 UseDrmVirtualEnginesForBcs = -1
 LimitEngineCountForVirtualBcs = -1

shared/source/debug_settings/debug_variables_base.inl

@@ -183,6 +183,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, ForceExtendedUSMBufferSize, -1, "-1: default, 0:
 DECLARE_DEBUG_VARIABLE(int32_t, ForceSimdMessageSizeInWalker, -1, "-1: default, >=0 Program given value in Walker command for SIMD size")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableRecoverablePageFaults, -1, "-1: default - ignore, 0: disable, 1: enable recoverable page faults on all VMs (on faultable hardware)")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableImplicitMigrationOnFaultableHardware, -1, "-1: default - ignore, 0: disable, 1: enable implicit migration on faultable hardware (for all allocations)")
+DECLARE_DEBUG_VARIABLE(int32_t, SetVmAdviseAtomicAttribute, -1, "-1: default - atomic system, 0: atomic none, 1: atomic device, 2: atomic system)")
 DECLARE_DEBUG_VARIABLE(int32_t, UseDrmVirtualEnginesForCcs, -1, "-1: default, 0: disable, 1: enable,  Combine all CCS nodes to single VE (per context)")
 DECLARE_DEBUG_VARIABLE(int32_t, UseDrmVirtualEnginesForBcs, -1, "-1: default, 0: disable, 1: enable,  Combine all BCS nodes to single VE (per context)")
 DECLARE_DEBUG_VARIABLE(int32_t, LimitEngineCountForVirtualBcs, -1, "-1: default, >0 Only use VirtualEngine with limited amount of engines, not max ")

shared/source/os_interface/linux/CMakeLists.txt

@@ -39,9 +39,7 @@ set(NEO_CORE_OS_INTERFACE_LINUX
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.h
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.h
-    ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_create_multi_host_allocation.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_local_memory.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_operations_handler_create.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_query.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config_drm.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h

shared/source/os_interface/linux/drm_memory_manager.cpp

@@ -21,9 +21,12 @@
 #include "shared/source/helpers/string.h"
 #include "shared/source/helpers/surface_format_info.h"
 #include "shared/source/memory_manager/host_ptr_manager.h"
+#include "shared/source/memory_manager/memory_banks.h"
+#include "shared/source/memory_manager/memory_pool.h"
 #include "shared/source/memory_manager/residency.h"
 #include "shared/source/os_interface/linux/allocator_helper.h"
 #include "shared/source/os_interface/linux/drm_memory_operations_handler.h"
+#include "shared/source/os_interface/linux/memory_info.h"
 #include "shared/source/os_interface/linux/os_context_linux.h"
 #include "shared/source/os_interface/os_interface.h"
 
@@ -35,6 +38,37 @@
 
 namespace NEO {
 
+namespace {
+void createMemoryRegionsForSharedAllocation(const HardwareInfo &hwInfo, MemoryInfo &memoryInfo, const AllocationData &allocationData, MemRegionsVec &memRegions) {
+    auto memoryBanks = allocationData.storageInfo.memoryBanks;
+
+    if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) {
+        //System memory region
+        auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo);
+        memRegions.push_back(regionClassAndInstance);
+    }
+
+    //All local memory regions
+    size_t currentBank = 0;
+    size_t i = 0;
+
+    while (i < memoryBanks.count()) {
+        if (memoryBanks.test(currentBank)) {
+            auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(1u << currentBank, hwInfo);
+            memRegions.push_back(regionClassAndInstance);
+            i++;
+        }
+        currentBank++;
+    }
+
+    if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::GPU) {
+        //System memory region
+        auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo);
+        memRegions.push_back(regionClassAndInstance);
+    }
+}
+} // namespace
+
 DrmMemoryManager::DrmMemoryManager(gemCloseWorkerMode mode,
                                    bool forcePinAllowed,
                                    bool validateHostPtrMemory,
@@ -129,6 +163,71 @@ DrmMemoryManager::~DrmMemoryManager() {
     }
 }
 
+DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) {
+    drm_prime_handle openFd = {0, 0, 0};
+    openFd.fd = handle;
+
+    auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
+    if (ret != 0) {
+        int err = this->getDrm(properties.rootDeviceIndex).getErrno();
+        PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
+        DEBUG_BREAK_IF(ret != 0);
+        return nullptr;
+    }
+
+    if (hasMappedPtr) {
+        auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount);
+        bo->setAddress(properties.gpuAddress);
+
+        return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
+                                 handle, MemoryPool::SystemCpuInaccessible);
+    }
+
+    auto boHandle = openFd.handle;
+    auto bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex);
+
+    void *cpuPointer = nullptr;
+    size_t size = lseekFunction(handle, 0, SEEK_END);
+
+    if (bo == nullptr) {
+        bo = new BufferObject(&getDrm(properties.rootDeviceIndex), boHandle, size, maxOsContextCount);
+        cpuPointer = this->mmapFunction(0, size, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+        bo->setAddress(reinterpret_cast<uintptr_t>(cpuPointer));
+
+        uint64_t offset = 0;
+        if (!retrieveMmapOffsetForBufferObject(properties.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
+            this->munmapFunction(cpuPointer, size);
+            delete bo;
+            return nullptr;
+        }
+
+        [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(properties.rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
+        DEBUG_BREAK_IF(retPtr != cpuPointer);
+
+        AllocationData allocationData = {};
+        allocationData.rootDeviceIndex = properties.rootDeviceIndex;
+        allocationData.size = size;
+        emitPinningRequest(bo, allocationData);
+
+        bo->setUnmapSize(size);
+        bo->setRootDeviceIndex(properties.rootDeviceIndex);
+
+        pushSharedBufferObject(bo);
+
+        DrmAllocation *drmAllocation = nullptr;
+        drmAllocation = new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), MemoryPool::System4KBPages);
+        drmAllocation->setMmapPtr(cpuPointer);
+        drmAllocation->setMmapSize(size);
+        drmAllocation->setReservedAddressRange(reinterpret_cast<void *>(cpuPointer), size);
+        drmAllocation->setCacheRegion(&this->getDrm(properties.rootDeviceIndex), static_cast<CacheRegion>(properties.cacheRegion));
+
+        return drmAllocation;
+    }
+
+    return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
+                             handle, MemoryPool::SystemCpuInaccessible);
+}
+
 void DrmMemoryManager::releaseDeviceSpecificMemResources(uint32_t rootDeviceIndex) {
     return releaseBufferObject(rootDeviceIndex);
 }
@@ -344,6 +443,109 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignmentFromUserptr(const Alloc
     return allocation.release();
 }
 
+DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress) {
+    bool useBooMmap = this->getDrm(allocationData.rootDeviceIndex).getMemoryInfo() && allocationData.useMmapObject;
+
+    if (DebugManager.flags.EnableBOMmapCreate.get() != -1) {
+        useBooMmap = DebugManager.flags.EnableBOMmapCreate.get();
+    }
+
+    if (useBooMmap) {
+        auto totalSizeToAlloc = alignedSize + alignment;
+        auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+
+        auto cpuBasePointer = cpuPointer;
+        cpuPointer = alignUp(cpuPointer, alignment);
+
+        auto pointerDiff = ptrDiff(cpuPointer, cpuBasePointer);
+        std::unique_ptr<BufferObject, BufferObject::Deleter> bo(this->createBufferObjectInMemoryRegion(&this->getDrm(allocationData.rootDeviceIndex), reinterpret_cast<uintptr_t>(cpuPointer), alignedSize, 0u, maxOsContextCount));
+
+        if (!bo) {
+            this->munmapFunction(cpuBasePointer, totalSizeToAlloc);
+            return nullptr;
+        }
+
+        uint64_t offset = 0;
+        if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
+            this->munmapFunction(cpuPointer, size);
+            return nullptr;
+        }
+
+        [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, alignedSize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(allocationData.rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
+        DEBUG_BREAK_IF(retPtr != cpuPointer);
+
+        obtainGpuAddress(allocationData, bo.get(), gpuAddress);
+        emitPinningRequest(bo.get(), allocationData);
+
+        auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), alignedSize, MemoryPool::System4KBPages);
+        allocation->setMmapPtr(cpuPointer);
+        allocation->setMmapSize(alignedSize);
+        if (pointerDiff != 0) {
+            allocation->registerMemoryToUnmap(cpuBasePointer, pointerDiff, this->munmapFunction);
+        }
+        [[maybe_unused]] int retCode = this->munmapFunction(ptrOffset(cpuPointer, alignedSize), alignment - pointerDiff);
+        DEBUG_BREAK_IF(retCode != 0);
+        allocation->setReservedAddressRange(reinterpret_cast<void *>(gpuAddress), alignedSize);
+        if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast<CacheRegion>(allocationData.cacheRegion))) {
+            if (pointerDiff == 0) {
+                allocation->registerMemoryToUnmap(cpuBasePointer, totalSizeToAlloc, this->munmapFunction);
+            }
+            return nullptr;
+        }
+
+        bo.release();
+
+        return allocation.release();
+    } else {
+        return createAllocWithAlignmentFromUserptr(allocationData, size, alignment, alignedSize, gpuAddress);
+    }
+}
+
+DrmAllocation *DrmMemoryManager::createMultiHostAllocation(const AllocationData &allocationData) {
+    if (!isAligned<MemoryConstants::pageSize>(allocationData.size)) {
+        return nullptr;
+    }
+
+    auto numTiles = allocationData.storageInfo.getNumBanks();
+    auto sizePerTile = allocationData.size;
+    auto hostSizeToAllocate = numTiles * sizePerTile;
+
+    auto cpuBasePointer = alignedMallocWrapper(hostSizeToAllocate, MemoryConstants::pageSize);
+    if (!cpuBasePointer) {
+        return nullptr;
+    }
+
+    zeroCpuMemoryIfRequested(allocationData, cpuBasePointer, hostSizeToAllocate);
+
+    auto gpuAddress = acquireGpuRange(sizePerTile, allocationData.rootDeviceIndex, HeapIndex::HEAP_STANDARD);
+    auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, numTiles, allocationData.type,
+                                        nullptr /*bo*/, cpuBasePointer, gpuAddress, sizePerTile, MemoryPool::System4KBPages);
+
+    allocation->storageInfo = allocationData.storageInfo;
+    allocation->setFlushL3Required(true);
+    allocation->setUncacheable(true);
+    allocation->setReservedAddressRange(reinterpret_cast<void *>(gpuAddress), sizePerTile);
+    allocation->setDriverAllocatedCpuPtr(cpuBasePointer);
+
+    for (auto tile = 0u, currentBank = 0u; tile < numTiles; ++tile, ++currentBank) {
+        while (!allocationData.storageInfo.memoryBanks.test(currentBank)) {
+            ++currentBank;
+        }
+
+        auto boHostPtr = static_cast<uint8_t *>(cpuBasePointer) + tile * sizePerTile;
+        auto bo = allocUserptr(reinterpret_cast<uintptr_t>(boHostPtr), sizePerTile, 0, allocationData.rootDeviceIndex);
+        if (!bo) {
+            freeGraphicsMemoryImpl(allocation);
+            return nullptr;
+        }
+
+        bo->setAddress(gpuAddress);
+        allocation->getBufferObjectToModify(currentBank) = bo;
+    }
+
+    return allocation;
+}
+
 void DrmMemoryManager::obtainGpuAddress(const AllocationData &allocationData, BufferObject *bo, uint64_t gpuAddress) {
     if ((isLimitedRange(allocationData.rootDeviceIndex) || allocationData.type == AllocationType::SVM_CPU) &&
         !allocationData.flags.isUSMHostAllocation) {
@@ -435,6 +637,69 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryWithGpuVa(const Allo
     return allocation;
 }
 
+GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) {
+    auto &drm = this->getDrm(allocationData.rootDeviceIndex);
+    auto ioctlHelper = drm.getIoctlHelper();
+
+    if (!ioctlHelper->hasGemVmAdviseSupport()) {
+        return nullptr;
+    }
+
+    auto memoryInfo = drm.getMemoryInfo();
+    if (!memoryInfo) {
+        return nullptr;
+    }
+
+    auto size = allocationData.size;
+    auto alignment = allocationData.alignment;
+
+    auto pHwInfo = drm.getRootDeviceEnvironment().getHardwareInfo();
+
+    MemRegionsVec memRegions;
+    createMemoryRegionsForSharedAllocation(*pHwInfo, *memoryInfo, allocationData, memRegions);
+
+    uint32_t handle = 0;
+    auto ret = memoryInfo->createGemExt(&drm, memRegions, size, handle);
+
+    if (ret) {
+        return nullptr;
+    }
+
+    std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new BufferObject(&drm, handle, size, maxOsContextCount));
+
+    const auto atomicAttr = ioctlHelper->getVmAdviseAtomicAttribute();
+    if (!ioctlHelper->setVmBoAdvise(&drm, bo->peekHandle(), atomicAttr, nullptr)) {
+        return nullptr;
+    }
+
+    uint64_t offset = 0;
+    if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
+        return nullptr;
+    }
+
+    auto totalSizeToAlloc = size + alignment;
+    auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+
+    auto cpuBasePointer = cpuPointer;
+    cpuPointer = alignUp(cpuPointer, alignment);
+
+    this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, drm.getFileDescriptor(), static_cast<off_t>(offset));
+
+    bo->setAddress(reinterpret_cast<uintptr_t>(cpuPointer));
+
+    auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), size, MemoryPool::System4KBPages);
+    allocation->setMmapPtr(cpuBasePointer);
+    allocation->setMmapSize(totalSizeToAlloc);
+    if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast<CacheRegion>(allocationData.cacheRegion))) {
+        this->munmapFunction(cpuPointer, totalSizeToAlloc);
+        return nullptr;
+    }
+
+    bo.release();
+
+    return allocation.release();
+}
+
 DrmAllocation *DrmMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) {
     if (allocationData.size == 0 || !allocationData.hostPtr)
         return nullptr;
@@ -1143,6 +1408,26 @@ void *DrmMemoryManager::lockResourceInLocalMemoryImpl(GraphicsAllocation &graphi
     return lockResourceInLocalMemoryImpl(bo);
 }
 
+void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) {
+    if (bo == nullptr) {
+        return nullptr;
+    }
+
+    auto rootDeviceIndex = this->getRootDeviceIndex(bo->peekDrm());
+
+    uint64_t offset = 0;
+    if (!retrieveMmapOffsetForBufferObject(rootDeviceIndex, *bo, I915_MMAP_OFFSET_WC, offset)) {
+        return nullptr;
+    }
+
+    auto addr = mmapFunction(nullptr, bo->peekSize(), PROT_WRITE | PROT_READ, MAP_SHARED, getDrm(rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
+    DEBUG_BREAK_IF(addr == MAP_FAILED);
+
+    bo->setLockedAddress(addr);
+
+    return bo->peekLockedAddress();
+}
+
 bool DrmMemoryManager::copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy) {
     if (graphicsAllocation->getUnderlyingBuffer() || !isLocalMemorySupported(graphicsAllocation->getRootDeviceIndex())) {
         return MemoryManager::copyMemoryToAllocation(graphicsAllocation, destinationOffset, memoryToCopy, sizeToCopy);

shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp

@@ -1,14 +0,0 @@
-/*
- * Copyright (C) 2021 Intel Corporation
- *
- * SPDX-License-Identifier: MIT
- *
- */
-
-#include "shared/source/os_interface/linux/drm_memory_manager.h"
-
-namespace NEO {
-DrmAllocation *DrmMemoryManager::createMultiHostAllocation(const AllocationData &allocationData) {
-    return allocateGraphicsMemoryWithAlignmentImpl(allocationData);
-}
-} // namespace NEO

shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp

@@ -1,109 +0,0 @@
-/*
- * Copyright (C) 2019-2022 Intel Corporation
- *
- * SPDX-License-Identifier: MIT
- *
- */
-
-#include "shared/source/helpers/basic_math.h"
-#include "shared/source/os_interface/linux/drm_memory_manager.h"
-
-namespace NEO {
-
-DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) {
-    drm_prime_handle openFd = {0, 0, 0};
-    openFd.fd = handle;
-
-    auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
-
-    if (ret != 0) {
-        int err = this->getDrm(properties.rootDeviceIndex).getErrno();
-        PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
-        DEBUG_BREAK_IF(ret != 0);
-        return nullptr;
-    }
-
-    auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount);
-    bo->setAddress(properties.gpuAddress);
-
-    return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
-                             handle, MemoryPool::SystemCpuInaccessible);
-}
-
-DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress) {
-    bool useBooMmap = this->getDrm(allocationData.rootDeviceIndex).getMemoryInfo() && allocationData.useMmapObject;
-
-    if (DebugManager.flags.EnableBOMmapCreate.get() != -1) {
-        useBooMmap = DebugManager.flags.EnableBOMmapCreate.get();
-    }
-
-    if (useBooMmap) {
-        auto totalSizeToAlloc = alignedSize + alignment;
-        auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-
-        auto cpuBasePointer = cpuPointer;
-        cpuPointer = alignUp(cpuPointer, alignment);
-
-        auto pointerDiff = ptrDiff(cpuPointer, cpuBasePointer);
-        std::unique_ptr<BufferObject, BufferObject::Deleter> bo(this->createBufferObjectInMemoryRegion(&this->getDrm(allocationData.rootDeviceIndex), reinterpret_cast<uintptr_t>(cpuPointer), alignedSize, 0u, maxOsContextCount));
-
-        if (!bo) {
-            this->munmapFunction(cpuBasePointer, totalSizeToAlloc);
-            return nullptr;
-        }
-
-        uint64_t offset = 0;
-        if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
-            this->munmapFunction(cpuPointer, size);
-            return nullptr;
-        }
-
-        [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, alignedSize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(allocationData.rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
-        DEBUG_BREAK_IF(retPtr != cpuPointer);
-
-        obtainGpuAddress(allocationData, bo.get(), gpuAddress);
-        emitPinningRequest(bo.get(), allocationData);
-
-        auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), alignedSize, MemoryPool::System4KBPages);
-        allocation->setMmapPtr(cpuPointer);
-        allocation->setMmapSize(alignedSize);
-        if (pointerDiff != 0) {
-            allocation->registerMemoryToUnmap(cpuBasePointer, pointerDiff, this->munmapFunction);
-        }
-        [[maybe_unused]] int retCode = this->munmapFunction(ptrOffset(cpuPointer, alignedSize), alignment - pointerDiff);
-        DEBUG_BREAK_IF(retCode != 0);
-        allocation->setReservedAddressRange(reinterpret_cast<void *>(gpuAddress), alignedSize);
-
-        bo.release();
-
-        return allocation;
-    } else {
-        return createAllocWithAlignmentFromUserptr(allocationData, size, alignment, alignedSize, gpuAddress);
-    }
-}
-
-GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) {
-    return nullptr;
-}
-
-void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) {
-    if (bo == nullptr) {
-        return nullptr;
-    }
-
-    auto rootDeviceIndex = this->getRootDeviceIndex(bo->peekDrm());
-
-    uint64_t offset = 0;
-    if (!retrieveMmapOffsetForBufferObject(rootDeviceIndex, *bo, I915_MMAP_OFFSET_WC, offset)) {
-        return nullptr;
-    }
-
-    auto addr = mmapFunction(nullptr, bo->peekSize(), PROT_WRITE | PROT_READ, MAP_SHARED, getDrm(rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
-    DEBUG_BREAK_IF(addr == MAP_FAILED);
-
-    bo->setLockedAddress(addr);
-
-    return bo->peekLockedAddress();
-}
-
-} // namespace NEO

shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp

@@ -1,15 +1,24 @@
 /*
- * Copyright (C) 2020-2021 Intel Corporation
+ * Copyright (C) 2020-2022 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
  */
 
+#include "shared/source/execution_environment/root_device_environment.h"
+#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h"
 #include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h"
+#include "shared/source/os_interface/linux/drm_neo.h"
 
 namespace NEO {
 
 std::unique_ptr<DrmMemoryOperationsHandler> DrmMemoryOperationsHandler::create(Drm &drm, uint32_t rootDeviceIndex) {
+    bool useVmBind = drm.isVmBindAvailable();
+
+    if (useVmBind) {
+        return std::make_unique<DrmMemoryOperationsHandlerBind>(drm.getRootDeviceEnvironment(), rootDeviceIndex);
+    }
+
     return std::make_unique<DrmMemoryOperationsHandlerDefault>();
 }
 

shared/source/os_interface/linux/ioctl_helper.h

@@ -85,6 +85,8 @@ class IoctlHelper {
     virtual int execBuffer(Drm *drm, drm_i915_gem_execbuffer2 *execBuffer, uint64_t completionGpuAddress, uint32_t counterValue) = 0;
     virtual bool completionFenceExtensionSupported(Drm &drm, const HardwareInfo &hwInfo) = 0;
     virtual std::optional<int> getHasPageFaultParamId() = 0;
+    virtual bool hasGemVmAdviseSupport() = 0;
+    virtual uint32_t getVmAdviseAtomicAttribute() = 0;
 };
 
 class IoctlHelperUpstream : public IoctlHelper {
@@ -112,6 +114,8 @@ class IoctlHelperUpstream : public IoctlHelper {
     int execBuffer(Drm *drm, drm_i915_gem_execbuffer2 *execBuffer, uint64_t completionGpuAddress, uint32_t counterValue) override;
     bool completionFenceExtensionSupported(Drm &drm, const HardwareInfo &hwInfo) override;
     std::optional<int> getHasPageFaultParamId() override;
+    bool hasGemVmAdviseSupport() override;
+    uint32_t getVmAdviseAtomicAttribute() override;
 };
 
 template <PRODUCT_FAMILY gfxProduct>
@@ -152,6 +156,8 @@ class IoctlHelperPrelim20 : public IoctlHelper {
     int execBuffer(Drm *drm, drm_i915_gem_execbuffer2 *execBuffer, uint64_t completionGpuAddress, uint32_t counterValue) override;
     bool completionFenceExtensionSupported(Drm &drm, const HardwareInfo &hwInfo) override;
     std::optional<int> getHasPageFaultParamId() override;
+    bool hasGemVmAdviseSupport() override;
+    uint32_t getVmAdviseAtomicAttribute() override;
 };
 
 } // namespace NEO

shared/source/os_interface/linux/ioctl_helper_prelim.cpp

@@ -251,4 +251,19 @@ std::optional<int> IoctlHelperPrelim20::getHasPageFaultParamId() {
     return PRELIM_I915_PARAM_HAS_PAGE_FAULT;
 };
 
+bool IoctlHelperPrelim20::hasGemVmAdviseSupport() {
+    return true;
+}
+
+uint32_t IoctlHelperPrelim20::getVmAdviseAtomicAttribute() {
+    switch (NEO::DebugManager.flags.SetVmAdviseAtomicAttribute.get()) {
+    case 0:
+        return PRELIM_I915_VM_ADVISE_ATOMIC_NONE;
+    case 1:
+        return PRELIM_I915_VM_ADVISE_ATOMIC_DEVICE;
+    default:
+        return PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM;
+    }
+}
+
 } // namespace NEO

shared/source/os_interface/linux/ioctl_helper_upstream.cpp

@@ -148,6 +148,14 @@ bool IoctlHelperUpstream::completionFenceExtensionSupported(Drm &drm, const Hard
 
 std::optional<int> IoctlHelperUpstream::getHasPageFaultParamId() {
     return std::nullopt;
-};
+}
+
+bool IoctlHelperUpstream::hasGemVmAdviseSupport() {
+    return false;
+}
+
+uint32_t IoctlHelperUpstream::getVmAdviseAtomicAttribute() {
+    return 0;
+}
 
 } // namespace NEO