From dec634c2cdc3f1a9cb1b68b9bf3dba7e41a6fe23 Mon Sep 17 00:00:00 2001 From: Daniel Chabrowski Date: Fri, 11 Feb 2022 14:33:45 +0000 Subject: [PATCH] Extend DRM memory manager Signed-off-by: Daniel Chabrowski daniel.chabrowski@intel.com Related-To: NEO-6591 --- .../test/unit_test/test_files/igdrcl.config | 1 + .../debug_settings/debug_variables_base.inl | 1 + .../source/os_interface/linux/CMakeLists.txt | 4 +- .../os_interface/linux/drm_memory_manager.cpp | 285 ++++++++++++++++++ ...y_manager_create_multi_host_allocation.cpp | 14 - .../linux/drm_memory_manager_local_memory.cpp | 109 ------- .../drm_memory_operations_handler_create.cpp | 11 +- .../source/os_interface/linux/ioctl_helper.h | 6 + .../linux/ioctl_helper_prelim.cpp | 15 + .../linux/ioctl_helper_upstream.cpp | 10 +- 10 files changed, 328 insertions(+), 128 deletions(-) delete mode 100644 shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp delete mode 100644 shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config index 06ac519bf3..2b46ce7f53 100644 --- a/opencl/test/unit_test/test_files/igdrcl.config +++ b/opencl/test/unit_test/test_files/igdrcl.config @@ -378,6 +378,7 @@ ForceSimdMessageSizeInWalker = -1 UseNewQueryTopoIoctl = 1 EnableRecoverablePageFaults = -1 EnableImplicitMigrationOnFaultableHardware = -1 +SetVmAdviseAtomicAttribute = -1 UseDrmVirtualEnginesForCcs = -1 UseDrmVirtualEnginesForBcs = -1 LimitEngineCountForVirtualBcs = -1 diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 535f5aaf44..4d27fbfd23 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -183,6 +183,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, ForceExtendedUSMBufferSize, -1, "-1: default, 0: DECLARE_DEBUG_VARIABLE(int32_t, ForceSimdMessageSizeInWalker, -1, "-1: default, >=0 Program given value in Walker command for SIMD size") DECLARE_DEBUG_VARIABLE(int32_t, EnableRecoverablePageFaults, -1, "-1: default - ignore, 0: disable, 1: enable recoverable page faults on all VMs (on faultable hardware)") DECLARE_DEBUG_VARIABLE(int32_t, EnableImplicitMigrationOnFaultableHardware, -1, "-1: default - ignore, 0: disable, 1: enable implicit migration on faultable hardware (for all allocations)") +DECLARE_DEBUG_VARIABLE(int32_t, SetVmAdviseAtomicAttribute, -1, "-1: default - atomic system, 0: atomic none, 1: atomic device, 2: atomic system)") DECLARE_DEBUG_VARIABLE(int32_t, UseDrmVirtualEnginesForCcs, -1, "-1: default, 0: disable, 1: enable, Combine all CCS nodes to single VE (per context)") DECLARE_DEBUG_VARIABLE(int32_t, UseDrmVirtualEnginesForBcs, -1, "-1: default, 0: disable, 1: enable, Combine all BCS nodes to single VE (per context)") DECLARE_DEBUG_VARIABLE(int32_t, LimitEngineCountForVirtualBcs, -1, "-1: default, >0 Only use VirtualEngine with limited amount of engines, not max ") diff --git a/shared/source/os_interface/linux/CMakeLists.txt b/shared/source/os_interface/linux/CMakeLists.txt index 23f9f555bd..5d50abe814 100644 --- a/shared/source/os_interface/linux/CMakeLists.txt +++ b/shared/source/os_interface/linux/CMakeLists.txt @@ -39,9 +39,7 @@ set(NEO_CORE_OS_INTERFACE_LINUX ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.h ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.h - ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_create_multi_host_allocation.cpp - ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_local_memory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_operations_handler_create.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_query.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config_drm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 7557751553..d66a8b9716 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -21,9 +21,12 @@ #include "shared/source/helpers/string.h" #include "shared/source/helpers/surface_format_info.h" #include "shared/source/memory_manager/host_ptr_manager.h" +#include "shared/source/memory_manager/memory_banks.h" +#include "shared/source/memory_manager/memory_pool.h" #include "shared/source/memory_manager/residency.h" #include "shared/source/os_interface/linux/allocator_helper.h" #include "shared/source/os_interface/linux/drm_memory_operations_handler.h" +#include "shared/source/os_interface/linux/memory_info.h" #include "shared/source/os_interface/linux/os_context_linux.h" #include "shared/source/os_interface/os_interface.h" @@ -35,6 +38,37 @@ namespace NEO { +namespace { +void createMemoryRegionsForSharedAllocation(const HardwareInfo &hwInfo, MemoryInfo &memoryInfo, const AllocationData &allocationData, MemRegionsVec &memRegions) { + auto memoryBanks = allocationData.storageInfo.memoryBanks; + + if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) { + //System memory region + auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo); + memRegions.push_back(regionClassAndInstance); + } + + //All local memory regions + size_t currentBank = 0; + size_t i = 0; + + while (i < memoryBanks.count()) { + if (memoryBanks.test(currentBank)) { + auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(1u << currentBank, hwInfo); + memRegions.push_back(regionClassAndInstance); + i++; + } + currentBank++; + } + + if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::GPU) { + //System memory region + auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo); + memRegions.push_back(regionClassAndInstance); + } +} +} // namespace + DrmMemoryManager::DrmMemoryManager(gemCloseWorkerMode mode, bool forcePinAllowed, bool validateHostPtrMemory, @@ -129,6 +163,71 @@ DrmMemoryManager::~DrmMemoryManager() { } } +DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) { + drm_prime_handle openFd = {0, 0, 0}; + openFd.fd = handle; + + auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd); + if (ret != 0) { + int err = this->getDrm(properties.rootDeviceIndex).getErrno(); + PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err)); + DEBUG_BREAK_IF(ret != 0); + return nullptr; + } + + if (hasMappedPtr) { + auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount); + bo->setAddress(properties.gpuAddress); + + return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), + handle, MemoryPool::SystemCpuInaccessible); + } + + auto boHandle = openFd.handle; + auto bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex); + + void *cpuPointer = nullptr; + size_t size = lseekFunction(handle, 0, SEEK_END); + + if (bo == nullptr) { + bo = new BufferObject(&getDrm(properties.rootDeviceIndex), boHandle, size, maxOsContextCount); + cpuPointer = this->mmapFunction(0, size, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + bo->setAddress(reinterpret_cast(cpuPointer)); + + uint64_t offset = 0; + if (!retrieveMmapOffsetForBufferObject(properties.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) { + this->munmapFunction(cpuPointer, size); + delete bo; + return nullptr; + } + + [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(properties.rootDeviceIndex).getFileDescriptor(), static_cast(offset)); + DEBUG_BREAK_IF(retPtr != cpuPointer); + + AllocationData allocationData = {}; + allocationData.rootDeviceIndex = properties.rootDeviceIndex; + allocationData.size = size; + emitPinningRequest(bo, allocationData); + + bo->setUnmapSize(size); + bo->setRootDeviceIndex(properties.rootDeviceIndex); + + pushSharedBufferObject(bo); + + DrmAllocation *drmAllocation = nullptr; + drmAllocation = new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), MemoryPool::System4KBPages); + drmAllocation->setMmapPtr(cpuPointer); + drmAllocation->setMmapSize(size); + drmAllocation->setReservedAddressRange(reinterpret_cast(cpuPointer), size); + drmAllocation->setCacheRegion(&this->getDrm(properties.rootDeviceIndex), static_cast(properties.cacheRegion)); + + return drmAllocation; + } + + return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), + handle, MemoryPool::SystemCpuInaccessible); +} + void DrmMemoryManager::releaseDeviceSpecificMemResources(uint32_t rootDeviceIndex) { return releaseBufferObject(rootDeviceIndex); } @@ -344,6 +443,109 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignmentFromUserptr(const Alloc return allocation.release(); } +DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress) { + bool useBooMmap = this->getDrm(allocationData.rootDeviceIndex).getMemoryInfo() && allocationData.useMmapObject; + + if (DebugManager.flags.EnableBOMmapCreate.get() != -1) { + useBooMmap = DebugManager.flags.EnableBOMmapCreate.get(); + } + + if (useBooMmap) { + auto totalSizeToAlloc = alignedSize + alignment; + auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + auto cpuBasePointer = cpuPointer; + cpuPointer = alignUp(cpuPointer, alignment); + + auto pointerDiff = ptrDiff(cpuPointer, cpuBasePointer); + std::unique_ptr bo(this->createBufferObjectInMemoryRegion(&this->getDrm(allocationData.rootDeviceIndex), reinterpret_cast(cpuPointer), alignedSize, 0u, maxOsContextCount)); + + if (!bo) { + this->munmapFunction(cpuBasePointer, totalSizeToAlloc); + return nullptr; + } + + uint64_t offset = 0; + if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) { + this->munmapFunction(cpuPointer, size); + return nullptr; + } + + [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, alignedSize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(allocationData.rootDeviceIndex).getFileDescriptor(), static_cast(offset)); + DEBUG_BREAK_IF(retPtr != cpuPointer); + + obtainGpuAddress(allocationData, bo.get(), gpuAddress); + emitPinningRequest(bo.get(), allocationData); + + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), alignedSize, MemoryPool::System4KBPages); + allocation->setMmapPtr(cpuPointer); + allocation->setMmapSize(alignedSize); + if (pointerDiff != 0) { + allocation->registerMemoryToUnmap(cpuBasePointer, pointerDiff, this->munmapFunction); + } + [[maybe_unused]] int retCode = this->munmapFunction(ptrOffset(cpuPointer, alignedSize), alignment - pointerDiff); + DEBUG_BREAK_IF(retCode != 0); + allocation->setReservedAddressRange(reinterpret_cast(gpuAddress), alignedSize); + if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { + if (pointerDiff == 0) { + allocation->registerMemoryToUnmap(cpuBasePointer, totalSizeToAlloc, this->munmapFunction); + } + return nullptr; + } + + bo.release(); + + return allocation.release(); + } else { + return createAllocWithAlignmentFromUserptr(allocationData, size, alignment, alignedSize, gpuAddress); + } +} + +DrmAllocation *DrmMemoryManager::createMultiHostAllocation(const AllocationData &allocationData) { + if (!isAligned(allocationData.size)) { + return nullptr; + } + + auto numTiles = allocationData.storageInfo.getNumBanks(); + auto sizePerTile = allocationData.size; + auto hostSizeToAllocate = numTiles * sizePerTile; + + auto cpuBasePointer = alignedMallocWrapper(hostSizeToAllocate, MemoryConstants::pageSize); + if (!cpuBasePointer) { + return nullptr; + } + + zeroCpuMemoryIfRequested(allocationData, cpuBasePointer, hostSizeToAllocate); + + auto gpuAddress = acquireGpuRange(sizePerTile, allocationData.rootDeviceIndex, HeapIndex::HEAP_STANDARD); + auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, numTiles, allocationData.type, + nullptr /*bo*/, cpuBasePointer, gpuAddress, sizePerTile, MemoryPool::System4KBPages); + + allocation->storageInfo = allocationData.storageInfo; + allocation->setFlushL3Required(true); + allocation->setUncacheable(true); + allocation->setReservedAddressRange(reinterpret_cast(gpuAddress), sizePerTile); + allocation->setDriverAllocatedCpuPtr(cpuBasePointer); + + for (auto tile = 0u, currentBank = 0u; tile < numTiles; ++tile, ++currentBank) { + while (!allocationData.storageInfo.memoryBanks.test(currentBank)) { + ++currentBank; + } + + auto boHostPtr = static_cast(cpuBasePointer) + tile * sizePerTile; + auto bo = allocUserptr(reinterpret_cast(boHostPtr), sizePerTile, 0, allocationData.rootDeviceIndex); + if (!bo) { + freeGraphicsMemoryImpl(allocation); + return nullptr; + } + + bo->setAddress(gpuAddress); + allocation->getBufferObjectToModify(currentBank) = bo; + } + + return allocation; +} + void DrmMemoryManager::obtainGpuAddress(const AllocationData &allocationData, BufferObject *bo, uint64_t gpuAddress) { if ((isLimitedRange(allocationData.rootDeviceIndex) || allocationData.type == AllocationType::SVM_CPU) && !allocationData.flags.isUSMHostAllocation) { @@ -435,6 +637,69 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryWithGpuVa(const Allo return allocation; } +GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) { + auto &drm = this->getDrm(allocationData.rootDeviceIndex); + auto ioctlHelper = drm.getIoctlHelper(); + + if (!ioctlHelper->hasGemVmAdviseSupport()) { + return nullptr; + } + + auto memoryInfo = drm.getMemoryInfo(); + if (!memoryInfo) { + return nullptr; + } + + auto size = allocationData.size; + auto alignment = allocationData.alignment; + + auto pHwInfo = drm.getRootDeviceEnvironment().getHardwareInfo(); + + MemRegionsVec memRegions; + createMemoryRegionsForSharedAllocation(*pHwInfo, *memoryInfo, allocationData, memRegions); + + uint32_t handle = 0; + auto ret = memoryInfo->createGemExt(&drm, memRegions, size, handle); + + if (ret) { + return nullptr; + } + + std::unique_ptr bo(new BufferObject(&drm, handle, size, maxOsContextCount)); + + const auto atomicAttr = ioctlHelper->getVmAdviseAtomicAttribute(); + if (!ioctlHelper->setVmBoAdvise(&drm, bo->peekHandle(), atomicAttr, nullptr)) { + return nullptr; + } + + uint64_t offset = 0; + if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) { + return nullptr; + } + + auto totalSizeToAlloc = size + alignment; + auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + auto cpuBasePointer = cpuPointer; + cpuPointer = alignUp(cpuPointer, alignment); + + this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, drm.getFileDescriptor(), static_cast(offset)); + + bo->setAddress(reinterpret_cast(cpuPointer)); + + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), size, MemoryPool::System4KBPages); + allocation->setMmapPtr(cpuBasePointer); + allocation->setMmapSize(totalSizeToAlloc); + if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { + this->munmapFunction(cpuPointer, totalSizeToAlloc); + return nullptr; + } + + bo.release(); + + return allocation.release(); +} + DrmAllocation *DrmMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) { if (allocationData.size == 0 || !allocationData.hostPtr) return nullptr; @@ -1143,6 +1408,26 @@ void *DrmMemoryManager::lockResourceInLocalMemoryImpl(GraphicsAllocation &graphi return lockResourceInLocalMemoryImpl(bo); } +void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) { + if (bo == nullptr) { + return nullptr; + } + + auto rootDeviceIndex = this->getRootDeviceIndex(bo->peekDrm()); + + uint64_t offset = 0; + if (!retrieveMmapOffsetForBufferObject(rootDeviceIndex, *bo, I915_MMAP_OFFSET_WC, offset)) { + return nullptr; + } + + auto addr = mmapFunction(nullptr, bo->peekSize(), PROT_WRITE | PROT_READ, MAP_SHARED, getDrm(rootDeviceIndex).getFileDescriptor(), static_cast(offset)); + DEBUG_BREAK_IF(addr == MAP_FAILED); + + bo->setLockedAddress(addr); + + return bo->peekLockedAddress(); +} + bool DrmMemoryManager::copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy) { if (graphicsAllocation->getUnderlyingBuffer() || !isLocalMemorySupported(graphicsAllocation->getRootDeviceIndex())) { return MemoryManager::copyMemoryToAllocation(graphicsAllocation, destinationOffset, memoryToCopy, sizeToCopy); diff --git a/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp b/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp deleted file mode 100644 index 8f947151f3..0000000000 --- a/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2021 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/os_interface/linux/drm_memory_manager.h" - -namespace NEO { -DrmAllocation *DrmMemoryManager::createMultiHostAllocation(const AllocationData &allocationData) { - return allocateGraphicsMemoryWithAlignmentImpl(allocationData); -} -} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp b/shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp deleted file mode 100644 index daef3a8a21..0000000000 --- a/shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2019-2022 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/helpers/basic_math.h" -#include "shared/source/os_interface/linux/drm_memory_manager.h" - -namespace NEO { - -DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) { - drm_prime_handle openFd = {0, 0, 0}; - openFd.fd = handle; - - auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd); - - if (ret != 0) { - int err = this->getDrm(properties.rootDeviceIndex).getErrno(); - PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err)); - DEBUG_BREAK_IF(ret != 0); - return nullptr; - } - - auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount); - bo->setAddress(properties.gpuAddress); - - return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), - handle, MemoryPool::SystemCpuInaccessible); -} - -DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress) { - bool useBooMmap = this->getDrm(allocationData.rootDeviceIndex).getMemoryInfo() && allocationData.useMmapObject; - - if (DebugManager.flags.EnableBOMmapCreate.get() != -1) { - useBooMmap = DebugManager.flags.EnableBOMmapCreate.get(); - } - - if (useBooMmap) { - auto totalSizeToAlloc = alignedSize + alignment; - auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - - auto cpuBasePointer = cpuPointer; - cpuPointer = alignUp(cpuPointer, alignment); - - auto pointerDiff = ptrDiff(cpuPointer, cpuBasePointer); - std::unique_ptr bo(this->createBufferObjectInMemoryRegion(&this->getDrm(allocationData.rootDeviceIndex), reinterpret_cast(cpuPointer), alignedSize, 0u, maxOsContextCount)); - - if (!bo) { - this->munmapFunction(cpuBasePointer, totalSizeToAlloc); - return nullptr; - } - - uint64_t offset = 0; - if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) { - this->munmapFunction(cpuPointer, size); - return nullptr; - } - - [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, alignedSize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(allocationData.rootDeviceIndex).getFileDescriptor(), static_cast(offset)); - DEBUG_BREAK_IF(retPtr != cpuPointer); - - obtainGpuAddress(allocationData, bo.get(), gpuAddress); - emitPinningRequest(bo.get(), allocationData); - - auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), alignedSize, MemoryPool::System4KBPages); - allocation->setMmapPtr(cpuPointer); - allocation->setMmapSize(alignedSize); - if (pointerDiff != 0) { - allocation->registerMemoryToUnmap(cpuBasePointer, pointerDiff, this->munmapFunction); - } - [[maybe_unused]] int retCode = this->munmapFunction(ptrOffset(cpuPointer, alignedSize), alignment - pointerDiff); - DEBUG_BREAK_IF(retCode != 0); - allocation->setReservedAddressRange(reinterpret_cast(gpuAddress), alignedSize); - - bo.release(); - - return allocation; - } else { - return createAllocWithAlignmentFromUserptr(allocationData, size, alignment, alignedSize, gpuAddress); - } -} - -GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) { - return nullptr; -} - -void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) { - if (bo == nullptr) { - return nullptr; - } - - auto rootDeviceIndex = this->getRootDeviceIndex(bo->peekDrm()); - - uint64_t offset = 0; - if (!retrieveMmapOffsetForBufferObject(rootDeviceIndex, *bo, I915_MMAP_OFFSET_WC, offset)) { - return nullptr; - } - - auto addr = mmapFunction(nullptr, bo->peekSize(), PROT_WRITE | PROT_READ, MAP_SHARED, getDrm(rootDeviceIndex).getFileDescriptor(), static_cast(offset)); - DEBUG_BREAK_IF(addr == MAP_FAILED); - - bo->setLockedAddress(addr); - - return bo->peekLockedAddress(); -} - -} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp index a55523c1fe..46cac14807 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp @@ -1,15 +1,24 @@ /* - * Copyright (C) 2020-2021 Intel Corporation + * Copyright (C) 2020-2022 Intel Corporation * * SPDX-License-Identifier: MIT * */ +#include "shared/source/execution_environment/root_device_environment.h" +#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h" #include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h" +#include "shared/source/os_interface/linux/drm_neo.h" namespace NEO { std::unique_ptr DrmMemoryOperationsHandler::create(Drm &drm, uint32_t rootDeviceIndex) { + bool useVmBind = drm.isVmBindAvailable(); + + if (useVmBind) { + return std::make_unique(drm.getRootDeviceEnvironment(), rootDeviceIndex); + } + return std::make_unique(); } diff --git a/shared/source/os_interface/linux/ioctl_helper.h b/shared/source/os_interface/linux/ioctl_helper.h index 130beab4b0..eb780934d7 100644 --- a/shared/source/os_interface/linux/ioctl_helper.h +++ b/shared/source/os_interface/linux/ioctl_helper.h @@ -85,6 +85,8 @@ class IoctlHelper { virtual int execBuffer(Drm *drm, drm_i915_gem_execbuffer2 *execBuffer, uint64_t completionGpuAddress, uint32_t counterValue) = 0; virtual bool completionFenceExtensionSupported(Drm &drm, const HardwareInfo &hwInfo) = 0; virtual std::optional getHasPageFaultParamId() = 0; + virtual bool hasGemVmAdviseSupport() = 0; + virtual uint32_t getVmAdviseAtomicAttribute() = 0; }; class IoctlHelperUpstream : public IoctlHelper { @@ -112,6 +114,8 @@ class IoctlHelperUpstream : public IoctlHelper { int execBuffer(Drm *drm, drm_i915_gem_execbuffer2 *execBuffer, uint64_t completionGpuAddress, uint32_t counterValue) override; bool completionFenceExtensionSupported(Drm &drm, const HardwareInfo &hwInfo) override; std::optional getHasPageFaultParamId() override; + bool hasGemVmAdviseSupport() override; + uint32_t getVmAdviseAtomicAttribute() override; }; template @@ -152,6 +156,8 @@ class IoctlHelperPrelim20 : public IoctlHelper { int execBuffer(Drm *drm, drm_i915_gem_execbuffer2 *execBuffer, uint64_t completionGpuAddress, uint32_t counterValue) override; bool completionFenceExtensionSupported(Drm &drm, const HardwareInfo &hwInfo) override; std::optional getHasPageFaultParamId() override; + bool hasGemVmAdviseSupport() override; + uint32_t getVmAdviseAtomicAttribute() override; }; } // namespace NEO diff --git a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp index d7ae7b4211..ce121f8c85 100644 --- a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp @@ -251,4 +251,19 @@ std::optional IoctlHelperPrelim20::getHasPageFaultParamId() { return PRELIM_I915_PARAM_HAS_PAGE_FAULT; }; +bool IoctlHelperPrelim20::hasGemVmAdviseSupport() { + return true; +} + +uint32_t IoctlHelperPrelim20::getVmAdviseAtomicAttribute() { + switch (NEO::DebugManager.flags.SetVmAdviseAtomicAttribute.get()) { + case 0: + return PRELIM_I915_VM_ADVISE_ATOMIC_NONE; + case 1: + return PRELIM_I915_VM_ADVISE_ATOMIC_DEVICE; + default: + return PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM; + } +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp index cbb1ffa0bc..87630bcfa3 100644 --- a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp @@ -148,6 +148,14 @@ bool IoctlHelperUpstream::completionFenceExtensionSupported(Drm &drm, const Hard std::optional IoctlHelperUpstream::getHasPageFaultParamId() { return std::nullopt; -}; +} + +bool IoctlHelperUpstream::hasGemVmAdviseSupport() { + return false; +} + +uint32_t IoctlHelperUpstream::getVmAdviseAtomicAttribute() { + return 0; +} } // namespace NEO