From 28b867f883094e277d7523483e8bf313224783bb Mon Sep 17 00:00:00 2001 From: Daniel Chabrowski Date: Mon, 21 Mar 2022 16:02:12 +0000 Subject: [PATCH] Move and extend local memory DRM allocations Signed-off-by: Daniel Chabrowski daniel.chabrowski@intel.com Related-To: NEO-6591 --- .../test/unit_test/test_files/igdrcl.config | 1 + .../debug_settings/debug_variables_base.inl | 1 + .../source/os_interface/linux/CMakeLists.txt | 3 +- .../os_interface/linux/drm_memory_manager.cpp | 180 ++++++++++++++++++ .../linux/drm_memory_manager_local_memory.cpp | 37 ---- .../source/os_interface/linux/ioctl_helper.h | 3 + .../linux/ioctl_helper_prelim.cpp | 11 ++ .../linux/ioctl_helper_upstream.cpp | 4 + 8 files changed, 201 insertions(+), 39 deletions(-) delete mode 100644 shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config index b673f79712..7d0328ffd6 100644 --- a/opencl/test/unit_test/test_files/igdrcl.config +++ b/opencl/test/unit_test/test_files/igdrcl.config @@ -402,4 +402,5 @@ DisableScratchPages = 0 ForceAllResourcesUncached = 0 ForcePreParserEnabledForMiArbCheck = -1 BatchBufferStartPrepatchingWaEnabled = -1 +SetVmAdviseAtomicAttribute = -1 DirectSubmissionForceLocalMemoryStorageMode = -1 diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index b1bda921aa..df9e7ad870 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -196,6 +196,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, UseTileMemoryBankInVirtualMemoryCreation, -1, "- DECLARE_DEBUG_VARIABLE(int32_t, OverrideTimestampEvents, -1, "-1: default (based on user settings), 0: Force disable timestamp events (no timestamps will be reported), 1: Force enable timestamp events") DECLARE_DEBUG_VARIABLE(int32_t, ForcePreParserEnabledForMiArbCheck, -1, "-1: default , 0: PreParser disabled, 1: PreParser enabled") DECLARE_DEBUG_VARIABLE(int32_t, BatchBufferStartPrepatchingWaEnabled, -1, "-1: default , 0: disabled, 1: enabled. WA applies valid VA pointing to 'self' instead of 0x0. This mitigates incorrect VA preparsing.") +DECLARE_DEBUG_VARIABLE(int32_t, SetVmAdviseAtomicAttribute, -1, "-1: default - atomic system, 0: atomic none, 1: atomic device, 2: atomic system)") DECLARE_DEBUG_VARIABLE(bool, DisableScratchPages, false, "Disable scratch pages during VM creations") /*LOGGING FLAGS*/ DECLARE_DEBUG_VARIABLE(int32_t, PrintDriverDiagnostics, -1, "prints driver diagnostics messages to standard output, value corresponds to hint level") diff --git a/shared/source/os_interface/linux/CMakeLists.txt b/shared/source/os_interface/linux/CMakeLists.txt index 523ace60bf..ec1c7bae30 100644 --- a/shared/source/os_interface/linux/CMakeLists.txt +++ b/shared/source/os_interface/linux/CMakeLists.txt @@ -38,11 +38,10 @@ set(NEO_CORE_OS_INTERFACE_LINUX ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler.h ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.h + ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.h ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_manager_create_multi_host_allocation.cpp - ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_local_memory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_query.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config_drm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 5b0eed7454..a1eb609888 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -21,6 +21,8 @@ #include "shared/source/helpers/string.h" #include "shared/source/helpers/surface_format_info.h" #include "shared/source/memory_manager/host_ptr_manager.h" +#include "shared/source/memory_manager/memory_banks.h" +#include "shared/source/memory_manager/memory_pool.h" #include "shared/source/memory_manager/residency.h" #include "shared/source/os_interface/linux/allocator_helper.h" #include "shared/source/os_interface/linux/drm_memory_operations_handler.h" @@ -510,6 +512,7 @@ GraphicsAllocation *DrmMemoryManager::allocateMemoryByKMD(const AllocationData & allocation->setDefaultGmm(gmm.release()); allocation->setReservedAddressRange(reinterpret_cast(gpuRange), bufferSize); + bo.release(); return allocation; } @@ -1628,4 +1631,181 @@ void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) { return bo->peekLockedAddress(); } +void createMemoryRegionsForSharedAllocation(const HardwareInfo &hwInfo, MemoryInfo &memoryInfo, const AllocationData &allocationData, MemRegionsVec &memRegions) { + auto memoryBanks = allocationData.storageInfo.memoryBanks; + + if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) { + //System memory region + auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo); + memRegions.push_back(regionClassAndInstance); + } + + //All local memory regions + size_t currentBank = 0; + size_t i = 0; + + while (i < memoryBanks.count()) { + if (memoryBanks.test(currentBank)) { + auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(1u << currentBank, hwInfo); + memRegions.push_back(regionClassAndInstance); + i++; + } + currentBank++; + } + + if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::GPU) { + //System memory region + auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo); + memRegions.push_back(regionClassAndInstance); + } +} + +GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) { + auto &drm = this->getDrm(allocationData.rootDeviceIndex); + + const auto vmAdviseAttribute = drm.getIoctlHelper()->getVmAdviseAtomicAttribute(); + if (vmAdviseAttribute == 0) { + return nullptr; + } + + auto memoryInfo = drm.getMemoryInfo(); + const bool useBooMmap = memoryInfo && allocationData.useMmapObject; + + if (not useBooMmap) { + return nullptr; + } + + auto size = allocationData.size; + auto alignment = allocationData.alignment; + + auto pHwInfo = drm.getRootDeviceEnvironment().getHardwareInfo(); + + MemRegionsVec memRegions; + createMemoryRegionsForSharedAllocation(*pHwInfo, *memoryInfo, allocationData, memRegions); + + uint32_t handle = 0; + auto ret = memoryInfo->createGemExt(&drm, memRegions, size, handle); + + if (ret) { + return nullptr; + } + + std::unique_ptr bo(new BufferObject(&drm, handle, size, maxOsContextCount)); + + if (!drm.getIoctlHelper()->setVmBoAdvise(&drm, bo->peekHandle(), vmAdviseAttribute, nullptr)) { + return nullptr; + } + + uint64_t offset = 0; + if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) { + return nullptr; + } + + auto totalSizeToAlloc = size + alignment; + auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + if (cpuPointer == MAP_FAILED) { + return nullptr; + } + + auto cpuBasePointer = cpuPointer; + cpuPointer = alignUp(cpuPointer, alignment); + + this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, drm.getFileDescriptor(), static_cast(offset)); + + bo->setAddress(reinterpret_cast(cpuPointer)); + + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), size, MemoryPool::System4KBPages); + allocation->setMmapPtr(cpuBasePointer); + allocation->setMmapSize(totalSizeToAlloc); + if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { + this->munmapFunction(cpuPointer, totalSizeToAlloc); + return nullptr; + } + + bo.release(); + + return allocation.release(); +} + +DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) { + drm_prime_handle openFd = {0, 0, 0}; + openFd.fd = handle; + + auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd); + if (ret != 0) { + int err = this->getDrm(properties.rootDeviceIndex).getErrno(); + PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err)); + DEBUG_BREAK_IF(ret != 0); + return nullptr; + } + + if (hasMappedPtr) { + auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount); + bo->setAddress(properties.gpuAddress); + + return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), + handle, MemoryPool::SystemCpuInaccessible); + } + + const bool useBooMmap = this->getDrm(properties.rootDeviceIndex).getMemoryInfo() && properties.useMmapObject; + if (not useBooMmap) { + auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount); + bo->setAddress(properties.gpuAddress); + + return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), + handle, MemoryPool::SystemCpuInaccessible); + } + + auto boHandle = openFd.handle; + auto bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex); + + if (bo == nullptr) { + void *cpuPointer = nullptr; + size_t size = lseekFunction(handle, 0, SEEK_END); + + bo = new BufferObject(&getDrm(properties.rootDeviceIndex), boHandle, size, maxOsContextCount); + cpuPointer = this->mmapFunction(0, size, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + if (cpuPointer == MAP_FAILED) { + delete bo; + return nullptr; + } + + bo->setAddress(reinterpret_cast(cpuPointer)); + + uint64_t offset = 0; + if (!retrieveMmapOffsetForBufferObject(properties.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) { + this->munmapFunction(cpuPointer, size); + delete bo; + return nullptr; + } + + [[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(properties.rootDeviceIndex).getFileDescriptor(), static_cast(offset)); + DEBUG_BREAK_IF(retPtr != cpuPointer); + + AllocationData allocationData = {}; + allocationData.rootDeviceIndex = properties.rootDeviceIndex; + allocationData.size = size; + emitPinningRequest(bo, allocationData); + + bo->setUnmapSize(size); + bo->setRootDeviceIndex(properties.rootDeviceIndex); + + pushSharedBufferObject(bo); + + DrmAllocation *drmAllocation = nullptr; + drmAllocation = new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), MemoryPool::System4KBPages); + drmAllocation->setMmapPtr(cpuPointer); + drmAllocation->setMmapSize(size); + drmAllocation->setReservedAddressRange(reinterpret_cast(cpuPointer), size); + drmAllocation->setCacheRegion(&this->getDrm(properties.rootDeviceIndex), static_cast(properties.cacheRegion)); + + return drmAllocation; + } + + return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), + handle, MemoryPool::SystemCpuInaccessible); +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp b/shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp deleted file mode 100644 index 12e48fd14b..0000000000 --- a/shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2019-2022 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/helpers/basic_math.h" -#include "shared/source/os_interface/linux/drm_memory_manager.h" - -namespace NEO { - -DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) { - drm_prime_handle openFd = {0, 0, 0}; - openFd.fd = handle; - - auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd); - - if (ret != 0) { - int err = this->getDrm(properties.rootDeviceIndex).getErrno(); - PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err)); - DEBUG_BREAK_IF(ret != 0); - return nullptr; - } - - auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount); - bo->setAddress(properties.gpuAddress); - - return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast(bo->peekAddress()), bo->peekSize(), - handle, MemoryPool::SystemCpuInaccessible); -} - -GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) { - return nullptr; -} - -} // namespace NEO diff --git a/shared/source/os_interface/linux/ioctl_helper.h b/shared/source/os_interface/linux/ioctl_helper.h index e4c3d2dd88..1a54844fb7 100644 --- a/shared/source/os_interface/linux/ioctl_helper.h +++ b/shared/source/os_interface/linux/ioctl_helper.h @@ -116,6 +116,7 @@ class IoctlHelper { virtual void fillVmBindExtSyncFence(const std::unique_ptr &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) = 0; virtual std::optional getCopyClassSaturatePCIECapability() = 0; virtual std::optional getCopyClassSaturateLinkCapability() = 0; + virtual uint32_t getVmAdviseAtomicAttribute() = 0; virtual int vmBind(Drm *drm, const VmBindParams &vmBindParams) = 0; virtual int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) = 0; virtual bool getEuStallProperties(std::array &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) = 0; @@ -167,6 +168,7 @@ class IoctlHelperUpstream : public IoctlHelper { void fillVmBindExtSyncFence(const std::unique_ptr &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) override; std::optional getCopyClassSaturatePCIECapability() override; std::optional getCopyClassSaturateLinkCapability() override; + uint32_t getVmAdviseAtomicAttribute() override; int vmBind(Drm *drm, const VmBindParams &vmBindParams) override; int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) override; bool getEuStallProperties(std::array &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) override; @@ -231,6 +233,7 @@ class IoctlHelperPrelim20 : public IoctlHelper { void fillVmBindExtSyncFence(const std::unique_ptr &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) override; std::optional getCopyClassSaturatePCIECapability() override; std::optional getCopyClassSaturateLinkCapability() override; + uint32_t getVmAdviseAtomicAttribute() override; int vmBind(Drm *drm, const VmBindParams &vmBindParams) override; int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) override; bool getEuStallProperties(std::array &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) override; diff --git a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp index ab15007615..678a63ba66 100644 --- a/shared/source/os_interface/linux/ioctl_helper_prelim.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_prelim.cpp @@ -470,6 +470,17 @@ std::optional IoctlHelperPrelim20::getCopyClassSaturateLinkCapability( return PRELIM_I915_COPY_CLASS_CAP_SATURATE_LINK; } +uint32_t IoctlHelperPrelim20::getVmAdviseAtomicAttribute() { + switch (NEO::DebugManager.flags.SetVmAdviseAtomicAttribute.get()) { + case 0: + return PRELIM_I915_VM_ADVISE_ATOMIC_NONE; + case 1: + return PRELIM_I915_VM_ADVISE_ATOMIC_DEVICE; + default: + return PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM; + } +} + prelim_drm_i915_gem_vm_bind translateVmBindParamsToPrelimStruct(const VmBindParams &vmBindParams) { prelim_drm_i915_gem_vm_bind vmBind{}; vmBind.vm_id = vmBindParams.vmId; diff --git a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp index 2a09db8b77..e52804e8ba 100644 --- a/shared/source/os_interface/linux/ioctl_helper_upstream.cpp +++ b/shared/source/os_interface/linux/ioctl_helper_upstream.cpp @@ -214,6 +214,10 @@ std::optional IoctlHelperUpstream::getCopyClassSaturateLinkCapability( return std::nullopt; } +uint32_t IoctlHelperUpstream::getVmAdviseAtomicAttribute() { + return 0; +} + int IoctlHelperUpstream::vmBind(Drm *drm, const VmBindParams &vmBindParams) { return 0; }