Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move and extend local memory DRM allocations 

Signed-off-by: Daniel Chabrowski daniel.chabrowski@intel.com
Related-To: NEO-6591
This commit is contained in:
Daniel Chabrowski 2022-02-17 10:58:04 +00:00 committed by Compute-Runtime-Automation
parent ad6fc15bc0
commit b84b0015d5
12 changed files with 221 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
level_zero/core/source/event/event.cpp
View File

@ -104,8 +104,14 @@ ze_result_t EventPoolImp::initialize(DriverHandle *driver, Context *context, uin
}
} else {
const bool isShareable = (eventPoolFlags & ZE_EVENT_POOL_FLAG_IPC);
if (isShareable) {
allocationType = NEO::AllocationType::TIMESTAMP_PACKET_TAG_BUFFER;
}
NEO::AllocationProperties allocationProperties{*rootDeviceIndices.begin(), alignedSize, allocationType, systemMemoryBitfield};
allocationProperties.alignment = eventAlignment;
allocationProperties.flags.shareable = isShareable;
std::vector<uint32_t> rootDeviceIndicesVector = {rootDeviceIndices.begin(), rootDeviceIndices.end()};
eventPoolPtr = driver->getMemoryManager()->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector,

2
level_zero/core/test/unit_tests/sources/event/test_event.cpp
View File

@ -258,7 +258,7 @@ TEST_F(EventPoolIPCHandleTests, whenGettingIpcHandleForEventPoolThenHandleAndNum
ze_event_pool_desc_t eventPoolDesc = {
ZE_STRUCTURE_TYPE_EVENT_POOL_DESC,
nullptr,
ZE_EVENT_POOL_FLAG_HOST_VISIBLE,
ZE_EVENT_POOL_FLAG_HOST_VISIBLE | ZE_EVENT_POOL_FLAG_IPC,
numEvents};
auto deviceHandle = device->toHandle();

28
opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp
View File

@ -1240,6 +1240,34 @@ TEST_F(DrmMemoryManagerTest, GivenShareableEnabledWhenAskedToCreateGraphicsAlloc
memoryManager->freeGraphicsMemory(allocation);
}
TEST_F(DrmMemoryManagerTest, GivenAllocationTypeThatRequiresCpuAccessForKmdAllocationThenLockTheResourceIsCalled) {
struct DrmMemoryManagerToTestLockResource : public DrmMemoryManager {
using DrmMemoryManager::allocateMemoryByKMD;
DrmMemoryManagerToTestLockResource(ExecutionEnvironment &executionEnvironment)
: DrmMemoryManager(gemCloseWorkerMode::gemCloseWorkerInactive, false, false, executionEnvironment) {
}
void *lockResourceImpl(GraphicsAllocation &allocation) override {
return reinterpret_cast<void *>(0xDEADBEEF);
}
};
DrmMemoryManagerToTestLockResource drmMemoryManager(*executionEnvironment);
mock->ioctl_expected.gemWait = 1;
mock->ioctl_expected.gemCreate = 1;
mock->ioctl_expected.gemClose = 1;
allocationData.type = AllocationType::TIMESTAMP_PACKET_TAG_BUFFER;
auto allocation = drmMemoryManager.allocateMemoryByKMD(allocationData);
EXPECT_NE(nullptr, allocation);
EXPECT_NE(0u, allocation->getGpuAddress());
EXPECT_EQ(0xDEADBEEF, reinterpret_cast<uintptr_t>(allocation->getUnderlyingBuffer()));
memoryManager->freeGraphicsMemory(allocation);
}
TEST_F(DrmMemoryManagerTest, GivenMisalignedHostPtrAndMultiplePagesSizeWhenAskedForGraphicsAllocationThenItContainsAllFragmentsWithProperGpuAdrresses) {
mock->ioctl_expected.gemUserptr = 3;
mock->ioctl_expected.gemWait = 3;

1
opencl/test/unit_test/test_files/igdrcl.config
View File

@ -389,3 +389,4 @@ AccessCountersGranularity = -1
OverridePatIndex = -1
UseTileMemoryBankInVirtualMemoryCreation = -1
DisableScratchPages = 0
SetVmAdviseAtomicAttribute = -1

1
shared/source/debug_settings/debug_variables_base.inl
View File

@ -192,6 +192,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, AccessCountersTrigger, -1, "-1: default - disabl
DECLARE_DEBUG_VARIABLE(int32_t, AccessCountersGranularity, -1, "-1: default - ACG_2MB, >= 0: granularites - 0: ACG_128K, 1: ACG_2M, 2: ACG_16M, 3: ACG_16M")
DECLARE_DEBUG_VARIABLE(int32_t, OverridePatIndex, -1, "-1: default, >=0: PatIndex to override")
DECLARE_DEBUG_VARIABLE(int32_t, UseTileMemoryBankInVirtualMemoryCreation, -1, "-1: default - on, 0: do not assign tile memory bank to virtual memory space, 1: assign tile memory bank to virtual memory space")
DECLARE_DEBUG_VARIABLE(int32_t, SetVmAdviseAtomicAttribute, -1, "-1: default - atomic system, 0: atomic none, 1: atomic device, 2: atomic system)")
DECLARE_DEBUG_VARIABLE(bool, DisableScratchPages, false, "Disable scratch pages during VM creations")
/*LOGGING FLAGS*/
DECLARE_DEBUG_VARIABLE(int32_t, PrintDriverDiagnostics, -1, "prints driver diagnostics messages to standard output, value corresponds to hint level")

2
shared/source/generated/gen12lp/hw_cmds_generated_gen12lp.inl
View File

@ -5062,7 +5062,7 @@ typedef struct tagMI_STORE_DATA_IMM {
ADDRESS_ALIGN_SIZE = 0x4,
} ADDRESS;
inline void setAddress(const uint64_t value) {
UNRECOVERABLE_IF(value > 0x3fffffffffffffffL);
UNRECOVERABLE_IF((value >> ADDRESS_BIT_SHIFT) > 0x3fffffffffffffffL);
TheStructure.Common.Address = value >> ADDRESS_BIT_SHIFT;
}
inline uint64_t getAddress() const {

3
shared/source/os_interface/linux/CMakeLists.txt
View File

@ -37,11 +37,10 @@ set(NEO_CORE_OS_INTERFACE_LINUX
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler.h
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.cpp
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.h
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.cpp
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.h
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_manager_create_multi_host_allocation.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_memory_manager_local_memory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_create.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_query.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config_drm.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h

164
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@ -21,6 +21,8 @@
#include "shared/source/helpers/string.h"
#include "shared/source/helpers/surface_format_info.h"
#include "shared/source/memory_manager/host_ptr_manager.h"
#include "shared/source/memory_manager/memory_banks.h"
#include "shared/source/memory_manager/memory_pool.h"
#include "shared/source/memory_manager/residency.h"
#include "shared/source/os_interface/linux/allocator_helper.h"
#include "shared/source/os_interface/linux/drm_memory_operations_handler.h"
@ -504,6 +506,12 @@ GraphicsAllocation *DrmMemoryManager::allocateMemoryByKMD(const AllocationData &
allocation->setDefaultGmm(gmm.release());
allocation->setReservedAddressRange(reinterpret_cast<void *>(gpuRange), bufferSize);
if (GraphicsAllocation::isCpuAccessRequired(allocationData.type)) {
auto cpuAddress = lockResource(allocation);
allocation->setCpuPtrAndGpuAddress(cpuAddress, gpuRange);
}
bo.release();
return allocation;
}
@ -1616,4 +1624,160 @@ void *DrmMemoryManager::lockResourceInLocalMemoryImpl(BufferObject *bo) {
return bo->peekLockedAddress();
}
void createMemoryRegionsForSharedAllocation(const HardwareInfo &hwInfo, MemoryInfo &memoryInfo, const AllocationData &allocationData, MemRegionsVec &memRegions) {
auto memoryBanks = allocationData.storageInfo.memoryBanks;
if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) {
//System memory region
auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo);
memRegions.push_back(regionClassAndInstance);
}
//All local memory regions
size_t currentBank = 0;
size_t i = 0;
while (i < memoryBanks.count()) {
if (memoryBanks.test(currentBank)) {
auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(1u << currentBank, hwInfo);
memRegions.push_back(regionClassAndInstance);
i++;
}
currentBank++;
}
if (allocationData.usmInitialPlacement == GraphicsAllocation::UsmInitialPlacement::GPU) {
//System memory region
auto regionClassAndInstance = memoryInfo.getMemoryRegionClassAndInstance(0u, hwInfo);
memRegions.push_back(regionClassAndInstance);
}
}
GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) {
auto &drm = this->getDrm(allocationData.rootDeviceIndex);
const auto vmAdviseAttribute = drm.getIoctlHelper()->getVmAdviseAtomicAttribute();
if (vmAdviseAttribute == 0) {
return nullptr;
}
auto memoryInfo = drm.getMemoryInfo();
if (!memoryInfo) {
return nullptr;
}
auto size = allocationData.size;
auto alignment = allocationData.alignment;
auto pHwInfo = drm.getRootDeviceEnvironment().getHardwareInfo();
MemRegionsVec memRegions;
createMemoryRegionsForSharedAllocation(*pHwInfo, *memoryInfo, allocationData, memRegions);
uint32_t handle = 0;
auto ret = memoryInfo->createGemExt(&drm, memRegions, size, handle);
if (ret) {
return nullptr;
}
std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new BufferObject(&drm, handle, size, maxOsContextCount));
if (!drm.getIoctlHelper()->setVmBoAdvise(&drm, bo->peekHandle(), vmAdviseAttribute, nullptr)) {
return nullptr;
}
uint64_t offset = 0;
if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
return nullptr;
}
auto totalSizeToAlloc = size + alignment;
auto cpuPointer = this->mmapFunction(0, totalSizeToAlloc, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
auto cpuBasePointer = cpuPointer;
cpuPointer = alignUp(cpuPointer, alignment);
this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, drm.getFileDescriptor(), static_cast<off_t>(offset));
bo->setAddress(reinterpret_cast<uintptr_t>(cpuPointer));
auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), size, MemoryPool::System4KBPages);
allocation->setMmapPtr(cpuBasePointer);
allocation->setMmapSize(totalSizeToAlloc);
if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast<CacheRegion>(allocationData.cacheRegion))) {
this->munmapFunction(cpuPointer, totalSizeToAlloc);
return nullptr;
}
bo.release();
return allocation.release();
}
DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) {
drm_prime_handle openFd = {0, 0, 0};
openFd.fd = handle;
auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
if (ret != 0) {
int err = this->getDrm(properties.rootDeviceIndex).getErrno();
PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
DEBUG_BREAK_IF(ret != 0);
return nullptr;
}
if (hasMappedPtr) {
auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount);
bo->setAddress(properties.gpuAddress);
return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible);
}
auto boHandle = openFd.handle;
auto bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex);
void *cpuPointer = nullptr;
size_t size = lseekFunction(handle, 0, SEEK_END);
if (bo == nullptr) {
bo = new BufferObject(&getDrm(properties.rootDeviceIndex), boHandle, size, maxOsContextCount);
cpuPointer = this->mmapFunction(0, size, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
bo->setAddress(reinterpret_cast<uintptr_t>(cpuPointer));
uint64_t offset = 0;
if (!retrieveMmapOffsetForBufferObject(properties.rootDeviceIndex, *bo, I915_MMAP_OFFSET_WB, offset)) {
this->munmapFunction(cpuPointer, size);
delete bo;
return nullptr;
}
[[maybe_unused]] auto retPtr = this->mmapFunction(cpuPointer, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, getDrm(properties.rootDeviceIndex).getFileDescriptor(), static_cast<off_t>(offset));
DEBUG_BREAK_IF(retPtr != cpuPointer);
AllocationData allocationData = {};
allocationData.rootDeviceIndex = properties.rootDeviceIndex;
allocationData.size = size;
emitPinningRequest(bo, allocationData);
bo->setUnmapSize(size);
bo->setRootDeviceIndex(properties.rootDeviceIndex);
pushSharedBufferObject(bo);
DrmAllocation *drmAllocation = nullptr;
drmAllocation = new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), MemoryPool::System4KBPages);
drmAllocation->setMmapPtr(cpuPointer);
drmAllocation->setMmapSize(size);
drmAllocation->setReservedAddressRange(reinterpret_cast<void *>(cpuPointer), size);
drmAllocation->setCacheRegion(&this->getDrm(properties.rootDeviceIndex), static_cast<CacheRegion>(properties.cacheRegion));
return drmAllocation;
}
return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible);
}
} // namespace NEO

37
shared/source/os_interface/linux/drm_memory_manager_local_memory.cpp
View File

@ -1,37 +0,0 @@
/*
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/basic_math.h"
#include "shared/source/os_interface/linux/drm_memory_manager.h"
namespace NEO {
DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr) {
drm_prime_handle openFd = {0, 0, 0};
openFd.fd = handle;
auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
if (ret != 0) {
int err = this->getDrm(properties.rootDeviceIndex).getErrno();
PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
DEBUG_BREAK_IF(ret != 0);
return nullptr;
}
auto bo = new BufferObject(&getDrm(properties.rootDeviceIndex), openFd.handle, properties.size, maxOsContextCount);
bo->setAddress(properties.gpuAddress);
return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible);
}
GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) {
return nullptr;
}
} // namespace NEO

3
shared/source/os_interface/linux/ioctl_helper.h
View File

@ -110,6 +110,7 @@ class IoctlHelper {
virtual void fillVmBindExtSyncFence(const std::unique_ptr<uint8_t[]> &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) = 0;
virtual std::optional<uint64_t> getCopyClassSaturatePCIECapability() = 0;
virtual std::optional<uint64_t> getCopyClassSaturateLinkCapability() = 0;
virtual uint32_t getVmAdviseAtomicAttribute() = 0;
virtual int vmBind(Drm *drm, const VmBindParams &vmBindParams) = 0;
virtual int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) = 0;
virtual bool getEuStallProperties(std::array<uint64_t, 10u> &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) = 0;
@ -155,6 +156,7 @@ class IoctlHelperUpstream : public IoctlHelper {
void fillVmBindExtSyncFence(const std::unique_ptr<uint8_t[]> &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) override;
std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
uint32_t getVmAdviseAtomicAttribute() override;
int vmBind(Drm *drm, const VmBindParams &vmBindParams) override;
int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) override;
bool getEuStallProperties(std::array<uint64_t, 10u> &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) override;
@ -213,6 +215,7 @@ class IoctlHelperPrelim20 : public IoctlHelper {
void fillVmBindExtSyncFence(const std::unique_ptr<uint8_t[]> &vmBindExtSyncFence, uint64_t fenceAddress, uint64_t fenceValue, uint64_t nextExtension) override;
std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
uint32_t getVmAdviseAtomicAttribute() override;
int vmBind(Drm *drm, const VmBindParams &vmBindParams) override;
int vmUnbind(Drm *drm, const VmBindParams &vmBindParams) override;
bool getEuStallProperties(std::array<uint64_t, 10u> &properties, uint64_t dssBufferSize, uint64_t samplingRate, uint64_t pollPeriod, uint64_t engineInstance) override;

11
shared/source/os_interface/linux/ioctl_helper_prelim.cpp
View File

@ -451,6 +451,17 @@ std::optional<uint64_t> IoctlHelperPrelim20::getCopyClassSaturateLinkCapability(
return PRELIM_I915_COPY_CLASS_CAP_SATURATE_LINK;
}
uint32_t IoctlHelperPrelim20::getVmAdviseAtomicAttribute() {
switch (NEO::DebugManager.flags.SetVmAdviseAtomicAttribute.get()) {
case 0:
return PRELIM_I915_VM_ADVISE_ATOMIC_NONE;
case 1:
return PRELIM_I915_VM_ADVISE_ATOMIC_DEVICE;
default:
return PRELIM_I915_VM_ADVISE_ATOMIC_SYSTEM;
}
}
prelim_drm_i915_gem_vm_bind translateVmBindParamsToPrelimStruct(const VmBindParams &vmBindParams) {
prelim_drm_i915_gem_vm_bind vmBind{};
vmBind.vm_id = vmBindParams.vmId;

4
shared/source/os_interface/linux/ioctl_helper_upstream.cpp
View File

@ -204,6 +204,10 @@ std::optional<uint64_t> IoctlHelperUpstream::getCopyClassSaturateLinkCapability(
return std::nullopt;
}
uint32_t IoctlHelperUpstream::getVmAdviseAtomicAttribute() {
return 0;
}
int IoctlHelperUpstream::vmBind(Drm *drm, const VmBindParams &vmBindParams) {
return 0;
}