mirror of
https://github.com/intel/compute-runtime.git
synced 2025-12-25 13:33:02 +08:00
Move OsAgnosticMemoryManager to shared
Resolves: NEO-5188 -move MockGraphicsAllocation to shared Change-Id: I17e90af48183f003151fab075a46e40d7df3f160 Signed-off-by: Mateusz Hoppe <mateusz.hoppe@intel.com>
This commit is contained in:
Mateusz Hoppe
committed by
sys_ocldev
sys_ocldev
parent
78dc305291
commit
4d2750161d
@@ -41,6 +41,9 @@ set(NEO_CORE_MEMORY_MANAGER
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/memory_pool.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/multi_graphics_allocation.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/multi_graphics_allocation.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_agnostic_memory_manager.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_agnostic_memory_manager.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/os_agnostic_memory_manager_allocate_in_device_pool.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/physical_address_allocator.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/residency.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/residency.h
|
||||
|
||||
392
shared/source/memory_manager/os_agnostic_memory_manager.cpp
Normal file
392
shared/source/memory_manager/os_agnostic_memory_manager.cpp
Normal file
@@ -0,0 +1,392 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
|
||||
|
||||
#include "shared/source/aub/aub_center.h"
|
||||
#include "shared/source/execution_environment/execution_environment.h"
|
||||
#include "shared/source/execution_environment/root_device_environment.h"
|
||||
#include "shared/source/gmm_helper/gmm.h"
|
||||
#include "shared/source/gmm_helper/gmm_helper.h"
|
||||
#include "shared/source/gmm_helper/resource_info.h"
|
||||
#include "shared/source/helpers/aligned_memory.h"
|
||||
#include "shared/source/helpers/basic_math.h"
|
||||
#include "shared/source/helpers/heap_assigner.h"
|
||||
#include "shared/source/helpers/hw_helper.h"
|
||||
#include "shared/source/helpers/hw_info.h"
|
||||
#include "shared/source/helpers/ptr_math.h"
|
||||
#include "shared/source/helpers/surface_format_info.h"
|
||||
#include "shared/source/memory_manager/host_ptr_manager.h"
|
||||
#include "shared/source/memory_manager/residency.h"
|
||||
#include "shared/source/os_interface/os_memory.h"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
OsAgnosticMemoryManager::OsAgnosticMemoryManager(bool aubUsage, ExecutionEnvironment &executionEnvironment) : MemoryManager(executionEnvironment) {
|
||||
initialize(aubUsage);
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::initialize(bool aubUsage) {
|
||||
// 4 x sizeof(Heap32) + 2 x sizeof(Standard/Standard64k)
|
||||
size_t reservedCpuAddressRangeSize = (4 * 4 + 2 * (aubUsage ? 32 : 4)) * GB;
|
||||
|
||||
for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < gfxPartitions.size(); ++rootDeviceIndex) {
|
||||
auto gpuAddressSpace = executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->getHardwareInfo()->capabilityTable.gpuAddressSpace;
|
||||
if (!getGfxPartition(rootDeviceIndex)->init(gpuAddressSpace, reservedCpuAddressRangeSize, rootDeviceIndex, gfxPartitions.size(), heapAssigner.apiAllowExternalHeapForSshAndDsh)) {
|
||||
initialized = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
OsAgnosticMemoryManager::~OsAgnosticMemoryManager() = default;
|
||||
|
||||
struct OsHandle {
|
||||
};
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) {
|
||||
auto sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize);
|
||||
MemoryAllocation *memoryAllocation = nullptr;
|
||||
|
||||
if (fakeBigAllocations && allocationData.size > bigAllocation) {
|
||||
memoryAllocation = createMemoryAllocation(
|
||||
allocationData.type, nullptr, (void *)dummyAddress, static_cast<uint64_t>(dummyAddress), allocationData.size, counter,
|
||||
MemoryPool::System4KBPages, allocationData.rootDeviceIndex, allocationData.flags.uncacheable, allocationData.flags.flushL3, false);
|
||||
counter++;
|
||||
return memoryAllocation;
|
||||
}
|
||||
auto ptr = allocateSystemMemory(sizeAligned, allocationData.alignment ? alignUp(allocationData.alignment, MemoryConstants::pageSize) : MemoryConstants::pageSize);
|
||||
if (ptr != nullptr) {
|
||||
memoryAllocation = createMemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast<uint64_t>(ptr), allocationData.size,
|
||||
counter, MemoryPool::System4KBPages, allocationData.rootDeviceIndex, allocationData.flags.uncacheable, allocationData.flags.flushL3, false);
|
||||
|
||||
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_CPU) {
|
||||
//add 2MB padding in case mapPtr is not 2MB aligned
|
||||
size_t reserveSize = sizeAligned + allocationData.alignment;
|
||||
void *gpuPtr = reserveCpuAddressRange(reserveSize, allocationData.rootDeviceIndex);
|
||||
if (!gpuPtr) {
|
||||
delete memoryAllocation;
|
||||
alignedFreeWrapper(ptr);
|
||||
return nullptr;
|
||||
}
|
||||
memoryAllocation->setReservedAddressRange(gpuPtr, reserveSize);
|
||||
gpuPtr = alignUp(gpuPtr, allocationData.alignment);
|
||||
memoryAllocation->setCpuPtrAndGpuAddress(ptr, reinterpret_cast<uint64_t>(gpuPtr));
|
||||
}
|
||||
}
|
||||
counter++;
|
||||
return memoryAllocation;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateUSMHostGraphicsMemory(const AllocationData &allocationData) {
|
||||
return allocateGraphicsMemoryWithHostPtr(allocationData);
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) {
|
||||
auto alignedPtr = alignDown(allocationData.hostPtr, MemoryConstants::pageSize);
|
||||
auto offsetInPage = ptrDiff(allocationData.hostPtr, alignedPtr);
|
||||
|
||||
auto memoryAllocation = createMemoryAllocation(allocationData.type, nullptr, const_cast<void *>(allocationData.hostPtr),
|
||||
reinterpret_cast<uint64_t>(alignedPtr), allocationData.size, counter,
|
||||
MemoryPool::System4KBPages, allocationData.rootDeviceIndex, false, allocationData.flags.flushL3, false);
|
||||
|
||||
memoryAllocation->setAllocationOffset(offsetInPage);
|
||||
|
||||
counter++;
|
||||
return memoryAllocation;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) {
|
||||
auto memoryAllocation = static_cast<MemoryAllocation *>(allocateGraphicsMemoryWithAlignment(allocationData));
|
||||
memoryAllocation->setCpuPtrAndGpuAddress(memoryAllocation->getUnderlyingBuffer(), allocationData.gpuAddress);
|
||||
return memoryAllocation;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemory64kb(const AllocationData &allocationData) {
|
||||
AllocationData allocationData64kb = allocationData;
|
||||
allocationData64kb.size = alignUp(allocationData.size, MemoryConstants::pageSize64k);
|
||||
allocationData64kb.alignment = MemoryConstants::pageSize64k;
|
||||
auto memoryAllocation = allocateGraphicsMemoryWithAlignment(allocationData64kb);
|
||||
if (memoryAllocation) {
|
||||
static_cast<MemoryAllocation *>(memoryAllocation)->overrideMemoryPool(MemoryPool::System64KBPages);
|
||||
}
|
||||
return memoryAllocation;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData, bool useLocalMemory) {
|
||||
auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo();
|
||||
auto heap = heapAssigner.get32BitHeapIndex(allocationData.type, useLocalMemory, *hwInfo, allocationData.flags.use32BitFrontWindow);
|
||||
auto gfxPartition = getGfxPartition(allocationData.rootDeviceIndex);
|
||||
if (allocationData.hostPtr) {
|
||||
auto allocationSize = alignSizeWholePage(allocationData.hostPtr, allocationData.size);
|
||||
auto gpuVirtualAddress = gfxPartition->heapAllocate(heap, allocationSize);
|
||||
if (!gpuVirtualAddress) {
|
||||
return nullptr;
|
||||
}
|
||||
uint64_t offset = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(allocationData.hostPtr) & MemoryConstants::pageMask);
|
||||
MemoryAllocation *memAlloc = new MemoryAllocation(
|
||||
allocationData.rootDeviceIndex, allocationData.type, nullptr, const_cast<void *>(allocationData.hostPtr),
|
||||
GmmHelper::canonize(gpuVirtualAddress + offset), allocationData.size,
|
||||
counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false, false, maxOsContextCount);
|
||||
|
||||
memAlloc->set32BitAllocation(true);
|
||||
memAlloc->setGpuBaseAddress(GmmHelper::canonize(gfxPartition->getHeapBase(heap)));
|
||||
memAlloc->sizeToFree = allocationSize;
|
||||
|
||||
counter++;
|
||||
return memAlloc;
|
||||
}
|
||||
|
||||
auto allocationSize = alignUp(allocationData.size, MemoryConstants::pageSize);
|
||||
void *ptrAlloc = nullptr;
|
||||
auto gpuAddress = gfxPartition->heapAllocate(heap, allocationSize);
|
||||
|
||||
if (allocationData.size < 0xfffff000) {
|
||||
if (fakeBigAllocations) {
|
||||
ptrAlloc = reinterpret_cast<void *>(dummyAddress);
|
||||
} else {
|
||||
ptrAlloc = alignedMallocWrapper(allocationSize, MemoryConstants::allocationAlignment);
|
||||
}
|
||||
}
|
||||
|
||||
MemoryAllocation *memoryAllocation = nullptr;
|
||||
if (ptrAlloc != nullptr) {
|
||||
memoryAllocation = new MemoryAllocation(allocationData.rootDeviceIndex, allocationData.type, ptrAlloc, ptrAlloc, GmmHelper::canonize(gpuAddress),
|
||||
allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing,
|
||||
false, allocationData.flags.flushL3, maxOsContextCount);
|
||||
|
||||
memoryAllocation->set32BitAllocation(true);
|
||||
memoryAllocation->setGpuBaseAddress(GmmHelper::canonize(gfxPartition->getHeapBase(heap)));
|
||||
memoryAllocation->sizeToFree = allocationSize;
|
||||
}
|
||||
counter++;
|
||||
return memoryAllocation;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) {
|
||||
auto graphicsAllocation = createMemoryAllocation(properties.allocationType, nullptr, reinterpret_cast<void *>(1), 1,
|
||||
4096u, static_cast<uint64_t>(handle), MemoryPool::SystemCpuInaccessible, properties.rootDeviceIndex,
|
||||
false, false, requireSpecificBitness);
|
||||
graphicsAllocation->setSharedHandle(handle);
|
||||
graphicsAllocation->set32BitAllocation(requireSpecificBitness);
|
||||
|
||||
if (properties.imgInfo) {
|
||||
Gmm *gmm = new Gmm(executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->getGmmClientContext(), *properties.imgInfo, createStorageInfoFromProperties(properties));
|
||||
graphicsAllocation->setDefaultGmm(gmm);
|
||||
}
|
||||
|
||||
return graphicsAllocation;
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) {
|
||||
FragmentStorage fragment = {};
|
||||
fragment.driverAllocation = true;
|
||||
fragment.fragmentCpuPointer = gfxAllocation->getUnderlyingBuffer();
|
||||
fragment.fragmentSize = alignUp(gfxAllocation->getUnderlyingBufferSize(), MemoryConstants::pageSize);
|
||||
fragment.osInternalStorage = new OsHandle();
|
||||
fragment.residency = new ResidencyData(maxOsContextCount);
|
||||
hostPtrManager->storeFragment(gfxAllocation->getRootDeviceIndex(), fragment);
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) {
|
||||
auto buffer = gfxAllocation->getUnderlyingBuffer();
|
||||
auto rootDeviceIndex = gfxAllocation->getRootDeviceIndex();
|
||||
auto fragment = hostPtrManager->getFragment({buffer, rootDeviceIndex});
|
||||
if (fragment && fragment->driverAllocation) {
|
||||
OsHandle *osStorageToRelease = fragment->osInternalStorage;
|
||||
ResidencyData *residencyDataToRelease = fragment->residency;
|
||||
if (hostPtrManager->releaseHostPtr(rootDeviceIndex, buffer)) {
|
||||
delete osStorageToRelease;
|
||||
delete residencyDataToRelease;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) {
|
||||
for (auto handleId = 0u; handleId < gfxAllocation->getNumGmms(); handleId++) {
|
||||
delete gfxAllocation->getGmm(handleId);
|
||||
}
|
||||
|
||||
if ((uintptr_t)gfxAllocation->getUnderlyingBuffer() == dummyAddress) {
|
||||
delete gfxAllocation;
|
||||
return;
|
||||
}
|
||||
|
||||
if (gfxAllocation->fragmentsStorage.fragmentCount) {
|
||||
cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
|
||||
delete gfxAllocation;
|
||||
return;
|
||||
}
|
||||
|
||||
auto memoryAllocation = static_cast<MemoryAllocation *>(gfxAllocation);
|
||||
auto sizeToFree = memoryAllocation->sizeToFree;
|
||||
|
||||
if (sizeToFree) {
|
||||
auto gpuAddressToFree = GmmHelper::decanonize(memoryAllocation->getGpuAddress()) & ~MemoryConstants::pageMask;
|
||||
auto gfxPartition = getGfxPartition(memoryAllocation->getRootDeviceIndex());
|
||||
gfxPartition->freeGpuAddressRange(gpuAddressToFree, sizeToFree);
|
||||
}
|
||||
|
||||
alignedFreeWrapper(gfxAllocation->getDriverAllocatedCpuPtr());
|
||||
if (gfxAllocation->getReservedAddressPtr()) {
|
||||
releaseReservedCpuAddressRange(gfxAllocation->getReservedAddressPtr(), gfxAllocation->getReservedAddressSize(), gfxAllocation->getRootDeviceIndex());
|
||||
}
|
||||
|
||||
auto rootDeviceIndex = gfxAllocation->getRootDeviceIndex();
|
||||
|
||||
if (executionEnvironment.rootDeviceEnvironments.size() > rootDeviceIndex) {
|
||||
auto aubCenter = executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->aubCenter.get();
|
||||
if (aubCenter && aubCenter->getAubManager() && DebugManager.flags.EnableFreeMemory.get()) {
|
||||
aubCenter->getAubManager()->freeMemory(gfxAllocation->getGpuAddress(), gfxAllocation->getUnderlyingBufferSize());
|
||||
}
|
||||
}
|
||||
delete gfxAllocation;
|
||||
}
|
||||
|
||||
uint64_t OsAgnosticMemoryManager::getSystemSharedMemory(uint32_t rootDeviceIndex) {
|
||||
return 16 * GB;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) {
|
||||
auto allocation = createMemoryAllocation(allocationData.type, nullptr, const_cast<void *>(allocationData.hostPtr),
|
||||
reinterpret_cast<uint64_t>(allocationData.hostPtr), allocationData.size, counter++,
|
||||
MemoryPool::System4KBPages, allocationData.rootDeviceIndex, false, allocationData.flags.flushL3, false);
|
||||
|
||||
allocation->fragmentsStorage = handleStorage;
|
||||
return allocation;
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::turnOnFakingBigAllocations() {
|
||||
this->fakeBigAllocations = true;
|
||||
}
|
||||
|
||||
MemoryManager::AllocationStatus OsAgnosticMemoryManager::populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) {
|
||||
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
||||
if (!handleStorage.fragmentStorageData[i].osHandleStorage && handleStorage.fragmentStorageData[i].cpuPtr) {
|
||||
handleStorage.fragmentStorageData[i].osHandleStorage = new OsHandle();
|
||||
handleStorage.fragmentStorageData[i].residency = new ResidencyData(maxOsContextCount);
|
||||
|
||||
FragmentStorage newFragment = {};
|
||||
newFragment.fragmentCpuPointer = const_cast<void *>(handleStorage.fragmentStorageData[i].cpuPtr);
|
||||
newFragment.fragmentSize = handleStorage.fragmentStorageData[i].fragmentSize;
|
||||
newFragment.osInternalStorage = handleStorage.fragmentStorageData[i].osHandleStorage;
|
||||
newFragment.residency = handleStorage.fragmentStorageData[i].residency;
|
||||
hostPtrManager->storeFragment(rootDeviceIndex, newFragment);
|
||||
}
|
||||
}
|
||||
return AllocationStatus::Success;
|
||||
}
|
||||
void OsAgnosticMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) {
|
||||
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
||||
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
|
||||
auto aubCenter = executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->aubCenter.get();
|
||||
if (aubCenter && aubCenter->getAubManager() && DebugManager.flags.EnableFreeMemory.get()) {
|
||||
aubCenter->getAubManager()->freeMemory((uint64_t)handleStorage.fragmentStorageData[i].cpuPtr, handleStorage.fragmentStorageData[i].fragmentSize);
|
||||
}
|
||||
delete handleStorage.fragmentStorageData[i].osHandleStorage;
|
||||
delete handleStorage.fragmentStorageData[i].residency;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateShareableMemory(const AllocationData &allocationData) {
|
||||
auto gmm = std::make_unique<Gmm>(executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmClientContext(), allocationData.hostPtr, allocationData.size, false);
|
||||
GraphicsAllocation *alloc = nullptr;
|
||||
|
||||
auto ptr = allocateSystemMemory(alignUp(allocationData.size, MemoryConstants::pageSize), MemoryConstants::pageSize);
|
||||
if (ptr != nullptr) {
|
||||
alloc = createMemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast<uint64_t>(ptr), allocationData.size,
|
||||
counter, MemoryPool::SystemCpuInaccessible, allocationData.rootDeviceIndex, allocationData.flags.uncacheable, allocationData.flags.flushL3, false);
|
||||
counter++;
|
||||
}
|
||||
|
||||
if (alloc) {
|
||||
alloc->setDefaultGmm(gmm.release());
|
||||
}
|
||||
|
||||
return alloc;
|
||||
}
|
||||
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) {
|
||||
GraphicsAllocation *alloc = nullptr;
|
||||
|
||||
if (allocationData.imgInfo->linearStorage && allocationData.imgInfo->mipCount == 0) {
|
||||
alloc = allocateGraphicsMemoryWithAlignment(allocationData);
|
||||
if (alloc) {
|
||||
alloc->setDefaultGmm(gmm.release());
|
||||
}
|
||||
return alloc;
|
||||
}
|
||||
|
||||
auto ptr = allocateSystemMemory(alignUp(allocationData.imgInfo->size, MemoryConstants::pageSize), MemoryConstants::pageSize);
|
||||
if (ptr != nullptr) {
|
||||
alloc = createMemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast<uint64_t>(ptr), allocationData.imgInfo->size,
|
||||
counter, MemoryPool::SystemCpuInaccessible, allocationData.rootDeviceIndex, allocationData.flags.uncacheable, allocationData.flags.flushL3, false);
|
||||
counter++;
|
||||
}
|
||||
|
||||
if (alloc) {
|
||||
alloc->setDefaultGmm(gmm.release());
|
||||
}
|
||||
|
||||
return alloc;
|
||||
}
|
||||
|
||||
void *OsAgnosticMemoryManager::reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) {
|
||||
void *reservePtr = allocateSystemMemory(size, MemoryConstants::preferredAlignment);
|
||||
return reservePtr;
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) {
|
||||
alignedFreeWrapper(reserved);
|
||||
}
|
||||
|
||||
MemoryAllocation *OsAgnosticMemoryManager::createMemoryAllocation(GraphicsAllocation::AllocationType allocationType, void *driverAllocatedCpuPointer,
|
||||
void *pMem, uint64_t gpuAddress, size_t memSize, uint64_t count,
|
||||
MemoryPool::Type pool, uint32_t rootDeviceIndex, bool uncacheable,
|
||||
bool flushL3Required, bool requireSpecificBitness) {
|
||||
if (!isLimitedRange(rootDeviceIndex)) {
|
||||
return new MemoryAllocation(rootDeviceIndex, allocationType, driverAllocatedCpuPointer, pMem, gpuAddress, memSize,
|
||||
count, pool, uncacheable, flushL3Required, maxOsContextCount);
|
||||
}
|
||||
|
||||
size_t alignedSize = alignSizeWholePage(pMem, memSize);
|
||||
|
||||
auto heap = (force32bitAllocations || requireSpecificBitness) ? HeapIndex::HEAP_EXTERNAL : HeapIndex::HEAP_STANDARD;
|
||||
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
uint64_t limitedGpuAddress = gfxPartition->heapAllocate(heap, alignedSize);
|
||||
|
||||
auto memoryAllocation = new MemoryAllocation(rootDeviceIndex, allocationType, driverAllocatedCpuPointer, pMem, limitedGpuAddress, memSize,
|
||||
count, pool, uncacheable, flushL3Required, maxOsContextCount);
|
||||
|
||||
if (heap == HeapIndex::HEAP_EXTERNAL) {
|
||||
memoryAllocation->setGpuBaseAddress(GmmHelper::canonize(gfxPartition->getHeapBase(heap)));
|
||||
}
|
||||
memoryAllocation->sizeToFree = alignedSize;
|
||||
|
||||
return memoryAllocation;
|
||||
}
|
||||
|
||||
AddressRange OsAgnosticMemoryManager::reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) {
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
auto gpuVa = GmmHelper::canonize(gfxPartition->heapAllocate(HeapIndex::HEAP_STANDARD, size));
|
||||
return AddressRange{gpuVa, size};
|
||||
}
|
||||
|
||||
void OsAgnosticMemoryManager::freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) {
|
||||
uint64_t graphicsAddress = addressRange.address;
|
||||
graphicsAddress = GmmHelper::decanonize(graphicsAddress);
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
gfxPartition->freeGpuAddressRange(graphicsAddress, addressRange.size);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
109
shared/source/memory_manager/os_agnostic_memory_manager.h
Normal file
109
shared/source/memory_manager/os_agnostic_memory_manager.h
Normal file
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "shared/source/helpers/basic_math.h"
|
||||
#include "shared/source/memory_manager/memory_manager.h"
|
||||
|
||||
namespace NEO {
|
||||
constexpr size_t bigAllocation = 1 * MB;
|
||||
constexpr uintptr_t dummyAddress = 0xFFFFF000u;
|
||||
class MemoryAllocation : public GraphicsAllocation {
|
||||
public:
|
||||
const unsigned long long id;
|
||||
size_t sizeToFree = 0;
|
||||
const bool uncacheable;
|
||||
|
||||
void setSharedHandle(osHandle handle) { sharingInfo.sharedHandle = handle; }
|
||||
|
||||
MemoryAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, uint64_t gpuAddress, uint64_t baseAddress, size_t sizeIn,
|
||||
MemoryPool::Type pool, size_t maxOsContextCount)
|
||||
: MemoryAllocation(rootDeviceIndex, 1, allocationType, cpuPtrIn, gpuAddress, baseAddress, sizeIn, pool, maxOsContextCount) {}
|
||||
|
||||
MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t gpuAddress, uint64_t baseAddress, size_t sizeIn,
|
||||
MemoryPool::Type pool, size_t maxOsContextCount)
|
||||
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, gpuAddress, baseAddress, sizeIn, pool, maxOsContextCount),
|
||||
id(0), uncacheable(false) {}
|
||||
|
||||
MemoryAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandleIn, MemoryPool::Type pool, size_t maxOsContextCount)
|
||||
: MemoryAllocation(rootDeviceIndex, 1, allocationType, cpuPtrIn, sizeIn, sharedHandleIn, pool, maxOsContextCount) {}
|
||||
|
||||
MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandleIn, MemoryPool::Type pool, size_t maxOsContextCount)
|
||||
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, sizeIn, sharedHandleIn, pool, maxOsContextCount),
|
||||
id(0), uncacheable(false) {}
|
||||
|
||||
MemoryAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize,
|
||||
uint64_t count, MemoryPool::Type pool, bool uncacheable, bool flushL3Required, size_t maxOsContextCount)
|
||||
: MemoryAllocation(rootDeviceIndex, 1, allocationType, driverAllocatedCpuPointer, pMem, gpuAddress, memSize,
|
||||
count, pool, uncacheable, flushL3Required, maxOsContextCount) {}
|
||||
|
||||
MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize,
|
||||
uint64_t count, MemoryPool::Type pool, bool uncacheable, bool flushL3Required, size_t maxOsContextCount)
|
||||
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, pMem, gpuAddress, 0u, memSize, pool, maxOsContextCount),
|
||||
id(count), uncacheable(uncacheable) {
|
||||
|
||||
this->driverAllocatedCpuPointer = driverAllocatedCpuPointer;
|
||||
overrideMemoryPool(pool);
|
||||
allocationInfo.flags.flushL3Required = flushL3Required;
|
||||
}
|
||||
|
||||
void overrideMemoryPool(MemoryPool::Type pool);
|
||||
};
|
||||
|
||||
class OsAgnosticMemoryManager : public MemoryManager {
|
||||
public:
|
||||
using MemoryManager::allocateGraphicsMemory;
|
||||
using MemoryManager::heapAssigner;
|
||||
|
||||
OsAgnosticMemoryManager(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(false, executionEnvironment) {}
|
||||
OsAgnosticMemoryManager(bool aubUsage, ExecutionEnvironment &executionEnvironment);
|
||||
void initialize(bool aubUsage);
|
||||
~OsAgnosticMemoryManager() override;
|
||||
GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) override;
|
||||
GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) override { return nullptr; }
|
||||
|
||||
void addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) override;
|
||||
void removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) override;
|
||||
void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override;
|
||||
|
||||
AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
|
||||
void cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
|
||||
|
||||
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override;
|
||||
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override;
|
||||
|
||||
void turnOnFakingBigAllocations();
|
||||
|
||||
void *reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) override;
|
||||
void releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) override;
|
||||
|
||||
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override;
|
||||
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
|
||||
|
||||
protected:
|
||||
GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateUSMHostGraphicsMemory(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateShareableMemory(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override;
|
||||
|
||||
void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override { return graphicsAllocation.getUnderlyingBuffer(); }
|
||||
void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override {}
|
||||
GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData, bool useLocalMemory) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
|
||||
MemoryAllocation *createMemoryAllocation(GraphicsAllocation::AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize,
|
||||
uint64_t count, MemoryPool::Type pool, uint32_t rootDeviceIndex, bool uncacheable, bool flushL3Required, bool requireSpecificBitness);
|
||||
|
||||
private:
|
||||
unsigned long long counter = 0;
|
||||
bool fakeBigAllocations = false;
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
|
||||
|
||||
namespace NEO {
|
||||
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) {
|
||||
status = AllocationStatus::Error;
|
||||
switch (allocationData.type) {
|
||||
case GraphicsAllocation::AllocationType::IMAGE:
|
||||
case GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY:
|
||||
break;
|
||||
default:
|
||||
if (!allocationData.flags.useSystemMemory && !(allocationData.flags.allow32Bit && this->force32bitAllocations)) {
|
||||
GraphicsAllocation *allocation = nullptr;
|
||||
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_GPU) {
|
||||
void *cpuAllocation = allocateSystemMemory(allocationData.size, allocationData.alignment);
|
||||
if (!cpuAllocation) {
|
||||
return nullptr;
|
||||
}
|
||||
uint64_t gpuAddress = reinterpret_cast<uint64_t>(allocationData.hostPtr);
|
||||
allocation = new MemoryAllocation(allocationData.rootDeviceIndex, allocationData.type, cpuAllocation,
|
||||
cpuAllocation, gpuAddress, allocationData.size, counter++, MemoryPool::LocalMemory, false, false, maxOsContextCount);
|
||||
} else {
|
||||
allocation = allocateGraphicsMemory(allocationData);
|
||||
}
|
||||
|
||||
if (allocation) {
|
||||
allocation->storageInfo = allocationData.storageInfo;
|
||||
allocation->setFlushL3Required(allocationData.flags.flushL3);
|
||||
status = AllocationStatus::Success;
|
||||
}
|
||||
return allocation;
|
||||
}
|
||||
}
|
||||
status = AllocationStatus::RetryInNonDevicePool;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint64_t OsAgnosticMemoryManager::getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) {
|
||||
return 0 * GB;
|
||||
}
|
||||
|
||||
void MemoryAllocation::overrideMemoryPool(MemoryPool::Type pool) {
|
||||
this->memoryPool = pool;
|
||||
}
|
||||
} // namespace NEO
|
||||
Reference in New Issue
Block a user