258 lines
11 KiB
C++
258 lines
11 KiB
C++
/*
|
|
* Copyright (C) 2017-2019 Intel Corporation
|
|
*
|
|
* SPDX-License-Identifier: MIT
|
|
*
|
|
*/
|
|
|
|
#include "runtime/execution_environment/execution_environment.h"
|
|
#include "runtime/memory_manager/os_agnostic_memory_manager.h"
|
|
#include "runtime/gmm_helper/gmm.h"
|
|
#include "runtime/gmm_helper/gmm_helper.h"
|
|
#include "runtime/gmm_helper/resource_info.h"
|
|
#include "runtime/helpers/aligned_memory.h"
|
|
#include "runtime/helpers/basic_math.h"
|
|
#include "runtime/helpers/options.h"
|
|
#include "runtime/helpers/ptr_math.h"
|
|
#include "runtime/helpers/surface_formats.h"
|
|
#include "runtime/memory_manager/host_ptr_manager.h"
|
|
#include <cassert>
|
|
|
|
namespace OCLRT {
|
|
|
|
OsAgnosticMemoryManager::~OsAgnosticMemoryManager() {
|
|
applyCommonCleanup();
|
|
}
|
|
|
|
struct OsHandle {
|
|
};
|
|
|
|
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) {
|
|
|
|
auto sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize);
|
|
MemoryAllocation *memoryAllocation = nullptr;
|
|
|
|
if (fakeBigAllocations && allocationData.size > bigAllocation) {
|
|
memoryAllocation = new MemoryAllocation(nullptr, (void *)dummyAddress, static_cast<uint64_t>(dummyAddress), allocationData.size, counter,
|
|
MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable);
|
|
counter++;
|
|
memoryAllocation->uncacheable = allocationData.flags.uncacheable;
|
|
return memoryAllocation;
|
|
}
|
|
auto ptr = allocateSystemMemory(sizeAligned, allocationData.alignment ? alignUp(allocationData.alignment, MemoryConstants::pageSize) : MemoryConstants::pageSize);
|
|
if (ptr != nullptr) {
|
|
memoryAllocation = new MemoryAllocation(ptr, ptr, reinterpret_cast<uint64_t>(ptr), allocationData.size, counter, MemoryPool::System4KBPages,
|
|
allocationData.flags.multiOsContextCapable);
|
|
if (!memoryAllocation) {
|
|
alignedFreeWrapper(ptr);
|
|
return nullptr;
|
|
}
|
|
memoryAllocation->uncacheable = allocationData.flags.uncacheable;
|
|
}
|
|
counter++;
|
|
return memoryAllocation;
|
|
}
|
|
|
|
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(size_t size, void *cpuPtr) {
|
|
MemoryAllocation *memoryAllocation = nullptr;
|
|
auto alignedPtr = alignDown(reinterpret_cast<char *>(cpuPtr), MemoryConstants::pageSize);
|
|
auto offsetInPage = reinterpret_cast<char *>(cpuPtr) - alignedPtr;
|
|
|
|
memoryAllocation = new MemoryAllocation(nullptr, cpuPtr, reinterpret_cast<uint64_t>(alignedPtr), size, counter,
|
|
MemoryPool::System4KBPages, false);
|
|
|
|
memoryAllocation->allocationOffset = offsetInPage;
|
|
memoryAllocation->uncacheable = false;
|
|
|
|
counter++;
|
|
return memoryAllocation;
|
|
}
|
|
|
|
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemory64kb(AllocationData allocationData) {
|
|
allocationData.size = alignUp(allocationData.size, MemoryConstants::pageSize64k);
|
|
allocationData.alignment = MemoryConstants::pageSize64k;
|
|
auto memoryAllocation = allocateGraphicsMemoryWithAlignment(allocationData);
|
|
if (memoryAllocation) {
|
|
reinterpret_cast<MemoryAllocation *>(memoryAllocation)->overrideMemoryPool(MemoryPool::System64KBPages);
|
|
}
|
|
return memoryAllocation;
|
|
}
|
|
|
|
GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) {
|
|
if (allocationData.hostPtr) {
|
|
auto allocationSize = alignSizeWholePage(allocationData.hostPtr, allocationData.size);
|
|
auto gpuVirtualAddress = allocator32Bit->allocate(allocationSize);
|
|
if (!gpuVirtualAddress) {
|
|
return nullptr;
|
|
}
|
|
uint64_t offset = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(allocationData.hostPtr) & MemoryConstants::pageMask);
|
|
MemoryAllocation *memAlloc = new MemoryAllocation(nullptr, const_cast<void *>(allocationData.hostPtr), GmmHelper::canonize(gpuVirtualAddress + offset), allocationData.size,
|
|
counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false);
|
|
memAlloc->is32BitAllocation = true;
|
|
memAlloc->gpuBaseAddress = GmmHelper::canonize(allocator32Bit->getBase());
|
|
memAlloc->sizeToFree = allocationSize;
|
|
|
|
counter++;
|
|
return memAlloc;
|
|
}
|
|
|
|
auto allocationSize = alignUp(allocationData.size, MemoryConstants::pageSize);
|
|
void *ptrAlloc = nullptr;
|
|
auto gpuAddress = allocator32Bit->allocate(allocationSize);
|
|
|
|
if (allocationData.size < 0xfffff000) {
|
|
ptrAlloc = alignedMallocWrapper(allocationSize, MemoryConstants::allocationAlignment);
|
|
}
|
|
|
|
MemoryAllocation *memoryAllocation = nullptr;
|
|
if (ptrAlloc != nullptr) {
|
|
memoryAllocation = new MemoryAllocation(ptrAlloc, ptrAlloc, GmmHelper::canonize(gpuAddress), allocationData.size, counter,
|
|
MemoryPool::System4KBPagesWith32BitGpuAddressing, false);
|
|
memoryAllocation->is32BitAllocation = true;
|
|
memoryAllocation->gpuBaseAddress = GmmHelper::canonize(allocator32Bit->getBase());
|
|
memoryAllocation->sizeToFree = allocationSize;
|
|
}
|
|
counter++;
|
|
return memoryAllocation;
|
|
}
|
|
|
|
GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness) {
|
|
auto graphicsAllocation = new MemoryAllocation(nullptr, reinterpret_cast<void *>(1), 1, 4096u, static_cast<uint64_t>(handle),
|
|
MemoryPool::SystemCpuInaccessible, false);
|
|
graphicsAllocation->setSharedHandle(handle);
|
|
graphicsAllocation->is32BitAllocation = requireSpecificBitness;
|
|
return graphicsAllocation;
|
|
}
|
|
|
|
void OsAgnosticMemoryManager::addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) {
|
|
FragmentStorage fragment = {};
|
|
fragment.driverAllocation = true;
|
|
fragment.fragmentCpuPointer = gfxAllocation->getUnderlyingBuffer();
|
|
fragment.fragmentSize = alignUp(gfxAllocation->getUnderlyingBufferSize(), MemoryConstants::pageSize);
|
|
fragment.osInternalStorage = new OsHandle();
|
|
fragment.residency = new ResidencyData();
|
|
hostPtrManager->storeFragment(fragment);
|
|
}
|
|
|
|
void OsAgnosticMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) {
|
|
auto buffer = gfxAllocation->getUnderlyingBuffer();
|
|
auto fragment = hostPtrManager->getFragment(buffer);
|
|
if (fragment && fragment->driverAllocation) {
|
|
OsHandle *osStorageToRelease = fragment->osInternalStorage;
|
|
ResidencyData *residencyDataToRelease = fragment->residency;
|
|
if (hostPtrManager->releaseHostPtr(buffer)) {
|
|
delete osStorageToRelease;
|
|
delete residencyDataToRelease;
|
|
}
|
|
}
|
|
}
|
|
|
|
void OsAgnosticMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) {
|
|
delete gfxAllocation->gmm;
|
|
|
|
if ((uintptr_t)gfxAllocation->getUnderlyingBuffer() == dummyAddress) {
|
|
delete gfxAllocation;
|
|
return;
|
|
}
|
|
|
|
if (gfxAllocation->fragmentsStorage.fragmentCount) {
|
|
cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
|
|
delete gfxAllocation;
|
|
return;
|
|
}
|
|
|
|
if (gfxAllocation->is32BitAllocation) {
|
|
auto gpuAddressToFree = gfxAllocation->getGpuAddress() & ~MemoryConstants::pageMask;
|
|
allocator32Bit->free(gpuAddressToFree, static_cast<MemoryAllocation *>(gfxAllocation)->sizeToFree);
|
|
}
|
|
|
|
alignedFreeWrapper(gfxAllocation->driverAllocatedCpuPointer);
|
|
delete gfxAllocation;
|
|
}
|
|
|
|
uint64_t OsAgnosticMemoryManager::getSystemSharedMemory() {
|
|
return 16 * GB;
|
|
}
|
|
|
|
uint64_t OsAgnosticMemoryManager::getMaxApplicationAddress() {
|
|
return MemoryConstants::max32BitAppAddress + static_cast<uint64_t>(is64bit) * (MemoryConstants::max64BitAppAddress - MemoryConstants::max32BitAppAddress);
|
|
}
|
|
|
|
uint64_t OsAgnosticMemoryManager::getInternalHeapBaseAddress() {
|
|
return this->allocator32Bit->getBase();
|
|
}
|
|
|
|
GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, size_t hostPtrSize, const void *hostPtr) {
|
|
auto allocation = new MemoryAllocation(nullptr, const_cast<void *>(hostPtr), reinterpret_cast<uint64_t>(hostPtr),
|
|
hostPtrSize, counter++, MemoryPool::System4KBPages, false);
|
|
allocation->fragmentsStorage = handleStorage;
|
|
return allocation;
|
|
}
|
|
|
|
void OsAgnosticMemoryManager::turnOnFakingBigAllocations() {
|
|
this->fakeBigAllocations = true;
|
|
}
|
|
|
|
MemoryManager::AllocationStatus OsAgnosticMemoryManager::populateOsHandles(OsHandleStorage &handleStorage) {
|
|
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
|
if (!handleStorage.fragmentStorageData[i].osHandleStorage && handleStorage.fragmentStorageData[i].cpuPtr) {
|
|
handleStorage.fragmentStorageData[i].osHandleStorage = new OsHandle();
|
|
handleStorage.fragmentStorageData[i].residency = new ResidencyData();
|
|
|
|
FragmentStorage newFragment = {};
|
|
newFragment.fragmentCpuPointer = const_cast<void *>(handleStorage.fragmentStorageData[i].cpuPtr);
|
|
newFragment.fragmentSize = handleStorage.fragmentStorageData[i].fragmentSize;
|
|
newFragment.osInternalStorage = handleStorage.fragmentStorageData[i].osHandleStorage;
|
|
newFragment.residency = handleStorage.fragmentStorageData[i].residency;
|
|
hostPtrManager->storeFragment(newFragment);
|
|
}
|
|
}
|
|
return AllocationStatus::Success;
|
|
}
|
|
void OsAgnosticMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage) {
|
|
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
|
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
|
|
delete handleStorage.fragmentStorageData[i].osHandleStorage;
|
|
delete handleStorage.fragmentStorageData[i].residency;
|
|
}
|
|
}
|
|
}
|
|
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) {
|
|
GraphicsAllocation *alloc = nullptr;
|
|
|
|
if (!GmmHelper::allowTiling(*allocationData.imgInfo->imgDesc) && allocationData.imgInfo->mipCount == 0) {
|
|
alloc = allocateGraphicsMemoryWithAlignment(allocationData);
|
|
alloc->gmm = gmm.release();
|
|
return alloc;
|
|
}
|
|
|
|
auto ptr = allocateSystemMemory(alignUp(allocationData.imgInfo->size, MemoryConstants::pageSize), MemoryConstants::pageSize);
|
|
if (ptr != nullptr) {
|
|
alloc = new MemoryAllocation(ptr, ptr, reinterpret_cast<uint64_t>(ptr), allocationData.imgInfo->size, counter,
|
|
MemoryPool::SystemCpuInaccessible, false);
|
|
counter++;
|
|
}
|
|
|
|
if (alloc) {
|
|
alloc->gmm = gmm.release();
|
|
}
|
|
|
|
return alloc;
|
|
}
|
|
|
|
Allocator32bit *OsAgnosticMemoryManager::create32BitAllocator(bool aubUsage) {
|
|
uint64_t allocatorSize = MemoryConstants::gigaByte - 2 * 4096;
|
|
uint64_t heap32Base = 0x80000000000ul;
|
|
|
|
if (is64bit && this->localMemorySupported && aubUsage) {
|
|
heap32Base = 0x40000000000ul;
|
|
}
|
|
|
|
if (is32bit) {
|
|
heap32Base = 0x0;
|
|
}
|
|
|
|
return new Allocator32bit(heap32Base, allocatorSize);
|
|
}
|
|
} // namespace OCLRT
|