mirror of
https://github.com/intel/compute-runtime.git
synced 2026-01-03 14:55:24 +08:00
Revert "Two commits related to pStart feature"
Related-To: NEO-15156, GSD-9939 Revert "test: fix minor errors" This reverts commit712dc8221e. Revert "feature: Support for pStart" This reverts commit47caeda487. Signed-off-by: Chandio, Bibrak Qamar <bibrak.qamar.chandio@intel.com>
This commit is contained in:
committed by
Compute-Runtime-Automation
parent
fed9d1c0a5
commit
6acfcb83b9
@@ -111,26 +111,24 @@ void GfxPartition::Heap::init(uint64_t base, uint64_t size, size_t allocationAli
|
||||
heapGranularity = GfxPartition::heapGranularity2MB;
|
||||
}
|
||||
|
||||
// Exclude very first and very last page from GPU address range allocation
|
||||
// Exclude very first and very last 64K from GPU address range allocation
|
||||
if (size > 2 * heapGranularity) {
|
||||
size -= 2 * heapGranularity;
|
||||
}
|
||||
|
||||
alloc = std::make_unique<HeapAllocator>(base + heapGranularity, size, allocationAlignment);
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
void GfxPartition::Heap::initExternalWithFrontWindow(uint64_t base, uint64_t size, size_t allocationAlignment) {
|
||||
void GfxPartition::Heap::initExternalWithFrontWindow(uint64_t base, uint64_t size) {
|
||||
this->base = base;
|
||||
this->size = size;
|
||||
|
||||
size -= GfxPartition::heapGranularity;
|
||||
|
||||
alloc = std::make_unique<HeapAllocator>(base, size, allocationAlignment, 0u);
|
||||
initialized = true;
|
||||
alloc = std::make_unique<HeapAllocator>(base, size, MemoryConstants::pageSize, 0u);
|
||||
}
|
||||
|
||||
void GfxPartition::Heap::initWithFrontWindow(uint64_t base, uint64_t size, uint64_t frontWindowSize, size_t allocationAlignment) {
|
||||
void GfxPartition::Heap::initWithFrontWindow(uint64_t base, uint64_t size, uint64_t frontWindowSize) {
|
||||
this->base = base;
|
||||
this->size = size;
|
||||
|
||||
@@ -138,38 +136,24 @@ void GfxPartition::Heap::initWithFrontWindow(uint64_t base, uint64_t size, uint6
|
||||
size -= GfxPartition::heapGranularity;
|
||||
size -= frontWindowSize;
|
||||
|
||||
alloc = std::make_unique<HeapAllocator>(base + frontWindowSize, size, allocationAlignment);
|
||||
initialized = true;
|
||||
alloc = std::make_unique<HeapAllocator>(base + frontWindowSize, size, MemoryConstants::pageSize);
|
||||
}
|
||||
|
||||
void GfxPartition::Heap::initFrontWindow(uint64_t base, uint64_t size, size_t allocationAlignment) {
|
||||
void GfxPartition::Heap::initFrontWindow(uint64_t base, uint64_t size) {
|
||||
this->base = base;
|
||||
this->size = size;
|
||||
|
||||
alloc = std::make_unique<HeapAllocator>(base, size, allocationAlignment, 0u);
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
size_t GfxPartition::Heap::getAllocAlignment() const {
|
||||
return alloc->getAllocationAlignment();
|
||||
alloc = std::make_unique<HeapAllocator>(base, size, MemoryConstants::pageSize, 0u);
|
||||
}
|
||||
|
||||
uint64_t GfxPartition::Heap::allocate(size_t &size) {
|
||||
return alloc->allocate(size);
|
||||
}
|
||||
|
||||
uint64_t GfxPartition::Heap::allocateWithStartAddressHint(const uint64_t requiredStartAddress, size_t &size) {
|
||||
return alloc->allocateWithStartAddressHint(requiredStartAddress, size);
|
||||
}
|
||||
|
||||
uint64_t GfxPartition::Heap::allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment) {
|
||||
return alloc->allocateWithCustomAlignment(sizeToAllocate, alignment);
|
||||
}
|
||||
|
||||
uint64_t GfxPartition::Heap::allocateWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, size_t &sizeToAllocate, size_t alignment) {
|
||||
return alloc->allocateWithCustomAlignmentWithStartAddressHint(requiredStartAddress, sizeToAllocate, alignment);
|
||||
}
|
||||
|
||||
void GfxPartition::Heap::free(uint64_t ptr, size_t size) {
|
||||
alloc->free(ptr, size);
|
||||
}
|
||||
@@ -264,7 +248,7 @@ bool GfxPartition::init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToRe
|
||||
auto cpuVirtualAddressSize = CpuInfo::getInstance().getVirtualAddressSize();
|
||||
if (cpuVirtualAddressSize == 48 && gpuAddressSpace == maxNBitValue(48)) {
|
||||
gfxBase = maxNBitValue(48 - 1) + 1;
|
||||
heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0ull, gfxBase, MemoryConstants::pageSize2M);
|
||||
heapInit(HeapIndex::heapSvm, 0ull, gfxBase);
|
||||
} else if (gpuAddressSpace == maxNBitValue(47)) {
|
||||
if (reservedCpuAddressRangeForHeapSvm.alignedPtr == nullptr) {
|
||||
if (cpuAddressRangeSizeToReserve == 0) {
|
||||
@@ -280,10 +264,10 @@ bool GfxPartition::init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToRe
|
||||
}
|
||||
gfxBase = reinterpret_cast<uint64_t>(reservedCpuAddressRangeForHeapSvm.alignedPtr);
|
||||
gfxTop = gfxBase + cpuAddressRangeSizeToReserve;
|
||||
heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0ull, gpuAddressSpace + 1, MemoryConstants::pageSize2M);
|
||||
heapInit(HeapIndex::heapSvm, 0ull, gpuAddressSpace + 1);
|
||||
} else if (gpuAddressSpace < maxNBitValue(47)) {
|
||||
gfxBase = 0ull;
|
||||
heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0ull, 0ull, MemoryConstants::pageSize2M);
|
||||
heapInit(HeapIndex::heapSvm, 0ull, 0ull);
|
||||
} else {
|
||||
if (!initAdditionalRange(cpuVirtualAddressSize, gpuAddressSpace, gfxBase, gfxTop, rootDeviceIndex, systemMemorySize)) {
|
||||
return false;
|
||||
@@ -293,14 +277,14 @@ bool GfxPartition::init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToRe
|
||||
|
||||
for (auto heap : GfxPartition::heap32Names) {
|
||||
if (useExternalFrontWindowPool && HeapAssigner::heapTypeExternalWithFrontWindowPool(heap)) {
|
||||
heapInitExternalWithFrontWindow(heap, gfxBase, gfxHeap32Size, MemoryConstants::pageSize);
|
||||
heapInitExternalWithFrontWindow(heap, gfxBase, gfxHeap32Size);
|
||||
size_t externalFrontWindowSize = GfxPartition::externalFrontWindowPoolSize;
|
||||
auto allocation = heapAllocate(heap, externalFrontWindowSize);
|
||||
heapInitExternalWithFrontWindow(HeapAssigner::mapExternalWindowIndex(heap), allocation,
|
||||
externalFrontWindowSize, MemoryConstants::pageSize);
|
||||
externalFrontWindowSize);
|
||||
} else if (HeapAssigner::isInternalHeap(heap)) {
|
||||
heapInitWithFrontWindow(heap, gfxBase, gfxHeap32Size, GfxPartition::internalFrontWindowPoolSize, MemoryConstants::pageSize);
|
||||
heapInitFrontWindow(HeapAssigner::mapInternalWindowIndex(heap), gfxBase, GfxPartition::internalFrontWindowPoolSize, MemoryConstants::pageSize);
|
||||
heapInitWithFrontWindow(heap, gfxBase, gfxHeap32Size, GfxPartition::internalFrontWindowPoolSize);
|
||||
heapInitFrontWindow(HeapAssigner::mapInternalWindowIndex(heap), gfxBase, GfxPartition::internalFrontWindowPoolSize);
|
||||
} else {
|
||||
heapInit(heap, gfxBase, gfxHeap32Size);
|
||||
}
|
||||
@@ -384,9 +368,9 @@ bool GfxPartition::initAdditionalRange(uint32_t cpuVirtualAddressSize, uint64_t
|
||||
gfxBase = castToUint64(reservedCpuAddressRangeForHeapSvm.alignedPtr);
|
||||
gfxTop = gfxBase + reservedCpuAddressRangeForHeapSvm.sizeToReserve;
|
||||
if (gpuAddressSpace == maxNBitValue(57)) {
|
||||
heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0ull, maxNBitValue(57 - 1) + 1, MemoryConstants::pageSize2M);
|
||||
heapInit(HeapIndex::heapSvm, 0ull, maxNBitValue(57 - 1) + 1);
|
||||
} else {
|
||||
heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0ull, maxNBitValue(48) + 1, MemoryConstants::pageSize2M);
|
||||
heapInit(HeapIndex::heapSvm, 0ull, maxNBitValue(48) + 1);
|
||||
}
|
||||
|
||||
if (gpuAddressSpace == maxNBitValue(57)) {
|
||||
@@ -400,7 +384,7 @@ bool GfxPartition::initAdditionalRange(uint32_t cpuVirtualAddressSize, uint64_t
|
||||
// On 48 bit CPU this range is reserved for OS usage, do not reserve
|
||||
gfxBase = maxNBitValue(48 - 1) + 1; // 0x800000000000
|
||||
gfxTop = maxNBitValue(48) + 1; // 0x1000000000000
|
||||
heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0ull, gfxBase, MemoryConstants::pageSize2M);
|
||||
heapInit(HeapIndex::heapSvm, 0ull, gfxBase);
|
||||
}
|
||||
|
||||
// Init HEAP_EXTENDED only for 57 bit GPU
|
||||
|
||||
@@ -49,34 +49,26 @@ class GfxPartition {
|
||||
getHeap(heapIndex).init(base, size, allocationAlignment);
|
||||
}
|
||||
|
||||
void heapInitExternalWithFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size, size_t allocationAlignment) {
|
||||
getHeap(heapIndex).initExternalWithFrontWindow(base, size, allocationAlignment);
|
||||
void heapInitExternalWithFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size) {
|
||||
getHeap(heapIndex).initExternalWithFrontWindow(base, size);
|
||||
}
|
||||
|
||||
void heapInitWithFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size, uint64_t frontWindowSize, size_t allocationAlignment) {
|
||||
getHeap(heapIndex).initWithFrontWindow(base, size, frontWindowSize, allocationAlignment);
|
||||
void heapInitWithFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size, uint64_t frontWindowSize) {
|
||||
getHeap(heapIndex).initWithFrontWindow(base, size, frontWindowSize);
|
||||
}
|
||||
|
||||
void heapInitFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size, size_t allocationAlignment) {
|
||||
getHeap(heapIndex).initFrontWindow(base, size, allocationAlignment);
|
||||
void heapInitFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size) {
|
||||
getHeap(heapIndex).initFrontWindow(base, size);
|
||||
}
|
||||
|
||||
MOCKABLE_VIRTUAL uint64_t heapAllocate(HeapIndex heapIndex, size_t &size) {
|
||||
return getHeap(heapIndex).allocate(size);
|
||||
}
|
||||
|
||||
MOCKABLE_VIRTUAL uint64_t heapAllocateWithStartAddressHint(const uint64_t requiredStartAddress, HeapIndex heapIndex, size_t &size) {
|
||||
return getHeap(heapIndex).allocateWithStartAddressHint(requiredStartAddress, size);
|
||||
}
|
||||
|
||||
MOCKABLE_VIRTUAL uint64_t heapAllocateWithCustomAlignment(HeapIndex heapIndex, size_t &size, size_t alignment) {
|
||||
return getHeap(heapIndex).allocateWithCustomAlignment(size, alignment);
|
||||
}
|
||||
|
||||
MOCKABLE_VIRTUAL uint64_t heapAllocateWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, HeapIndex heapIndex, size_t &size, size_t alignment) {
|
||||
return getHeap(heapIndex).allocateWithCustomAlignmentWithStartAddressHint(requiredStartAddress, size, alignment);
|
||||
}
|
||||
|
||||
MOCKABLE_VIRTUAL void heapFree(HeapIndex heapIndex, uint64_t ptr, size_t size) {
|
||||
getHeap(heapIndex).free(ptr, size);
|
||||
}
|
||||
@@ -91,31 +83,8 @@ class GfxPartition {
|
||||
return getHeap(heapIndex).getLimit();
|
||||
}
|
||||
|
||||
size_t getHeapAllocationAlignment(HeapIndex heapIndex) {
|
||||
return getHeap(heapIndex).getAllocAlignment();
|
||||
}
|
||||
|
||||
bool isHeapInitialized(HeapIndex heapIndex) {
|
||||
return getHeap(heapIndex).isInitialized();
|
||||
}
|
||||
|
||||
uint64_t getHeapMinimalAddress(HeapIndex heapIndex);
|
||||
|
||||
MOCKABLE_VIRTUAL bool getHeapIndexAndPageSizeBasedOnAddress(uint64_t ptr, HeapIndex &heapIndex, size_t &pageSize) {
|
||||
for (size_t index = 0; index < heaps.size(); ++index) {
|
||||
|
||||
if (!isHeapInitialized(static_cast<HeapIndex>(index))) {
|
||||
continue;
|
||||
}
|
||||
if (isAddressInHeapRange(static_cast<HeapIndex>(index), ptr)) {
|
||||
heapIndex = static_cast<HeapIndex>(index);
|
||||
pageSize = getHeapAllocationAlignment(heapIndex);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool isLimitedRange() { return getHeap(HeapIndex::heapSvm).getSize() == 0ull; }
|
||||
|
||||
static bool isAnyHeap32(HeapIndex heapIndex) {
|
||||
@@ -141,34 +110,25 @@ class GfxPartition {
|
||||
public:
|
||||
Heap() = default;
|
||||
void init(uint64_t base, uint64_t size, size_t allocationAlignment);
|
||||
void initExternalWithFrontWindow(uint64_t base, uint64_t size, size_t allocationAlignment);
|
||||
void initWithFrontWindow(uint64_t base, uint64_t size, uint64_t frontWindowSize, size_t allocationAlignment);
|
||||
void initFrontWindow(uint64_t base, uint64_t size, size_t allocationAlignment);
|
||||
void initExternalWithFrontWindow(uint64_t base, uint64_t size);
|
||||
void initWithFrontWindow(uint64_t base, uint64_t size, uint64_t frontWindowSize);
|
||||
void initFrontWindow(uint64_t base, uint64_t size);
|
||||
uint64_t getBase() const { return base; }
|
||||
uint64_t getSize() const { return size; }
|
||||
uint64_t getLimit() const { return size ? base + size - 1 : 0; }
|
||||
size_t getAllocAlignment() const;
|
||||
uint64_t allocate(size_t &size);
|
||||
uint64_t allocateWithStartAddressHint(const uint64_t requiredStartAddress, size_t &size);
|
||||
uint64_t allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment);
|
||||
uint64_t allocateWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, size_t &sizeToAllocate, size_t alignment);
|
||||
void free(uint64_t ptr, size_t size);
|
||||
bool isInitialized() const { return initialized; }
|
||||
|
||||
protected:
|
||||
uint64_t base = 0, size = 0;
|
||||
std::unique_ptr<HeapAllocator> alloc;
|
||||
bool initialized = false;
|
||||
};
|
||||
|
||||
Heap &getHeap(HeapIndex heapIndex) {
|
||||
return heaps[static_cast<uint32_t>(heapIndex)];
|
||||
}
|
||||
|
||||
bool isAddressInHeapRange(HeapIndex heapIndex, uint64_t ptr) {
|
||||
return (ptr >= getHeap(heapIndex).getBase()) && (ptr <= getHeap(heapIndex).getLimit());
|
||||
}
|
||||
|
||||
std::array<Heap, static_cast<uint32_t>(HeapIndex::totalHeaps)> heaps;
|
||||
|
||||
OSMemory::ReservedCpuAddressRange &reservedCpuAddressRangeForHeapSvm;
|
||||
|
||||
@@ -262,7 +262,6 @@ class MemoryManager {
|
||||
virtual AddressRange reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) = 0;
|
||||
virtual AddressRange reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) = 0;
|
||||
virtual size_t selectAlignmentAndHeap(size_t size, HeapIndex *heap) = 0;
|
||||
virtual size_t selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) = 0;
|
||||
virtual void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) = 0;
|
||||
virtual AddressRange reserveCpuAddress(const uint64_t requiredStartAddress, size_t size) = 0;
|
||||
AddressRange reserveCpuAddressWithZeroBaseRetry(const uint64_t requiredStartAddress, size_t size);
|
||||
|
||||
@@ -647,29 +647,12 @@ MemoryAllocation *OsAgnosticMemoryManager::createMemoryAllocation(AllocationType
|
||||
}
|
||||
|
||||
size_t OsAgnosticMemoryManager::selectAlignmentAndHeap(size_t size, HeapIndex *heap) {
|
||||
return selectAlignmentAndHeap(0ULL, size, heap);
|
||||
}
|
||||
|
||||
size_t OsAgnosticMemoryManager::selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) {
|
||||
|
||||
// Always default to HEAP STANDARD 2MB.
|
||||
*heap = HeapIndex::heapStandard2MB;
|
||||
size_t pageSizeAlignment = MemoryConstants::pageSize2M;
|
||||
|
||||
// If the user provides a start address, we try to find the heap and page size alignment based on that address.
|
||||
if (requiredStartAddress != 0ULL) {
|
||||
auto rootDeviceIndex = 0u;
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
if (gfxPartition->getHeapIndexAndPageSizeBasedOnAddress(requiredStartAddress, *heap, pageSizeAlignment)) {
|
||||
return pageSizeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
return pageSizeAlignment;
|
||||
*heap = HeapIndex::heapStandard;
|
||||
return MemoryConstants::pageSize64k;
|
||||
}
|
||||
|
||||
AddressRange OsAgnosticMemoryManager::reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) {
|
||||
return reserveGpuAddressOnHeap(requiredStartAddress, size, rootDeviceIndices, reservedOnRootDeviceIndex, HeapIndex::heapStandard2MB, MemoryConstants::pageSize2M);
|
||||
return reserveGpuAddressOnHeap(requiredStartAddress, size, rootDeviceIndices, reservedOnRootDeviceIndex, HeapIndex::heapStandard, MemoryConstants::pageSize64k);
|
||||
}
|
||||
|
||||
AddressRange OsAgnosticMemoryManager::reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) {
|
||||
@@ -678,7 +661,7 @@ AddressRange OsAgnosticMemoryManager::reserveGpuAddressOnHeap(const uint64_t req
|
||||
for (auto rootDeviceIndex : rootDeviceIndices) {
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
auto gmmHelper = getGmmHelper(rootDeviceIndex);
|
||||
gpuVa = requiredStartAddress == 0 ? gmmHelper->canonize(gfxPartition->heapAllocateWithCustomAlignment(heap, size, alignment)) : gmmHelper->canonize(gfxPartition->heapAllocateWithCustomAlignmentWithStartAddressHint(gmmHelper->decanonize(requiredStartAddress), heap, size, alignment));
|
||||
gpuVa = gmmHelper->canonize(gfxPartition->heapAllocate(heap, size));
|
||||
if (gpuVa != 0u) {
|
||||
*reservedOnRootDeviceIndex = rootDeviceIndex;
|
||||
break;
|
||||
|
||||
@@ -47,7 +47,6 @@ class OsAgnosticMemoryManager : public MemoryManager {
|
||||
AddressRange reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
|
||||
AddressRange reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) override;
|
||||
size_t selectAlignmentAndHeap(size_t size, HeapIndex *heap) override;
|
||||
size_t selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) override;
|
||||
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
|
||||
AddressRange reserveCpuAddress(const uint64_t requiredStartAddress, size_t size) override;
|
||||
void freeCpuAddress(AddressRange addressRange) override;
|
||||
|
||||
@@ -270,12 +270,6 @@ uint64_t DrmMemoryManager::acquireGpuRangeWithCustomAlignment(size_t &size, uint
|
||||
return gmmHelper->canonize(gfxPartition->heapAllocateWithCustomAlignment(heapIndex, size, alignment));
|
||||
}
|
||||
|
||||
uint64_t DrmMemoryManager::acquireGpuRangeWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, size_t &size, uint32_t rootDeviceIndex, HeapIndex heapIndex, size_t alignment) {
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
auto gmmHelper = getGmmHelper(rootDeviceIndex);
|
||||
return gmmHelper->canonize(gfxPartition->heapAllocateWithCustomAlignmentWithStartAddressHint(gmmHelper->decanonize(requiredStartAddress), heapIndex, size, alignment));
|
||||
}
|
||||
|
||||
void DrmMemoryManager::releaseGpuRange(void *address, size_t unmapSize, uint32_t rootDeviceIndex) {
|
||||
uint64_t graphicsAddress = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(address));
|
||||
auto gmmHelper = getGmmHelper(rootDeviceIndex);
|
||||
@@ -1725,62 +1719,46 @@ uint32_t DrmMemoryManager::getRootDeviceIndex(const Drm *drm) {
|
||||
}
|
||||
|
||||
size_t DrmMemoryManager::selectAlignmentAndHeap(size_t size, HeapIndex *heap) {
|
||||
return selectAlignmentAndHeap(0ULL, size, heap);
|
||||
}
|
||||
|
||||
size_t DrmMemoryManager::selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) {
|
||||
|
||||
// Always default to HEAP STANDARD 2MB.
|
||||
*heap = HeapIndex::heapStandard2MB;
|
||||
size_t pageSizeAlignment = MemoryConstants::pageSize2M;
|
||||
|
||||
// If the user provides a start address, we try to find the heap and page size alignment based on that address.
|
||||
if (requiredStartAddress != 0ULL) {
|
||||
auto rootDeviceIndex = 0u;
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
if (gfxPartition->getHeapIndexAndPageSizeBasedOnAddress(requiredStartAddress, *heap, pageSizeAlignment)) {
|
||||
return pageSizeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
// If all devices can support HEAP EXTENDED, then that heap is used.
|
||||
bool useExtendedHeap = true;
|
||||
AlignmentSelector::CandidateAlignment alignmentBase = alignmentSelector.selectAlignment(size);
|
||||
size_t pageSizeAlignment = alignmentBase.alignment;
|
||||
auto rootDeviceCount = this->executionEnvironment.rootDeviceEnvironments.size();
|
||||
|
||||
// If all devices can support HEAP EXTENDED, then that heap is used, otherwise the HEAP based on the size is used.
|
||||
for (auto rootDeviceIndex = 0u; rootDeviceIndex < rootDeviceCount; rootDeviceIndex++) {
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
if (!(gfxPartition->getHeapLimit(HeapIndex::heapExtended) > 0)) {
|
||||
useExtendedHeap = false;
|
||||
if (gfxPartition->getHeapLimit(HeapIndex::heapExtended) > 0) {
|
||||
auto alignSize = size >= 8 * MemoryConstants::gigaByte && Math::isPow2(size);
|
||||
if (debugManager.flags.UseHighAlignmentForHeapExtended.get() != -1) {
|
||||
alignSize = !!debugManager.flags.UseHighAlignmentForHeapExtended.get();
|
||||
}
|
||||
|
||||
if (alignSize) {
|
||||
pageSizeAlignment = Math::prevPowerOfTwo(size);
|
||||
}
|
||||
|
||||
*heap = HeapIndex::heapExtended;
|
||||
} else {
|
||||
pageSizeAlignment = alignmentBase.alignment;
|
||||
*heap = alignmentBase.heap;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (useExtendedHeap) {
|
||||
auto alignSize = size >= 8 * MemoryConstants::gigaByte && Math::isPow2(size);
|
||||
if (debugManager.flags.UseHighAlignmentForHeapExtended.get() != -1) {
|
||||
alignSize = !!debugManager.flags.UseHighAlignmentForHeapExtended.get();
|
||||
}
|
||||
|
||||
if (alignSize) {
|
||||
pageSizeAlignment = Math::prevPowerOfTwo(size);
|
||||
}
|
||||
|
||||
*heap = HeapIndex::heapExtended;
|
||||
}
|
||||
|
||||
return pageSizeAlignment;
|
||||
}
|
||||
|
||||
AddressRange DrmMemoryManager::reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) {
|
||||
return reserveGpuAddressOnHeap(requiredStartAddress, size, rootDeviceIndices, reservedOnRootDeviceIndex, HeapIndex::heapStandard2MB, MemoryConstants::pageSize2M);
|
||||
return reserveGpuAddressOnHeap(requiredStartAddress, size, rootDeviceIndices, reservedOnRootDeviceIndex, HeapIndex::heapStandard, MemoryConstants::pageSize64k);
|
||||
}
|
||||
|
||||
AddressRange DrmMemoryManager::reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) {
|
||||
uint64_t gpuVa = 0u;
|
||||
*reservedOnRootDeviceIndex = 0;
|
||||
|
||||
for (auto rootDeviceIndex : rootDeviceIndices) {
|
||||
|
||||
gpuVa = requiredStartAddress == 0 ? acquireGpuRangeWithCustomAlignment(size, rootDeviceIndex, heap, alignment) : acquireGpuRangeWithCustomAlignmentWithStartAddressHint(requiredStartAddress, size, rootDeviceIndex, heap, alignment);
|
||||
if (heap == HeapIndex::heapExtended) {
|
||||
gpuVa = acquireGpuRangeWithCustomAlignment(size, rootDeviceIndex, heap, alignment);
|
||||
} else {
|
||||
gpuVa = acquireGpuRange(size, rootDeviceIndex, heap);
|
||||
}
|
||||
if (gpuVa != 0u) {
|
||||
*reservedOnRootDeviceIndex = rootDeviceIndex;
|
||||
break;
|
||||
|
||||
@@ -77,7 +77,6 @@ class DrmMemoryManager : public MemoryManager {
|
||||
AddressRange reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
|
||||
AddressRange reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) override;
|
||||
size_t selectAlignmentAndHeap(size_t size, HeapIndex *heap) override;
|
||||
size_t selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) override;
|
||||
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
|
||||
AddressRange reserveCpuAddress(const uint64_t requiredStartAddress, size_t size) override;
|
||||
void freeCpuAddress(AddressRange addressRange) override;
|
||||
@@ -150,7 +149,6 @@ class DrmMemoryManager : public MemoryManager {
|
||||
void pushSharedBufferObject(BufferObject *bo);
|
||||
bool setDomainCpu(GraphicsAllocation &graphicsAllocation, bool writeEnable);
|
||||
MOCKABLE_VIRTUAL uint64_t acquireGpuRangeWithCustomAlignment(size_t &size, uint32_t rootDeviceIndex, HeapIndex heapIndex, size_t alignment);
|
||||
MOCKABLE_VIRTUAL uint64_t acquireGpuRangeWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, size_t &size, uint32_t rootDeviceIndex, HeapIndex heapIndex, size_t alignment);
|
||||
void emitPinningRequest(BufferObject *bo, const AllocationData &allocationData) const;
|
||||
uint32_t getDefaultDrmContextId(uint32_t rootDeviceIndex) const;
|
||||
OsContextLinux *getDefaultOsContext(uint32_t rootDeviceIndex) const;
|
||||
|
||||
@@ -1258,34 +1258,34 @@ bool Wddm::isGpuHangDetected(OsContext &osContext) {
|
||||
|
||||
void Wddm::initGfxPartition(GfxPartition &outGfxPartition, uint32_t rootDeviceIndex, size_t numRootDevices, bool useExternalFrontWindowPool) const {
|
||||
if (gfxPartition.SVM.Limit != 0) {
|
||||
outGfxPartition.heapInitWithAllocationAlignment(HeapIndex::heapSvm, gfxPartition.SVM.Base, gfxPartition.SVM.Limit - gfxPartition.SVM.Base + 1, MemoryConstants::pageSize64k);
|
||||
outGfxPartition.heapInit(HeapIndex::heapSvm, gfxPartition.SVM.Base, gfxPartition.SVM.Limit - gfxPartition.SVM.Base + 1);
|
||||
} else if (is32bit) {
|
||||
outGfxPartition.heapInitWithAllocationAlignment(HeapIndex::heapSvm, 0x0ull, 4 * MemoryConstants::gigaByte, MemoryConstants::pageSize64k);
|
||||
outGfxPartition.heapInit(HeapIndex::heapSvm, 0x0ull, 4 * MemoryConstants::gigaByte);
|
||||
}
|
||||
|
||||
outGfxPartition.heapInitWithAllocationAlignment(HeapIndex::heapStandard, gfxPartition.Standard.Base, gfxPartition.Standard.Limit - gfxPartition.Standard.Base + 1, MemoryConstants::pageSize64k);
|
||||
outGfxPartition.heapInit(HeapIndex::heapStandard, gfxPartition.Standard.Base, gfxPartition.Standard.Limit - gfxPartition.Standard.Base + 1);
|
||||
|
||||
// Split HEAP_STANDARD64K among root devices
|
||||
auto gfxStandard64KBSize = alignDown((gfxPartition.Standard64KB.Limit - gfxPartition.Standard64KB.Base + 1) / numRootDevices, GfxPartition::heapGranularity);
|
||||
outGfxPartition.heapInitWithAllocationAlignment(HeapIndex::heapStandard64KB, gfxPartition.Standard64KB.Base + rootDeviceIndex * gfxStandard64KBSize, gfxStandard64KBSize, MemoryConstants::pageSize64k);
|
||||
outGfxPartition.heapInit(HeapIndex::heapStandard64KB, gfxPartition.Standard64KB.Base + rootDeviceIndex * gfxStandard64KBSize, gfxStandard64KBSize);
|
||||
|
||||
for (auto heap : GfxPartition::heap32Names) {
|
||||
if (useExternalFrontWindowPool && HeapAssigner::heapTypeExternalWithFrontWindowPool(heap)) {
|
||||
outGfxPartition.heapInitExternalWithFrontWindow(heap, gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base,
|
||||
gfxPartition.Heap32[static_cast<uint32_t>(heap)].Limit - gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base + 1, MemoryConstants::pageSize64k);
|
||||
gfxPartition.Heap32[static_cast<uint32_t>(heap)].Limit - gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base + 1);
|
||||
size_t externalFrontWindowSize = GfxPartition::externalFrontWindowPoolSize;
|
||||
outGfxPartition.heapInitExternalWithFrontWindow(HeapAssigner::mapExternalWindowIndex(heap), outGfxPartition.heapAllocate(heap, externalFrontWindowSize),
|
||||
externalFrontWindowSize, MemoryConstants::pageSize64k);
|
||||
externalFrontWindowSize);
|
||||
} else if (HeapAssigner::isInternalHeap(heap)) {
|
||||
auto baseAddress = gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base >= minAddress ? gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base : minAddress;
|
||||
|
||||
outGfxPartition.heapInitWithFrontWindow(heap, baseAddress,
|
||||
gfxPartition.Heap32[static_cast<uint32_t>(heap)].Limit - baseAddress + 1,
|
||||
GfxPartition::internalFrontWindowPoolSize, MemoryConstants::pageSize64k);
|
||||
outGfxPartition.heapInitFrontWindow(HeapAssigner::mapInternalWindowIndex(heap), baseAddress, GfxPartition::internalFrontWindowPoolSize, MemoryConstants::pageSize64k);
|
||||
GfxPartition::internalFrontWindowPoolSize);
|
||||
outGfxPartition.heapInitFrontWindow(HeapAssigner::mapInternalWindowIndex(heap), baseAddress, GfxPartition::internalFrontWindowPoolSize);
|
||||
} else {
|
||||
outGfxPartition.heapInitWithAllocationAlignment(heap, gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base,
|
||||
gfxPartition.Heap32[static_cast<uint32_t>(heap)].Limit - gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base + 1, MemoryConstants::pageSize64k);
|
||||
outGfxPartition.heapInit(heap, gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base,
|
||||
gfxPartition.Heap32[static_cast<uint32_t>(heap)].Limit - gfxPartition.Heap32[static_cast<uint32_t>(heap)].Base + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -987,30 +987,13 @@ bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation, void *r
|
||||
}
|
||||
|
||||
size_t WddmMemoryManager::selectAlignmentAndHeap(size_t size, HeapIndex *heap) {
|
||||
return selectAlignmentAndHeap(0ULL, size, heap);
|
||||
}
|
||||
|
||||
size_t WddmMemoryManager::selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) {
|
||||
|
||||
// Always default to heapStandard64KB.
|
||||
*heap = HeapIndex::heapStandard64KB;
|
||||
|
||||
// If the user provides a start address, we try to find the heap and page size alignment based on that address.
|
||||
if (requiredStartAddress != 0ULL) {
|
||||
auto rootDeviceIndex = 0u;
|
||||
auto gfxPartition = getGfxPartition(rootDeviceIndex);
|
||||
size_t pageSizeAlignment = 0;
|
||||
if (gfxPartition->getHeapIndexAndPageSizeBasedOnAddress(requiredStartAddress, *heap, pageSizeAlignment)) {
|
||||
return pageSizeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
AlignmentSelector::CandidateAlignment alignment = alignmentSelector.selectAlignment(size);
|
||||
*heap = HeapIndex::heapStandard64KB;
|
||||
return alignment.alignment;
|
||||
}
|
||||
|
||||
AddressRange WddmMemoryManager::reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) {
|
||||
return reserveGpuAddressOnHeap(requiredStartAddress, size, rootDeviceIndices, reservedOnRootDeviceIndex, HeapIndex::heapStandard64KB, MemoryConstants::pageSize2M);
|
||||
return reserveGpuAddressOnHeap(requiredStartAddress, size, rootDeviceIndices, reservedOnRootDeviceIndex, HeapIndex::heapStandard64KB, MemoryConstants::pageSize64k);
|
||||
}
|
||||
|
||||
AddressRange WddmMemoryManager::reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) {
|
||||
|
||||
@@ -65,7 +65,6 @@ class WddmMemoryManager : public MemoryManager, NEO::NonCopyableAndNonMovableCla
|
||||
AddressRange reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
|
||||
AddressRange reserveGpuAddressOnHeap(const uint64_t requiredStartAddress, size_t size, const RootDeviceIndicesContainer &rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex, HeapIndex heap, size_t alignment) override;
|
||||
size_t selectAlignmentAndHeap(size_t size, HeapIndex *heap) override;
|
||||
size_t selectAlignmentAndHeap(const uint64_t requiredStartAddress, size_t size, HeapIndex *heap) override;
|
||||
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
|
||||
AddressRange reserveCpuAddress(const uint64_t requiredStartAddress, size_t size) override;
|
||||
void freeCpuAddress(AddressRange addressRange) override;
|
||||
|
||||
@@ -18,62 +18,6 @@ bool operator<(const HeapChunk &hc1, const HeapChunk &hc2) {
|
||||
return hc1.ptr < hc2.ptr;
|
||||
}
|
||||
|
||||
uint64_t HeapAllocator::allocateWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, size_t &sizeToAllocate, size_t alignment) {
|
||||
|
||||
if (alignment < this->allocationAlignment) {
|
||||
alignment = this->allocationAlignment;
|
||||
}
|
||||
|
||||
UNRECOVERABLE_IF(alignment % allocationAlignment != 0); // custom alignment have to be a multiple of allocator alignment
|
||||
sizeToAllocate = alignUp(sizeToAllocate, allocationAlignment);
|
||||
|
||||
uint64_t ptrReturn = 0llu;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mtx);
|
||||
DBG_LOG(LogAllocationMemoryPool, __FUNCTION__, "Allocator usage == ", this->getUsage());
|
||||
if (availableSize < sizeToAllocate) {
|
||||
return 0llu;
|
||||
}
|
||||
|
||||
if (requiredStartAddress >= pLeftBound && requiredStartAddress <= pRightBound) {
|
||||
|
||||
const uint64_t misalignment = requiredStartAddress - pLeftBound;
|
||||
if (pLeftBound + misalignment + sizeToAllocate <= pRightBound) {
|
||||
if (misalignment) {
|
||||
storeInFreedChunks(pLeftBound, static_cast<size_t>(misalignment), freedChunksBig);
|
||||
pLeftBound += misalignment;
|
||||
}
|
||||
ptrReturn = pLeftBound;
|
||||
pLeftBound += sizeToAllocate;
|
||||
availableSize -= sizeToAllocate;
|
||||
}
|
||||
} else { // Try to find in freed chunks
|
||||
|
||||
defragment();
|
||||
|
||||
if (requiredStartAddress < this->pLeftBound) {
|
||||
// If between baseAddress and pLeftBound, get from freedChunksBig
|
||||
ptrReturn = getFromFreedChunksWithStartAddressHint(requiredStartAddress, sizeToAllocate, freedChunksBig);
|
||||
} else {
|
||||
// If between pRightBound and heapLimit, get from freedChunksSmall
|
||||
ptrReturn = getFromFreedChunksWithStartAddressHint(requiredStartAddress, sizeToAllocate, freedChunksSmall);
|
||||
}
|
||||
|
||||
if (ptrReturn != 0llu) {
|
||||
availableSize -= sizeToAllocate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ptrReturn == 0llu) {
|
||||
return allocateWithCustomAlignment(sizeToAllocate, alignment);
|
||||
}
|
||||
|
||||
UNRECOVERABLE_IF(!isAligned(ptrReturn, alignment));
|
||||
return ptrReturn;
|
||||
}
|
||||
|
||||
uint64_t HeapAllocator::allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment) {
|
||||
if (alignment < this->allocationAlignment) {
|
||||
alignment = this->allocationAlignment;
|
||||
@@ -129,7 +73,7 @@ uint64_t HeapAllocator::allocateWithCustomAlignment(size_t &sizeToAllocate, size
|
||||
} else {
|
||||
availableSize -= sizeToAllocate;
|
||||
}
|
||||
UNRECOVERABLE_IF(!isAligned(ptrReturn, alignment));
|
||||
DEBUG_BREAK_IF(!isAligned(ptrReturn, alignment));
|
||||
return ptrReturn;
|
||||
}
|
||||
|
||||
@@ -171,43 +115,6 @@ double HeapAllocator::getUsage() const {
|
||||
return static_cast<double>(size - availableSize) / size;
|
||||
}
|
||||
|
||||
uint64_t HeapAllocator::getFromFreedChunksWithStartAddressHint(const uint64_t requiredStartAddress, size_t size, std::vector<HeapChunk> &freedChunks) {
|
||||
|
||||
for (size_t i = 0; i < freedChunks.size(); i++) {
|
||||
uint64_t chunkStart = freedChunks[i].ptr;
|
||||
uint64_t chunkEnd = chunkStart + freedChunks[i].size;
|
||||
|
||||
if (requiredStartAddress >= chunkStart && requiredStartAddress + size <= chunkEnd) {
|
||||
size_t leadingSize = static_cast<size_t>(requiredStartAddress - chunkStart);
|
||||
size_t trailingSize = static_cast<size_t>(chunkEnd - (requiredStartAddress + size));
|
||||
|
||||
// Chunk splitting
|
||||
if (leadingSize > 0) {
|
||||
|
||||
freedChunks[i].size = leadingSize;
|
||||
|
||||
if (trailingSize > 0) {
|
||||
freedChunks.emplace_back(requiredStartAddress + size, trailingSize);
|
||||
}
|
||||
} else {
|
||||
|
||||
if (trailingSize > 0) {
|
||||
|
||||
freedChunks[i].ptr = requiredStartAddress + size;
|
||||
freedChunks[i].size = trailingSize;
|
||||
} else {
|
||||
|
||||
freedChunks.erase(freedChunks.begin() + i);
|
||||
}
|
||||
}
|
||||
|
||||
return requiredStartAddress;
|
||||
}
|
||||
}
|
||||
|
||||
return 0llu;
|
||||
}
|
||||
|
||||
uint64_t HeapAllocator::getFromFreedChunks(size_t size, std::vector<HeapChunk> &freedChunks, size_t &sizeOfFreedChunk, size_t requiredAlignment) {
|
||||
size_t elements = freedChunks.size();
|
||||
size_t bestFitIndex = -1;
|
||||
|
||||
@@ -44,11 +44,6 @@ class HeapAllocator {
|
||||
return allocateWithCustomAlignment(sizeToAllocate, 0u);
|
||||
}
|
||||
|
||||
uint64_t allocateWithStartAddressHint(const uint64_t requiredStartAddress, size_t &sizeToAllocate) {
|
||||
return allocateWithCustomAlignmentWithStartAddressHint(requiredStartAddress, sizeToAllocate, 0u);
|
||||
}
|
||||
|
||||
uint64_t allocateWithCustomAlignmentWithStartAddressHint(const uint64_t requiredStartAddress, size_t &sizeToAllocate, size_t alignment);
|
||||
uint64_t allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment);
|
||||
|
||||
MOCKABLE_VIRTUAL void free(uint64_t ptr, size_t size);
|
||||
@@ -67,10 +62,6 @@ class HeapAllocator {
|
||||
return this->baseAddress;
|
||||
}
|
||||
|
||||
size_t getAllocationAlignment() const {
|
||||
return this->allocationAlignment;
|
||||
}
|
||||
|
||||
protected:
|
||||
const uint64_t baseAddress;
|
||||
const uint64_t size;
|
||||
@@ -85,7 +76,6 @@ class HeapAllocator {
|
||||
std::mutex mtx;
|
||||
|
||||
uint64_t getFromFreedChunks(size_t size, std::vector<HeapChunk> &freedChunks, size_t &sizeOfFreedChunk, size_t requiredAlignment);
|
||||
MOCKABLE_VIRTUAL uint64_t getFromFreedChunksWithStartAddressHint(const uint64_t requiredStartAddress, size_t size, std::vector<HeapChunk> &freedChunks);
|
||||
|
||||
void storeInFreedChunks(uint64_t ptr, size_t size, std::vector<HeapChunk> &freedChunks) {
|
||||
for (auto &freedChunk : freedChunks) {
|
||||
|
||||
Reference in New Issue
Block a user