intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2026-01-03 14:55:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: reorder members to reduce internal padding in structs

Browse Source 
Signed-off-by: Mateusz Jablonski <mateusz.jablonski@intel.com>
This commit is contained in:
Mateusz Jablonski
2024-03-25 14:44:36 +00:00
committed by Compute-Runtime-Automation
parent 9b715c3ffc
commit 78a4a92b44
61 changed files with 288 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
shared/source/command_stream/command_stream_receiver_simulated_hw.h
View File

@@ -111,7 +111,7 @@ class CommandStreamReceiverSimulatedHw : public CommandStreamReceiverSimulatedCo
auto gmm = graphicsAllocation.getDefaultGmm();
if (gmm) {
allocationParams.additionalParams.compressionEnabled = gmm->isCompressionEnabled;
allocationParams.additionalParams.compressionEnabled = gmm->isCompressionEnabled();
allocationParams.additionalParams.uncached = CacheSettingsHelper::isUncachedType(gmm->resourceParams.Usage);
}

10
shared/source/gmm_helper/gmm.cpp
View File

@@ -152,7 +152,7 @@ void Gmm::applyAuxFlagsForBuffer(bool preferCompression) {
gfxCoreHelper.applyRenderCompressionFlag(*this, 1);
resourceParams.Flags.Gpu.CCS = 1;
resourceParams.Flags.Gpu.UnifiedAuxSurface = 1;
isCompressionEnabled = true;
compressionEnabled = true;
}
if (debugManager.flags.PrintGmmCompressionParams.get()) {
@@ -160,7 +160,7 @@ void Gmm::applyAuxFlagsForBuffer(bool preferCompression) {
resourceParams.Flags.Gpu.CCS, resourceParams.Flags.Gpu.UnifiedAuxSurface, resourceParams.Flags.Info.RenderCompressed);
}
gfxCoreHelper.applyAdditionalCompressionSettings(*this, !isCompressionEnabled);
gfxCoreHelper.applyAdditionalCompressionSettings(*this, !compressionEnabled);
}
void Gmm::applyAuxFlagsForImage(ImageInfo &imgInfo, bool preferCompressed) {
@@ -199,7 +199,7 @@ void Gmm::applyAuxFlagsForImage(ImageInfo &imgInfo, bool preferCompressed) {
this->resourceParams.Flags.Gpu.CCS = 1;
this->resourceParams.Flags.Gpu.UnifiedAuxSurface = 1;
this->resourceParams.Flags.Gpu.IndirectClearColor = 1;
this->isCompressionEnabled = true;
this->compressionEnabled = true;
}
}
@@ -208,7 +208,7 @@ void Gmm::applyAuxFlagsForImage(ImageInfo &imgInfo, bool preferCompressed) {
resourceParams.Flags.Gpu.CCS, resourceParams.Flags.Gpu.UnifiedAuxSurface, resourceParams.Flags.Info.RenderCompressed);
}
gfxCoreHelper.applyAdditionalCompressionSettings(*this, !isCompressionEnabled);
gfxCoreHelper.applyAdditionalCompressionSettings(*this, !compressionEnabled);
}
void Gmm::queryImageParams(ImageInfo &imgInfo) {
@@ -371,7 +371,7 @@ void Gmm::applyMemoryFlags(const StorageInfo &storageInfo) {
if (extraMemoryFlagsRequired()) {
applyExtraMemoryFlags(storageInfo);
} else if (!storageInfo.isLockable) {
if (isCompressionEnabled || storageInfo.localOnlyRequired) {
if (isCompressionEnabled() || storageInfo.localOnlyRequired) {
resourceParams.Flags.Info.LocalOnly = 1;
}
}

5
shared/source/gmm_helper/gmm.h
View File

@@ -71,8 +71,8 @@ class Gmm {
std::unique_ptr<GmmResourceInfo> gmmResourceInfo;
const char *getUsageTypeString();
bool isCompressionEnabled = false;
void setCompressionEnabled(bool compresionEnabled) { this->compressionEnabled = compresionEnabled; }
bool isCompressionEnabled() const { return compressionEnabled; }
protected:
void applyAuxFlagsForImage(ImageInfo &imgInfo, bool preferCompressed);
@@ -82,6 +82,7 @@ class Gmm {
void applyDebugOverrides();
GmmHelper *gmmHelper = nullptr;
bool compressionEnabled = false;
bool preferNoCpuAccess = false;
};
} // namespace NEO

13
shared/source/memory_manager/definitions/storage_info.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2023 Intel Corporation
* Copyright (C) 2020-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -16,14 +16,18 @@ namespace NEO {
struct StorageInfo {
StorageInfo();
uint32_t getNumBanks() const;
uint32_t getMemoryBanks() const { return static_cast<uint32_t>(memoryBanks.to_ulong()); }
uint32_t getTotalBanksCnt() const;
size_t colouringGranularity = MemoryConstants::pageSize64k;
DeviceBitfield memoryBanks{};
DeviceBitfield pageTablesVisibility{};
DeviceBitfield subDeviceBitfield{};
ColouringPolicy colouringPolicy = ColouringPolicy::deviceCountBased;
uint32_t numOfChunks = 0;
bool cloningOfPageTables = true;
bool tileInstanced = false;
bool multiStorage = false;
ColouringPolicy colouringPolicy = ColouringPolicy::deviceCountBased;
size_t colouringGranularity = MemoryConstants::pageSize64k;
bool readOnlyMultiStorage = false;
bool cpuVisibleSegment = false;
bool isLockable = false;
@@ -31,9 +35,6 @@ struct StorageInfo {
bool systemMemoryPlacement = true;
bool systemMemoryForced = false;
char resourceTag[AppResourceDefines::maxStrLen + 1] = "";
uint32_t getMemoryBanks() const { return static_cast<uint32_t>(memoryBanks.to_ulong()); }
uint32_t getTotalBanksCnt() const;
bool isChunked = false;
uint32_t numOfChunks = 0;
};
} // namespace NEO

2
shared/source/memory_manager/graphics_allocation.cpp
View File

@@ -107,7 +107,7 @@ void GraphicsAllocation::setTbxWritable(bool writable, uint32_t banks) {
}
bool GraphicsAllocation::isCompressionEnabled() const {
return (getDefaultGmm() && getDefaultGmm()->isCompressionEnabled);
return (getDefaultGmm() && getDefaultGmm()->isCompressionEnabled());
}
bool GraphicsAllocation::isTbxWritable(uint32_t banks) const {

7
shared/source/memory_manager/graphics_allocation.h
View File

@@ -316,6 +316,8 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
return bindlessInfo;
}
MOCKABLE_VIRTUAL void updateCompletionDataForAllocationAndFragments(uint64_t newFenceValue, uint32_t contextId);
void setShareableHostMemory(bool shareableHostMemory) { this->shareableHostMemory = shareableHostMemory; }
bool isShareableHostMemory() const { return shareableHostMemory; }
OsHandleStorage fragmentsStorage;
StorageInfo storageInfo = {};
@@ -325,8 +327,8 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
constexpr static TaskCountType objectNotResident = std::numeric_limits<TaskCountType>::max();
constexpr static TaskCountType objectNotUsed = std::numeric_limits<TaskCountType>::max();
constexpr static TaskCountType objectAlwaysResident = std::numeric_limits<TaskCountType>::max() - 1;
std::atomic<uint32_t> hostPtrTaskCountAssignment{0};
bool isShareableHostMemory = false;
protected:
struct UsageInfo {
@@ -388,8 +390,9 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
AllocationType allocationType = AllocationType::unknown;
StackVec<UsageInfo, 32> usageInfos;
std::atomic<uint32_t> registeredContextsNum{0};
StackVec<Gmm *, EngineLimits::maxHandleCount> gmms;
ResidencyData residency;
std::atomic<uint32_t> registeredContextsNum{0};
bool shareableHostMemory = false;
};
} // namespace NEO

2
shared/source/os_interface/aub_memory_operations_handler.cpp
View File

@@ -48,7 +48,7 @@ MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device,
auto gmm = allocation->getDefaultGmm();
if (gmm) {
params.additionalParams.compressionEnabled = gmm->isCompressionEnabled;
params.additionalParams.compressionEnabled = gmm->isCompressionEnabled();
params.additionalParams.uncached = CacheSettingsHelper::isUncachedType(gmm->resourceParams.Usage);
}

4
shared/source/os_interface/linux/drm_allocation.h
View File

@@ -152,14 +152,14 @@ class DrmAllocation : public GraphicsAllocation {
OsContextLinux *osContext = nullptr;
BufferObjects bufferObjects{};
StackVec<uint32_t, 1> registeredBoBindHandles;
MemAdviseFlags enabledMemAdviseFlags{};
StackVec<MemoryToUnmap, 1> memoryToUnmap;
uint32_t numHandles = 0u;
std::vector<uint64_t> handles;
void *mmapPtr = nullptr;
void *importedMmapPtr = nullptr;
size_t mmapSize = 0u;
uint32_t numHandles = 0u;
MemAdviseFlags enabledMemAdviseFlags{};
bool usmHostAllocation = false;
};

2
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@@ -2306,7 +2306,7 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &
bo.release();
allocation->setDefaultGmm(gmm.release());
allocation->isShareableHostMemory = true;
allocation->setShareableHostMemory(true);
allocation->storageInfo = allocationData.storageInfo;
return allocation.release();
} else {

2
shared/source/os_interface/linux/drm_neo.cpp
View File

@@ -1251,7 +1251,7 @@ uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion c
if (gmm) {
resourceInfo = gmm->gmmResourceInfo->peekGmmResourceInfo();
usageType = gmm->resourceParams.Usage;
compressed = gmm->isCompressionEnabled;
compressed = gmm->isCompressionEnabled();
cachable = gmm->gmmResourceInfo->getResourceFlags()->Info.Cacheable;
}

6
shared/source/os_interface/windows/wddm/wddm.cpp
View File

@@ -575,7 +575,7 @@ bool Wddm::mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_A
kmDafListener->notifyMapGpuVA(featureTable->flags.ftrKmdDaf, getAdapter(), device, handle, mapGPUVA.VirtualAddress, getGdi()->escape);
bool ret = true;
auto &productHelper = rootDeviceEnvironment.getHelper<ProductHelper>();
if (gmm->isCompressionEnabled && productHelper.isPageTableManagerSupported(*rootDeviceEnvironment.getHardwareInfo())) {
if (gmm->isCompressionEnabled() && productHelper.isPageTableManagerSupported(*rootDeviceEnvironment.getHardwareInfo())) {
this->forEachContextWithinWddm([&](const EngineControl &engine) {
if (engine.commandStreamReceiver->pageTableManager.get()) {
ret &= engine.commandStreamReceiver->pageTableManager->updateAuxTable(gpuPtr, gmm, true);
@@ -834,7 +834,7 @@ bool Wddm::openSharedHandle(D3DKMT_HANDLE handle, WddmAllocation *alloc) {
DEBUG_BREAK_IF(status != STATUS_SUCCESS);
alloc->setDefaultHandle(allocationInfo[0].hAllocation);
alloc->resourceHandle = openResource.hResource;
alloc->setResourceHandle(openResource.hResource);
auto resourceInfo = const_cast<void *>(allocationInfo[0].pPrivateDriverData);
alloc->setDefaultGmm(new Gmm(rootDeviceEnvironment.getGmmHelper(), static_cast<GMM_RESOURCE_INFO *>(resourceInfo)));
@@ -883,7 +883,7 @@ bool Wddm::openNTHandle(HANDLE handle, WddmAllocation *alloc) {
alloc->setDefaultGmm(new Gmm(rootDeviceEnvironment.getGmmHelper(), static_cast<GMM_RESOURCE_INFO *>(resourceInfo), hwDeviceId->getUmKmDataTranslator()->enabled()));
alloc->setDefaultHandle(allocationInfo2[0].hAllocation);
alloc->resourceHandle = openResourceFromNtHandle.hResource;
alloc->setResourceHandle(openResourceFromNtHandle.hResource);
return true;
}

39
shared/source/os_interface/windows/wddm_allocation.h
View File

@@ -16,9 +16,9 @@ namespace NEO {
struct OsHandleWin : OsHandle {
~OsHandleWin() override = default;
D3DKMT_HANDLE handle = 0;
D3DGPU_VIRTUAL_ADDRESS gpuPtr = 0;
Gmm *gmm = nullptr;
D3DKMT_HANDLE handle = 0;
};
constexpr size_t trimListUnusedPosition = std::numeric_limits<size_t>::max();
@@ -36,7 +36,7 @@ class WddmAllocation : public GraphicsAllocation {
WddmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t canonizedAddress, size_t sizeIn,
void *reservedAddr, MemoryPool pool, uint32_t shareable, size_t maxOsContextCount)
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, canonizedAddress, 0llu, sizeIn, pool, maxOsContextCount),
shareable(shareable), trimCandidateListPositions(maxOsContextCount, trimListUnusedPosition) {
trimCandidateListPositions(maxOsContextCount, trimListUnusedPosition), shareable(shareable) {
reservedAddressRangeInfo.addressPtr = reservedAddr;
reservedAddressRangeInfo.rangeSize = sizeIn;
handles.resize(gmms.size());
@@ -105,18 +105,29 @@ class WddmAllocation : public GraphicsAllocation {
std::string getAllocationInfoString() const override;
std::string getPatIndexInfoString() const override;
uint64_t &getGpuAddressToModify() { return gpuAddress; }
bool needsMakeResidentBeforeLock() const { return makeResidentBeforeLockRequired; }
void setMakeResidentBeforeLockRequired(bool makeResidentBeforeLockRequired) { this->makeResidentBeforeLockRequired = makeResidentBeforeLockRequired; }
bool isAllocInFrontWindowPool() const { return allocInFrontWindowPool; }
void setAllocInFrontWindowPool(bool allocInFrontWindowPool) { this->allocInFrontWindowPool = allocInFrontWindowPool; }
bool isPhysicalMemoryReservation() const { return physicalMemoryReservation; }
void setPhysicalMemoryReservation(bool physicalMemoryReservation) { this->physicalMemoryReservation = physicalMemoryReservation; }
bool isMappedPhysicalMemoryReservation() const { return mappedPhysicalMemoryReservation; }
void setMappedPhysicalMemoryReservation(bool mappedPhysicalMemoryReservation) { this->mappedPhysicalMemoryReservation = mappedPhysicalMemoryReservation; }
// OS assigned fields
D3DKMT_HANDLE resourceHandle = 0u; // used by shared resources
bool needsMakeResidentBeforeLock = false;
D3DGPU_VIRTUAL_ADDRESS reservedGpuVirtualAddress = 0u;
uint64_t reservedSizeForGpuVirtualAddress = 0u;
uint32_t shareable = 0u;
bool allocInFrontWindowPool = false;
bool physicalMemoryReservation = false;
bool mappedPhysicalMemoryReservation = false;
D3DGPU_VIRTUAL_ADDRESS getReservedGpuVirtualAddress() const { return reservedGpuVirtualAddress; }
D3DGPU_VIRTUAL_ADDRESS &getReservedGpuVirtualAddressToModify() { return reservedGpuVirtualAddress; }
void setReservedGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS reservedGpuVirtualAddress) { this->reservedGpuVirtualAddress = reservedGpuVirtualAddress; }
uint64_t getReservedSizeForGpuVirtualAddress() const { return reservedSizeForGpuVirtualAddress; }
void setReservedSizeForGpuVirtualAddress(uint64_t reservedSizeForGpuVirtualAddress) { this->reservedSizeForGpuVirtualAddress = reservedSizeForGpuVirtualAddress; }
D3DKMT_HANDLE getResourceHandle() const { return resourceHandle; }
D3DKMT_HANDLE &getResourceHandleToModify() { return resourceHandle; }
void setResourceHandle(D3DKMT_HANDLE resourceHandle) { this->resourceHandle = resourceHandle; }
protected:
// OS assigned fields
D3DGPU_VIRTUAL_ADDRESS reservedGpuVirtualAddress = 0u;
uint64_t reservedSizeForGpuVirtualAddress = 0u;
uint64_t ntSecureHandle = 0u;
std::string getHandleInfoString() const {
std::stringstream ss;
@@ -127,5 +138,11 @@ class WddmAllocation : public GraphicsAllocation {
}
std::vector<size_t> trimCandidateListPositions;
StackVec<D3DKMT_HANDLE, EngineLimits::maxHandleCount> handles;
D3DKMT_HANDLE resourceHandle = 0u; // used by shared resources
uint32_t shareable = 0u;
bool allocInFrontWindowPool = false;
bool physicalMemoryReservation = false;
bool mappedPhysicalMemoryReservation = false;
bool makeResidentBeforeLockRequired = false;
};
} // namespace NEO

76
shared/source/os_interface/windows/wddm_memory_manager.cpp
View File

@@ -98,16 +98,16 @@ void WddmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physica
physicalAllocation->setCpuPtrAndGpuAddress(nullptr, 0u);
physicalAllocation->setReservedAddressRange(nullptr, 0u);
WddmAllocation *wddmAllocation = reinterpret_cast<WddmAllocation *>(physicalAllocation);
wddmAllocation->mappedPhysicalMemoryReservation = false;
wddmAllocation->setMappedPhysicalMemoryReservation(false);
}
bool WddmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) {
WddmAllocation *wddmAllocation = reinterpret_cast<WddmAllocation *>(physicalAllocation);
auto decanonizedAddress = getGmmHelper(physicalAllocation->getRootDeviceIndex())->decanonize(gpuRange);
wddmAllocation->mappedPhysicalMemoryReservation = mapGpuVirtualAddress(wddmAllocation, reinterpret_cast<void *>(decanonizedAddress));
wddmAllocation->setMappedPhysicalMemoryReservation(mapGpuVirtualAddress(wddmAllocation, reinterpret_cast<void *>(decanonizedAddress)));
physicalAllocation->setCpuPtrAndGpuAddress(nullptr, gpuRange);
physicalAllocation->setReservedAddressRange(reinterpret_cast<void *>(gpuRange), bufferSize);
return wddmAllocation->mappedPhysicalMemoryReservation;
return wddmAllocation->isMappedPhysicalMemoryReservation();
}
GraphicsAllocation *WddmMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) {
@@ -239,7 +239,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryUsingKmdAndMapItToC
return nullptr;
}
auto cpuPtr = gmm->isCompressionEnabled ? nullptr : lockResource(wddmAllocation.get());
auto cpuPtr = gmm->isCompressionEnabled() ? nullptr : lockResource(wddmAllocation.get());
[[maybe_unused]] auto status = true;
@@ -541,7 +541,7 @@ GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const All
wddmAllocation->setDriverAllocatedCpuPtr(pSysMem);
wddmAllocation->set32BitAllocation(true);
wddmAllocation->setAllocationOffset(offset);
wddmAllocation->allocInFrontWindowPool = allocationData.flags.use32BitFrontWindow;
wddmAllocation->setAllocInFrontWindowPool(allocationData.flags.use32BitFrontWindow);
auto &productHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHelper<ProductHelper>();
auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo();
@@ -668,19 +668,19 @@ void WddmMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *g
void *WddmMemoryManager::lockResourceImpl(GraphicsAllocation &graphicsAllocation) {
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
return getWddm(graphicsAllocation.getRootDeviceIndex()).lockResource(wddmAllocation.getDefaultHandle(), this->peekExecutionEnvironment().rootDeviceEnvironments[wddmAllocation.getRootDeviceIndex()]->getHelper<GfxCoreHelper>().makeResidentBeforeLockNeeded(wddmAllocation.needsMakeResidentBeforeLock), wddmAllocation.getAlignedSize());
return getWddm(graphicsAllocation.getRootDeviceIndex()).lockResource(wddmAllocation.getDefaultHandle(), this->peekExecutionEnvironment().rootDeviceEnvironments[wddmAllocation.getRootDeviceIndex()]->getHelper<GfxCoreHelper>().makeResidentBeforeLockNeeded(wddmAllocation.needsMakeResidentBeforeLock()), wddmAllocation.getAlignedSize());
}
void WddmMemoryManager::unlockResourceImpl(GraphicsAllocation &graphicsAllocation) {
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
getWddm(graphicsAllocation.getRootDeviceIndex()).unlockResource(wddmAllocation.getDefaultHandle());
if (wddmAllocation.needsMakeResidentBeforeLock) {
if (wddmAllocation.needsMakeResidentBeforeLock()) {
[[maybe_unused]] auto evictionStatus = getWddm(graphicsAllocation.getRootDeviceIndex()).getTemporaryResourcesContainer()->evictResource(wddmAllocation.getDefaultHandle());
DEBUG_BREAK_IF(evictionStatus == MemoryOperationsStatus::failed);
}
}
void WddmMemoryManager::freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) {
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
if (wddmAllocation.needsMakeResidentBeforeLock) {
if (wddmAllocation.needsMakeResidentBeforeLock()) {
for (auto i = 0u; i < wddmAllocation.getNumGmms(); i++) {
getWddm(graphicsAllocation.getRootDeviceIndex()).getTemporaryResourcesContainer()->removeResource(wddmAllocation.getHandles()[i]);
}
@@ -747,8 +747,8 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation
input->fragmentsStorage.fragmentCount > 0) {
cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
} else {
if (input->resourceHandle != 0) {
[[maybe_unused]] auto status = tryDeferDeletions(nullptr, 0, input->resourceHandle, gfxAllocation->getRootDeviceIndex());
if (input->getResourceHandle() != 0) {
[[maybe_unused]] auto status = tryDeferDeletions(nullptr, 0, input->getResourceHandle(), gfxAllocation->getRootDeviceIndex());
DEBUG_BREAK_IF(!status);
} else {
for (auto handle : input->getHandles()) {
@@ -762,12 +762,12 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation
if (input->getReservedAddressPtr()) {
releaseReservedCpuAddressRange(input->getReservedAddressPtr(), input->getReservedAddressSize(), gfxAllocation->getRootDeviceIndex());
}
if (input->reservedGpuVirtualAddress) {
getWddm(gfxAllocation->getRootDeviceIndex()).freeGpuVirtualAddress(input->reservedGpuVirtualAddress, input->reservedSizeForGpuVirtualAddress);
if (input->getReservedGpuVirtualAddress()) {
getWddm(gfxAllocation->getRootDeviceIndex()).freeGpuVirtualAddress(input->getReservedGpuVirtualAddressToModify(), input->getReservedSizeForGpuVirtualAddress());
}
if (input->allocInFrontWindowPool) {
auto heapIndex = selectHeap(input, false, is32bit || executionEnvironment.rootDeviceEnvironments[input->getRootDeviceIndex()]->isFullRangeSvm(), input->allocInFrontWindowPool);
if (input->isAllocInFrontWindowPool()) {
auto heapIndex = selectHeap(input, false, is32bit || executionEnvironment.rootDeviceEnvironments[input->getRootDeviceIndex()]->isFullRangeSvm(), input->isAllocInFrontWindowPool());
auto alignedSize = input->getAlignedSize();
auto gfxPartition = getGfxPartition(input->getRootDeviceIndex());
gfxPartition->heapFree(heapIndex, input->getGpuAddress(), alignedSize);
@@ -811,7 +811,7 @@ bool WddmMemoryManager::isMemoryBudgetExhausted() const {
bool WddmMemoryManager::validateAllocation(WddmAllocation *alloc) {
if (alloc == nullptr)
return false;
if (alloc->physicalMemoryReservation && !alloc->mappedPhysicalMemoryReservation) {
if (alloc->isPhysicalMemoryReservation() && !alloc->isMappedPhysicalMemoryReservation()) {
return true;
}
if (alloc->getGpuAddress() == 0u || (alloc->getDefaultHandle() == 0 && alloc->fragmentsStorage.fragmentCount == 0))
@@ -942,7 +942,7 @@ bool WddmMemoryManager::createPhysicalAllocation(WddmAllocation *allocation) {
if (!status) {
return false;
}
allocation->physicalMemoryReservation = true;
allocation->setPhysicalMemoryReservation(true);
return true;
}
@@ -995,15 +995,15 @@ void WddmMemoryManager::freeGpuAddress(AddressRange addressRange, uint32_t rootD
bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocation, const void *preferredGpuVirtualAddress) {
D3DGPU_VIRTUAL_ADDRESS addressToMap = castToUint64(preferredGpuVirtualAddress);
auto heapIndex = selectHeap(allocation, preferredGpuVirtualAddress != nullptr, is32bit || executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm(), allocation->allocInFrontWindowPool);
auto heapIndex = selectHeap(allocation, preferredGpuVirtualAddress != nullptr, is32bit || executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm(), allocation->isAllocInFrontWindowPool());
if (!executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm() && !is32bit) {
addressToMap = 0u;
}
if (allocation->reservedGpuVirtualAddress) {
addressToMap = allocation->reservedGpuVirtualAddress;
if (allocation->getReservedGpuVirtualAddress()) {
addressToMap = allocation->getReservedGpuVirtualAddress();
}
auto gfxPartition = getGfxPartition(allocation->getRootDeviceIndex());
if (allocation->allocInFrontWindowPool) {
if (allocation->isAllocInFrontWindowPool()) {
auto alignedSize = allocation->getAlignedSize();
addressToMap = gfxPartition->heapAllocate(heapIndex, alignedSize);
}
@@ -1017,10 +1017,10 @@ bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocatio
status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), minimumAddress, maximumAddress, addressToMap, allocation->getGpuAddressToModify());
}
if (!status) {
if (allocation->reservedGpuVirtualAddress) {
getWddm(allocation->getRootDeviceIndex()).freeGpuVirtualAddress(allocation->reservedGpuVirtualAddress, allocation->reservedSizeForGpuVirtualAddress);
if (allocation->getReservedGpuVirtualAddress()) {
getWddm(allocation->getRootDeviceIndex()).freeGpuVirtualAddress(allocation->getReservedGpuVirtualAddressToModify(), allocation->getReservedSizeForGpuVirtualAddress());
}
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(&allocation->getHandles()[0], allocation->getNumGmms(), allocation->resourceHandle);
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(&allocation->getHandles()[0], allocation->getNumGmms(), allocation->getResourceHandle());
return false;
}
@@ -1044,13 +1044,13 @@ bool WddmMemoryManager::mapMultiHandleAllocationWithRetry(WddmAllocation *alloca
addressToMap = castToUint64(preferredGpuVirtualAddress);
allocation->getGpuAddressToModify() = addressToMap;
} else {
allocation->reservedSizeForGpuVirtualAddress = alignUp(alignedSize, MemoryConstants::pageSize64k);
allocation->setReservedSizeForGpuVirtualAddress(alignUp(alignedSize, MemoryConstants::pageSize64k));
auto status = wddm.reserveGpuVirtualAddress(0ull, gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex),
allocation->reservedSizeForGpuVirtualAddress, &allocation->reservedGpuVirtualAddress);
allocation->getReservedSizeForGpuVirtualAddress(), &allocation->getReservedGpuVirtualAddressToModify());
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
auto gmmHelper = getGmmHelper(allocation->getRootDeviceIndex());
allocation->getGpuAddressToModify() = gmmHelper->canonize(allocation->reservedGpuVirtualAddress);
addressToMap = allocation->reservedGpuVirtualAddress;
allocation->getGpuAddressToModify() = gmmHelper->canonize(allocation->getReservedGpuVirtualAddress());
addressToMap = allocation->getReservedGpuVirtualAddress();
}
for (auto currentHandle = 0u; currentHandle < allocation->getNumGmms(); currentHandle++) {
@@ -1064,10 +1064,10 @@ bool WddmMemoryManager::mapMultiHandleAllocationWithRetry(WddmAllocation *alloca
gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, gpuAddress);
}
if (!status) {
if (allocation->reservedGpuVirtualAddress) {
wddm.freeGpuVirtualAddress(allocation->reservedGpuVirtualAddress, allocation->reservedSizeForGpuVirtualAddress);
if (allocation->getReservedGpuVirtualAddress()) {
wddm.freeGpuVirtualAddress(allocation->getReservedGpuVirtualAddressToModify(), allocation->getReservedSizeForGpuVirtualAddress());
}
wddm.destroyAllocations(&allocation->getHandles()[0], allocation->getNumGmms(), allocation->resourceHandle);
wddm.destroyAllocations(&allocation->getHandles()[0], allocation->getNumGmms(), allocation->getResourceHandle());
return false;
}
gpuAddress = getGmmHelper(allocation->getRootDeviceIndex())->decanonize(gpuAddress);
@@ -1080,13 +1080,13 @@ bool WddmMemoryManager::mapMultiHandleAllocationWithRetry(WddmAllocation *alloca
bool WddmMemoryManager::createGpuAllocationsWithRetry(WddmAllocation *allocation) {
for (auto handleId = 0u; handleId < allocation->getNumGmms(); handleId++) {
auto gmm = allocation->getGmm(handleId);
auto status = getWddm(allocation->getRootDeviceIndex()).createAllocation(gmm->gmmResourceInfo->getSystemMemPointer(), gmm, allocation->getHandleToModify(handleId), allocation->resourceHandle, allocation->getSharedHandleToModify());
auto status = getWddm(allocation->getRootDeviceIndex()).createAllocation(gmm->gmmResourceInfo->getSystemMemPointer(), gmm, allocation->getHandleToModify(handleId), allocation->getResourceHandleToModify(), allocation->getSharedHandleToModify());
if (status == STATUS_GRAPHICS_NO_VIDEO_MEMORY && deferredDeleter) {
deferredDeleter->drain(true);
status = getWddm(allocation->getRootDeviceIndex()).createAllocation(gmm->gmmResourceInfo->getSystemMemPointer(), gmm, allocation->getHandleToModify(handleId), allocation->resourceHandle, allocation->getSharedHandleToModify());
status = getWddm(allocation->getRootDeviceIndex()).createAllocation(gmm->gmmResourceInfo->getSystemMemPointer(), gmm, allocation->getHandleToModify(handleId), allocation->getResourceHandleToModify(), allocation->getSharedHandleToModify());
}
if (status != STATUS_SUCCESS) {
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(&allocation->getHandles()[0], handleId, allocation->resourceHandle);
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(&allocation->getHandles()[0], handleId, allocation->getResourceHandle());
return false;
}
}
@@ -1197,7 +1197,7 @@ bool WddmMemoryManager::copyMemoryToAllocationBanks(GraphicsAllocation *graphics
if (!handleMask.test(handleId)) {
continue;
}
auto ptr = wddm.lockResource(wddmAllocation->getHandles()[handleId], wddmAllocation->needsMakeResidentBeforeLock, wddmAllocation->getAlignedSize());
auto ptr = wddm.lockResource(wddmAllocation->getHandles()[handleId], wddmAllocation->needsMakeResidentBeforeLock(), wddmAllocation->getAlignedSize());
if (!ptr) {
return false;
}
@@ -1335,7 +1335,7 @@ GraphicsAllocation *WddmMemoryManager::allocatePhysicalLocalDeviceMemory(const A
}
wddmAllocation->storageInfo = allocationData.storageInfo;
wddmAllocation->setFlushL3Required(allocationData.flags.flushL3);
wddmAllocation->needsMakeResidentBeforeLock = true;
wddmAllocation->setMakeResidentBeforeLockRequired(true);
auto &wddm = getWddm(allocationData.rootDeviceIndex);
@@ -1421,9 +1421,9 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const
}
wddmAllocation->storageInfo = allocationData.storageInfo;
wddmAllocation->setFlushL3Required(allocationData.flags.flushL3);
wddmAllocation->needsMakeResidentBeforeLock = true;
wddmAllocation->setMakeResidentBeforeLockRequired(true);
if (heapAssigners[allocationData.rootDeviceIndex]->use32BitHeap(allocationData.type)) {
wddmAllocation->allocInFrontWindowPool = allocationData.flags.use32BitFrontWindow;
wddmAllocation->setAllocInFrontWindowPool(allocationData.flags.use32BitFrontWindow);
}
void *requiredGpuVa = nullptr;
@@ -1468,7 +1468,7 @@ bool WddmMemoryManager::mapGpuVirtualAddress(WddmAllocation *allocation, const v
return mapTileInstancedAllocation(allocation,
requiredPtr,
&getWddm(allocation->getRootDeviceIndex()),
selectHeap(allocation, requiredPtr != nullptr, executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm(), allocation->allocInFrontWindowPool),
selectHeap(allocation, requiredPtr != nullptr, executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm(), allocation->isAllocInFrontWindowPool()),
*getGfxPartition(allocation->getRootDeviceIndex()));
} else if (allocation->storageInfo.multiStorage) {
return mapMultiHandleAllocationWithRetry(allocation, requiredPtr);