Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
compute-runtime/runtime/os_interface/windows/wddm_memory_manager.cpp

750 lines
29 KiB
C++
Raw Blame History

/*
* Copyright (c) 2017, Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "runtime/helpers/aligned_memory.h"
#include "runtime/helpers/ptr_math.h"
#include "runtime/gmm_helper/gmm_helper.h"
#include "runtime/gmm_helper/resource_info.h"
#include "runtime/helpers/surface_formats.h"
#include "runtime/memory_manager/deferred_deleter.h"
#include "runtime/memory_manager/deferrable_deletion.h"
#include "runtime/os_interface/windows/wddm_memory_manager.h"
#include "runtime/os_interface/windows/wddm_allocation.h"
#include "runtime/os_interface/windows/wddm.h"
#include <algorithm>
#pragma warning(push)
#pragma warning(disable : 4005)
#include <ntstatus.h>
#pragma warning(pop)
#undef max
namespace OCLRT {
WddmMemoryManager::~WddmMemoryManager() {
applyCommonCleanup();
delete this->wddm;
}
WddmMemoryManager::WddmMemoryManager(bool enable64kbPages, Wddm *wddm) : MemoryManager(enable64kbPages), residencyLock(false) {
DEBUG_BREAK_IF(wddm == nullptr);
this->wddm = wddm;
allocator32Bit = std::unique_ptr<Allocator32bit>(new Allocator32bit(wddm->getHeap32Base(), wddm->getHeap32Size()));
wddm->registerTrimCallback(trimCallback, this);
asyncDeleterEnabled = DebugManager.flags.EnableDeferredDeleter.get();
if (asyncDeleterEnabled)
deferredDeleter = createDeferredDeleter();
}
void APIENTRY WddmMemoryManager::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) {
WddmMemoryManager *wddmMemMngr = (WddmMemoryManager *)trimNotification->Context;
DEBUG_BREAK_IF(wddmMemMngr == nullptr);
wddmMemMngr->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) {
if (!Gmm::allowTiling(*imgInfo.imgDesc)) {
delete gmm;
return allocateGraphicsMemory(imgInfo.size, MemoryConstants::preferredAlignment);
}
WddmAllocation allocation(nullptr, imgInfo.size);
allocation.gmm = gmm;
auto status = wddm->createAllocation(&allocation);
if (status == STATUS_GRAPHICS_NO_VIDEO_MEMORY && deferredDeleter) {
deferredDeleter->drain(true);
status = wddm->createAllocation(&allocation);
}
allocation.setGpuAddress(allocation.gpuPtr);
if (status == STATUS_SUCCESS) {
auto *wddmAllocation = new WddmAllocation(allocation);
return wddmAllocation;
}
return nullptr;
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory64kb(size_t size, size_t alignment, bool forcePin) {
size_t sizeAligned = alignUp(size, MemoryConstants::pageSize64k);
bool success = true;
Gmm *gmm = nullptr;
WddmAllocation allocation(nullptr, sizeAligned, nullptr, sizeAligned);
gmm = Gmm::create(nullptr, sizeAligned, false);
while (success) {
allocation.gmm = gmm;
success = wddm->createAllocation64k(&allocation);
if (!success)
break;
auto *wddmAllocation = new WddmAllocation(allocation);
auto cpuPtr = lockResource(wddmAllocation);
wddmAllocation->setLocked(true);
wddmAllocation->setAlignedCpuPtr(cpuPtr);
// 64kb map is not needed
wddm->mapGpuVirtualAddress(wddmAllocation, cpuPtr, sizeAligned, false, false);
wddmAllocation->setCpuPtrAndGpuAddress(cpuPtr, (uint64_t)wddmAllocation->gpuPtr);
return wddmAllocation;
}
delete gmm;
return nullptr;
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory(size_t size, size_t alignment, bool forcePin, bool uncacheable) {
size_t newAlignment = alignment ? alignUp(alignment, MemoryConstants::pageSize) : MemoryConstants::pageSize;
size_t sizeAligned = size ? alignUp(size, MemoryConstants::pageSize) : MemoryConstants::pageSize;
void *pSysMem = allocateSystemMemory(sizeAligned, newAlignment);
bool success = true;
Gmm *gmm = nullptr;
if (pSysMem == nullptr) {
return nullptr;
}
WddmAllocation allocation(pSysMem, sizeAligned, pSysMem, sizeAligned);
allocation.cpuPtrAllocated = true;
gmm = Gmm::create(pSysMem, sizeAligned, uncacheable);
while (success) {
allocation.gmm = gmm;
success = wddm->createAllocation(&allocation) == STATUS_SUCCESS;
if (!success)
break;
auto *wddmAllocation = new WddmAllocation(allocation);
return wddmAllocation;
}
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory(size_t size, const void *ptrArg) {
void *ptr = const_cast<void *>(ptrArg);
if (ptr == nullptr) {
DEBUG_BREAK_IF(true);
return nullptr;
}
return MemoryManager::allocateGraphicsMemory(size, ptr);
}
GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemory(size_t size, void *ptr) {
GraphicsAllocation *graphicsAllocation = nullptr;
bool success = true;
Gmm *gmm = nullptr;
const void *ptrAligned = nullptr;
size_t sizeAligned = size;
void *pSysMem = nullptr;
size_t offset = 0;
bool cpuPtrAllocated = false;
if (ptr) {
ptrAligned = alignDown(ptr, MemoryConstants::allocationAlignment);
sizeAligned = alignSizeWholePage(ptr, size);
offset = ptrDiff(ptr, ptrAligned);
} else {
sizeAligned = alignUp(size, MemoryConstants::allocationAlignment);
pSysMem = allocateSystemMemory(sizeAligned, MemoryConstants::allocationAlignment);
if (pSysMem == nullptr) {
return nullptr;
}
ptrAligned = pSysMem;
cpuPtrAllocated = true;
}
WddmAllocation allocation((void *)ptrAligned, sizeAligned, (void *)ptrAligned, sizeAligned);
allocation.cpuPtrAllocated = cpuPtrAllocated;
allocation.is32BitAllocation = true;
gmm = Gmm::create(ptrAligned, sizeAligned, false);
while (success) {
allocation.gmm = gmm;
success = wddm->createAllocation(&allocation) == STATUS_SUCCESS;
if (!success)
break;
allocation.setGpuAddress(allocation.gpuPtr);
allocation.allocationOffset = offset;
auto *wddmAllocation = new WddmAllocation(allocation);
graphicsAllocation = wddmAllocation;
graphicsAllocation->is32BitAllocation = true;
graphicsAllocation->gpuBaseAddress = Gmm::canonize(allocator32Bit->getBase());
return graphicsAllocation;
}
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;
}
GraphicsAllocation *WddmMemoryManager::createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle) {
WddmAllocation allocation(nullptr, 0, handle);
bool is32BitAllocation = false;
if (ntHandle) {
wddm->openNTHandle((HANDLE)((UINT_PTR)handle), &allocation);
} else {
if (wddm->openSharedHandle(handle, &allocation) == false) {
return nullptr;
}
}
// Shared objects are passed without size
size_t size = allocation.gmm->gmmResourceInfo->getSizeAllocation();
allocation.setSize(size);
void *ptr = nullptr;
if (is32bit) {
ptr = (void *)VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE);
} else if (requireSpecificBitness && this->force32bitAllocations) {
is32BitAllocation = true;
allocation.is32BitAllocation = true;
allocation.gpuBaseAddress = Gmm::canonize(allocator32Bit->getBase());
}
wddm->mapGpuVirtualAddress(&allocation, ptr, size, is32BitAllocation, false);
allocation.setGpuAddress(allocation.gpuPtr);
return new WddmAllocation(allocation);
}
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness, bool /*isReused*/) {
return createAllocationFromHandle(handle, requireSpecificBitness, false);
}
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromNTHandle(void *handle) {
return createAllocationFromHandle((osHandle)((UINT_PTR)handle), false, true);
}
void *WddmMemoryManager::lockResource(GraphicsAllocation *graphicsAllocation) {
return wddm->lockResource(static_cast<WddmAllocation *>(graphicsAllocation));
};
void WddmMemoryManager::unlockResource(GraphicsAllocation *graphicsAllocation) {
wddm->unlockResource(static_cast<WddmAllocation *>(graphicsAllocation));
};
void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) {
WddmAllocation *input = static_cast<WddmAllocation *>(gfxAllocation);
auto status = validateAllocation(input);
DEBUG_BREAK_IF(!status);
acquireResidencyLock();
if (input->getTrimCandidateListPosition() != trimListUnusedPosition) {
removeFromTrimCandidateList(gfxAllocation, true);
}
releaseResidencyLock();
if (input->gmm) {
if (input->gmm->isRenderCompressed) {
status = unmapAuxVA(input->gmm, input->gpuPtr);
DEBUG_BREAK_IF(!status);
}
delete input->gmm;
}
if (input->peekSharedHandle() == false &&
input->cpuPtrAllocated == false &&
input->fragmentsStorage.fragmentCount > 0) {
cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
} else {
D3DKMT_HANDLE *allocationHandles = nullptr;
uint32_t allocationCount = 0;
D3DKMT_HANDLE resourceHandle = 0;
void *cpuPtr = nullptr;
void *gpuPtr = nullptr;
if (input->peekSharedHandle()) {
resourceHandle = input->resourceHandle;
if (is32bit) {
gpuPtr = (void *)input->gpuPtr;
}
} else {
allocationHandles = &input->handle;
allocationCount = 1;
if (input->cpuPtrAllocated) {
cpuPtr = input->getAlignedCpuPtr();
}
}
if (input->isLocked()) {
unlockResource(input);
input->setLocked(false);
}
status = tryDeferDeletions(allocationHandles, allocationCount, input->getResidencyData().lastFence, resourceHandle, cpuPtr, gpuPtr);
DEBUG_BREAK_IF(!status);
}
delete gfxAllocation;
}
bool WddmMemoryManager::tryDeferDeletions(D3DKMT_HANDLE *handles, uint32_t allocationCount, uint64_t lastFenceValue, D3DKMT_HANDLE resourceHandle, void *cpuPtr, void *gpuPtr) {
bool status = true;
if (deferredDeleter) {
deferredDeleter->deferDeletion(DeferrableDeletion::create(wddm, handles, allocationCount, lastFenceValue, resourceHandle, cpuPtr, gpuPtr));
} else {
status = wddm->destroyAllocations(handles, allocationCount, lastFenceValue, resourceHandle);
::alignedFree(cpuPtr);
wddm->releaseGpuPtr(gpuPtr);
}
return status;
}
bool WddmMemoryManager::validateAllocation(WddmAllocation *alloc) {
if (alloc == nullptr)
return false;
auto size = alloc->getUnderlyingBufferSize();
if (alloc->getGpuAddress() == 0u || size == 0 || (alloc->handle == 0 && alloc->fragmentsStorage.fragmentCount == 0))
return false;
return true;
}
bool WddmMemoryManager::populateOsHandles(OsHandleStorage &handleStorage) {
for (unsigned int i = 0; i < max_fragments_count; i++) {
// If no fragment is present it means it already exists.
if (!handleStorage.fragmentStorageData[i].osHandleStorage && handleStorage.fragmentStorageData[i].cpuPtr) {
handleStorage.fragmentStorageData[i].osHandleStorage = new OsHandle();
handleStorage.fragmentStorageData[i].residency = new ResidencyData();
handleStorage.fragmentStorageData[i].osHandleStorage->gmm = Gmm::create(handleStorage.fragmentStorageData[i].cpuPtr, handleStorage.fragmentStorageData[i].fragmentSize, false);
hostPtrManager.storeFragment(handleStorage.fragmentStorageData[i]);
}
}
wddm->createAllocationsAndMapGpuVa(handleStorage);
return true;
}
void WddmMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage) {
D3DKMT_HANDLE handles[max_fragments_count] = {0};
auto allocationCount = 0;
uint64_t lastFenceValue = 0;
for (unsigned int i = 0; i < max_fragments_count; i++) {
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
handles[allocationCount] = handleStorage.fragmentStorageData[i].osHandleStorage->handle;
handleStorage.fragmentStorageData[i].residency->resident = false;
allocationCount++;
lastFenceValue = std::max(handleStorage.fragmentStorageData[i].residency->lastFence, lastFenceValue);
}
}
bool success = tryDeferDeletions(handles, allocationCount, lastFenceValue, 0, nullptr, nullptr);
for (unsigned int i = 0; i < max_fragments_count; i++) {
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
if (success) {
handleStorage.fragmentStorageData[i].osHandleStorage->handle = 0;
}
delete handleStorage.fragmentStorageData[i].osHandleStorage->gmm;
delete handleStorage.fragmentStorageData[i].osHandleStorage;
delete handleStorage.fragmentStorageData[i].residency;
}
}
}
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, size_t hostPtrSize, const void *hostPtr) {
auto allocation = new WddmAllocation(const_cast<void *>(hostPtr), hostPtrSize, const_cast<void *>(hostPtr), hostPtrSize);
allocation->fragmentsStorage = handleStorage;
return allocation;
}
uint64_t WddmMemoryManager::getSystemSharedMemory() {
return wddm->getSystemSharedMemory();
}
uint64_t WddmMemoryManager::getMaxApplicationAddress() {
return wddm->getMaxApplicationAddress();
}
bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer *allocationsForResidency) {
auto &residencyAllocations = allocationsForResidency ? *allocationsForResidency : this->residencyAllocations;
size_t residencyCount = residencyAllocations.size();
std::unique_ptr<D3DKMT_HANDLE[]> handlesForResidency(new D3DKMT_HANDLE[residencyCount * max_fragments_count]);
uint32_t totalHandlesCount = 0;
acquireResidencyLock();
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", wddm->getMonitoredFence().currentFenceValue);
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = reinterpret_cast<WddmAllocation *>(residencyAllocations[i]);
bool mainResidency = false;
bool fragmentResidency[3] = {false, false, false};
mainResidency = allocation->getResidencyData().resident;
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, mainResidency ? "resident" : "not resident");
if (allocation->getTrimCandidateListPosition() != trimListUnusedPosition) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, "on trimCandidateList");
removeFromTrimCandidateList(allocation);
} else {
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
fragmentResidency[allocationId] = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident;
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "fragment handle =",
allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle,
fragmentResidency[allocationId] ? "resident" : "not resident");
}
}
if (allocation->fragmentsStorage.fragmentCount == 0) {
if (!mainResidency)
handlesForResidency[totalHandlesCount++] = allocation->handle;
} else {
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
if (!fragmentResidency[allocationId])
handlesForResidency[totalHandlesCount++] = allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
}
}
}
bool result = true;
if (totalHandlesCount) {
uint64_t bytesToTrim = 0;
while ((result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) {
this->memoryBudgetExhausted = true;
bool trimmingDone = trimResidencyToBudget(bytesToTrim);
bool cantTrimFurther = !trimmingDone;
if (cantTrimFurther) {
result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim);
break;
}
}
}
if (result == true) {
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = reinterpret_cast<WddmAllocation *>(residencyAllocations[i]);
// Update fence value not to early destroy / evict allocation
allocation->getResidencyData().lastFence = wddm->getMonitoredFence().currentFenceValue;
allocation->getResidencyData().resident = true;
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = true;
// Update fence value not to remove the fragment referenced by different GA in trimming callback
allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->lastFence = wddm->getMonitoredFence().currentFenceValue;
}
}
}
releaseResidencyLock();
return result;
}
void WddmMemoryManager::makeNonResidentEvictionAllocations() {
acquireResidencyLock();
size_t residencyCount = evictionAllocations.size();
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = reinterpret_cast<WddmAllocation *>(evictionAllocations[i]);
addToTrimCandidateList(allocation);
}
releaseResidencyLock();
}
void WddmMemoryManager::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) {
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
size_t position = wddmAllocation->getTrimCandidateListPosition();
DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1)));
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
trimCandidatesCount--;
trimCandidateList[position] = nullptr;
checkTrimCandidateCount();
if (position == trimCandidateList.size() - 1) {
size_t erasePosition = position;
if (position == 0) {
trimCandidateList.resize(0);
} else {
while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) {
erasePosition--;
}
size_t sizeRemaining = erasePosition + 1;
if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) {
sizeRemaining = 0;
}
trimCandidateList.resize(sizeRemaining);
}
}
wddmAllocation->setTrimCandidateListPosition(trimListUnusedPosition);
if (compactList && checkTrimCandidateListCompaction()) {
compactTrimCandidateList();
}
checkTrimCandidateCount();
}
void WddmMemoryManager::addToTrimCandidateList(GraphicsAllocation *allocation) {
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
size_t position = trimCandidateList.size();
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
if (wddmAllocation->getTrimCandidateListPosition() == trimListUnusedPosition) {
trimCandidatesCount++;
trimCandidateList.push_back(allocation);
wddmAllocation->setTrimCandidateListPosition(position);
}
checkTrimCandidateCount();
}
void WddmMemoryManager::compactTrimCandidateList() {
size_t size = trimCandidateList.size();
size_t freePosition = 0;
if (size == 0 || size == trimCandidatesCount) {
return;
}
DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr));
uint32_t previousCount = trimCandidatesCount;
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr)
freePosition++;
for (uint32_t i = 1; i < size; i++) {
if (trimCandidateList[i] != nullptr && freePosition < i) {
trimCandidateList[freePosition] = trimCandidateList[i];
trimCandidateList[i] = nullptr;
((WddmAllocation *)trimCandidateList[freePosition])->setTrimCandidateListPosition(freePosition);
freePosition++;
// Last element was moved, erase elements from freePosition
if (i == size - 1) {
trimCandidateList.resize(freePosition);
}
}
}
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
DEBUG_BREAK_IF(trimCandidatesCount != previousCount);
checkTrimCandidateCount();
}
void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
if (flags.PeriodicTrim) {
bool periodicTrimDone = false;
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
uint64_t sizeToTrim = 0;
acquireResidencyLock();
size_t size = trimCandidateList.size();
WddmAllocation *wddmAllocation = nullptr;
while ((wddmAllocation = getTrimCandidateHead()) != nullptr) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastPeriodicTrimFenceValue);
// allocation was not used from last periodic trim
if ((wddmAllocation)->getResidencyData().lastFence <= lastPeriodicTrimFenceValue) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().lastFence);
size_t fragmentsSizeToEvict = 0;
uint32_t fragmentsToEvict = 0;
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().lastFence);
wddm->evict(&wddmAllocation->handle, 1, sizeToTrim);
}
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->lastFence <= lastPeriodicTrimFenceValue) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->lastFence);
fragmentEvictHandles[fragmentsToEvict++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = false;
}
}
if (fragmentsToEvict != 0) {
wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
}
wddmAllocation->getResidencyData().resident = false;
removeFromTrimCandidateList(wddmAllocation);
} else {
periodicTrimDone = true;
break;
}
}
if (checkTrimCandidateListCompaction()) {
compactTrimCandidateList();
}
releaseResidencyLock();
}
if (flags.TrimToBudget) {
acquireResidencyLock();
trimResidencyToBudget(bytes);
releaseResidencyLock();
}
if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
lastPeriodicTrimFenceValue = *wddm->getMonitoredFence().cpuAddress;
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastPeriodicTrimFenceValue);
}
}
void WddmMemoryManager::checkTrimCandidateCount() {
if (DebugManager.flags.ResidencyDebugEnable.get()) {
uint32_t sum = 0;
for (size_t i = 0; i < trimCandidateList.size(); i++) {
if (trimCandidateList[i] != nullptr) {
sum++;
}
}
DEBUG_BREAK_IF(sum != trimCandidatesCount);
}
}
bool WddmMemoryManager::checkTrimCandidateListCompaction() {
if (2 * trimCandidatesCount <= trimCandidateList.size()) {
return true;
}
return false;
}
bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) {
bool trimToBudgetDone = false;
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
uint64_t numberOfBytesToTrim = bytes;
WddmAllocation *wddmAllocation = nullptr;
trimToBudgetDone = (numberOfBytesToTrim == 0);
while (!trimToBudgetDone) {
uint64_t lastFence = 0;
wddmAllocation = getTrimCandidateHead();
if (wddmAllocation == nullptr) {
break;
}
lastFence = wddmAllocation->getResidencyData().lastFence;
if (lastFence <= wddm->getMonitoredFence().lastSubmittedFence) {
uint32_t fragmentsToEvict = 0;
uint64_t sizeEvicted = 0;
uint64_t sizeToTrim = 0;
if (lastFence > *wddm->getMonitoredFence().cpuAddress) {
wddm->waitFromCpu(lastFence);
}
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
wddm->evict(&wddmAllocation->handle, 1, sizeToTrim);
sizeEvicted = wddmAllocation->getUnderlyingBufferSize();
} else {
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->lastFence <= wddm->getMonitoredFence().lastSubmittedFence) {
fragmentEvictHandles[fragmentsToEvict++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
}
}
if (fragmentsToEvict != 0) {
wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->lastFence <= wddm->getMonitoredFence().lastSubmittedFence) {
wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = false;
sizeEvicted += wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].fragmentSize;
}
}
}
}
if (sizeEvicted >= numberOfBytesToTrim) {
numberOfBytesToTrim = 0;
} else {
numberOfBytesToTrim -= sizeEvicted;
}
wddmAllocation->getResidencyData().resident = false;
removeFromTrimCandidateList(wddmAllocation);
trimToBudgetDone = (numberOfBytesToTrim == 0);
} else {
trimToBudgetDone = true;
}
}
if (bytes > numberOfBytesToTrim && checkTrimCandidateListCompaction()) {
compactTrimCandidateList();
}
return numberOfBytesToTrim == 0;
}
bool WddmMemoryManager::unmapAuxVA(Gmm *gmm, D3DGPU_VIRTUAL_ADDRESS &gpuVA) {
GMM_DDI_UPDATEAUXTABLE ddiUpdateAuxTable = {};
ddiUpdateAuxTable.BaseGpuVA = gpuVA;
ddiUpdateAuxTable.BaseResInfo = gmm->gmmResourceInfo->peekHandle();
ddiUpdateAuxTable.DoNotWait = true;
ddiUpdateAuxTable.Map = false;
return wddm->updateAuxTable(ddiUpdateAuxTable);
}
} // namespace OCLRT
Reference in New Issue View Git Blame Copy Permalink