performance: remove trim candidate list

Related-To: NEO-11755

Removing trim candidate list reduces overhead
caused by residency handling. Allocations required
for eviction are placed in eviction container managed
by CSR.

Signed-off-by: Szymon Morek <szymon.morek@intel.com>
This commit is contained in:
Szymon Morek
2024-08-22 10:25:49 +00:00
committed by Compute-Runtime-Automation
parent 6455d4648c
commit b8f181d50e
24 changed files with 184 additions and 599 deletions

View File

@@ -35,7 +35,7 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
void makeResidentExternal(AllocationView &allocationView);
void makeNonResidentExternal(uint64_t gpuAddress);

View File

@@ -756,7 +756,7 @@ bool AUBCommandStreamReceiverHw<GfxFamily>::expectMemory(const void *gfxAddress,
}
template <typename GfxFamily>
SubmissionStatus AUBCommandStreamReceiverHw<GfxFamily>::processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) {
SubmissionStatus AUBCommandStreamReceiverHw<GfxFamily>::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) {
if (subCaptureManager->isSubCaptureMode()) {
if (!subCaptureManager->isSubCaptureEnabled()) {
return SubmissionStatus::success;

View File

@@ -99,6 +99,7 @@ CommandStreamReceiver::CommandStreamReceiver(ExecutionEnvironment &executionEnvi
auto &compilerProductHelper = rootDeviceEnvironment.getHelper<CompilerProductHelper>();
this->heaplessModeEnabled = compilerProductHelper.isHeaplessModeEnabled();
this->evictionAllocations.reserve(2 * MemoryConstants::kiloByte);
}
CommandStreamReceiver::~CommandStreamReceiver() {
@@ -182,7 +183,7 @@ void CommandStreamReceiver::processEviction() {
void CommandStreamReceiver::makeNonResident(GraphicsAllocation &gfxAllocation) {
if (gfxAllocation.isResident(osContext->getContextId())) {
if (gfxAllocation.peekEvictable() && !gfxAllocation.isAlwaysResident(osContext->getContextId())) {
this->getEvictionAllocations().push_back(&gfxAllocation);
this->addToEvictionContainer(gfxAllocation);
} else {
gfxAllocation.setEvictable(true);
}
@@ -203,7 +204,7 @@ void CommandStreamReceiver::makeSurfacePackNonResident(ResidencyContainer &alloc
this->processEviction();
}
SubmissionStatus CommandStreamReceiver::processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) {
SubmissionStatus CommandStreamReceiver::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) {
return SubmissionStatus::success;
}
@@ -1168,5 +1169,9 @@ void CommandStreamReceiver::ensurePrimaryCsrInitialized(Device &device) {
csrToInitialize->initializeDeviceWithFirstSubmission(device);
}
void CommandStreamReceiver::addToEvictionContainer(GraphicsAllocation &gfxAllocation) {
this->getEvictionAllocations().push_back(&gfxAllocation);
}
std::function<void()> CommandStreamReceiver::debugConfirmationFunction = []() { std::cin.get(); };
} // namespace NEO

View File

@@ -126,7 +126,7 @@ class CommandStreamReceiver {
MOCKABLE_VIRTUAL void makeResident(GraphicsAllocation &gfxAllocation);
virtual void makeNonResident(GraphicsAllocation &gfxAllocation);
MOCKABLE_VIRTUAL void makeSurfacePackNonResident(ResidencyContainer &allocationsForResidency, bool clearAllocations);
virtual SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId);
virtual SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId);
virtual void processEviction();
void makeResidentHostPtrAllocation(GraphicsAllocation *gfxAllocation);
@@ -554,6 +554,8 @@ class CommandStreamReceiver {
bool checkImplicitFlushForGpuIdle();
void downloadTagAllocation(TaskCountType taskCountToWait);
void printTagAddressContent(TaskCountType taskCountToWait, int64_t waitTimeout, bool start);
virtual void addToEvictionContainer(GraphicsAllocation &gfxAllocation);
[[nodiscard]] MOCKABLE_VIRTUAL std::unique_lock<MutexType> obtainHostPtrSurfaceCreationLock();
std::vector<void *> registeredClients;

View File

@@ -50,7 +50,7 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
void removeDownloadAllocation(GraphicsAllocation *alloc) override;
void processEviction() override;
SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
void writeMemory(uint64_t gpuAddress, void *cpuAddress, size_t size, uint32_t memoryBank, uint64_t entryBits) override;
bool writeMemory(GraphicsAllocation &gfxAllocation, bool isChunkCopy, uint64_t gpuVaChunkOffset, size_t chunkSize) override;
void writeMMIO(uint32_t offset, uint32_t value) override;

View File

@@ -532,7 +532,7 @@ void TbxCommandStreamReceiverHw<GfxFamily>::processEviction() {
}
template <typename GfxFamily>
SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) {
SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) {
for (auto &gfxAllocation : allocationsForResidency) {
if (dumpTbxNonWritable) {
this->setTbxWritable(true, *gfxAllocation);

View File

@@ -45,7 +45,7 @@ class DrmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily> {
~DrmCommandStreamReceiver() override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
void makeNonResident(GraphicsAllocation &gfxAllocation) override;
bool waitForFlushStamp(FlushStamp &flushStampToWait) override;
bool isKmdWaitModeActive() override;
@@ -69,7 +69,7 @@ class DrmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily> {
using CommandStreamReceiver::pageTableManager;
protected:
MOCKABLE_VIRTUAL SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency);
MOCKABLE_VIRTUAL SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency);
MOCKABLE_VIRTUAL int exec(const BatchBuffer &batchBuffer, uint32_t vmHandleId, uint32_t drmContextId, uint32_t index);
MOCKABLE_VIRTUAL void readBackAllocation(void *source);
bool isUserFenceWaitActive();

View File

@@ -253,7 +253,7 @@ int DrmCommandStreamReceiver<GfxFamily>::exec(const BatchBuffer &batchBuffer, ui
}
template <typename GfxFamily>
SubmissionStatus DrmCommandStreamReceiver<GfxFamily>::processResidency(const ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) {
SubmissionStatus DrmCommandStreamReceiver<GfxFamily>::processResidency(ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) {
if (drm->isVmBindAvailable()) {
return SubmissionStatus::success;
}
@@ -325,7 +325,7 @@ bool DrmCommandStreamReceiver<GfxFamily>::waitForFlushStamp(FlushStamp &flushSta
}
template <typename GfxFamily>
SubmissionStatus DrmCommandStreamReceiver<GfxFamily>::flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) {
SubmissionStatus DrmCommandStreamReceiver<GfxFamily>::flushInternal(const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
if (drm->useVMBindImmediate()) {
auto osContextLinux = static_cast<OsContextLinux *>(this->osContext);
osContextLinux->waitForPagingFence();

View File

@@ -5,6 +5,7 @@
*
*/
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/os_interface/windows/gdi_interface.h"
#include "shared/source/os_interface/windows/wddm/wddm.h"
#include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h"
@@ -46,6 +47,7 @@ void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, V
void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) {
auto residencyController = static_cast<WddmResidencyController *>(trimNotification->Context);
DEBUG_BREAK_IF(residencyController == nullptr);
UNRECOVERABLE_IF(residencyController->csr == nullptr);
auto lock = residencyController->acquireTrimCallbackLock();
residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
@@ -57,16 +59,19 @@ void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS
if (flags.PeriodicTrim) {
uint64_t sizeToTrim = 0;
handlesToEvict.clear();
handlesToEvict.reserve(trimCandidatesCount);
auto lock = this->acquireLock();
WddmAllocation *wddmAllocation = nullptr;
while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
auto &allocations = csr->getEvictionAllocations();
handlesToEvict.clear();
handlesToEvict.reserve(allocations.size());
for (auto allocationIter = allocations.begin(); allocationIter != allocations.end();) {
wddmAllocation = reinterpret_cast<WddmAllocation *>(*allocationIter);
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastTrimFenceValue);
if (wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) {
break;
allocationIter++;
continue;
}
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
@@ -89,12 +94,9 @@ void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId));
wddmAllocation->getResidencyData().resident[osContextId] = false;
this->removeFromTrimCandidateList(wddmAllocation, false);
allocationIter = allocations.erase(allocationIter);
}
if (this->checkTrimCandidateListCompaction()) {
this->compactTrimCandidateList();
}
lock.unlock();
this->wddm.evict(handlesToEvict.data(), static_cast<uint32_t>(handlesToEvict.size()), sizeToTrim, false);
}
@@ -116,16 +118,20 @@ bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes, std::unique_
uint64_t sizeToTrim = 0;
uint64_t numberOfBytesToTrim = bytes;
WddmAllocation *wddmAllocation = nullptr;
handlesToEvict.clear();
handlesToEvict.reserve(trimCandidatesCount);
perfLogResidencyTrimToBudget(wddm.getResidencyLogger(), bytes, this);
while (numberOfBytesToTrim > 0 && (wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
auto &allocations = csr->getEvictionAllocations();
auto allocationIter = allocations.begin();
handlesToEvict.clear();
handlesToEvict.reserve(allocations.size());
while (numberOfBytesToTrim > 0 && allocationIter != allocations.end()) {
wddmAllocation = reinterpret_cast<WddmAllocation *>(*allocationIter);
uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId);
auto &monitoredFence = this->getMonitoredFence();
if (lastFence > monitoredFence.lastSubmittedFence) {
break;
allocationIter++;
continue;
}
uint64_t sizeEvicted = 0;
@@ -158,11 +164,7 @@ bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes, std::unique_
}
wddmAllocation->getResidencyData().resident[osContextId] = false;
this->removeFromTrimCandidateList(wddmAllocation, false);
}
if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) {
this->compactTrimCandidateList();
allocationIter = allocations.erase(allocationIter);
}
lock.unlock();

View File

@@ -21,8 +21,6 @@ struct OsHandleWin : OsHandle {
D3DKMT_HANDLE handle = 0;
};
constexpr size_t trimListUnusedPosition = std::numeric_limits<size_t>::max();
class WddmAllocation : public GraphicsAllocation {
public:
struct RegistrationData {
@@ -33,7 +31,7 @@ class WddmAllocation : public GraphicsAllocation {
WddmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t canonizedAddress, size_t sizeIn,
void *reservedAddr, MemoryPool pool, uint32_t shareable, size_t maxOsContextCount)
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, canonizedAddress, 0llu, sizeIn, pool, maxOsContextCount),
trimCandidateListPositions(maxOsContextCount, trimListUnusedPosition), shareable(shareable) {
shareable(shareable) {
reservedAddressRangeInfo.addressPtr = reservedAddr;
reservedAddressRangeInfo.rangeSize = sizeIn;
handles.resize(gmms.size());
@@ -41,8 +39,7 @@ class WddmAllocation : public GraphicsAllocation {
WddmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn,
osHandle sharedHandle, MemoryPool pool, size_t maxOsContextCount, uint64_t canonizedGpuAddress)
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, sizeIn, sharedHandle, pool, maxOsContextCount, canonizedGpuAddress),
trimCandidateListPositions(maxOsContextCount, trimListUnusedPosition) {
: GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, sizeIn, sharedHandle, pool, maxOsContextCount, canonizedGpuAddress) {
handles.resize(gmms.size());
}
@@ -81,17 +78,6 @@ class WddmAllocation : public GraphicsAllocation {
return nullptr;
}
void setTrimCandidateListPosition(uint32_t osContextId, size_t position) {
trimCandidateListPositions[osContextId] = position;
}
size_t getTrimCandidateListPosition(uint32_t osContextId) const {
if (osContextId < trimCandidateListPositions.size()) {
return trimCandidateListPositions[osContextId];
}
return trimListUnusedPosition;
}
void setGpuAddress(uint64_t graphicsAddress) { this->gpuAddress = graphicsAddress; }
void setCpuAddress(void *cpuPtr) { this->cpuPtr = cpuPtr; }
@@ -129,7 +115,7 @@ class WddmAllocation : public GraphicsAllocation {
}
return ss.str();
}
std::vector<size_t> trimCandidateListPositions;
StackVec<D3DKMT_HANDLE, EngineLimits::maxHandleCount> handles;
D3DKMT_HANDLE resourceHandle = 0u; // used by shared resources
uint32_t shareable = 0u;

View File

@@ -26,7 +26,7 @@ class WddmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily>
~WddmCommandStreamReceiver() override;
SubmissionStatus flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override;
SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) override;
void processEviction() override;
bool waitForFlushStamp(FlushStamp &flushStampToWait) override;
bool isTlbFlushRequiredForStateCacheFlush() override;
@@ -37,11 +37,13 @@ class WddmCommandStreamReceiver : public DeviceCommandStreamReceiver<GfxFamily>
}
GmmPageTableMngr *createPageTableManager() override;
void flushMonitorFence() override;
void setupContext(OsContext &osContext) override;
using CommandStreamReceiver::pageTableManager;
protected:
void kmDafLockAllocations(ResidencyContainer &allocationsForResidency);
void addToEvictionContainer(GraphicsAllocation &gfxAllocation) override;
Wddm *wddm;
COMMAND_BUFFER_HEADER_REC *commandBufferHeader;

View File

@@ -146,14 +146,12 @@ SubmissionStatus WddmCommandStreamReceiver<GfxFamily>::flush(BatchBuffer &batchB
}
template <typename GfxFamily>
SubmissionStatus WddmCommandStreamReceiver<GfxFamily>::processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) {
SubmissionStatus WddmCommandStreamReceiver<GfxFamily>::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) {
return static_cast<OsContextWin *>(this->osContext)->getResidencyController().makeResidentResidencyAllocations(allocationsForResidency, this->requiresBlockingResidencyHandling) ? SubmissionStatus::success : SubmissionStatus::outOfMemory;
}
template <typename GfxFamily>
void WddmCommandStreamReceiver<GfxFamily>::processEviction() {
static_cast<OsContextWin *>(this->osContext)->getResidencyController().makeNonResidentEvictionAllocations(this->getEvictionAllocations());
this->getEvictionAllocations().clear();
}
template <typename GfxFamily>
@@ -217,4 +215,17 @@ CommandStreamReceiver *createWddmDeviceCommandStreamReceiver(bool withAubDump,
}
}
template <typename GfxFamily>
void WddmCommandStreamReceiver<GfxFamily>::setupContext(OsContext &osContext) {
this->osContext = &osContext;
static_cast<OsContextWin *>(this->osContext)->getResidencyController().setCommandStreamReceiver(this);
}
template <typename GfxFamily>
void WddmCommandStreamReceiver<GfxFamily>::addToEvictionContainer(GraphicsAllocation &gfxAllocation) {
// Eviction allocations are shared with trim callback thread.
auto lock = static_cast<OsContextWin *>(this->osContext)->getResidencyController().acquireLock();
this->getEvictionAllocations().push_back(&gfxAllocation);
}
} // namespace NEO

View File

@@ -721,7 +721,11 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation
for (auto &engine : registeredEngines) {
auto &residencyController = static_cast<OsContextWin *>(engine.osContext)->getResidencyController();
auto lock = residencyController.acquireLock();
residencyController.removeFromTrimCandidateListIfUsed(input, true);
auto &evictContainer = engine.commandStreamReceiver->getEvictionAllocations();
auto iter = std::find(evictContainer.begin(), evictContainer.end(), gfxAllocation);
if (iter != evictContainer.end()) {
evictContainer.erase(iter);
}
}
auto defaultGmm = gfxAllocation->getDefaultGmm();

View File

@@ -39,132 +39,6 @@ std::unique_lock<SpinLock> WddmResidencyController::acquireTrimCallbackLock() {
return std::unique_lock<SpinLock>{this->trimCallbackLock};
}
WddmAllocation *WddmResidencyController::getTrimCandidateHead() {
uint32_t i = 0;
const size_t size = trimCandidateList.size();
if (size == 0) {
return nullptr;
}
while ((trimCandidateList[i] == nullptr) && (i < size))
i++;
return static_cast<WddmAllocation *>(trimCandidateList[i]);
}
void WddmResidencyController::addToTrimCandidateList(GraphicsAllocation *allocation) {
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
size_t position = trimCandidateList.size();
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
if (wddmAllocation->getTrimCandidateListPosition(this->osContextId) == trimListUnusedPosition) {
trimCandidatesCount++;
trimCandidateList.push_back(allocation);
wddmAllocation->setTrimCandidateListPosition(this->osContextId, position);
}
checkTrimCandidateCount();
}
void WddmResidencyController::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) {
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
size_t position = wddmAllocation->getTrimCandidateListPosition(this->osContextId);
DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1)));
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
trimCandidatesCount--;
trimCandidateList[position] = nullptr;
checkTrimCandidateCount();
if (position == trimCandidateList.size() - 1) {
size_t erasePosition = position;
if (position == 0) {
trimCandidateList.resize(0);
} else {
while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) {
erasePosition--;
}
size_t sizeRemaining = erasePosition + 1;
if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) {
sizeRemaining = 0;
}
trimCandidateList.resize(sizeRemaining);
}
}
wddmAllocation->setTrimCandidateListPosition(this->osContextId, trimListUnusedPosition);
if (compactList && checkTrimCandidateListCompaction()) {
compactTrimCandidateList();
}
checkTrimCandidateCount();
}
void WddmResidencyController::removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList) {
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
this->removeFromTrimCandidateList(allocation, true);
}
}
void WddmResidencyController::checkTrimCandidateCount() {
if (debugManager.flags.ResidencyDebugEnable.get()) {
[[maybe_unused]] uint32_t sum = 0;
for (auto trimCandidate : trimCandidateList) {
if (trimCandidate != nullptr) {
sum++;
}
}
DEBUG_BREAK_IF(sum != trimCandidatesCount);
}
}
bool WddmResidencyController::checkTrimCandidateListCompaction() {
return 2 * trimCandidatesCount <= trimCandidateList.size();
}
void WddmResidencyController::compactTrimCandidateList() {
size_t size = trimCandidateList.size();
size_t freePosition = 0;
if (size == 0 || size == trimCandidatesCount) {
return;
}
DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr));
[[maybe_unused]] uint32_t previousCount = trimCandidatesCount;
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr)
freePosition++;
for (uint32_t i = 1; i < size; i++) {
if (trimCandidateList[i] != nullptr && freePosition < i) {
trimCandidateList[freePosition] = trimCandidateList[i];
trimCandidateList[i] = nullptr;
static_cast<WddmAllocation *>(trimCandidateList[freePosition])->setTrimCandidateListPosition(this->osContextId, freePosition);
freePosition++;
// Last element was moved, erase elements from freePosition
if (i == size - 1) {
trimCandidateList.resize(freePosition);
}
}
}
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
DEBUG_BREAK_IF(trimCandidatesCount != previousCount);
checkTrimCandidateCount();
}
void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress) {
monitoredFence.lastSubmittedFence = 0;
monitoredFence.currentFenceValue = 1;
@@ -180,17 +54,18 @@ void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, u
* @param[out] requiresBlockingResidencyHandling flag indicating whether wait for paging fence must be handled in user thread.
* Setting to false means that it can be handled in background thread, which will signal semaphore after paging fence reaches required value.
*
* @note This method skips allocations which are already resident.
* @note This method filters allocations which are already resident. After calling this method, passed allocationsForResidency will contain
* only allocations which were not resident before.
*
* @return returns true if all allocations either succeeded or are pending to be resident
*/
bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling) {
const size_t residencyCount = allocationsForResidency.size();
bool WddmResidencyController::makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling) {
constexpr uint32_t stackAllocations = 64;
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount * stackAllocations;
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForResidency;
uint32_t totalHandlesCount = 0;
size_t totalSize = 0;
const auto currentFence = this->getMonitoredFence().currentFenceValue;
requiresBlockingResidencyHandling = false;
if (debugManager.flags.WaitForPagingFenceInController.get() != -1) {
requiresBlockingResidencyHandling = !debugManager.flags.WaitForPagingFenceInController.get();
@@ -199,36 +74,31 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo
auto lock = this->acquireLock();
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", this->getMonitoredFence().currentFenceValue);
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
auto iter = allocationsForResidency.begin();
while (iter != allocationsForResidency.end()) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(*iter);
ResidencyData &residencyData = allocation->getResidencyData();
static constexpr int maxFragments = 3;
bool fragmentResidency[maxFragments] = {false, false, false};
const auto fragmentCount = allocation->fragmentsStorage.fragmentCount;
UNRECOVERABLE_IF(fragmentCount > maxFragments);
totalSize += allocation->getAlignedSize();
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation, gpu address = ", std::hex, allocation->getGpuAddress(), residencyData.resident[osContextId] ? "resident" : "not resident");
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation, gpu address = ", std::hex, allocation->getGpuAddress(), "fence value to reach for eviction = ", currentFence);
const auto fragmentCount = allocation->fragmentsStorage.fragmentCount;
UNRECOVERABLE_IF(fragmentCount > maxFragments);
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation, gpu address = ", std::hex, allocation->getGpuAddress(), "on trimCandidateList");
this->removeFromTrimCandidateList(allocation, false);
} else {
for (uint32_t allocationId = 0; allocationId < fragmentCount; allocationId++) {
fragmentResidency[allocationId] = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident[osContextId];
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "fragment handle =",
static_cast<OsHandleWin *>(allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage)->handle,
fragmentResidency[allocationId] ? "resident" : "not resident");
}
}
allocation->getResidencyData().updateCompletionData(currentFence, this->osContextId);
bool isAlreadyResident = true;
if (fragmentCount > 0) {
for (uint32_t allocationId = 0; allocationId < fragmentCount; allocationId++) {
auto residencyData = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency;
residencyData->updateCompletionData(currentFence, this->osContextId);
if (!fragmentResidency[allocationId]) {
handlesForResidency.push_back(static_cast<OsHandleWin *>(allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage)->handle);
totalHandlesCount++;
requiresBlockingResidencyHandling |= (allocation->getAllocationType() != AllocationType::buffer && allocation->getAllocationType() != AllocationType::bufferHostMemory);
isAlreadyResident = false;
}
}
} else if (!residencyData.resident[osContextId]) {
@@ -236,8 +106,15 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo
handlesForResidency.push_back(allocation->getHandle(gmmId));
totalHandlesCount++;
requiresBlockingResidencyHandling |= (allocation->getAllocationType() != AllocationType::buffer && allocation->getAllocationType() != AllocationType::bufferHostMemory);
isAlreadyResident = false;
}
}
if (isAlreadyResident) {
iter = allocationsForResidency.erase(iter);
} else {
iter++;
}
}
bool result = true;
@@ -261,36 +138,21 @@ bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyCo
}
if (result == true) {
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
// Update fence value not to early destroy / evict allocation
const auto currentFence = this->getMonitoredFence().currentFenceValue;
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation, gpu address = ", std::hex, allocation->getGpuAddress(), "fence value to reach for eviction = ", currentFence);
allocation->getResidencyData().updateCompletionData(currentFence, this->osContextId);
iter = allocationsForResidency.begin();
while (iter != allocationsForResidency.end()) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(*iter);
allocation->getResidencyData().resident[osContextId] = true;
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
auto residencyData = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency;
// Update fence value not to remove the fragment referenced by different GA in trimming callback
residencyData->updateCompletionData(currentFence, this->osContextId);
residencyData->resident[osContextId] = true;
}
iter++;
}
}
return result;
}
void WddmResidencyController::makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations) {
auto lock = this->acquireLock();
const size_t residencyCount = evictionAllocations.size();
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(evictionAllocations[i]);
this->addToTrimCandidateList(allocation);
}
}
bool WddmResidencyController::isInitialized() const {
bool requiresTrimCallbacks = OSInterface::requiresSupportForWddmTrimNotification;
requiresTrimCallbacks = requiresTrimCallbacks && (false == debugManager.flags.DoNotRegisterTrimCallback.get());
@@ -300,4 +162,8 @@ bool WddmResidencyController::isInitialized() const {
return true;
}
void WddmResidencyController::setCommandStreamReceiver(CommandStreamReceiver *csr) {
this->csr = csr;
}
} // namespace NEO

View File

@@ -24,6 +24,7 @@ namespace NEO {
class GraphicsAllocation;
class WddmAllocation;
class Wddm;
class CommandStreamReceiver;
class WddmResidencyController {
public:
@@ -35,19 +36,8 @@ class WddmResidencyController {
[[nodiscard]] MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock();
[[nodiscard]] std::unique_lock<SpinLock> acquireTrimCallbackLock();
WddmAllocation *getTrimCandidateHead();
void addToTrimCandidateList(GraphicsAllocation *allocation);
void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList);
void removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList);
void checkTrimCandidateCount();
MOCKABLE_VIRTUAL bool checkTrimCandidateListCompaction();
void compactTrimCandidateList();
bool wasAllocationUsedSinceLastTrim(uint64_t fenceValue) { return fenceValue > lastTrimFenceValue; }
void updateLastTrimFenceValue() { lastTrimFenceValue = *this->getMonitoredFence().cpuAddress; }
const ResidencyContainer &peekTrimCandidateList() const { return trimCandidateList; }
uint32_t peekTrimCandidatesCount() const { return trimCandidatesCount; }
MonitoredFence &getMonitoredFence() { return monitoredFence; }
void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress);
@@ -60,15 +50,15 @@ class WddmResidencyController {
bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; }
void setMemoryBudgetExhausted() { memoryBudgetExhausted = true; }
bool makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling);
void makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations);
bool makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling);
bool isInitialized() const;
void setCommandStreamReceiver(CommandStreamReceiver *csr);
protected:
MonitoredFence monitoredFence = {};
ResidencyContainer trimCandidateList;
std::vector<D3DKMT_HANDLE> handlesToEvict;
SpinLock lock;
@@ -80,8 +70,9 @@ class WddmResidencyController {
VOID *trimCallbackHandle = nullptr;
uint32_t osContextId;
uint32_t trimCandidatesCount = 0;
bool memoryBudgetExhausted = false;
CommandStreamReceiver *csr;
};
} // namespace NEO

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2023 Intel Corporation
* Copyright (C) 2019-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -77,7 +77,7 @@ void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(void *ptr, CommandStrea
auto &residencyController = static_cast<OsContextWin *>(&csr.getOsContext())->getResidencyController();
auto lock = residencyController.acquireLock();
residencyController.addToTrimCandidateList(allocData->cpuAllocation);
csr.getEvictionAllocations().push_back(allocData->cpuAllocation);
}
}

View File

@@ -77,7 +77,7 @@ class TestedDrmCommandStreamReceiver : public DrmCommandStreamReceiver<GfxFamily
return this->submissionAggregator.get();
}
ADDMETHOD(processResidency, SubmissionStatus, true, SubmissionStatus::success, (const ResidencyContainer &allocationsForResidency, uint32_t handleId), (allocationsForResidency, handleId));
ADDMETHOD(processResidency, SubmissionStatus, true, SubmissionStatus::success, (ResidencyContainer & allocationsForResidency, uint32_t handleId), (allocationsForResidency, handleId));
ADDMETHOD(exec, int, true, 0, (const BatchBuffer &batchBuffer, uint32_t vmHandleId, uint32_t drmContextId, uint32_t index), (batchBuffer, vmHandleId, drmContextId, index));
@@ -117,7 +117,7 @@ class TestedDrmCommandStreamReceiver : public DrmCommandStreamReceiver<GfxFamily
}
}
ADDMETHOD(flushInternal, SubmissionStatus, true, SubmissionStatus::success, (const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency), (batchBuffer, allocationsForResidency));
ADDMETHOD(flushInternal, SubmissionStatus, true, SubmissionStatus::success, (const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency), (batchBuffer, allocationsForResidency));
void readBackAllocation(void *source) override {
latestReadBackAddress = source;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2023 Intel Corporation
* Copyright (C) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -14,22 +14,15 @@ class MockWddmResidencyController : public WddmResidencyController {
using WddmResidencyController::lastTrimFenceValue;
using WddmResidencyController::lock;
using WddmResidencyController::trimCallbackHandle;
using WddmResidencyController::trimCandidateList;
using WddmResidencyController::trimCandidatesCount;
using WddmResidencyController::trimResidency;
using WddmResidencyController::trimResidencyToBudget;
using WddmResidencyController::WddmResidencyController;
uint32_t acquireLockCallCount = 0u;
bool forceTrimCandidateListCompaction = false;
std::unique_lock<SpinLock> acquireLock() override {
acquireLockCallCount++;
return WddmResidencyController::acquireLock();
}
bool checkTrimCandidateListCompaction() override {
return forceTrimCandidateListCompaction || WddmResidencyController::checkTrimCandidateListCompaction();
}
};
} // namespace NEO

View File

@@ -57,7 +57,7 @@ struct MyMockCsr : UltCommandStreamReceiver<DEFAULT_TEST_FAMILY_NAME> {
gfxAllocation.updateResidencyTaskCount(1, osContext->getContextId());
}
SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override {
SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) override {
processResidencyParameterization.wasCalled = true;
processResidencyParameterization.receivedAllocationsForResidency = &allocationsForResidency;
return SubmissionStatus::success;

View File

@@ -1423,7 +1423,7 @@ struct MockMergeResidencyContainerMemoryOperationsHandler : public DrmMemoryOper
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMergeWithResidencyContainerFailsThenFlushReturnsError) {
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return SubmissionStatus::success;
}
};
@@ -1455,7 +1455,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMergeWithResidencyContaine
HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMergeWithResidencyContainerReturnsOutOfMemoryThenFlushReturnsError) {
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
return SubmissionStatus::success;
}
};
@@ -1490,7 +1490,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenNoAllocsInMemoryOperationH
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
auto memoryOperationsInterface = static_cast<MockDrmMemoryOperationsHandler *>(this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface.get());
EXPECT_TRUE(memoryOperationsInterface->mutex.try_lock());
memoryOperationsInterface->mutex.unlock();
@@ -1522,7 +1522,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocsInMemoryOperationHan
struct MockDrmCsr : public DrmCommandStreamReceiver<FamilyType> {
using DrmCommandStreamReceiver<FamilyType>::DrmCommandStreamReceiver;
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override {
SubmissionStatus flushInternal(const BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) override {
auto memoryOperationsInterface = static_cast<MockDrmMemoryOperationsHandler *>(this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface.get());
EXPECT_FALSE(memoryOperationsInterface->mutex.try_lock());
return SubmissionStatus::success;

View File

@@ -363,7 +363,7 @@ class DrmCommandStreamForceTileTest : public ::testing::Test {
: DrmCommandStreamReceiver<GfxFamily>(executionEnvironment, rootDeviceIndex, deviceBitfield, mode), expectedHandleId(inputHandleId) {
}
SubmissionStatus processResidency(const ResidencyContainer &allocationsForResidency, uint32_t handleId) override {
SubmissionStatus processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) override {
EXPECT_EQ(handleId, expectedHandleId);
return DrmCommandStreamReceiver<GfxFamily>::processResidency(allocationsForResidency, handleId);
}
@@ -678,7 +678,7 @@ HWCMDTEST_F(IGFX_XE_HP_CORE, DrmImplicitScalingCommandStreamTest, givenDisabledI
execCalled++;
return 0;
}
SubmissionStatus processResidency(const ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) override {
SubmissionStatus processResidency(ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) override {
EXPECT_EQ(0u, processResidencyCalled);
EXPECT_EQ(0u, handleId);
processResidencyCalled++;
@@ -715,7 +715,7 @@ HWCMDTEST_F(IGFX_XE_HP_CORE, DrmImplicitScalingCommandStreamTest, givenMultiTile
execCalled++;
return 0;
}
SubmissionStatus processResidency(const ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) override {
SubmissionStatus processResidency(ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) override {
EXPECT_EQ(execCalled, handleId);
return SubmissionStatus::success;
}

View File

@@ -687,26 +687,7 @@ TEST_F(WddmCommandStreamTest, WhenMakingNonResidentThenAllocationIsPlacedInEvict
memoryManager->freeGraphicsMemory(commandBuffer);
}
TEST_F(WddmCommandStreamTest, WhenProcessingEvictionThenAllAllocationsArePlacedOnTrimCandidateList) {
GraphicsAllocation *allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
GraphicsAllocation *allocation2 = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
ASSERT_NE(nullptr, allocation);
ASSERT_NE(nullptr, allocation2);
csr->getEvictionAllocations().push_back(allocation);
csr->getEvictionAllocations().push_back(allocation2);
EXPECT_EQ(2u, csr->getEvictionAllocations().size());
csr->processEviction();
EXPECT_EQ(2u, static_cast<OsContextWin &>(csr->getOsContext()).getResidencyController().peekTrimCandidateList().size());
memoryManager->freeGraphicsMemory(allocation);
memoryManager->freeGraphicsMemory(allocation2);
}
TEST_F(WddmCommandStreamTest, WhenProcesssingEvictionThenEvictionAllocationsListIsCleared) {
TEST_F(WddmCommandStreamTest, WhenProcesssingEvictionThenEvictionAllocationsListIsNotCleared) {
GraphicsAllocation *allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize});
ASSERT_NE(nullptr, allocation);
@@ -716,7 +697,7 @@ TEST_F(WddmCommandStreamTest, WhenProcesssingEvictionThenEvictionAllocationsList
csr->processEviction();
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
memoryManager->freeGraphicsMemory(allocation);
}
@@ -888,16 +869,12 @@ HWTEST_TEMPLATED_F(WddmCommandStreamMockGdiTest, WhenMakingResidentThenResidency
EXPECT_EQ(1u, csr->getResidencyAllocations().size());
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
EXPECT_EQ(trimListUnusedPosition, static_cast<WddmAllocation *>(commandBuffer)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
csr->processResidency(csr->getResidencyAllocations(), 0u);
csr->makeSurfacePackNonResident(csr->getResidencyAllocations(), true);
EXPECT_EQ(0u, csr->getResidencyAllocations().size());
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
EXPECT_EQ(0u, static_cast<WddmAllocation *>(commandBuffer)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
memoryManager->freeGraphicsMemory(commandBuffer);
}
@@ -980,12 +957,22 @@ HWTEST_TEMPLATED_F(WddmCommandStreamMockGdiTest, givenRecordedCommandBufferWhenI
EXPECT_TRUE(found);
}
EXPECT_NE(trimListUnusedPosition, static_cast<WddmAllocation *>(tagAllocation)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
EXPECT_NE(trimListUnusedPosition, static_cast<WddmAllocation *>(commandBuffer)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
EXPECT_EQ(trimListUnusedPosition, static_cast<WddmAllocation *>(dshAlloc)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
EXPECT_EQ(trimListUnusedPosition, static_cast<WddmAllocation *>(iohAlloc)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
EXPECT_NE(trimListUnusedPosition, static_cast<WddmAllocation *>(sshAlloc)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
EXPECT_NE(trimListUnusedPosition, static_cast<WddmAllocation *>(csrCommandStream)->getTrimCandidateListPosition(csr->getOsContext().getContextId()));
struct {
GraphicsAllocation *gfxAlloc;
bool expectEviction;
} evictAllocs[] = {
{tagAllocation, true},
{commandBuffer, true},
{sshAlloc, true},
{csrCommandStream, true},
{dshAlloc, false},
{iohAlloc, false}};
for (auto &alloc : evictAllocs) {
// If eviction is required then allocation should be added to container
auto iter = std::find(csr->getEvictionAllocations().begin(), csr->getEvictionAllocations().end(), alloc.gfxAlloc);
EXPECT_EQ(alloc.expectEviction, iter != csr->getEvictionAllocations().end());
}
memoryManager->freeGraphicsMemory(dshAlloc);
memoryManager->freeGraphicsMemory(iohAlloc);

View File

@@ -4281,34 +4281,6 @@ TEST(WddmMemoryManagerTest3, WhenWddmMemoryManagerIsCreatedThenItIsNonAssignable
EXPECT_FALSE(std::is_copy_assignable<WddmMemoryManager>::value);
}
TEST(WddmMemoryManagerTest3, givenAllocationIsTrimCandidateInOneOsContextWhenGettingTrimCandidatePositionThenReturnItsPositionAndUnusedPositionInOtherContexts) {
auto executionEnvironment = std::unique_ptr<ExecutionEnvironment>(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u));
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
MockWddmAllocation allocation(executionEnvironment->rootDeviceEnvironments[0]->getGmmHelper());
MockOsContext osContext(1u, EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_RCS, EngineUsage::regular},
PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo)));
allocation.setTrimCandidateListPosition(osContext.getContextId(), 700u);
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(0u));
EXPECT_EQ(700u, allocation.getTrimCandidateListPosition(1u));
}
TEST(WddmMemoryManagerTest3, givenAllocationCreatedWithOsContextCountOneWhenItIsCreatedThenMaxOsContextCountIsUsedInstead) {
auto executionEnvironment = std::unique_ptr<ExecutionEnvironment>(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u));
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
MockWddmAllocation allocation(executionEnvironment->rootDeviceEnvironments[0]->getGmmHelper());
allocation.setTrimCandidateListPosition(1u, 700u);
EXPECT_EQ(700u, allocation.getTrimCandidateListPosition(1u));
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(0u));
}
TEST(WddmMemoryManagerTest3, givenRequestedContextIdTooLargeWhenGettingTrimCandidateListPositionThenReturnUnusedPosition) {
auto executionEnvironment = std::unique_ptr<ExecutionEnvironment>(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u));
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
MockWddmAllocation allocation(executionEnvironment->rootDeviceEnvironments[0]->getGmmHelper());
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(1u));
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(1000u));
}
TEST(WddmMemoryManagerTest3, givenAllocationTypeWhenPassedToWddmAllocationConstructorThenAllocationTypeIsStored) {
WddmAllocation allocation{0, 1u /*num gmms*/, AllocationType::commandBuffer, nullptr, 0, 0, nullptr, MemoryPool::memoryNull, 0u, 1u};
EXPECT_EQ(AllocationType::commandBuffer, allocation.getAllocationType());

View File

@@ -39,46 +39,27 @@
using namespace NEO;
class MockWddmResidencyController : public WddmResidencyController {
public:
using WddmResidencyController::lastTrimFenceValue;
using WddmResidencyController::trimCallbackHandle;
using WddmResidencyController::trimCandidateList;
using WddmResidencyController::trimCandidatesCount;
using WddmResidencyController::trimResidency;
using WddmResidencyController::trimResidencyToBudget;
using WddmResidencyController::WddmResidencyController;
uint32_t acquireLockCallCount = 0u;
bool forceTrimCandidateListCompaction = false;
std::unique_lock<SpinLock> acquireLock() override {
acquireLockCallCount++;
return WddmResidencyController::acquireLock();
}
bool checkTrimCandidateListCompaction() override {
return forceTrimCandidateListCompaction || WddmResidencyController::checkTrimCandidateListCompaction();
}
};
struct WddmResidencyControllerTest : ::testing::Test {
const uint32_t osContextId = 0u;
void SetUp() override {
executionEnvironment = std::make_unique<MockExecutionEnvironment>();
executionEnvironment->initializeMemoryManager();
rootDeviceEnvironment = std::make_unique<RootDeviceEnvironment>(*executionEnvironment);
wddm = static_cast<WddmMock *>(Wddm::createWddm(nullptr, *rootDeviceEnvironment));
wddm->init();
mockOsContextWin = std::make_unique<MockOsContextWin>(*wddm, 0, osContextId, EngineDescriptorHelper::getDefaultDescriptor());
wddm->getWddmInterface()->createMonitoredFence(*mockOsContextWin);
residencyController = &mockOsContextWin->mockResidencyController;
csr.reset(createCommandStream(*executionEnvironment, 0u, 1));
residencyController->setCommandStreamReceiver(csr.get());
}
std::unique_ptr<MockExecutionEnvironment> executionEnvironment;
std::unique_ptr<RootDeviceEnvironment> rootDeviceEnvironment;
WddmMock *wddm = nullptr;
std::unique_ptr<MockOsContextWin> mockOsContextWin;
std::unique_ptr<CommandStreamReceiver> csr;
NEO::MockWddmResidencyController *residencyController = nullptr;
};
@@ -87,6 +68,7 @@ struct WddmResidencyControllerWithGdiTest : ::testing::Test {
void SetUp() override {
executionEnvironment = std::make_unique<MockExecutionEnvironment>();
executionEnvironment->initializeMemoryManager();
rootDeviceEnvironment = std::make_unique<RootDeviceEnvironment>(*executionEnvironment);
wddm = static_cast<WddmMock *>(Wddm::createWddm(nullptr, *rootDeviceEnvironment));
gdi = new MockGdi();
@@ -95,7 +77,9 @@ struct WddmResidencyControllerWithGdiTest : ::testing::Test {
mockOsContextWin = std::make_unique<MockOsContextWin>(*wddm, 0, osContextId, EngineDescriptorHelper::getDefaultDescriptor());
wddm->getWddmInterface()->createMonitoredFence(*mockOsContextWin);
csr.reset(createCommandStream(*executionEnvironment, 0u, 1));
residencyController = &mockOsContextWin->mockResidencyController;
residencyController->setCommandStreamReceiver(csr.get());
residencyController->registerCallback();
}
@@ -103,6 +87,7 @@ struct WddmResidencyControllerWithGdiTest : ::testing::Test {
std::unique_ptr<RootDeviceEnvironment> rootDeviceEnvironment;
WddmMock *wddm = nullptr;
std::unique_ptr<MockOsContextWin> mockOsContextWin;
std::unique_ptr<CommandStreamReceiver> csr;
NEO::MockWddmResidencyController *residencyController = nullptr;
MockGdi *gdi = nullptr;
};
@@ -123,10 +108,12 @@ struct WddmResidencyControllerWithMockWddmTest : public WddmResidencyControllerT
auto &gfxCoreHelper = executionEnvironment.rootDeviceEnvironments[0]->getHelper<GfxCoreHelper>();
osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0],
preemptionMode));
csr->setupContext(*osContext);
osContext->ensureContextInitialized(false);
osContext->incRefInternal();
residencyController = &static_cast<OsContextWin *>(osContext)->getResidencyController();
residencyController->setCommandStreamReceiver(csr.get());
gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper();
}
@@ -165,6 +152,7 @@ struct WddmResidencyControllerWithGdiAndMemoryManagerTest : ::testing::Test {
osContext->incRefInternal();
residencyController = &static_cast<OsContextWin *>(osContext)->getResidencyController();
residencyController->setCommandStreamReceiver(csr.get());
gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper();
}
@@ -259,232 +247,13 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenWddmResidencyControllerWhenItIsD
EXPECT_EQ(nullptr, gdi->getUnregisterTrimNotificationArg().Handle);
}
TEST_F(WddmResidencyControllerTest, givenUsedAllocationWhenCallingRemoveFromTrimCandidateListIfUsedThenRemoveIt) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation);
residencyController->removeFromTrimCandidateListIfUsed(&allocation, false);
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, givenWddmResidencyControllerWhenIsMemoryExhaustedIsCalledThenReturnCorrectResult) {
EXPECT_FALSE(residencyController->isMemoryBudgetExhausted());
residencyController->setMemoryBudgetExhausted();
EXPECT_TRUE(residencyController->isMemoryBudgetExhausted());
}
TEST_F(WddmResidencyControllerTest, givenUnusedAllocationWhenCallingRemoveFromTrimCandidateListIfUsedThenIgnore) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->removeFromTrimCandidateListIfUsed(&allocation, false);
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, WhenAddingToTrimCandidateListThenAllocationIsPlacedInContainerAndAssignedPosition) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation);
EXPECT_NE(0u, residencyController->trimCandidateList.size());
EXPECT_NE(trimListUnusedPosition, allocation.getTrimCandidateListPosition(osContextId));
size_t position = allocation.getTrimCandidateListPosition(osContextId);
ASSERT_LT(position, residencyController->trimCandidateList.size());
EXPECT_EQ(&allocation, residencyController->trimCandidateList[position]);
}
TEST_F(WddmResidencyControllerTest, WhenAddingToTrimCandidateListThenDoNotInsertAllocationAlreadyOnTheList) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->trimCandidateList.resize(0);
residencyController->addToTrimCandidateList(&allocation);
EXPECT_NE(trimListUnusedPosition, allocation.getTrimCandidateListPosition(osContextId));
size_t position = allocation.getTrimCandidateListPosition(osContextId);
ASSERT_LT(position, residencyController->trimCandidateList.size());
EXPECT_EQ(&allocation, residencyController->trimCandidateList[position]);
size_t previousSize = residencyController->trimCandidateList.size();
residencyController->addToTrimCandidateList(&allocation);
EXPECT_EQ(previousSize, residencyController->trimCandidateList.size());
EXPECT_EQ(position, allocation.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, WhenRemovingFromTrimCandidateListThenUnusedPositionIsAssigned) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation);
residencyController->removeFromTrimCandidateList(&allocation, false);
EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, WhenRemovingFromTrimCandidateListThenAllocationInAssignedPositionIsRemoved) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation);
size_t position = allocation.getTrimCandidateListPosition(osContextId);
residencyController->removeFromTrimCandidateList(&allocation, false);
if (residencyController->trimCandidateList.size() > position) {
EXPECT_NE(&allocation, residencyController->trimCandidateList[position]);
}
}
TEST_F(WddmResidencyControllerTest, GivenOneAllocationWhenRemovingFromTrimCandidateListThenTrimCandidateListIsEmpty) {
MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper());
residencyController->trimCandidateList.resize(0);
residencyController->addToTrimCandidateList(&allocation);
residencyController->removeFromTrimCandidateList(&allocation, false);
EXPECT_EQ(0u, residencyController->trimCandidateList.size());
}
TEST_F(WddmResidencyControllerTest, WhenRemovingFromTrimCandidateListThenLastAllocationAndAllPreviousEmptyEntriesAreRemoved) {
MockWddmAllocation allocation1(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation2(rootDeviceEnvironment->getGmmHelper());
residencyController->trimCandidateList.resize(0);
residencyController->addToTrimCandidateList(&allocation1);
residencyController->trimCandidateList.push_back(nullptr);
residencyController->trimCandidateList.push_back(nullptr);
residencyController->trimCandidateList.push_back(nullptr);
residencyController->addToTrimCandidateList(&allocation2);
EXPECT_EQ(5u, residencyController->trimCandidateList.size());
residencyController->removeFromTrimCandidateList(&allocation2, false);
EXPECT_EQ(1u, residencyController->trimCandidateList.size());
}
TEST_F(WddmResidencyControllerTest, WhenAddingToTrimCandidateListThenSuccessivePositionIsAssigned) {
MockWddmAllocation allocation1(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation2(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation3(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
EXPECT_EQ(3u, residencyController->trimCandidateList.size());
EXPECT_NE(allocation1.getTrimCandidateListPosition(osContextId), allocation2.getTrimCandidateListPosition(osContextId));
EXPECT_NE(allocation2.getTrimCandidateListPosition(osContextId), allocation3.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, GivenAllocationThatIsNotLastWhenRemovingFromTrimCandidateListAndCompactingThenRemoveEntry) {
MockWddmAllocation allocation1(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation2(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation3(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
residencyController->forceTrimCandidateListCompaction = true;
residencyController->removeFromTrimCandidateList(&allocation2, true);
EXPECT_EQ(2u, residencyController->trimCandidateList.size());
EXPECT_EQ(1u, allocation3.getTrimCandidateListPosition(osContextId));
EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, GivenAllocationThatIsNotLastWhenRemovingFromTrimCandidateListThenReplaceWithNullEntry) {
MockWddmAllocation allocation1(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation2(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation3(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
size_t position2 = allocation2.getTrimCandidateListPosition(osContextId);
size_t position3 = allocation3.getTrimCandidateListPosition(osContextId);
residencyController->removeFromTrimCandidateList(&allocation2, false);
EXPECT_EQ(3u, residencyController->trimCandidateList.size());
EXPECT_EQ(2u, position3);
EXPECT_EQ(nullptr, residencyController->trimCandidateList[position2]);
}
TEST_F(WddmResidencyControllerTest, WhenCompactingTrimCandidateListThenInitialNullEntriesAreRemovedAndPositionsAreUpdated) {
MockWddmAllocation allocation1(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation2(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation3(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation4(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
residencyController->addToTrimCandidateList(&allocation4);
allocation3.getTrimCandidateListPosition(osContextId);
allocation4.getTrimCandidateListPosition(osContextId);
residencyController->removeFromTrimCandidateList(&allocation2, false);
residencyController->removeFromTrimCandidateList(&allocation1, false);
EXPECT_EQ(4u, residencyController->trimCandidateList.size());
residencyController->compactTrimCandidateList();
EXPECT_EQ(2u, residencyController->trimCandidateList.size());
EXPECT_EQ(residencyController->trimCandidateList[0], &allocation3);
EXPECT_EQ(0u, allocation3.getTrimCandidateListPosition(osContextId));
EXPECT_EQ(residencyController->trimCandidateList[1], &allocation4);
EXPECT_EQ(1u, allocation4.getTrimCandidateListPosition(osContextId));
}
TEST_F(WddmResidencyControllerTest, WhenCompactingTrimCandidateListThenNonNullEntriesAreNotRemoved) {
MockWddmAllocation allocation1(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation2(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation3(rootDeviceEnvironment->getGmmHelper());
MockWddmAllocation allocation4(rootDeviceEnvironment->getGmmHelper());
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
residencyController->addToTrimCandidateList(&allocation4);
EXPECT_EQ(4u, residencyController->trimCandidateList.size());
residencyController->compactTrimCandidateList();
EXPECT_EQ(4u, residencyController->trimCandidateList.size());
}
TEST_F(WddmResidencyControllerTest, GivenListSizeLessThenDoubleCandidateCountWhenCheckingTrimCandidateListCompactionThenCompactionIsRequired) {
bool comapactionRequired;
residencyController->trimCandidatesCount = 10;
residencyController->trimCandidateList.resize(20);
comapactionRequired = residencyController->checkTrimCandidateListCompaction();
EXPECT_TRUE(comapactionRequired);
residencyController->trimCandidatesCount = 5;
residencyController->trimCandidateList.resize(20);
comapactionRequired = residencyController->checkTrimCandidateListCompaction();
EXPECT_TRUE(comapactionRequired);
residencyController->trimCandidatesCount = 18;
residencyController->trimCandidateList.resize(20);
comapactionRequired = residencyController->checkTrimCandidateListCompaction();
EXPECT_FALSE(comapactionRequired);
}
TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPeriodicTrimWhenTrimResidencyPeriodicTrimIsCalledThenAllocationsAreEvictedMarkedAndRemovedFromTrimCandidateList) {
TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPeriodicTrimWhenTrimResidencyPeriodicTrimIsCalledThenAllocationsAreEvictedMarkedAndRemovedFromEvictionContainer) {
DebugManagerStateRestore restorer{};
debugManager.flags.PlaformSupportEvictIfNecessaryFlag.set(1);
@@ -513,16 +282,15 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPe
wddm->evictResult.called = 0;
wddm->callBaseEvict = true;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim);
// Single evict called
EXPECT_EQ(1u, wddm->evictResult.called);
EXPECT_EQ(1u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary);
// removed from trim candidate list
EXPECT_EQ(0u, residencyController->peekTrimCandidateList().size());
EXPECT_TRUE(csr->getEvictionAllocations().empty());
// marked nonresident
EXPECT_FALSE(allocation1.getResidencyData().resident[osContextId]);
EXPECT_FALSE(allocation2.getResidencyData().resident[osContextId]);
@@ -549,15 +317,14 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenOneUsedAllocationFromPreviousPer
wddm->evictResult.called = 0;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim);
// 1 allocation evicted
EXPECT_EQ(1u, wddm->evictResult.called);
// removed from trim candidate list
EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContextId));
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
// marked nonresident
EXPECT_FALSE(allocation1.getResidencyData().resident[osContextId]);
@@ -606,7 +373,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, givenTripleAllocation
wddm->evictResult.called = 0;
wddm->callBaseEvict = true;
residencyController->addToTrimCandidateList(allocationTriple);
csr->getEvictionAllocations().push_back(allocationTriple);
residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim);
@@ -654,7 +421,6 @@ TEST_F(WddmResidencyControllerWithGdiTest, GivenZeroWhenTrimmingToBudgetThenTrue
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
bool status = residencyController->trimResidencyToBudget(0, lock);
EXPECT_TRUE(status);
}
@@ -681,23 +447,20 @@ TEST_F(WddmResidencyControllerWithGdiTest, WhenTrimmingToBudgetThenAllDoneAlloca
wddm->evictResult.called = 0;
wddm->callBaseEvict = true;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
csr->getEvictionAllocations().push_back(&allocation3);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
residencyController->trimResidencyToBudget(3 * 4096, lock);
EXPECT_EQ(1u, wddm->evictResult.called);
EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary);
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
EXPECT_EQ(1u, residencyController->peekTrimCandidatesCount());
residencyController->compactTrimCandidateList();
EXPECT_EQ(1u, residencyController->peekTrimCandidateList().size());
EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContextId));
EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContextId));
EXPECT_NE(trimListUnusedPosition, allocation3.getTrimCandidateListPosition(osContextId));
auto iter = std::find(csr->getEvictionAllocations().begin(), csr->getEvictionAllocations().end(), &allocation3);
EXPECT_NE(iter, csr->getEvictionAllocations().end());
}
TEST_F(WddmResidencyControllerWithGdiTest, GivenNumBytesToTrimIsNotZeroWhenTrimmingToBudgetThenFalseIsReturned) {
@@ -713,13 +476,14 @@ TEST_F(WddmResidencyControllerWithGdiTest, GivenNumBytesToTrimIsNotZeroWhenTrimm
wddm->evictResult.called = 0;
residencyController->addToTrimCandidateList(&allocation1);
csr->getEvictionAllocations().push_back(&allocation1);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
bool status = residencyController->trimResidencyToBudget(3 * 4096, lock);
EXPECT_EQ(1u, wddm->evictResult.called);
EXPECT_EQ(0u, residencyController->peekTrimCandidateList().size());
EXPECT_TRUE(csr->getEvictionAllocations().empty());
EXPECT_FALSE(status);
}
@@ -747,20 +511,19 @@ TEST_F(WddmResidencyControllerWithGdiTest, GivenNumBytesToTrimIsZeroWhenTrimming
wddm->evictResult.called = 0;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
csr->getEvictionAllocations().push_back(&allocation3);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
bool status = residencyController->trimResidencyToBudget(3 * 4096, lock);
EXPECT_TRUE(status);
EXPECT_EQ(1u, wddm->evictResult.called);
EXPECT_EQ(1u, residencyController->peekTrimCandidateList().size());
EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContextId));
EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContextId));
EXPECT_NE(trimListUnusedPosition, allocation3.getTrimCandidateListPosition(osContextId));
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
auto iter = std::find(csr->getEvictionAllocations().begin(), csr->getEvictionAllocations().end(), &allocation3);
EXPECT_NE(iter, csr->getEvictionAllocations().end());
}
TEST_F(WddmResidencyControllerWithGdiTest, WhenTrimmingToBudgetThenEvictedAllocationIsMarkedNonResident) {
@@ -785,9 +548,10 @@ TEST_F(WddmResidencyControllerWithGdiTest, WhenTrimmingToBudgetThenEvictedAlloca
wddm->evictResult.called = 0;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
csr->getEvictionAllocations().push_back(&allocation3);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
residencyController->trimResidencyToBudget(3 * 4096, lock);
@@ -812,7 +576,7 @@ TEST_F(WddmResidencyControllerWithGdiTest, GivenLastFenceIsGreaterThanMonitoredW
wddm->evictResult.called = 0;
wddm->waitFromCpuResult.called = 0;
residencyController->addToTrimCandidateList(&allocation1);
csr->getEvictionAllocations().push_back(&allocation1);
gdi->getWaitFromCpuArg().hDevice = 0;
std::mutex mtx;
@@ -858,9 +622,9 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenTrimmingToBudgetT
// This should not be evicted
allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->updateCompletionData(2, osContextId);
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(allocationTriple);
residencyController->addToTrimCandidateList(&allocation2);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(allocationTriple);
csr->getEvictionAllocations().push_back(&allocation2);
*residencyController->getMonitoredFence().cpuAddress = 1;
residencyController->getMonitoredFence().lastSubmittedFence = 1;
@@ -868,6 +632,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenTrimmingToBudgetT
wddm->evictResult.called = 0;
wddm->callBaseEvict = true;
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
residencyController->trimResidencyToBudget(3 * 4096, lock);
@@ -914,9 +679,10 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenThreeAllocationsAlignedSizeBigge
wddm->evictResult.called = 0;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
csr->getEvictionAllocations().push_back(&allocation3);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
bool status = residencyController->trimResidencyToBudget(budget, lock);
@@ -1120,8 +886,6 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB
wddm->makeResidentNumberOfBytesToTrim = allocationSize;
wddm->makeResidentResults = {false, true};
residencyController->addToTrimCandidateList(&allocationToTrim);
ResidencyContainer residencyPack{&allocation1};
bool requiresBlockingResidencyHandling = true;
bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling);
@@ -1175,6 +939,7 @@ struct WddmResidencyControllerWithMockWddmMakeResidentTest : public WddmResidenc
osContext->incRefInternal();
residencyController = &static_cast<OsContextWin *>(osContext)->getResidencyController();
residencyController->setCommandStreamReceiver(csr.get());
gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper();
}
};
@@ -1268,15 +1033,14 @@ TEST_F(WddmResidencyControllerWithGdiTest, GivenResidencyLoggingEnabledWhenTrimm
wddm->evictResult.called = 0;
wddm->callBaseEvict = true;
residencyController->addToTrimCandidateList(&allocation1);
residencyController->addToTrimCandidateList(&allocation2);
residencyController->addToTrimCandidateList(&allocation3);
csr->getEvictionAllocations().push_back(&allocation1);
csr->getEvictionAllocations().push_back(&allocation2);
csr->getEvictionAllocations().push_back(&allocation3);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
constexpr uint64_t trimBudgetSize = 3 * 4096;
residencyController->trimResidencyToBudget(trimBudgetSize, lock);
residencyController->compactTrimCandidateList();
EXPECT_EQ(trimBudgetSize, logger->numBytesToTrimSave);
EXPECT_EQ(residencyController, logger->controllerObjectSave);