Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Create WddmResidencyController to encapsulate basic residency data 

Change-Id: I6dd94405ca5ea3a13b137c1e0ead72e615d36e00
Signed-off-by: Maciej Dziuban <maciej.dziuban@intel.com>
This commit is contained in:
Maciej Dziuban 2018-10-04 00:52:51 +02:00 committed by sys_ocldev
parent 88bd89cb58
commit b2dd8a9a1d
9 changed files with 111 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime/os_interface/windows/CMakeLists.txt
View File

@ -55,6 +55,8 @@ set(RUNTIME_SRCS_OS_INTERFACE_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.h
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.h
${CMAKE_CURRENT_SOURCE_DIR}/windows_defs.h
${CMAKE_CURRENT_SOURCE_DIR}/windows_inc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/windows_wrapper.h

53
runtime/os_interface/windows/wddm_memory_manager.cpp
View File

@ -18,6 +18,7 @@
#include "runtime/memory_manager/deferred_deleter.h"
#include "runtime/os_interface/windows/wddm/wddm.h"
#include "runtime/os_interface/windows/wddm_allocation.h"
#include "runtime/os_interface/windows/wddm_residency_controller.h"
#include "runtime/os_interface/windows/os_context_win.h"
#include "runtime/platform/platform.h"
#include <algorithm>
@ -28,7 +29,7 @@ WddmMemoryManager::~WddmMemoryManager() {
applyCommonCleanup();
}
WddmMemoryManager::WddmMemoryManager(bool enable64kbPages, bool enableLocalMemory, Wddm *wddm, ExecutionEnvironment &executionEnvironment) : MemoryManager(enable64kbPages, enableLocalMemory, executionEnvironment), residencyLock(false) {
WddmMemoryManager::WddmMemoryManager(bool enable64kbPages, bool enableLocalMemory, Wddm *wddm, ExecutionEnvironment &executionEnvironment) : MemoryManager(enable64kbPages, enableLocalMemory, executionEnvironment) {
DEBUG_BREAK_IF(wddm == nullptr);
this->wddm = wddm;
allocator32Bit = std::unique_ptr<Allocator32bit>(new Allocator32bit(wddm->getHeap32Base(), wddm->getHeap32Size()));
@ -292,12 +293,22 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation
if (gfxAllocation == nullptr) {
return;
}
acquireResidencyLock();
for (const auto &residencyController : this->residencyControllers) {
if (residencyController) {
residencyController->acquireLock();
}
}
if (input->getTrimCandidateListPosition() != trimListUnusedPosition) {
removeFromTrimCandidateList(gfxAllocation, true);
}
releaseResidencyLock();
for (const auto &residencyController : this->residencyControllers) {
if (residencyController) {
residencyController->releaseLock();
}
}
UNRECOVERABLE_IF(DebugManager.flags.CreateMultipleDevices.get() == 0 &&
gfxAllocation->taskCount != ObjectNotUsed && this->executionEnvironment.commandStreamReceivers.size() > 0 &&
@ -418,7 +429,8 @@ void WddmMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage) {
void WddmMemoryManager::registerOsContext(OsContext *contextToRegister) {
MemoryManager::registerOsContext(contextToRegister);
this->lastPeriodicTrimFenceValues.resize(this->registeredOsContexts.size());
this->residencyControllers.resize(this->getOsContextCount());
this->residencyControllers[contextToRegister->getContextId()] = std::make_unique<WddmResidencyController>();
}
void WddmMemoryManager::obtainGpuAddresFromFragments(WddmAllocation *allocation, OsHandleStorage &handleStorage) {
@ -463,7 +475,7 @@ bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer &all
uint32_t totalHandlesCount = 0;
acquireResidencyLock();
this->residencyControllers[osContext.getContextId()]->acquireLock();
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", osContext.get()->getMonitoredFence().currentFenceValue);
@ -533,14 +545,14 @@ bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer &all
}
}
releaseResidencyLock();
this->residencyControllers[osContext.getContextId()]->releaseLock();
return result;
}
void WddmMemoryManager::makeNonResidentEvictionAllocations(ResidencyContainer &evictionAllocations, OsContext &osContext) {
acquireResidencyLock();
this->residencyControllers[osContext.getContextId()]->acquireLock();
size_t residencyCount = evictionAllocations.size();
@ -550,7 +562,7 @@ void WddmMemoryManager::makeNonResidentEvictionAllocations(ResidencyContainer &e
addToTrimCandidateList(allocation);
}
releaseResidencyLock();
this->residencyControllers[osContext.getContextId()]->releaseLock();
}
void WddmMemoryManager::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) {
@ -646,21 +658,21 @@ void WddmMemoryManager::compactTrimCandidateList() {
}
void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
OsContext *osContext = nullptr;
OsContext &osContext = *getRegisteredOsContext(0);
if (flags.PeriodicTrim) {
bool periodicTrimDone = false;
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
uint64_t sizeToTrim = 0;
acquireResidencyLock();
this->residencyControllers[osContext.getContextId()]->acquireLock();
WddmAllocation *wddmAllocation = nullptr;
while ((wddmAllocation = getTrimCandidateHead()) != nullptr) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastPeriodicTrimFenceValues[0]);
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", this->residencyControllers[osContext.getContextId()]->getLastTrimFenceValue());
// allocation was not used from last periodic trim
if (wddmAllocation->getResidencyData().getFenceValueForContextId(0) <= lastPeriodicTrimFenceValues[0]) {
if (wddmAllocation->getResidencyData().getFenceValueForContextId(0) <= this->residencyControllers[osContext.getContextId()]->getLastTrimFenceValue()) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0));
@ -672,7 +684,7 @@ void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint6
}
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= lastPeriodicTrimFenceValues[0]) {
if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= this->residencyControllers[osContext.getContextId()]->getLastTrimFenceValue()) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0));
@ -686,7 +698,6 @@ void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint6
}
wddmAllocation->getResidencyData().resident = false;
osContext = wddmAllocation->getResidencyData().getOsContextFromId(0);
removeFromTrimCandidateList(wddmAllocation);
} else {
periodicTrimDone = true;
@ -698,24 +709,22 @@ void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint6
compactTrimCandidateList();
}
releaseResidencyLock();
this->residencyControllers[osContext.getContextId()]->releaseLock();
}
if (flags.TrimToBudget) {
acquireResidencyLock();
this->residencyControllers[osContext.getContextId()]->acquireLock();
trimResidencyToBudget(bytes);
releaseResidencyLock();
this->residencyControllers[osContext.getContextId()]->releaseLock();
}
if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
if (!osContext) {
osContext = platform()->getDevice(0)->getOsContext();
}
lastPeriodicTrimFenceValues[0] = *osContext->get()->getMonitoredFence().cpuAddress;
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastPeriodicTrimFenceValues[0]);
const auto newPeriodicTrimFenceValue = *osContext.get()->getMonitoredFence().cpuAddress;
this->residencyControllers[osContext.getContextId()]->setLastTrimFenceValue(newPeriodicTrimFenceValue);
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", newPeriodicTrimFenceValue);
}
}

15
runtime/os_interface/windows/wddm_memory_manager.h
View File

@ -19,6 +19,7 @@
namespace OCLRT {
class Gmm;
class Wddm;
class WddmResidencyController;
using OsContextWin = OsContext::OsContextImpl;
@ -56,6 +57,7 @@ class WddmMemoryManager : public MemoryManager {
void cleanOsHandles(OsHandleStorage &handleStorage) override;
void registerOsContext(OsContext *contextToRegister) override;
OsContext *getRegisteredOsContext(uint32_t osContextId) { return registeredOsContexts[osContextId]; }
void obtainGpuAddresFromFragments(WddmAllocation *allocation, OsHandleStorage &handleStorage);
@ -70,16 +72,6 @@ class WddmMemoryManager : public MemoryManager {
static void APIENTRY trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification);
void acquireResidencyLock() {
bool previousLockValue = false;
while (!residencyLock.compare_exchange_weak(previousLockValue, true))
previousLockValue = false;
}
void releaseResidencyLock() {
residencyLock = false;
}
bool tryDeferDeletions(D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
bool isMemoryBudgetExhausted() const override { return memoryBudgetExhausted; }
@ -113,9 +105,8 @@ class WddmMemoryManager : public MemoryManager {
bool createWddmAllocation(WddmAllocation *allocation, AllocationOrigin origin);
ResidencyContainer trimCandidateList;
std::mutex trimCandidateListMutex;
std::atomic<bool> residencyLock;
std::vector<uint64_t> lastPeriodicTrimFenceValues;
uint32_t trimCandidatesCount = 0;
std::vector<std::unique_ptr<WddmResidencyController>> residencyControllers;
bool memoryBudgetExhausted = false;
AlignedMallocRestrictions mallocRestrictions;

23
runtime/os_interface/windows/wddm_residency_controller.cpp Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (C) 2018 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "wddm_residency_controller.h"
namespace OCLRT {
WddmResidencyController::WddmResidencyController() : lock(false), lastTrimFenceValue(0u) {}
void WddmResidencyController::acquireLock() {
bool previousLockValue = false;
while (!lock.compare_exchange_weak(previousLockValue, true))
previousLockValue = false;
}
void WddmResidencyController::releaseLock() {
lock = false;
}
} // namespace OCLRT

28
runtime/os_interface/windows/wddm_residency_controller.h Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright (C) 2018 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <atomic>
namespace OCLRT {
class WddmResidencyController {
public:
WddmResidencyController();
void acquireLock();
void releaseLock();
uint64_t getLastTrimFenceValue() { return lastTrimFenceValue; }
void setLastTrimFenceValue(uint64_t value) { lastTrimFenceValue = value; }
protected:
std::atomic<bool> lock;
uint64_t lastTrimFenceValue;
};
} // namespace OCLRT

3
unit_tests/os_interface/windows/device_command_stream_tests.cpp
View File

@ -67,6 +67,7 @@ class WddmCommandStreamFixture {
mockWddmMM = new MockWddmMemoryManager(wddm, *executionEnvironment);
memManager.reset(mockWddmMM);
memManager->registerOsContext(new OsContext(executionEnvironment->osInterface.get(), 0u));
csr->setMemoryManager(memManager.get());
ASSERT_NE(nullptr, memManager);
@ -266,6 +267,7 @@ TEST(WddmPreemptionHeaderTests, givenWddmCommandStreamReceiverWhenPreemptionIsOf
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, cs.getUsed(), &cs};
OsContext *osContext = new OsContext(executionEnvironment.osInterface.get(), 0u);
osContext->incRefInternal();
executionEnvironment.memoryManager->registerOsContext(osContext);
executionEnvironment.commandStreamReceivers[0u]->flush(batchBuffer, EngineType::ENGINE_RCS, executionEnvironment.commandStreamReceivers[0u]->getResidencyAllocations(), *osContext);
auto commandHeader = wddm->submitResult.commandHeaderSubmitted;
COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandHeader);
@ -293,6 +295,7 @@ TEST(WddmPreemptionHeaderTests, givenWddmCommandStreamReceiverWhenPreemptionIsOn
BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, cs.getUsed(), &cs};
OsContext *osContext = new OsContext(executionEnvironment.osInterface.get(), 0u);
osContext->incRefInternal();
executionEnvironment.memoryManager->registerOsContext(osContext);
executionEnvironment.commandStreamReceivers[0u]->flush(batchBuffer, EngineType::ENGINE_RCS, executionEnvironment.commandStreamReceivers[0u]->getResidencyAllocations(), *osContext);
auto commandHeader = wddm->submitResult.commandHeaderSubmitted;
COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandHeader);

2
unit_tests/os_interface/windows/mock_wddm_memory_manager.h
View File

@ -18,8 +18,8 @@ class MockWddmMemoryManager : public WddmMemoryManager {
using BaseClass::checkTrimCandidateListCompaction;
using BaseClass::compactTrimCandidateList;
using BaseClass::createWddmAllocation;
using BaseClass::lastPeriodicTrimFenceValues;
using BaseClass::removeFromTrimCandidateList;
using BaseClass::residencyControllers;
using BaseClass::trimCandidateList;
using BaseClass::trimCandidatesCount;
using BaseClass::trimResidency;

37
unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp
View File

@ -12,6 +12,7 @@
#include "runtime/mem_obj/image.h"
#include "runtime/os_interface/os_library.h"
#include "runtime/os_interface/windows/os_context_win.h"
#include "runtime/os_interface/windows/wddm_residency_controller.h"
#include "runtime/platform/platform.h"
#include "runtime/utilities/tag_allocator.h"
@ -54,22 +55,22 @@ TEST(WddmMemoryManager, NonAssignable) {
EXPECT_FALSE(std::is_copy_assignable<WddmMemoryManager>::value);
}
TEST_F(WddmMemoryManagerTest, givenWddmMemoryManagerWhenRegisteringOsContextThenLastPeriodicTrimFenceValuesIsResizedAccordinglyToContextId) {
TEST_F(WddmMemoryManagerTest, givenWddmMemoryManagerWhenRegisteringOsContextThenResidencyControllersIsResizedAccordinglyToContextId) {
memoryManager->registerOsContext(new OsContext(osInterface.get(), 0u));
EXPECT_EQ(1, memoryManager->lastPeriodicTrimFenceValues.size());
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[0]);
EXPECT_EQ(1u, memoryManager->residencyControllers.size());
EXPECT_EQ(0u, memoryManager->residencyControllers[0]->getLastTrimFenceValue());
memoryManager->registerOsContext(new OsContext(osInterface.get(), 2u));
EXPECT_EQ(3, memoryManager->lastPeriodicTrimFenceValues.size());
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[0]);
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[1]);
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[2]);
EXPECT_EQ(3u, memoryManager->residencyControllers.size());
EXPECT_EQ(0u, memoryManager->residencyControllers[0]->getLastTrimFenceValue());
EXPECT_EQ(nullptr, memoryManager->residencyControllers[1]);
EXPECT_EQ(0u, memoryManager->residencyControllers[2]->getLastTrimFenceValue());
memoryManager->registerOsContext(new OsContext(osInterface.get(), 1u));
EXPECT_EQ(3, memoryManager->lastPeriodicTrimFenceValues.size());
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[0]);
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[1]);
EXPECT_EQ(0, memoryManager->lastPeriodicTrimFenceValues[2]);
EXPECT_EQ(3u, memoryManager->residencyControllers.size());
EXPECT_EQ(0u, memoryManager->residencyControllers[0]->getLastTrimFenceValue());
EXPECT_EQ(0u, memoryManager->residencyControllers[1]->getLastTrimFenceValue());
EXPECT_EQ(0u, memoryManager->residencyControllers[2]->getLastTrimFenceValue());
}
TEST(WddmMemoryManagerAllocator32BitTest, allocator32BitIsCreatedWithCorrectBase) {
@ -1137,7 +1138,7 @@ TEST_F(WddmMemoryManagerResidencyTest, givenNotUsedAllocationsFromPreviousPeriod
allocation2.getResidencyData().resident = true;
// Set last periodic fence value
memoryManager->lastPeriodicTrimFenceValues[0] = 10;
memoryManager->residencyControllers[0]->setLastTrimFenceValue(10);
// Set current fence value to greater value
osContext->get()->getMonitoredFence().currentFenceValue = 20;
@ -1173,7 +1174,7 @@ TEST_F(WddmMemoryManagerResidencyTest, givenOneUsedAllocationFromPreviousPeriodi
allocation2.getResidencyData().resident = true;
// Set last periodic fence value
memoryManager->lastPeriodicTrimFenceValues[0] = 10;
memoryManager->residencyControllers[0]->setLastTrimFenceValue(10);
// Set current fence value to greater value
osContext->get()->getMonitoredFence().currentFenceValue = 20;
@ -1219,7 +1220,7 @@ TEST_F(WddmMemoryManagerResidencyTest, givenTripleAllocationWithUsedAndUnusedFra
allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident = true;
// Set last periodic fence value
memoryManager->lastPeriodicTrimFenceValues[0] = 10;
memoryManager->residencyControllers[0]->setLastTrimFenceValue(10);
// Set current fence value to greater value
osContext->get()->getMonitoredFence().currentFenceValue = 20;
@ -1248,14 +1249,14 @@ TEST_F(WddmMemoryManagerResidencyTest, givenPeriodicTrimWhenTrimCallbackCalledTh
trimNotification.NumBytesToTrim = 0;
// Set last periodic fence value
memoryManager->lastPeriodicTrimFenceValues[0] = 10;
memoryManager->residencyControllers[0]->setLastTrimFenceValue(10);
// Set current fence value to greater value
*osContext->get()->getMonitoredFence().cpuAddress = 20;
memoryManager->trimCandidateList.resize(0);
memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim);
EXPECT_EQ(20u, memoryManager->lastPeriodicTrimFenceValues[0]);
EXPECT_EQ(20u, memoryManager->residencyControllers[0]->getLastTrimFenceValue());
}
TEST_F(WddmMemoryManagerResidencyTest, givenRestartPeriodicTrimWhenTrimCallbackCalledThenLastPeriodicTrimFenceIsSetToCurrentFenceValue) {
@ -1266,14 +1267,14 @@ TEST_F(WddmMemoryManagerResidencyTest, givenRestartPeriodicTrimWhenTrimCallbackC
trimNotification.NumBytesToTrim = 0;
// Set last periodic fence value
memoryManager->lastPeriodicTrimFenceValues[0] = 10;
memoryManager->residencyControllers[0]->setLastTrimFenceValue(10);
// Set current fence value to greater value
*osContext->get()->getMonitoredFence().cpuAddress = 20;
memoryManager->trimCandidateList.resize(0);
memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim);
EXPECT_EQ(20u, memoryManager->lastPeriodicTrimFenceValues[0]);
EXPECT_EQ(20u, memoryManager->residencyControllers[0]->getLastTrimFenceValue());
}
TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetWithZeroSizeReturnsTrue) {

1
unit_tests/os_interface/windows/wddm_memory_manager_tests.h
View File

@ -123,6 +123,7 @@ class WddmMemoryManagerFixtureWithGmockWddm : public GmmEnvironmentFixture {
memoryManager = new (std::nothrow) MockWddmMemoryManager(wddm, executionEnvironment);
//assert we have memory manager
ASSERT_NE(nullptr, memoryManager);
memoryManager->registerOsContext(osContext);
ON_CALL(*wddm, createAllocationsAndMapGpuVa(::testing::_)).WillByDefault(::testing::Invoke(wddm, &GmockWddm::baseCreateAllocationAndMapGpuVa));
}