intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-30 09:58:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

L0: add support for deferred memory free as per ContextImp::freeMemExt

Browse Source 
Signed-off-by: John Falkowski <john.falkowski@intel.com>
This commit is contained in:
John Falkowski
2023-01-03 18:39:11 +00:00
committed by Compute-Runtime-Automation
parent 889c2fe4e9
commit 01017a5df3
10 changed files with 597 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
shared/source/memory_manager/memory_manager.cpp
View File

@@ -780,6 +780,19 @@ void MemoryManager::waitForEnginesCompletion(GraphicsAllocation &graphicsAllocat
}
}
bool MemoryManager::allocInUse(GraphicsAllocation &graphicsAllocation) {
for (auto &engine : getRegisteredEngines()) {
auto osContextId = engine.osContext->getContextId();
auto allocationTaskCount = graphicsAllocation.getTaskCount(osContextId);
if (graphicsAllocation.isUsedByOsContext(osContextId) &&
engine.commandStreamReceiver->getTagAllocation() != nullptr &&
allocationTaskCount > *engine.commandStreamReceiver->getTagAddress()) {
return true;
}
}
return false;
}
void MemoryManager::cleanTemporaryAllocationListOnAllEngines(bool waitForCompletion) {
for (auto &engine : getRegisteredEngines()) {
auto csr = engine.commandStreamReceiver;

1
shared/source/memory_manager/memory_manager.h
View File

@@ -166,6 +166,7 @@ class MemoryManager {
void waitForDeletions();
MOCKABLE_VIRTUAL void waitForEnginesCompletion(GraphicsAllocation &graphicsAllocation);
MOCKABLE_VIRTUAL bool allocInUse(GraphicsAllocation &graphicsAllocation);
void cleanTemporaryAllocationListOnAllEngines(bool waitForCompletion);
bool isAsyncDeleterEnabled() const;

77
shared/source/memory_manager/unified_memory_manager.cpp
View File

@@ -62,7 +62,7 @@ void SVMAllocsManager::SvmAllocationCache::trim(SVMAllocsManager *svmAllocsManag
for (auto &cachedAllocationInfo : this->allocations) {
SvmAllocationData *svmData = svmAllocsManager->getSVMAlloc(cachedAllocationInfo.allocation);
DEBUG_BREAK_IF(nullptr == svmData);
svmAllocsManager->freeSVMAllocImpl(cachedAllocationInfo.allocation, false, svmData);
svmAllocsManager->freeSVMAllocImpl(cachedAllocationInfo.allocation, FreePolicyType::POLICY_NONE, svmData);
}
this->allocations.clear();
}
@@ -391,6 +391,11 @@ SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) {
return SVMAllocs.get(ptr);
}
SvmAllocationData *SVMAllocsManager::getSVMDeferFreeAlloc(const void *ptr) {
std::shared_lock<std::shared_mutex> lock(mtx);
return SVMDeferFreeAllocs.get(ptr);
}
void SVMAllocsManager::insertSVMAlloc(const SvmAllocationData &svmAllocData) {
std::unique_lock<std::shared_mutex> lock(mtx);
SVMAllocs.insert(svmAllocData);
@@ -402,6 +407,11 @@ void SVMAllocsManager::removeSVMAlloc(const SvmAllocationData &svmAllocData) {
}
bool SVMAllocsManager::freeSVMAlloc(void *ptr, bool blocking) {
if (SVMDeferFreeAllocs.allocations.size() > 0) {
this->freeSVMAllocDeferImpl();
}
SvmAllocationData *svmData = getSVMAlloc(ptr);
if (svmData) {
if (InternalMemoryType::DEVICE_UNIFIED_MEMORY == svmData->memoryType &&
@@ -409,16 +419,39 @@ bool SVMAllocsManager::freeSVMAlloc(void *ptr, bool blocking) {
this->usmDeviceAllocationsCache.insert(svmData->size, ptr);
return true;
}
this->freeSVMAllocImpl(ptr, blocking, svmData);
if (blocking) {
this->freeSVMAllocImpl(ptr, FreePolicyType::POLICY_BLOCKING, svmData);
} else {
this->freeSVMAllocImpl(ptr, FreePolicyType::POLICY_NONE, svmData);
}
return true;
}
return false;
}
void SVMAllocsManager::freeSVMAllocImpl(void *ptr, bool blocking, SvmAllocationData *svmData) {
bool SVMAllocsManager::freeSVMAllocDefer(void *ptr) {
if (SVMDeferFreeAllocs.allocations.size() > 0) {
this->freeSVMAllocDeferImpl();
}
SvmAllocationData *svmData = getSVMAlloc(ptr);
if (svmData) {
if (InternalMemoryType::DEVICE_UNIFIED_MEMORY == svmData->memoryType &&
this->usmDeviceAllocationsCacheEnabled) {
this->usmDeviceAllocationsCache.insert(svmData->size, ptr);
return true;
}
this->freeSVMAllocImpl(ptr, FreePolicyType::POLICY_DEFER, svmData);
return true;
}
return false;
}
void SVMAllocsManager::freeSVMAllocImpl(void *ptr, FreePolicyType policy, SvmAllocationData *svmData) {
this->prepareIndirectAllocationForDestruction(svmData);
if (blocking) {
if (policy == FreePolicyType::POLICY_BLOCKING) {
if (svmData->cpuAllocation) {
this->memoryManager->waitForEnginesCompletion(*svmData->cpuAllocation);
}
@@ -428,8 +461,26 @@ void SVMAllocsManager::freeSVMAllocImpl(void *ptr, bool blocking, SvmAllocationD
this->memoryManager->waitForEnginesCompletion(*gpuAllocation);
}
}
} else if (policy == FreePolicyType::POLICY_DEFER) {
if (svmData->cpuAllocation) {
if (this->memoryManager->allocInUse(*svmData->cpuAllocation)) {
if (getSVMDeferFreeAlloc(svmData) == nullptr) {
this->SVMDeferFreeAllocs.insert(*svmData);
}
return;
}
}
for (auto &gpuAllocation : svmData->gpuAllocations.getGraphicsAllocations()) {
if (gpuAllocation) {
if (this->memoryManager->allocInUse(*gpuAllocation)) {
if (getSVMDeferFreeAlloc(svmData) == nullptr) {
this->SVMDeferFreeAllocs.insert(*svmData);
}
return;
}
}
}
}
auto pageFaultManager = this->memoryManager->getPageFaultManager();
if (svmData->cpuAllocation && pageFaultManager) {
pageFaultManager->removeAllocation(svmData->cpuAllocation->getUnderlyingBuffer());
@@ -441,6 +492,22 @@ void SVMAllocsManager::freeSVMAllocImpl(void *ptr, bool blocking, SvmAllocationD
}
}
void SVMAllocsManager::freeSVMAllocDeferImpl() {
std::vector<void *> freedPtr;
for (auto iter = SVMDeferFreeAllocs.allocations.begin(); iter != SVMDeferFreeAllocs.allocations.end(); ++iter) {
void *ptr = reinterpret_cast<void *>(iter->second.gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress());
this->freeSVMAllocImpl(ptr, FreePolicyType::POLICY_DEFER, this->getSVMAlloc(ptr));
if (this->getSVMAlloc(ptr) == nullptr) {
freedPtr.push_back(ptr);
}
}
for (uint32_t i = 0; i < freedPtr.size(); ++i) {
SVMDeferFreeAllocs.allocations.erase(freedPtr[i]);
}
}
void SVMAllocsManager::trimUSMDeviceAllocCache() {
this->usmDeviceAllocationsCache.trim(this);
}

13
shared/source/memory_manager/unified_memory_manager.h
View File

@@ -145,6 +145,12 @@ class SVMAllocsManager {
std::mutex mtx;
};
enum class FreePolicyType : uint32_t {
POLICY_NONE = 0,
POLICY_BLOCKING = 1,
POLICY_DEFER = 2
};
SVMAllocsManager(MemoryManager *memoryManager, bool multiOsContextSupport);
MOCKABLE_VIRTUAL ~SVMAllocsManager();
void *createSVMAlloc(size_t size,
@@ -163,13 +169,17 @@ class SVMAllocsManager {
const UnifiedMemoryProperties &unifiedMemoryProperties);
void setUnifiedAllocationProperties(GraphicsAllocation *allocation, const SvmAllocationProperties &svmProperties);
SvmAllocationData *getSVMAlloc(const void *ptr);
SvmAllocationData *getSVMDeferFreeAlloc(const void *ptr);
MOCKABLE_VIRTUAL bool freeSVMAlloc(void *ptr, bool blocking);
MOCKABLE_VIRTUAL void freeSVMAllocImpl(void *ptr, bool blocking, SvmAllocationData *svmData);
MOCKABLE_VIRTUAL bool freeSVMAllocDefer(void *ptr);
MOCKABLE_VIRTUAL void freeSVMAllocDeferImpl();
MOCKABLE_VIRTUAL void freeSVMAllocImpl(void *ptr, FreePolicyType policy, SvmAllocationData *svmData);
bool freeSVMAlloc(void *ptr) { return freeSVMAlloc(ptr, false); }
void trimUSMDeviceAllocCache();
void insertSVMAlloc(const SvmAllocationData &svmData);
void removeSVMAlloc(const SvmAllocationData &svmData);
size_t getNumAllocs() const { return SVMAllocs.getNumAllocs(); }
MOCKABLE_VIRTUAL size_t getNumDeferFreeAllocs() const { return SVMDeferFreeAllocs.getNumAllocs(); }
MapBasedAllocationTracker *getSVMAllocs() { return &SVMAllocs; }
MOCKABLE_VIRTUAL void insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap);
@@ -206,6 +216,7 @@ class SVMAllocsManager {
MapBasedAllocationTracker SVMAllocs;
MapOperationsTracker svmMapOperations;
MapBasedAllocationTracker SVMDeferFreeAllocs;
MemoryManager *memoryManager;
std::shared_mutex mtx;
std::mutex mtxForIndirectAccess;