intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-25 21:42:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Store indirect residency at command queue level

Browse Source 
Signed-off-by: Maciej Plewka <maciej.plewka@intel.com>

Related-To: NEO-7211
This commit is contained in:
Maciej Plewka
2022-09-14 09:39:13 +00:00
committed by Compute-Runtime-Automation
parent fc9352cfcb
commit ffad5c6c09
18 changed files with 336 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
shared/source/memory_manager/unified_memory_manager.cpp
View File

@@ -107,7 +107,7 @@ SvmMapOperation *SVMAllocsManager::MapOperationsTracker::get(const void *regionP
void SVMAllocsManager::addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,
ResidencyContainer &residencyContainer,
uint32_t requestedTypesMask) {
std::shared_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
for (auto &allocation : this->SVMAllocs.allocations) {
if (rootDeviceIndex >= allocation.second.gpuAllocations.getGraphicsAllocations().size()) {
continue;
@@ -124,7 +124,7 @@ void SVMAllocsManager::addInternalAllocationsToResidencyContainer(uint32_t rootD
}
void SVMAllocsManager::makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t requestedTypesMask) {
std::shared_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
for (auto &allocation : this->SVMAllocs.allocations) {
if (allocation.second.memoryType & requestedTypesMask) {
auto gpuAllocation = allocation.second.gpuAllocations.getGraphicsAllocation(commandStreamReceiver.getRootDeviceIndex());
@@ -211,7 +211,7 @@ void *SVMAllocsManager::createHostUnifiedMemoryAllocation(size_t size,
allocData.pageSizeForAlignment = pageSizeForAlignment;
allocData.setAllocId(this->allocationsCounter++);
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
this->SVMAllocs.insert(allocData);
return usmPtr;
@@ -288,7 +288,7 @@ void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size,
allocData.device = memoryProperties.device;
allocData.setAllocId(this->allocationsCounter++);
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
this->SVMAllocs.insert(allocData);
return reinterpret_cast<void *>(unifiedMemoryAllocation->getGpuAddress());
}
@@ -370,7 +370,7 @@ void *SVMAllocsManager::createUnifiedKmdMigratedAllocation(size_t size, const Sv
allocData.pageSizeForAlignment = pageSizeForAlignment;
allocData.setAllocId(this->allocationsCounter++);
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
this->SVMAllocs.insert(allocData);
return allocationGpu->getUnderlyingBuffer();
}
@@ -381,17 +381,17 @@ void SVMAllocsManager::setUnifiedAllocationProperties(GraphicsAllocation *alloca
}
SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) {
std::shared_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
return SVMAllocs.get(ptr);
}
void SVMAllocsManager::insertSVMAlloc(const SvmAllocationData &svmAllocData) {
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
SVMAllocs.insert(svmAllocData);
}
void SVMAllocsManager::removeSVMAlloc(const SvmAllocationData &svmAllocData) {
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
SVMAllocs.remove(svmAllocData);
}
@@ -428,7 +428,7 @@ void SVMAllocsManager::freeSVMAllocImpl(void *ptr, bool blocking, SvmAllocationD
if (pageFaultManager) {
pageFaultManager->removeAllocation(ptr);
}
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
if (svmData->gpuAllocations.getAllocationType() == AllocationType::SVM_ZERO_COPY) {
freeZeroCopySvmAllocation(svmData);
} else {
@@ -470,7 +470,7 @@ void *SVMAllocsManager::createZeroCopySvmAllocation(size_t size, const SvmAlloca
}
allocData.size = size;
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
this->SVMAllocs.insert(allocData);
return usmPtr;
}
@@ -534,7 +534,7 @@ void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(size_t size, co
allocData.size = size;
allocData.setAllocId(this->allocationsCounter++);
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
this->SVMAllocs.insert(allocData);
return svmPtr;
}
@@ -564,7 +564,7 @@ void SVMAllocsManager::freeSvmAllocationWithDeviceStorage(SvmAllocationData *svm
}
bool SVMAllocsManager::hasHostAllocations() {
std::shared_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
for (auto &allocation : this->SVMAllocs.allocations) {
if (allocation.second.memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY) {
return true;
@@ -574,7 +574,7 @@ bool SVMAllocsManager::hasHostAllocations() {
}
void SVMAllocsManager::makeIndirectAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t taskCount) {
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
bool parseAllAllocations = false;
auto entry = indirectAllocationsResidency.find(&commandStreamReceiver);
@@ -608,7 +608,7 @@ void SVMAllocsManager::makeIndirectAllocationsResident(CommandStreamReceiver &co
}
void SVMAllocsManager::prepareIndirectAllocationForDestruction(SvmAllocationData *allocationData) {
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
if (this->indirectAllocationsResidency.size() > 0u) {
for (auto &internalAllocationsHandling : this->indirectAllocationsResidency) {
auto commandStreamReceiver = internalAllocationsHandling.first;
@@ -627,7 +627,7 @@ void SVMAllocsManager::prepareIndirectAllocationForDestruction(SvmAllocationData
}
SvmMapOperation *SVMAllocsManager::getSvmMapOperation(const void *ptr) {
std::shared_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
return svmMapOperations.get(ptr);
}
@@ -638,12 +638,12 @@ void SVMAllocsManager::insertSvmMapOperation(void *regionSvmPtr, size_t regionSi
svmMapOperation.offset = offset;
svmMapOperation.regionSize = regionSize;
svmMapOperation.readOnlyMap = readOnlyMap;
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
svmMapOperations.insert(svmMapOperation);
}
void SVMAllocsManager::removeSvmMapOperation(const void *regionSvmPtr) {
std::unique_lock<std::shared_mutex> lock(mtx);
std::unique_lock<std::recursive_mutex> lock(mtx);
svmMapOperations.remove(regionSvmPtr);
}
@@ -686,4 +686,7 @@ void SVMAllocsManager::prefetchMemory(Device &device, SvmAllocationData &svmData
}
}
std::unique_lock<std::recursive_mutex> SVMAllocsManager::obtainOwnership() {
return std::unique_lock<std::recursive_mutex>(mtx);
}
} // namespace NEO

11
shared/source/memory_manager/unified_memory_manager.h
View File

@@ -173,17 +173,18 @@ class SVMAllocsManager {
MOCKABLE_VIRTUAL void insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap);
void removeSvmMapOperation(const void *regionSvmPtr);
SvmMapOperation *getSvmMapOperation(const void *regionPtr);
void addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,
ResidencyContainer &residencyContainer,
uint32_t requestedTypesMask);
MOCKABLE_VIRTUAL void addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,
ResidencyContainer &residencyContainer,
uint32_t requestedTypesMask);
void makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t requestedTypesMask);
void *createUnifiedAllocationWithDeviceStorage(size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties);
void freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData);
bool hasHostAllocations();
std::atomic<uint32_t> allocationsCounter = 0;
void makeIndirectAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t taskCount);
MOCKABLE_VIRTUAL void makeIndirectAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t taskCount);
void prepareIndirectAllocationForDestruction(SvmAllocationData *);
void prefetchMemory(Device &device, SvmAllocationData &svmData);
std::unique_lock<std::recursive_mutex> obtainOwnership();
std::map<CommandStreamReceiver *, InternalAllocationsTracker> indirectAllocationsResidency;
@@ -203,9 +204,9 @@ class SVMAllocsManager {
MapBasedAllocationTracker SVMAllocs;
MapOperationsTracker svmMapOperations;
MemoryManager *memoryManager;
std::shared_mutex mtx;
bool multiOsContextSupport;
SvmAllocationCache usmDeviceAllocationsCache;
bool usmDeviceAllocationsCacheEnabled = false;
std::recursive_mutex mtx;
};
} // namespace NEO