mirror of
https://github.com/intel/compute-runtime.git
synced 2026-01-01 12:33:12 +08:00
feature: adapt usm pool manager
Change behavior to continue allocating usm pools as needed. Intended to replace singular usm pools. Related-To: NEO-16084 Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>
This commit is contained in:
Dominik Dabek
committed by
Compute-Runtime-Automation
Compute-Runtime-Automation
parent
841dab8e2b
commit
f54e3fda41
@@ -35,10 +35,10 @@ bool UsmMemAllocPool::initialize(SVMAllocsManager *svmMemoryManager, void *ptr,
|
||||
svmData->size,
|
||||
chunkAlignment,
|
||||
maxServicedSize / 2));
|
||||
this->poolSize = svmData->size;
|
||||
this->poolMemoryType = svmData->memoryType;
|
||||
this->minServicedSize = minServicedSize;
|
||||
this->maxServicedSize = maxServicedSize;
|
||||
this->poolInfo.minServicedSize = minServicedSize;
|
||||
this->poolInfo.maxServicedSize = maxServicedSize;
|
||||
this->poolInfo.poolSize = svmData->size;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ bool UsmMemAllocPool::isInitialized() const {
|
||||
}
|
||||
|
||||
size_t UsmMemAllocPool::getPoolSize() const {
|
||||
return this->poolSize;
|
||||
return this->poolInfo.poolSize;
|
||||
}
|
||||
|
||||
void UsmMemAllocPool::cleanup() {
|
||||
@@ -57,7 +57,7 @@ void UsmMemAllocPool::cleanup() {
|
||||
this->svmMemoryManager = nullptr;
|
||||
this->pool = nullptr;
|
||||
this->poolEnd = nullptr;
|
||||
this->poolSize = 0u;
|
||||
this->poolInfo.poolSize = 0u;
|
||||
this->poolMemoryType = InternalMemoryType::notSpecified;
|
||||
}
|
||||
}
|
||||
@@ -67,7 +67,7 @@ bool UsmMemAllocPool::alignmentIsAllowed(size_t alignment) {
|
||||
}
|
||||
|
||||
bool UsmMemAllocPool::sizeIsAllowed(size_t size) {
|
||||
return size >= minServicedSize && size <= maxServicedSize;
|
||||
return size >= poolInfo.minServicedSize && size <= poolInfo.maxServicedSize;
|
||||
}
|
||||
|
||||
bool UsmMemAllocPool::flagsAreAllowed(const UnifiedMemoryProperties &memoryProperties) {
|
||||
@@ -121,7 +121,7 @@ bool UsmMemAllocPool::isInPool(const void *ptr) const {
|
||||
return ptr >= this->pool && ptr < this->poolEnd;
|
||||
}
|
||||
|
||||
bool UsmMemAllocPool::isEmpty() {
|
||||
bool UsmMemAllocPool::isEmpty() const {
|
||||
return 0u == this->allocations.getNumAllocs();
|
||||
}
|
||||
|
||||
@@ -171,38 +171,23 @@ uint64_t UsmMemAllocPool::getPoolAddress() const {
|
||||
return castToUint64(this->pool);
|
||||
}
|
||||
|
||||
bool UsmMemAllocPoolsManager::PoolInfo::isPreallocated() const {
|
||||
return 0u != preallocateSize;
|
||||
UsmMemAllocPool::PoolInfo UsmMemAllocPool::getPoolInfo() const {
|
||||
return poolInfo;
|
||||
}
|
||||
|
||||
bool UsmMemAllocPoolsManager::ensureInitialized(SVMAllocsManager *svmMemoryManager) {
|
||||
bool UsmMemAllocPoolsManager::initialize(SVMAllocsManager *svmMemoryManager) {
|
||||
DEBUG_BREAK_IF(poolMemoryType != InternalMemoryType::deviceUnifiedMemory &&
|
||||
poolMemoryType != InternalMemoryType::hostUnifiedMemory);
|
||||
if (isInitialized()) {
|
||||
return true;
|
||||
}
|
||||
std::unique_lock<std::mutex> lock(mtx);
|
||||
if (isInitialized()) {
|
||||
return true;
|
||||
}
|
||||
bool allPoolAllocationsSucceeded = true;
|
||||
this->totalSize = 0u;
|
||||
SVMAllocsManager::UnifiedMemoryProperties poolsMemoryProperties(poolMemoryType, MemoryConstants::pageSize2M, rootDeviceIndices, deviceBitFields);
|
||||
poolsMemoryProperties.device = device;
|
||||
DEBUG_BREAK_IF(device == nullptr && poolMemoryType == InternalMemoryType::deviceUnifiedMemory);
|
||||
this->svmMemoryManager = svmMemoryManager;
|
||||
for (const auto &poolInfo : this->poolInfos) {
|
||||
this->pools[poolInfo] = std::vector<std::unique_ptr<UsmMemAllocPool>>();
|
||||
if (poolInfo.isPreallocated()) {
|
||||
auto pool = std::make_unique<UsmMemAllocPool>();
|
||||
allPoolAllocationsSucceeded &= pool->initialize(svmMemoryManager, poolsMemoryProperties, poolInfo.preallocateSize, poolInfo.minServicedSize, poolInfo.maxServicedSize);
|
||||
this->pools[poolInfo].push_back(std::move(pool));
|
||||
this->totalSize += poolInfo.preallocateSize;
|
||||
auto pool = tryAddPool(poolInfo);
|
||||
if (nullptr == pool) {
|
||||
cleanup();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (false == allPoolAllocationsSucceeded) {
|
||||
cleanup();
|
||||
return false;
|
||||
}
|
||||
this->svmMemoryManager = svmMemoryManager;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -210,34 +195,13 @@ bool UsmMemAllocPoolsManager::isInitialized() const {
|
||||
return nullptr != this->svmMemoryManager;
|
||||
}
|
||||
|
||||
void UsmMemAllocPoolsManager::trim() {
|
||||
std::unique_lock<std::mutex> lock(mtx);
|
||||
for (const auto &poolInfo : this->poolInfos) {
|
||||
if (false == poolInfo.isPreallocated()) {
|
||||
trim(this->pools[poolInfo]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void UsmMemAllocPoolsManager::trim(std::vector<std::unique_ptr<UsmMemAllocPool>> &poolVector) {
|
||||
auto poolIterator = poolVector.begin();
|
||||
while (poolIterator != poolVector.end()) {
|
||||
if ((*poolIterator)->isEmpty()) {
|
||||
this->totalSize -= (*poolIterator)->getPoolSize();
|
||||
(*poolIterator)->cleanup();
|
||||
poolIterator = poolVector.erase(poolIterator);
|
||||
} else {
|
||||
++poolIterator;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void UsmMemAllocPoolsManager::cleanup() {
|
||||
for (const auto &poolInfo : this->poolInfos) {
|
||||
for (const auto &pool : this->pools[poolInfo]) {
|
||||
for (auto &[_, bucket] : this->pools) {
|
||||
for (const auto &pool : bucket) {
|
||||
pool->cleanup();
|
||||
}
|
||||
}
|
||||
this->pools.clear();
|
||||
this->svmMemoryManager = nullptr;
|
||||
}
|
||||
|
||||
@@ -247,83 +211,90 @@ void *UsmMemAllocPoolsManager::createUnifiedMemoryAllocation(size_t size, const
|
||||
return nullptr;
|
||||
}
|
||||
std::unique_lock<std::mutex> lock(mtx);
|
||||
void *ptr = nullptr;
|
||||
for (const auto &poolInfo : this->poolInfos) {
|
||||
if (size <= poolInfo.maxServicedSize) {
|
||||
for (auto &pool : this->pools[poolInfo]) {
|
||||
if (void *ptr = pool->createUnifiedMemoryAllocation(size, memoryProperties)) {
|
||||
return ptr;
|
||||
if (nullptr != (ptr = pool->createUnifiedMemoryAllocation(size, memoryProperties))) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (nullptr == ptr) {
|
||||
if (auto pool = tryAddPool(poolInfo)) {
|
||||
ptr = pool->createUnifiedMemoryAllocation(size, memoryProperties);
|
||||
DEBUG_BREAK_IF(nullptr == ptr);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
UsmMemAllocPool *UsmMemAllocPoolsManager::tryAddPool(PoolInfo poolInfo) {
|
||||
UsmMemAllocPool *poolPtr = nullptr;
|
||||
if (canAddPool(poolInfo)) {
|
||||
auto pool = std::make_unique<UsmMemAllocPool>();
|
||||
if (pool->initialize(svmMemoryManager, poolMemoryProperties, poolInfo.poolSize, poolInfo.minServicedSize, poolInfo.maxServicedSize)) {
|
||||
poolPtr = pool.get();
|
||||
this->totalSize += pool->getPoolSize();
|
||||
this->pools[poolInfo].push_back(std::move(pool));
|
||||
}
|
||||
}
|
||||
return poolPtr;
|
||||
}
|
||||
bool UsmMemAllocPoolsManager::canAddPool(PoolInfo poolInfo) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void UsmMemAllocPoolsManager::trimEmptyPools(PoolInfo poolInfo) {
|
||||
std::lock_guard lock(mtx);
|
||||
auto &bucket = pools[poolInfo];
|
||||
auto firstEmptyPoolIt = std::partition(bucket.begin(), bucket.end(), [](std::unique_ptr<UsmMemAllocPool> &pool) {
|
||||
return !pool->isEmpty();
|
||||
});
|
||||
const auto emptyPoolsCount = static_cast<size_t>(std::distance(firstEmptyPoolIt, bucket.end()));
|
||||
if (emptyPoolsCount > maxEmptyPoolsPerBucket) {
|
||||
std::advance(firstEmptyPoolIt, maxEmptyPoolsPerBucket);
|
||||
for (auto it = firstEmptyPoolIt; it != bucket.end(); ++it) {
|
||||
(*it)->cleanup();
|
||||
}
|
||||
bucket.erase(firstEmptyPoolIt, bucket.end());
|
||||
}
|
||||
}
|
||||
|
||||
bool UsmMemAllocPoolsManager::freeSVMAlloc(const void *ptr, bool blocking) {
|
||||
if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) {
|
||||
return pool->freeSVMAlloc(ptr, blocking);
|
||||
bool allocFreed = false;
|
||||
if (auto pool = this->getPoolContainingAlloc(ptr); pool) {
|
||||
allocFreed = pool->freeSVMAlloc(ptr, blocking);
|
||||
if (allocFreed && pool->isEmpty()) {
|
||||
trimEmptyPools(pool->getPoolInfo());
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return allocFreed;
|
||||
}
|
||||
|
||||
size_t UsmMemAllocPoolsManager::getPooledAllocationSize(const void *ptr) {
|
||||
if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) {
|
||||
if (auto pool = this->getPoolContainingAlloc(ptr); pool) {
|
||||
return pool->getPooledAllocationSize(ptr);
|
||||
}
|
||||
return 0u;
|
||||
}
|
||||
|
||||
void *UsmMemAllocPoolsManager::getPooledAllocationBasePtr(const void *ptr) {
|
||||
if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) {
|
||||
if (auto pool = this->getPoolContainingAlloc(ptr); pool) {
|
||||
return pool->getPooledAllocationBasePtr(ptr);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t UsmMemAllocPoolsManager::getOffsetInPool(const void *ptr) {
|
||||
if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) {
|
||||
if (auto pool = this->getPoolContainingAlloc(ptr); pool) {
|
||||
return pool->getOffsetInPool(ptr);
|
||||
}
|
||||
return 0u;
|
||||
}
|
||||
|
||||
uint64_t UsmMemAllocPoolsManager::getFreeMemory() {
|
||||
const auto isIntegrated = device->getHardwareInfo().capabilityTable.isIntegratedDevice;
|
||||
const uint64_t deviceMemory = isIntegrated ? device->getDeviceInfo().globalMemSize : device->getDeviceInfo().localMemSize;
|
||||
const uint64_t usedMemory = memoryManager->getUsedLocalMemorySize(device->getRootDeviceIndex());
|
||||
DEBUG_BREAK_IF(usedMemory > deviceMemory);
|
||||
const uint64_t freeMemory = deviceMemory - usedMemory;
|
||||
return freeMemory;
|
||||
}
|
||||
|
||||
bool UsmMemAllocPoolsManager::recycleSVMAlloc(void *ptr, bool blocking) {
|
||||
if (false == isInitialized()) {
|
||||
return false;
|
||||
}
|
||||
auto svmData = this->svmMemoryManager->getSVMAlloc(ptr);
|
||||
DEBUG_BREAK_IF(svmData->memoryType != this->poolMemoryType);
|
||||
if (svmData->size > maxPoolableSize || belongsInPreallocatedPool(svmData->size)) {
|
||||
return false;
|
||||
}
|
||||
std::unique_lock<std::mutex> lock(mtx);
|
||||
if (this->totalSize + svmData->size > getFreeMemory() * UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(svmData->memoryType)) {
|
||||
return false;
|
||||
}
|
||||
for (auto poolInfoIndex = firstNonPreallocatedIndex; poolInfoIndex < this->poolInfos.size(); ++poolInfoIndex) {
|
||||
const auto &poolInfo = this->poolInfos[poolInfoIndex];
|
||||
if (svmData->size <= poolInfo.maxServicedSize) {
|
||||
auto pool = std::make_unique<UsmMemAllocPool>();
|
||||
pool->initialize(this->svmMemoryManager, ptr, svmData, poolInfo.minServicedSize, svmData->size);
|
||||
this->pools[poolInfo].push_back(std::move(pool));
|
||||
this->totalSize += svmData->size;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
DEBUG_BREAK_IF(true);
|
||||
return false;
|
||||
}
|
||||
|
||||
UsmMemAllocPool *UsmMemAllocPoolsManager::getPoolContainingAlloc(const void *ptr) {
|
||||
std::unique_lock<std::mutex> lock(mtx);
|
||||
for (const auto &poolInfo : this->poolInfos) {
|
||||
|
||||
Reference in New Issue
Block a user