intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2026-01-05 09:09:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(ocl): allocation info from pool svm ptr

Browse Source 
Fix querying allocation info from pooled svm ptr.
Handle requested allocation alignment.
Refactor sorted vector usage.
Do not associate device with host pool allocation.

Related-To: NEO-9700

Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>
This commit is contained in:
Dominik Dabek
2024-01-03 13:15:24 +00:00
committed by Compute-Runtime-Automation
parent bbb8193be7
commit af1620a308
9 changed files with 309 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
shared/source/memory_manager/unified_memory_manager.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2023 Intel Corporation
* Copyright (C) 2019-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -85,13 +85,7 @@ struct SvmMapOperation {
class SVMAllocsManager {
public:
struct CompareAcceptOffsetSvmPointers {
bool operator()(const std::unique_ptr<SvmAllocationData> &svmData, const void *ptr, const void *otherPtr) {
return ptr == otherPtr || (otherPtr < ptr &&
(reinterpret_cast<uintptr_t>(ptr) < (reinterpret_cast<uintptr_t>(otherPtr) + svmData->size)));
}
};
using SortedVectorBasedAllocationTracker = BaseSortedPointerWithValueVector<SvmAllocationData, CompareAcceptOffsetSvmPointers>;
using SortedVectorBasedAllocationTracker = BaseSortedPointerWithValueVector<SvmAllocationData>;
class MapBasedAllocationTracker {
friend class SVMAllocsManager;

47
shared/source/memory_manager/unified_memory_pooling.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2023 Intel Corporation
* Copyright (C) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -30,7 +30,7 @@ bool UsmMemAllocPool::initialize(SVMAllocsManager *svmMemoryManager, const Unifi
}
bool UsmMemAllocPool::isInitialized() {
return this->svmMemoryManager && this->pool;
return this->pool;
}
void UsmMemAllocPool::cleanup() {
@@ -44,18 +44,27 @@ void UsmMemAllocPool::cleanup() {
}
}
bool UsmMemAllocPool::canBePooled(size_t size, const UnifiedMemoryProperties &memoryProperties) {
return size <= allocationThreshold && memoryProperties.memoryType == this->poolMemoryType;
bool UsmMemAllocPool::alignmentIsAllowed(size_t alignment) {
return alignment % chunkAlignment == 0;
}
void *UsmMemAllocPool::createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties &memoryProperties) {
bool UsmMemAllocPool::canBePooled(size_t size, const UnifiedMemoryProperties &memoryProperties) {
return size <= allocationThreshold &&
alignmentIsAllowed(memoryProperties.alignment) &&
memoryProperties.memoryType == this->poolMemoryType &&
memoryProperties.allocationFlags.allFlags == 0u &&
memoryProperties.allocationFlags.allAllocFlags == 0u;
}
void *UsmMemAllocPool::createUnifiedMemoryAllocation(size_t requestedSize, const UnifiedMemoryProperties &memoryProperties) {
void *pooledPtr = nullptr;
if (isInitialized()) {
if (false == canBePooled(size, memoryProperties)) {
if (false == canBePooled(requestedSize, memoryProperties)) {
return nullptr;
}
std::unique_lock<std::mutex> lock(mtx);
size_t offset = static_cast<size_t>(this->chunkAllocator->allocate(size));
auto actualSize = requestedSize;
size_t offset = static_cast<size_t>(this->chunkAllocator->allocateWithCustomAlignment(actualSize, memoryProperties.alignment));
if (offset == 0) {
return nullptr;
}
@@ -63,7 +72,7 @@ void *UsmMemAllocPool::createUnifiedMemoryAllocation(size_t size, const UnifiedM
DEBUG_BREAK_IF(offset >= poolSize);
pooledPtr = ptrOffset(this->pool, offset);
{
this->allocations.insert(pooledPtr, AllocationInfo{offset, size});
this->allocations.insert(pooledPtr, AllocationInfo{offset, actualSize, requestedSize});
}
++this->svmMemoryManager->allocationsCounter;
}
@@ -93,4 +102,26 @@ bool UsmMemAllocPool::freeSVMAlloc(void *ptr, bool blocking) {
return false;
}
size_t UsmMemAllocPool::getPooledAllocationSize(const void *ptr) {
if (isInitialized() && isInPool(ptr)) {
std::unique_lock<std::mutex> lock(mtx);
auto allocationInfo = allocations.get(ptr);
if (allocationInfo) {
return allocationInfo->requestedSize;
}
}
return 0u;
}
void *UsmMemAllocPool::getPooledAllocationBasePtr(const void *ptr) {
if (isInitialized() && isInPool(ptr)) {
std::unique_lock<std::mutex> lock(mtx);
auto allocationInfo = allocations.get(ptr);
if (allocationInfo) {
return ptrOffset(this->pool, allocationInfo->offset);
}
}
return nullptr;
}
} // namespace NEO

20
shared/source/memory_manager/unified_memory_pooling.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2023 Intel Corporation
* Copyright (C) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -12,35 +12,31 @@
#include "shared/source/utilities/sorted_vector.h"
namespace NEO {
class SVMAllocsManager;
class HeapAllocator;
class UsmMemAllocPool {
public:
using UnifiedMemoryProperties = SVMAllocsManager::UnifiedMemoryProperties;
struct AllocationInfo {
size_t offset;
size_t size;
size_t requestedSize;
};
struct CompareAcceptEqualPointers {
bool operator()(const std::unique_ptr<AllocationInfo> &svmData, const void *ptr, const void *otherPtr) {
return ptr == otherPtr;
}
};
using AllocationsInfoStorage = BaseSortedPointerWithValueVector<AllocationInfo, CompareAcceptEqualPointers>;
using AllocationsInfoStorage = BaseSortedPointerWithValueVector<AllocationInfo>;
public:
UsmMemAllocPool() = default;
bool initialize(SVMAllocsManager *svmMemoryManager, const UnifiedMemoryProperties &memoryProperties, size_t poolSize);
bool isInitialized();
void cleanup();
bool alignmentIsAllowed(size_t alignment);
bool canBePooled(size_t size, const UnifiedMemoryProperties &memoryProperties);
void *createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties &memoryProperties);
bool isInPool(const void *ptr);
bool freeSVMAlloc(void *ptr, bool blocking);
size_t getPooledAllocationSize(const void *ptr);
void *getPooledAllocationBasePtr(const void *ptr);
static constexpr auto allocationThreshold = 1 * MemoryConstants::megaByte;
static constexpr auto chunkAlignment = 512u;
static constexpr auto startingOffset = chunkAlignment;
static constexpr auto startingOffset = 2 * allocationThreshold;
protected:
size_t poolSize{};

26
shared/source/utilities/sorted_vector.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2023 Intel Corporation
* Copyright (C) 2018-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -16,13 +16,22 @@
namespace NEO {
template <typename ValueType, class Compare>
template <typename ValueType>
class BaseSortedPointerWithValueVector {
public:
using PointerPair = std::pair<const void *, std::unique_ptr<ValueType>>;
using Container = std::vector<PointerPair>;
BaseSortedPointerWithValueVector() : compareFunctor(Compare()){};
BaseSortedPointerWithValueVector() = default;
bool comparePointers(size_t allowedOffset, const void *ptr, const void *otherPtr) {
return ptr == otherPtr || (allowedOffset > 0u && (otherPtr < ptr &&
(reinterpret_cast<uintptr_t>(ptr) < (reinterpret_cast<uintptr_t>(otherPtr) + allowedOffset))));
}
size_t getAllocationSize(const std::unique_ptr<ValueType> &data) {
return data->size;
}
void insert(const void *ptr, const ValueType &value) {
allocations.push_back(std::make_pair(ptr, std::make_unique<ValueType>(value)));
@@ -42,7 +51,7 @@ class BaseSortedPointerWithValueVector {
allocations.erase(removeIt);
}
typename Container::iterator getImpl(const void *ptr) {
typename Container::iterator getImpl(const void *ptr, bool allowOffset) {
if (allocations.size() == 0) {
return allocations.end();
}
@@ -58,8 +67,8 @@ class BaseSortedPointerWithValueVector {
while (end >= begin) {
int currentPos = (begin + end) / 2;
const auto &allocation = allocations[currentPos];
if (compareFunctor(allocation.second, ptr, allocation.first)) {
const size_t allowedOffset = allowOffset ? getAllocationSize(allocation.second) : 0u;
if (comparePointers(allowedOffset, ptr, allocation.first)) {
return allocations.begin() + currentPos;
} else if (ptr < allocation.first) {
end = currentPos - 1;
@@ -74,7 +83,7 @@ class BaseSortedPointerWithValueVector {
std::unique_ptr<ValueType> extract(const void *ptr) {
std::unique_ptr<ValueType> retVal{};
auto it = getImpl(ptr);
auto it = getImpl(ptr, false);
if (it != allocations.end()) {
retVal.swap(it->second);
allocations.erase(it);
@@ -83,7 +92,7 @@ class BaseSortedPointerWithValueVector {
}
ValueType *get(const void *ptr) {
auto it = getImpl(ptr);
auto it = getImpl(ptr, true);
if (it != allocations.end()) {
return it->second.get();
}
@@ -93,6 +102,5 @@ class BaseSortedPointerWithValueVector {
size_t getNumAllocs() const { return allocations.size(); }
Container allocations;
Compare compareFunctor;
};
} // namespace NEO

97
shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2023 Intel Corporation
* Copyright (C) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -16,6 +16,7 @@
#include "gtest/gtest.h"
#include <array>
using namespace NEO;
using UnifiedMemoryPoolingTest = Test<SVMMemoryAllocatorFixture<true>>;
@@ -73,11 +74,20 @@ TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenDifferentAllocationSizesWhe
EXPECT_TRUE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold, memoryProperties));
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold + 1, memoryProperties));
memoryProperties.memoryType = InternalMemoryType::sharedUnifiedMemory;
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold, memoryProperties));
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold + 1, memoryProperties));
memoryProperties.memoryType = InternalMemoryType::deviceUnifiedMemory;
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold, memoryProperties));
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold + 1, memoryProperties));
memoryProperties.memoryType = InternalMemoryType::sharedUnifiedMemory;
memoryProperties.memoryType = InternalMemoryType::hostUnifiedMemory;
memoryProperties.allocationFlags.allFlags = 1u;
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold, memoryProperties));
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold + 1, memoryProperties));
memoryProperties.allocationFlags.allFlags = 0u;
memoryProperties.allocationFlags.allAllocFlags = 1u;
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold, memoryProperties));
EXPECT_FALSE(usmMemAllocPool.canBePooled(UsmMemAllocPool::allocationThreshold + 1, memoryProperties));
}
@@ -92,6 +102,13 @@ TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenVariousPointersWhenCallingI
EXPECT_FALSE(usmMemAllocPool.isInPool(usmMemAllocPool.poolEnd));
}
TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenAlignmentsWhenCallingAlignmentIsAllowedThenCorrectValueIsReturned) {
EXPECT_FALSE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment / 2));
EXPECT_TRUE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment));
EXPECT_FALSE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment + UsmMemAllocPool::chunkAlignment / 2));
EXPECT_TRUE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment * 2));
}
TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenPoolableAllocationWhenUsingPoolThenAllocationIsPooledUnlessPoolIsFull) {
SVMAllocsManager::UnifiedMemoryProperties memoryProperties(InternalMemoryType::hostUnifiedMemory, MemoryConstants::pageSize64k, rootDeviceIndices, deviceBitfields);
const auto allocationSize = UsmMemAllocPool::allocationThreshold;
@@ -127,10 +144,82 @@ TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenPoolableAllocationWhenUsing
EXPECT_NE(nullptr, usmMemAllocPool.createUnifiedMemoryAllocation(allocationSize, memoryProperties));
}
TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenVariousAlignmentsWhenUsingPoolThenAddressIsAligned) {
SVMAllocsManager::UnifiedMemoryProperties memoryProperties(InternalMemoryType::hostUnifiedMemory, 0u, rootDeviceIndices, deviceBitfields);
const auto allocationSize = UsmMemAllocPool::allocationThreshold;
std::array<size_t, 8> alignmentsToCheck = {UsmMemAllocPool::chunkAlignment,
UsmMemAllocPool::chunkAlignment * 2,
UsmMemAllocPool::chunkAlignment * 4,
UsmMemAllocPool::chunkAlignment * 8,
UsmMemAllocPool::chunkAlignment * 16,
UsmMemAllocPool::chunkAlignment * 32,
UsmMemAllocPool::chunkAlignment * 64,
UsmMemAllocPool::chunkAlignment * 128};
for (const auto &alignment : alignmentsToCheck) {
if (alignment > UsmMemAllocPool::allocationThreshold) {
break;
}
memoryProperties.alignment = alignment;
auto allocFromPool = usmMemAllocPool.createUnifiedMemoryAllocation(allocationSize, memoryProperties);
EXPECT_NE(nullptr, allocFromPool);
EXPECT_TRUE(usmMemAllocPool.isInPool(allocFromPool));
auto address = castToUint64(allocFromPool);
EXPECT_EQ(0u, address % alignment);
EXPECT_TRUE(usmMemAllocPool.freeSVMAlloc(allocFromPool, true));
}
}
TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenPoolableAllocationWhenGettingSizeAndBasePtrThenCorrectValuesAreReturned) {
const auto bogusPtr = reinterpret_cast<void *>(0x1);
EXPECT_EQ(nullptr, usmMemAllocPool.getPooledAllocationBasePtr(bogusPtr));
EXPECT_EQ(0u, usmMemAllocPool.getPooledAllocationSize(bogusPtr));
const auto ptrInPoolButNotAllocated = usmMemAllocPool.pool;
EXPECT_EQ(nullptr, usmMemAllocPool.getPooledAllocationBasePtr(ptrInPoolButNotAllocated));
EXPECT_EQ(0u, usmMemAllocPool.getPooledAllocationSize(ptrInPoolButNotAllocated));
SVMAllocsManager::UnifiedMemoryProperties memoryProperties(InternalMemoryType::hostUnifiedMemory, MemoryConstants::pageSize64k, rootDeviceIndices, deviceBitfields);
const auto allocationSize = 1 * MemoryConstants::kiloByte;
EXPECT_GT(usmMemAllocPool.allocationThreshold, allocationSize + usmMemAllocPool.chunkAlignment);
// we want an allocation from the middle of the pool for testing
auto unusedAlloc = usmMemAllocPool.createUnifiedMemoryAllocation(allocationSize, memoryProperties);
EXPECT_NE(nullptr, unusedAlloc);
EXPECT_TRUE(usmMemAllocPool.isInPool(unusedAlloc));
auto allocFromPool = usmMemAllocPool.createUnifiedMemoryAllocation(allocationSize, memoryProperties);
EXPECT_NE(nullptr, allocFromPool);
EXPECT_TRUE(usmMemAllocPool.isInPool(allocFromPool));
EXPECT_TRUE(usmMemAllocPool.freeSVMAlloc(unusedAlloc, true));
auto offsetPointer = ptrOffset(allocFromPool, allocationSize - 1);
auto pastEndPointer = ptrOffset(allocFromPool, allocationSize);
EXPECT_TRUE(usmMemAllocPool.isInPool(offsetPointer));
EXPECT_TRUE(usmMemAllocPool.isInPool(pastEndPointer));
EXPECT_EQ(0u, usmMemAllocPool.getPooledAllocationSize(bogusPtr));
EXPECT_EQ(0u, usmMemAllocPool.getPooledAllocationSize(usmMemAllocPool.pool));
EXPECT_EQ(allocationSize, usmMemAllocPool.getPooledAllocationSize(allocFromPool));
EXPECT_EQ(allocationSize, usmMemAllocPool.getPooledAllocationSize(offsetPointer));
EXPECT_EQ(0u, usmMemAllocPool.getPooledAllocationSize(pastEndPointer));
EXPECT_EQ(nullptr, usmMemAllocPool.getPooledAllocationBasePtr(bogusPtr));
EXPECT_EQ(nullptr, usmMemAllocPool.getPooledAllocationBasePtr(usmMemAllocPool.pool));
EXPECT_EQ(allocFromPool, usmMemAllocPool.getPooledAllocationBasePtr(allocFromPool));
EXPECT_EQ(allocFromPool, usmMemAllocPool.getPooledAllocationBasePtr(offsetPointer));
EXPECT_EQ(nullptr, usmMemAllocPool.getPooledAllocationBasePtr(pastEndPointer));
}
using InitializationFailedUnifiedMemoryPoolingTest = InitializedUnifiedMemoryPoolingTest<InternalMemoryType::hostUnifiedMemory, true>;
TEST_F(InitializationFailedUnifiedMemoryPoolingTest, givenNotInitializedPoolWhenUsingPoolThenMethodsSucceed) {
SVMAllocsManager::UnifiedMemoryProperties memoryProperties(InternalMemoryType::hostUnifiedMemory, MemoryConstants::pageSize64k, rootDeviceIndices, deviceBitfields);
const auto allocationSize = UsmMemAllocPool::allocationThreshold;
EXPECT_EQ(nullptr, usmMemAllocPool.createUnifiedMemoryAllocation(allocationSize, memoryProperties));
EXPECT_FALSE(usmMemAllocPool.freeSVMAlloc(reinterpret_cast<void *>(0x1), true));
}
const auto bogusPtr = reinterpret_cast<void *>(0x1);
EXPECT_FALSE(usmMemAllocPool.freeSVMAlloc(bogusPtr, true));
EXPECT_EQ(0u, usmMemAllocPool.getPooledAllocationSize(bogusPtr));
EXPECT_EQ(nullptr, usmMemAllocPool.getPooledAllocationBasePtr(bogusPtr));
}

26
shared/test/unit_test/utilities/sorted_vector_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2023 Intel Corporation
* Copyright (C) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -9,12 +9,10 @@
#include "gtest/gtest.h"
struct Comparator {
bool operator()(const std::unique_ptr<size_t> &svmData, const void *ptr, const void *otherPtr) {
return ptr == otherPtr;
}
struct Data {
size_t size;
};
using TestedSortedVector = NEO::BaseSortedPointerWithValueVector<size_t, Comparator>;
using TestedSortedVector = NEO::BaseSortedPointerWithValueVector<Data>;
TEST(SortedVectorTest, givenBaseSortedVectorWhenGettingNullptrThenNullptrIsReturned) {
TestedSortedVector testedVector;
@@ -24,19 +22,19 @@ TEST(SortedVectorTest, givenBaseSortedVectorWhenGettingNullptrThenNullptrIsRetur
TEST(SortedVectorTest, givenBaseSortedVectorWhenCallingExtractThenCorrectValueIsReturned) {
TestedSortedVector testedVector;
void *ptr = reinterpret_cast<void *>(0x1);
testedVector.insert(ptr, 1u);
testedVector.insert(ptr, Data{1u});
EXPECT_EQ(nullptr, testedVector.extract(nullptr));
auto valuePtr = testedVector.extract(ptr);
EXPECT_EQ(1u, *valuePtr);
EXPECT_EQ(1u, valuePtr->size);
EXPECT_EQ(nullptr, testedVector.extract(ptr));
testedVector.insert(reinterpret_cast<void *>(0x1), 1u);
testedVector.insert(reinterpret_cast<void *>(0x2), 2u);
testedVector.insert(reinterpret_cast<void *>(0x3), 3u);
testedVector.insert(reinterpret_cast<void *>(0x4), 4u);
testedVector.insert(reinterpret_cast<void *>(0x5), 5u);
testedVector.insert(reinterpret_cast<void *>(0x1), Data{1u});
testedVector.insert(reinterpret_cast<void *>(0x2), Data{2u});
testedVector.insert(reinterpret_cast<void *>(0x3), Data{3u});
testedVector.insert(reinterpret_cast<void *>(0x4), Data{4u});
testedVector.insert(reinterpret_cast<void *>(0x5), Data{5u});
valuePtr = testedVector.extract(reinterpret_cast<void *>(0x1));
EXPECT_EQ(1u, *valuePtr);
EXPECT_EQ(1u, valuePtr->size);
}