diff --git a/level_zero/core/source/event/event.cpp b/level_zero/core/source/event/event.cpp index ff241f2191..e2b2b9573f 100644 --- a/level_zero/core/source/event/event.cpp +++ b/level_zero/core/source/event/event.cpp @@ -20,7 +20,8 @@ #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/memory_manager.h" #include "shared/source/memory_manager/memory_operations_handler.h" -#include "shared/source/utilities/timestamp_pool_allocator.h" +#include "shared/source/os_interface/device_factory.h" +#include "shared/source/utilities/pool_allocators.h" #include "level_zero/core/source/cmdlist/cmdlist.h" #include "level_zero/core/source/cmdlist/cmdlist_imp.h" @@ -110,9 +111,17 @@ ze_result_t EventPool::initialize(DriverHandle *driver, Context *context, uint32 if (this->isDeviceEventPoolAllocation) { this->isHostVisibleEventPoolAllocation = !(isEventPoolDeviceAllocationFlagSet()); - if (neoDevice->getDeviceTimestampPoolAllocator().isEnabled() && + const auto isTimestampPoolingEnabled = [neoDevice]() -> bool { + if (NEO::debugManager.flags.EnableTimestampPoolAllocator.get() != -1) { + return NEO::debugManager.flags.EnableTimestampPoolAllocator.get(); + } + return NEO::DeviceFactory::isHwModeSelected() && + neoDevice->getProductHelper().is2MBLocalMemAlignmentEnabled(); + }(); + + if (isTimestampPoolingEnabled && !isIpcPoolFlagSet()) { - auto sharedTsAlloc = neoDevice->getDeviceTimestampPoolAllocator().requestGraphicsAllocationForTimestamp(this->eventPoolSize); + auto sharedTsAlloc = neoDevice->getDeviceTimestampPoolAllocator().requestGraphicsAllocation(this->eventPoolSize); if (sharedTsAlloc) { this->sharedTimestampAllocation.reset(sharedTsAlloc); eventPoolAllocations->addAllocation(this->sharedTimestampAllocation->getGraphicsAllocation()); @@ -171,7 +180,7 @@ EventPool::~EventPool() { } if (this->sharedTimestampAllocation) { auto neoDevice = devices[0]->getNEODevice(); - neoDevice->getDeviceTimestampPoolAllocator().freeSharedTimestampAllocation(this->sharedTimestampAllocation.release()); + neoDevice->getDeviceTimestampPoolAllocator().freeSharedAllocation(this->sharedTimestampAllocation.release()); } } diff --git a/level_zero/core/source/event/event.h b/level_zero/core/source/event/event.h index 9e6ee02233..d93d61f815 100644 --- a/level_zero/core/source/event/event.h +++ b/level_zero/core/source/event/event.h @@ -13,7 +13,7 @@ #include "shared/source/helpers/timestamp_packet_container.h" #include "shared/source/memory_manager/multi_graphics_allocation.h" #include "shared/source/os_interface/os_time.h" -#include "shared/source/utilities/timestamp_pool_allocator.h" +#include "shared/source/utilities/pool_allocators.h" #include "level_zero/core/source/helpers/api_handle_helper.h" @@ -477,7 +477,7 @@ struct EventPool : _ze_event_pool_handle_t { inline ze_event_pool_handle_t toHandle() { return this; } MOCKABLE_VIRTUAL NEO::MultiGraphicsAllocation &getAllocation() { return *eventPoolAllocations; } - std::unique_ptr &getSharedTimestampAllocation() { + std::unique_ptr &getSharedTimestampAllocation() { return sharedTimestampAllocation; } @@ -533,7 +533,7 @@ struct EventPool : _ze_event_pool_handle_t { std::vector devices; std::unique_ptr eventPoolAllocations; - std::unique_ptr sharedTimestampAllocation; + std::unique_ptr sharedTimestampAllocation; void *eventPoolPtr = nullptr; ContextImp *context = nullptr; diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index db03caa48c..02d7340da9 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -1701,8 +1701,6 @@ HWTEST_F(EventCreate, GivenEnabledTimestampPoolAllocatorAndForcedEventAllocateIn std::unique_ptr l0GfxCoreHelperBackup(static_cast(&mockL0GfxCoreHelper)); device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->apiGfxCoreHelper.swap(l0GfxCoreHelperBackup); - ASSERT_TRUE(device->getNEODevice()->getDeviceTimestampPoolAllocator().isEnabled()); - ze_device_handle_t devices[] = {device->toHandle()}; std::vector> eventPools; @@ -1768,9 +1766,6 @@ HWTEST_F(EventPoolCreateMultiDevice, GivenEnabledTimestampPoolAllocatorAndForced auto neoDevice0 = device0->getNEODevice(); auto neoDevice1 = device1->getNEODevice(); - ASSERT_TRUE(neoDevice0->getDeviceTimestampPoolAllocator().isEnabled()); - ASSERT_TRUE(neoDevice1->getDeviceTimestampPoolAllocator().isEnabled()); - VariableBackup> backupApiGfxCoreHelper0(&neoDevice0->getExecutionEnvironment()->rootDeviceEnvironments[0]->apiGfxCoreHelper, std::make_unique>()); VariableBackup> backupApiGfxCoreHelper1(&neoDevice1->getExecutionEnvironment()->rootDeviceEnvironments[1]->apiGfxCoreHelper, std::make_unique>()); diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp index c98821388c..8541c28813 100644 --- a/shared/source/device/device.cpp +++ b/shared/source/device/device.cpp @@ -34,6 +34,7 @@ #include "shared/source/release_helper/release_helper.h" #include "shared/source/sip_external_lib/sip_external_lib.h" #include "shared/source/unified_memory/usm_memory_support.h" +#include "shared/source/utilities/generic_pool_allocator.inl" #include "shared/source/utilities/software_tags_manager.h" namespace NEO { diff --git a/shared/source/device/device.h b/shared/source/device/device.h index c0312c803d..1d68a724ec 100644 --- a/shared/source/device/device.h +++ b/shared/source/device/device.h @@ -17,8 +17,8 @@ #include "shared/source/os_interface/performance_counters.h" #include "shared/source/os_interface/product_helper.h" #include "shared/source/utilities/isa_pool_allocator.h" +#include "shared/source/utilities/pool_allocators.h" #include "shared/source/utilities/reference_tracked_object.h" -#include "shared/source/utilities/timestamp_pool_allocator.h" #include #include diff --git a/shared/source/utilities/CMakeLists.txt b/shared/source/utilities/CMakeLists.txt index 745d7c7538..2718779bef 100644 --- a/shared/source/utilities/CMakeLists.txt +++ b/shared/source/utilities/CMakeLists.txt @@ -18,6 +18,8 @@ set(NEO_CORE_UTILITIES ${CMAKE_CURRENT_SOURCE_DIR}/debug_settings_reader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/debug_settings_reader.h ${CMAKE_CURRENT_SOURCE_DIR}/directory.h + ${CMAKE_CURRENT_SOURCE_DIR}/generic_pool_allocator.h + ${CMAKE_CURRENT_SOURCE_DIR}/generic_pool_allocator.inl ${CMAKE_CURRENT_SOURCE_DIR}/heap_allocator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/heap_allocator.h ${CMAKE_CURRENT_SOURCE_DIR}/hw_timestamps.h @@ -34,6 +36,10 @@ set(NEO_CORE_UTILITIES ${CMAKE_CURRENT_SOURCE_DIR}/perf_counter.h ${CMAKE_CURRENT_SOURCE_DIR}/perf_profiler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/perf_profiler.h + ${CMAKE_CURRENT_SOURCE_DIR}/pool_allocator_traits.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pool_allocator_traits.h + ${CMAKE_CURRENT_SOURCE_DIR}/pool_allocators.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pool_allocators.h ${CMAKE_CURRENT_SOURCE_DIR}/reference_tracked_object.h ${CMAKE_CURRENT_SOURCE_DIR}/shared_pool_allocation.h ${CMAKE_CURRENT_SOURCE_DIR}/software_tags.cpp @@ -48,8 +54,6 @@ set(NEO_CORE_UTILITIES ${CMAKE_CURRENT_SOURCE_DIR}/tag_allocator.inl ${CMAKE_CURRENT_SOURCE_DIR}/time_measure_wrapper.h ${CMAKE_CURRENT_SOURCE_DIR}/timer_util.h - ${CMAKE_CURRENT_SOURCE_DIR}/timestamp_pool_allocator.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/timestamp_pool_allocator.h ${CMAKE_CURRENT_SOURCE_DIR}/wait_util.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wait_util.h ${CMAKE_CURRENT_SOURCE_DIR}/isa_pool_allocator.cpp diff --git a/shared/source/utilities/buffer_pool_allocator.h b/shared/source/utilities/buffer_pool_allocator.h index 8168c1dd1b..57556f6fe5 100644 --- a/shared/source/utilities/buffer_pool_allocator.h +++ b/shared/source/utilities/buffer_pool_allocator.h @@ -56,7 +56,7 @@ struct AbstractBuffersPool : public NonCopyableClass { // a BufferType-dependent function reserving chunks within `mainStorage`. // Example: see `NEO::Context::BufferPool::allocate()` using AllocsVecCRef = const StackVec &; - using OnChunkFreeCallback = void (PoolT::*)(uint64_t offset, size_t size); + using OnChunkFreeCallback = std::function; AbstractBuffersPool(MemoryManager *memoryManager, OnChunkFreeCallback onChunkFreeCallback); AbstractBuffersPool(MemoryManager *memoryManager, OnChunkFreeCallback onChunkFreeCallback, const SmallBuffersParams ¶ms); diff --git a/shared/source/utilities/buffer_pool_allocator.inl b/shared/source/utilities/buffer_pool_allocator.inl index 9a06f1c37e..b50d5e1950 100644 --- a/shared/source/utilities/buffer_pool_allocator.inl +++ b/shared/source/utilities/buffer_pool_allocator.inl @@ -60,8 +60,8 @@ void AbstractBuffersPool::drain() { } for (auto &chunk : this->chunksToFree) { this->chunkAllocator->free(chunk.first + params.startingOffset, chunk.second); - if (static_cast(this)->onChunkFreeCallback) { - (static_cast(this)->*onChunkFreeCallback)(chunk.first, chunk.second); + if (this->onChunkFreeCallback) { + this->onChunkFreeCallback(static_cast(this), chunk.first, chunk.second); } } this->chunksToFree.clear(); diff --git a/shared/source/utilities/generic_pool_allocator.h b/shared/source/utilities/generic_pool_allocator.h new file mode 100644 index 0000000000..0c0c68ab62 --- /dev/null +++ b/shared/source/utilities/generic_pool_allocator.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/utilities/buffer_pool_allocator.h" +#include "shared/source/utilities/pool_allocator_traits.h" +#include "shared/source/utilities/shared_pool_allocation.h" +#include "shared/source/utilities/stackvec.h" + +#include +#include + +namespace NEO { + +class Device; +class GraphicsAllocation; + +template +class GenericPool : public AbstractBuffersPool, GraphicsAllocation> { + using BaseType = AbstractBuffersPool, GraphicsAllocation>; + + public: + GenericPool(Device *device, size_t poolSize); + + GenericPool(const GenericPool &) = delete; + GenericPool &operator=(const GenericPool &) = delete; + + GenericPool(GenericPool &&pool) noexcept; + GenericPool &operator=(GenericPool &&) = delete; + + ~GenericPool() override; + + SharedPoolAllocation *allocate(size_t size); + const StackVec &getAllocationsVector(); + + private: + Device *device; + StackVec stackVec; + std::unique_ptr mtx; +}; + +template +class GenericPoolAllocator : public AbstractBuffersAllocator, GraphicsAllocation> { + public: + explicit GenericPoolAllocator(Device *device); + + SharedPoolAllocation *requestGraphicsAllocation(size_t size); + void freeSharedAllocation(SharedPoolAllocation *sharedPoolAllocation); + + size_t getDefaultPoolSize() const { return Traits::defaultPoolSize; } + + private: + SharedPoolAllocation *allocateFromPools(size_t size); + size_t alignToPoolSize(size_t size) const; + + Device *device; + std::mutex allocatorMtx; +}; + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/generic_pool_allocator.inl b/shared/source/utilities/generic_pool_allocator.inl new file mode 100644 index 0000000000..cc993656f1 --- /dev/null +++ b/shared/source/utilities/generic_pool_allocator.inl @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/device/device.h" +#include "shared/source/helpers/aligned_memory.h" +#include "shared/source/memory_manager/memory_manager.h" +#include "shared/source/utilities/buffer_pool_allocator.inl" +#include "shared/source/utilities/generic_pool_allocator.h" +#include "shared/source/utilities/heap_allocator.h" + +namespace NEO { + +template +GenericPool::GenericPool(Device *device, size_t poolSize) + : BaseType(device->getMemoryManager(), nullptr), device(device) { + DEBUG_BREAK_IF(device->getProductHelper().is2MBLocalMemAlignmentEnabled() && + !isAligned(poolSize, MemoryConstants::pageSize2M)); + + auto properties = Traits::createAllocationProperties(device, poolSize); + auto graphicsAllocation = this->memoryManager->allocateGraphicsMemoryWithProperties(properties); + + this->chunkAllocator.reset(new NEO::HeapAllocator(this->params.startingOffset, + graphicsAllocation ? graphicsAllocation->getUnderlyingBufferSize() : 0u, + MemoryConstants::pageSize, + 0u)); + this->mainStorage.reset(graphicsAllocation); + this->mtx = std::make_unique(); + stackVec.push_back(graphicsAllocation); +} + +template +GenericPool::GenericPool(GenericPool &&pool) noexcept : BaseType(std::move(pool)) { + mtx.reset(pool.mtx.release()); + this->stackVec = std::move(pool.stackVec); + this->device = pool.device; +} + +template +GenericPool::~GenericPool() { + if (this->mainStorage) { + device->getMemoryManager()->freeGraphicsMemory(this->mainStorage.release()); + } +} + +template +SharedPoolAllocation *GenericPool::allocate(size_t size) { + auto offset = static_cast(this->chunkAllocator->allocate(size)); + if (offset == 0) { + return nullptr; + } + return new SharedPoolAllocation{this->mainStorage.get(), + offset - this->params.startingOffset, + size, + mtx.get()}; +} + +template +const StackVec &GenericPool::getAllocationsVector() { + return stackVec; +} + +template +GenericPoolAllocator::GenericPoolAllocator(Device *device) : device(device) {} + +template +SharedPoolAllocation *GenericPoolAllocator::requestGraphicsAllocation(size_t size) { + if (size > Traits::maxAllocationSize) { + return nullptr; + } + + std::lock_guard lock(allocatorMtx); + + if (this->bufferPools.empty()) { + this->addNewBufferPool(GenericPool(device, alignToPoolSize(Traits::defaultPoolSize))); + } + + auto allocFromPool = allocateFromPools(size); + if (allocFromPool != nullptr) { + return allocFromPool; + } + + this->drain(); + + allocFromPool = allocateFromPools(size); + if (allocFromPool != nullptr) { + return allocFromPool; + } + + this->addNewBufferPool(GenericPool(device, alignToPoolSize(Traits::defaultPoolSize))); + return allocateFromPools(size); +} + +template +void GenericPoolAllocator::freeSharedAllocation(SharedPoolAllocation *sharedPoolAllocation) { + std::unique_lock lock(allocatorMtx); + this->tryFreeFromPoolBuffer(sharedPoolAllocation->getGraphicsAllocation(), + sharedPoolAllocation->getOffset(), + sharedPoolAllocation->getSize()); + delete sharedPoolAllocation; +} + +template +SharedPoolAllocation *GenericPoolAllocator::allocateFromPools(size_t size) { + for (auto &pool : this->bufferPools) { + if (auto allocation = pool.allocate(size)) { + return allocation; + } + } + return nullptr; +} + +template +size_t GenericPoolAllocator::alignToPoolSize(size_t size) const { + return alignUp(size, Traits::poolAlignment); +} + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/pool_allocator_traits.cpp b/shared/source/utilities/pool_allocator_traits.cpp new file mode 100644 index 0000000000..2eab29971c --- /dev/null +++ b/shared/source/utilities/pool_allocator_traits.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/utilities/pool_allocator_traits.h" + +#include "shared/source/device/device.h" +#include "shared/source/os_interface/device_factory.h" + +namespace NEO { + +AllocationProperties TimestampPoolTraits::createAllocationProperties(Device *device, size_t poolSize) { + return AllocationProperties{device->getRootDeviceIndex(), + poolSize, + allocationType, + device->getDeviceBitfield()}; +} + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/pool_allocator_traits.h b/shared/source/utilities/pool_allocator_traits.h new file mode 100644 index 0000000000..a528b9026a --- /dev/null +++ b/shared/source/utilities/pool_allocator_traits.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/helpers/constants.h" +#include "shared/source/memory_manager/allocation_properties.h" + +#include + +namespace NEO { + +class Device; + +template +concept PoolTraits = requires(Device *d, size_t s) { + { T::allocationType } -> std::convertible_to; + { T::maxAllocationSize } -> std::convertible_to; + { T::defaultPoolSize } -> std::convertible_to; + { T::poolAlignment } -> std::convertible_to; + { T::createAllocationProperties(d, s) } -> std::same_as; + }; + +struct TimestampPoolTraits { + static constexpr AllocationType allocationType = AllocationType::gpuTimestampDeviceBuffer; + static constexpr size_t maxAllocationSize = 2 * MemoryConstants::megaByte; + static constexpr size_t defaultPoolSize = 4 * MemoryConstants::megaByte; + static constexpr size_t poolAlignment = MemoryConstants::pageSize2M; + + static AllocationProperties createAllocationProperties(Device *device, size_t poolSize); +}; + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/pool_allocators.cpp b/shared/source/utilities/pool_allocators.cpp new file mode 100644 index 0000000000..9036d94044 --- /dev/null +++ b/shared/source/utilities/pool_allocators.cpp @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/utilities/pool_allocators.h" + +#include "shared/source/helpers/non_copyable_or_moveable.h" +#include "shared/source/utilities/generic_pool_allocator.inl" + +namespace NEO { + +template class GenericPool; +template class GenericPoolAllocator; + +static_assert(NEO::NonCopyable); + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/pool_allocators.h b/shared/source/utilities/pool_allocators.h new file mode 100644 index 0000000000..c181c75392 --- /dev/null +++ b/shared/source/utilities/pool_allocators.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/utilities/generic_pool_allocator.h" +#include "shared/source/utilities/pool_allocator_traits.h" + +namespace NEO { + +using TimestampPool = GenericPool; +using TimestampPoolAllocator = GenericPoolAllocator; + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/timestamp_pool_allocator.cpp b/shared/source/utilities/timestamp_pool_allocator.cpp deleted file mode 100644 index 318ac7afa4..0000000000 --- a/shared/source/utilities/timestamp_pool_allocator.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/utilities/timestamp_pool_allocator.h" - -#include "shared/source/device/device.h" -#include "shared/source/helpers/aligned_memory.h" -#include "shared/source/memory_manager/allocation_properties.h" -#include "shared/source/memory_manager/memory_manager.h" -#include "shared/source/os_interface/device_factory.h" -#include "shared/source/utilities/buffer_pool_allocator.inl" - -namespace NEO { -TimestampPool::TimestampPool(Device *device, size_t poolSize) - : BaseType(device->getMemoryManager(), nullptr), device(device) { - DEBUG_BREAK_IF(device->getProductHelper().is2MBLocalMemAlignmentEnabled() && - !isAligned(poolSize, MemoryConstants::pageSize2M)); - - AllocationProperties properties{device->getRootDeviceIndex(), - poolSize, - AllocationType::gpuTimestampDeviceBuffer, - device->getDeviceBitfield()}; - auto graphicsAllocation = memoryManager->allocateGraphicsMemoryWithProperties(properties); - - this->mainStorage.reset(graphicsAllocation); - this->chunkAllocator.reset(new HeapAllocator(params.startingOffset, poolSize, MemoryConstants::pageSize, 0u)); - stackVec.push_back(graphicsAllocation); - this->mtx = std::make_unique(); -} - -TimestampPool::TimestampPool(TimestampPool &&pool) noexcept : BaseType(std::move(pool)) { - mtx.reset(pool.mtx.release()); - this->stackVec = std::move(pool.stackVec); - this->device = pool.device; -} - -TimestampPool::~TimestampPool() { - if (mainStorage) { - device->getMemoryManager()->freeGraphicsMemory(mainStorage.release()); - } -} - -SharedTimestampAllocation *TimestampPool::allocate(size_t size) { - auto offset = static_cast(this->chunkAllocator->allocate(size)); - if (offset == 0) { - return nullptr; - } - return new SharedTimestampAllocation{this->mainStorage.get(), offset - params.startingOffset, size, mtx.get()}; -} - -const StackVec &TimestampPool::getAllocationsVector() { - return stackVec; -} - -TimestampPoolAllocator::TimestampPoolAllocator(Device *device) : device(device) {} - -bool TimestampPoolAllocator::isEnabled() const { - if (NEO::debugManager.flags.EnableTimestampPoolAllocator.get() != -1) { - return NEO::debugManager.flags.EnableTimestampPoolAllocator.get(); - } - - return NEO::DeviceFactory::isHwModeSelected() && device->getProductHelper().is2MBLocalMemAlignmentEnabled(); -} - -SharedTimestampAllocation *TimestampPoolAllocator::requestGraphicsAllocationForTimestamp(size_t size) { - if (size > maxAllocationSize) { - return nullptr; - } - - std::lock_guard lock(allocatorMtx); - - if (bufferPools.empty()) { - addNewBufferPool(TimestampPool(device, alignToPoolSize(defaultPoolSize))); - } - - auto allocFromPool = allocateFromPools(size); - if (allocFromPool != nullptr) { - return allocFromPool; - } - - this->drain(); - - allocFromPool = allocateFromPools(size); - if (allocFromPool != nullptr) { - return allocFromPool; - } - - addNewBufferPool(TimestampPool(device, alignToPoolSize(defaultPoolSize))); - return allocateFromPools(size); -} - -void TimestampPoolAllocator::freeSharedTimestampAllocation(SharedTimestampAllocation *sharedTimestampAllocation) { - std::unique_lock lock(allocatorMtx); - tryFreeFromPoolBuffer(sharedTimestampAllocation->getGraphicsAllocation(), sharedTimestampAllocation->getOffset(), sharedTimestampAllocation->getSize()); - delete sharedTimestampAllocation; -} - -SharedTimestampAllocation *TimestampPoolAllocator::allocateFromPools(size_t size) { - for (auto &pool : bufferPools) { - if (auto allocation = pool.allocate(size)) { - return allocation; - } - } - return nullptr; -} - -size_t TimestampPoolAllocator::alignToPoolSize(size_t size) const { - return alignUp(size, poolAlignment); -} - -} // namespace NEO diff --git a/shared/source/utilities/timestamp_pool_allocator.h b/shared/source/utilities/timestamp_pool_allocator.h deleted file mode 100644 index aaedaf9ad5..0000000000 --- a/shared/source/utilities/timestamp_pool_allocator.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2025 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#pragma once - -#include "shared/source/helpers/constants.h" -#include "shared/source/helpers/non_copyable_or_moveable.h" -#include "shared/source/utilities/buffer_pool_allocator.h" -#include "shared/source/utilities/shared_pool_allocation.h" -#include "shared/source/utilities/stackvec.h" - -#include - -namespace NEO { -class GraphicsAllocation; -class Device; - -using SharedTimestampAllocation = SharedPoolAllocation; - -class TimestampPool : public AbstractBuffersPool { - using BaseType = AbstractBuffersPool; - - public: - TimestampPool(Device *device, size_t poolSize); - - TimestampPool(const TimestampPool &) = delete; - TimestampPool &operator=(const TimestampPool &) = delete; - - TimestampPool(TimestampPool &&pool) noexcept; - TimestampPool &operator=(TimestampPool &&) = delete; - - ~TimestampPool() override; - - SharedTimestampAllocation *allocate(size_t size); - const StackVec &getAllocationsVector(); - - private: - Device *device; - StackVec stackVec; - std::unique_ptr mtx; -}; - -class TimestampPoolAllocator : public AbstractBuffersAllocator { - public: - TimestampPoolAllocator(Device *device); - - bool isEnabled() const; - - SharedTimestampAllocation *requestGraphicsAllocationForTimestamp(size_t size); - void freeSharedTimestampAllocation(SharedTimestampAllocation *sharedTimestampAllocation); - - size_t getDefaultPoolSize() const { return defaultPoolSize; } - - private: - SharedTimestampAllocation *allocateFromPools(size_t size); - size_t alignToPoolSize(size_t size) const; - - const size_t maxAllocationSize = 2 * MemoryConstants::megaByte; - const size_t defaultPoolSize = 4 * MemoryConstants::megaByte; - const size_t poolAlignment = MemoryConstants::pageSize2M; - - Device *device; - std::mutex allocatorMtx; -}; - -static_assert(NEO::NonCopyable); - -} // namespace NEO diff --git a/shared/test/unit_test/utilities/timestamp_pool_allocator_tests.cpp b/shared/test/unit_test/utilities/timestamp_pool_allocator_tests.cpp index e183720b0b..63e123c044 100644 --- a/shared/test/unit_test/utilities/timestamp_pool_allocator_tests.cpp +++ b/shared/test/unit_test/utilities/timestamp_pool_allocator_tests.cpp @@ -18,7 +18,7 @@ using namespace NEO; using TimestampPoolAllocatorTest = Test; namespace { -void verifySharedTimestampAllocation(const SharedTimestampAllocation *sharedAllocation, +void verifySharedTimestampAllocation(const SharedPoolAllocation *sharedAllocation, size_t expectedOffset, size_t expectedSize) { ASSERT_NE(nullptr, sharedAllocation); @@ -32,19 +32,19 @@ TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenNoAllocationsT auto ×tampAllocator = pDevice->getDeviceTimestampPoolAllocator(); constexpr size_t requestAllocationSize = MemoryConstants::pageSize; - auto allocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(allocation, 0ul, requestAllocationSize); EXPECT_EQ(AllocationType::gpuTimestampDeviceBuffer, allocation->getGraphicsAllocation()->getAllocationType()); - timestampAllocator.freeSharedTimestampAllocation(allocation); + timestampAllocator.freeSharedAllocation(allocation); } TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenAllocationsExistThenReuseAllocation) { auto ×tampAllocator = pDevice->getDeviceTimestampPoolAllocator(); constexpr size_t requestAllocationSize = MemoryConstants::pageSize; - auto allocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(allocation, 0ul, requestAllocationSize); auto allocationSize = allocation->getGraphicsAllocation()->getUnderlyingBufferSize(); @@ -52,19 +52,19 @@ TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenAllocationsExi // Perform requests until allocation is full for (auto i = 1u; i < numOfSharedAllocations; i++) { - auto tempSharedAllocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto tempSharedAllocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(tempSharedAllocation, requestAllocationSize * i, requestAllocationSize); EXPECT_EQ(allocation->getGraphicsAllocation(), tempSharedAllocation->getGraphicsAllocation()); - timestampAllocator.freeSharedTimestampAllocation(tempSharedAllocation); + timestampAllocator.freeSharedAllocation(tempSharedAllocation); } // Verify that draining freed chunks is correct and allocation can be reused - auto newAllocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto newAllocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(newAllocation, requestAllocationSize, requestAllocationSize); EXPECT_EQ(allocation->getGraphicsAllocation(), newAllocation->getGraphicsAllocation()); - timestampAllocator.freeSharedTimestampAllocation(newAllocation); - timestampAllocator.freeSharedTimestampAllocation(allocation); + timestampAllocator.freeSharedAllocation(newAllocation); + timestampAllocator.freeSharedAllocation(allocation); } TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenPoolIsFullThenCreateNewPool) { @@ -79,99 +79,43 @@ TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenPoolIsFullThen size_t requestAllocationSize = timestampAllocator.getDefaultPoolSize() / 2; // First allocation - should come from first pool - auto allocation1 = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation1 = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(allocation1, 0, requestAllocationSize); // Second allocation - should come from first pool but with offset - auto allocation2 = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation2 = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(allocation2, requestAllocationSize, requestAllocationSize); EXPECT_EQ(allocation1->getGraphicsAllocation(), allocation2->getGraphicsAllocation()); // Third allocation - should create new pool because first one is full - auto allocation3 = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation3 = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); verifySharedTimestampAllocation(allocation3, 0, requestAllocationSize); EXPECT_NE(allocation1->getGraphicsAllocation(), allocation3->getGraphicsAllocation()); - timestampAllocator.freeSharedTimestampAllocation(allocation1); - timestampAllocator.freeSharedTimestampAllocation(allocation2); - timestampAllocator.freeSharedTimestampAllocation(allocation3); + timestampAllocator.freeSharedAllocation(allocation1); + timestampAllocator.freeSharedAllocation(allocation2); + timestampAllocator.freeSharedAllocation(allocation3); } TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenRequestExceedsMaxSizeThenReturnNull) { auto ×tampAllocator = pDevice->getDeviceTimestampPoolAllocator(); constexpr size_t requestAllocationSize = 3 * MemoryConstants::megaByte; // Larger than maxAllocationSize - auto allocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); EXPECT_EQ(nullptr, allocation); } -TEST_F(TimestampPoolAllocatorTest, whenCheckingIsEnabledWithDifferentSettingsThenReturnsExpectedValue) { - DebugManagerStateRestore restorer; - - auto mockProductHelper = new MockProductHelper; - pDevice->getRootDeviceEnvironmentRef().productHelper.reset(mockProductHelper); - auto ×tampAllocator = pDevice->getDeviceTimestampPoolAllocator(); - - constexpr int32_t csrHwMode = static_cast(CommandStreamReceiverType::hardware); - constexpr int32_t csrNonHwMode = static_cast(CommandStreamReceiverType::tbx); - auto setHwMode = [&](bool hwMode) { - debugManager.flags.SetCommandStreamReceiver.set(hwMode ? csrHwMode : csrNonHwMode); - }; - - { - debugManager.flags.EnableTimestampPoolAllocator.set(0); - mockProductHelper->is2MBLocalMemAlignmentEnabledResult = true; - setHwMode(true); - - EXPECT_FALSE(timestampAllocator.isEnabled()); - } - { - debugManager.flags.EnableTimestampPoolAllocator.set(1); - mockProductHelper->is2MBLocalMemAlignmentEnabledResult = false; - setHwMode(false); - - EXPECT_TRUE(timestampAllocator.isEnabled()); - } - - debugManager.flags.EnableTimestampPoolAllocator.set(-1); - - { - mockProductHelper->is2MBLocalMemAlignmentEnabledResult = false; - setHwMode(false); - - EXPECT_FALSE(timestampAllocator.isEnabled()); - } - { - mockProductHelper->is2MBLocalMemAlignmentEnabledResult = false; - setHwMode(true); - - EXPECT_FALSE(timestampAllocator.isEnabled()); - } - { - mockProductHelper->is2MBLocalMemAlignmentEnabledResult = true; - setHwMode(false); - - EXPECT_FALSE(timestampAllocator.isEnabled()); - } - { - mockProductHelper->is2MBLocalMemAlignmentEnabledResult = true; - setHwMode(true); - - EXPECT_TRUE(timestampAllocator.isEnabled()); - } -} - TEST_F(TimestampPoolAllocatorTest, givenTimestampPoolAllocatorWhenPoolSizeAlignmentRequestedThenReturnsAlignedSize) { auto ×tampAllocator = pDevice->getDeviceTimestampPoolAllocator(); constexpr size_t requestAllocationSize = MemoryConstants::pageSize; - auto allocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); ASSERT_NE(nullptr, allocation); auto allocationSize = allocation->getGraphicsAllocation()->getUnderlyingBufferSize(); EXPECT_EQ(0u, allocationSize % MemoryConstants::pageSize2M); - timestampAllocator.freeSharedTimestampAllocation(allocation); + timestampAllocator.freeSharedAllocation(allocation); } TEST_F(TimestampPoolAllocatorTest, givenFailingMemoryManagerWhenRequestingAllocationThenReturnNull) { @@ -182,10 +126,10 @@ TEST_F(TimestampPoolAllocatorTest, givenFailingMemoryManagerWhenRequestingAlloca memoryManager->forceFailureInPrimaryAllocation = true; size_t requestAllocationSize = timestampAllocator.getDefaultPoolSize() / 2; - auto allocation = timestampAllocator.requestGraphicsAllocationForTimestamp(requestAllocationSize); + auto allocation = timestampAllocator.requestGraphicsAllocation(requestAllocationSize); EXPECT_EQ(nullptr, allocation); if (allocation) { - timestampAllocator.freeSharedTimestampAllocation(allocation); + timestampAllocator.freeSharedAllocation(allocation); } }