From 803bbb89d2e60e3a609e9369457a0f3443ef4485 Mon Sep 17 00:00:00 2001 From: "Warchulski, Jaroslaw" Date: Thu, 15 Dec 2022 16:01:37 +0000 Subject: [PATCH] Cleanup includes 13 Cleaned up files: shared/source/helpers/blit_commands_helper.h shared/source/helpers/heap_assigner.h shared/source/memory_manager/alignment_selector.h shared/source/memory_manager/gfx_partition.h shared/source/memory_manager/memory_manager.h shared/source/os_interface/os_memory.h shared/source/utilities/heap_allocator.h Related-To: NEO-5548 Signed-off-by: Warchulski, Jaroslaw --- .../sources/context/test_context.cpp | 1 + .../unit_tests/sources/driver/test_driver.cpp | 1 + .../unit_tests/sources/event/test_event.cpp | 1 + .../sources/helper/heap_assigner_l0_tests.cpp | 1 + .../unit_tests/sources/memory/test_memory.cpp | 1 + .../sources/memory/test_memory_wddm.cpp | 2 + opencl/source/gtpin/gtpin_helpers.cpp | 1 + .../gl/windows/gl_texture_windows.cpp | 1 + opencl/test/unit_test/mocks/mock_buffer.h | 1 + .../aub_command_stream_receiver.cpp | 1 + .../command_stream_receiver_simulated_hw.h | 1 + shared/source/device/device_caps.cpp | 1 + .../execution_environment.cpp | 1 + shared/source/helpers/blit_commands_helper.h | 2 +- shared/source/helpers/heap_assigner.cpp | 1 + shared/source/helpers/heap_assigner.h | 3 +- .../memory_manager/alignment_selector.cpp | 1 + .../memory_manager/alignment_selector.h | 7 +- .../definitions/storage_info.cpp | 2 + .../source/memory_manager/gfx_partition.cpp | 22 ++++ shared/source/memory_manager/gfx_partition.h | 23 +--- .../memory_manager/host_ptr_manager.cpp | 1 + .../source/memory_manager/memory_manager.cpp | 37 ++++++ shared/source/memory_manager/memory_manager.h | 29 ++--- .../os_agnostic_memory_manager.cpp | 1 + .../os_interface/linux/drm_allocation.cpp | 1 + .../os_interface/linux/drm_memory_manager.cpp | 1 + ...y_manager_create_multi_host_allocation.cpp | 2 + shared/source/os_interface/linux/drm_neo.cpp | 1 + shared/source/os_interface/os_memory.h | 3 +- ...igure_device_address_space_drm_or_wddm.cpp | 1 + .../source/os_interface/windows/wddm/wddm.cpp | 1 + .../windows/wddm_memory_manager.cpp | 1 + shared/source/utilities/heap_allocator.cpp | 118 ++++++++++++++++- shared/source/utilities/heap_allocator.h | 120 +----------------- .../test/common/mocks/mock_memory_manager.h | 1 + .../stream_properties_tests_common.cpp | 1 + .../unit_test/device/neo_device_tests.cpp | 1 + .../kernel/local_ids_cache_tests.cpp | 1 + .../alignment_selector_tests.cpp | 3 +- ..._device_address_space_drm_or_wddm_test.cpp | 1 + .../utilities/heap_allocator_tests.cpp | 1 + 42 files changed, 235 insertions(+), 166 deletions(-) diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index 2074ba27d8..be8aa5024c 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -5,6 +5,7 @@ * */ +#include "shared/source/gmm_helper/gmm.h" #include "shared/test/common/mocks/mock_command_stream_receiver.h" #include "shared/test/common/mocks/mock_compilers.h" #include "shared/test/common/mocks/mock_cpu_page_fault_manager.h" diff --git a/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp b/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp index c232e684de..b1652fd687 100644 --- a/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp +++ b/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp @@ -5,6 +5,7 @@ * */ +#include "shared/source/gmm_helper/gmm.h" #include "shared/source/helpers/string.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/os_interface/device_factory.h" diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index 7b558f057f..5446b280c3 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -5,6 +5,7 @@ * */ +#include "shared/source/gmm_helper/gmm.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/helpers/variable_backup.h" #include "shared/test/common/mocks/mock_compilers.h" diff --git a/level_zero/core/test/unit_tests/sources/helper/heap_assigner_l0_tests.cpp b/level_zero/core/test/unit_tests/sources/helper/heap_assigner_l0_tests.cpp index 9490a8728b..4829c462fa 100644 --- a/level_zero/core/test/unit_tests/sources/helper/heap_assigner_l0_tests.cpp +++ b/level_zero/core/test/unit_tests/sources/helper/heap_assigner_l0_tests.cpp @@ -7,6 +7,7 @@ #include "shared/source/helpers/heap_assigner.h" #include "shared/source/memory_manager/allocation_properties.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/os_interface/hw_info_config.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/test_macros/hw_test.h" diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 5eb26837a8..c670700fc3 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -5,6 +5,7 @@ * */ +#include "shared/source/gmm_helper/gmm.h" #include "shared/source/helpers/file_io.h" #include "shared/source/helpers/string.h" #include "shared/source/memory_manager/graphics_allocation.h" diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp index e7a70afca0..f5269386e8 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp @@ -5,6 +5,8 @@ * */ +#include "shared/source/gmm_helper/gmm.h" + #include "level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h" namespace L0 { diff --git a/opencl/source/gtpin/gtpin_helpers.cpp b/opencl/source/gtpin/gtpin_helpers.cpp index 78cf84cd16..fa4682bde0 100644 --- a/opencl/source/gtpin/gtpin_helpers.cpp +++ b/opencl/source/gtpin/gtpin_helpers.cpp @@ -7,6 +7,7 @@ #include "gtpin_helpers.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/memory_manager/memory_manager.h" #include "shared/source/memory_manager/unified_memory_manager.h" diff --git a/opencl/source/sharings/gl/windows/gl_texture_windows.cpp b/opencl/source/sharings/gl/windows/gl_texture_windows.cpp index d038e13429..e2dcb55981 100644 --- a/opencl/source/sharings/gl/windows/gl_texture_windows.cpp +++ b/opencl/source/sharings/gl/windows/gl_texture_windows.cpp @@ -9,6 +9,7 @@ #include "shared/source/gmm_helper/gmm.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/gmm_helper/resource_info.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/get_info.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/hw_info.h" diff --git a/opencl/test/unit_test/mocks/mock_buffer.h b/opencl/test/unit_test/mocks/mock_buffer.h index 21fba2e341..60594f76e5 100644 --- a/opencl/test/unit_test/mocks/mock_buffer.h +++ b/opencl/test/unit_test/mocks/mock_buffer.h @@ -6,6 +6,7 @@ */ #pragma once +#include "shared/source/gmm_helper/gmm.h" #include "shared/source/helpers/aligned_memory.h" #include "shared/test/common/mocks/mock_device.h" #include "shared/test/common/mocks/mock_graphics_allocation.h" diff --git a/shared/source/command_stream/aub_command_stream_receiver.cpp b/shared/source/command_stream/aub_command_stream_receiver.cpp index 0485871ff8..1483d90610 100644 --- a/shared/source/command_stream/aub_command_stream_receiver.cpp +++ b/shared/source/command_stream/aub_command_stream_receiver.cpp @@ -7,6 +7,7 @@ #include "shared/source/command_stream/aub_command_stream_receiver.h" +#include "shared/source/debug_settings/debug_settings_manager.h" #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/helpers/debug_helpers.h" diff --git a/shared/source/command_stream/command_stream_receiver_simulated_hw.h b/shared/source/command_stream/command_stream_receiver_simulated_hw.h index 4d86ae2247..d6276bfd47 100644 --- a/shared/source/command_stream/command_stream_receiver_simulated_hw.h +++ b/shared/source/command_stream/command_stream_receiver_simulated_hw.h @@ -9,6 +9,7 @@ #include "shared/source/aub/aub_helper.h" #include "shared/source/command_stream/command_stream_receiver_simulated_common_hw.h" #include "shared/source/gmm_helper/cache_settings_helper.h" +#include "shared/source/gmm_helper/gmm.h" #include "shared/source/helpers/hardware_context_controller.h" #include "shared/source/memory_manager/physical_address_allocator.h" #include "shared/source/os_interface/os_context.h" diff --git a/shared/source/device/device_caps.cpp b/shared/source/device/device_caps.cpp index 924e6ed8d7..c6cfd93976 100644 --- a/shared/source/device/device_caps.cpp +++ b/shared/source/device/device_caps.cpp @@ -11,6 +11,7 @@ #include "shared/source/debugger/debugger.h" #include "shared/source/device/device.h" #include "shared/source/execution_environment/root_device_environment.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/hw_helper.h" diff --git a/shared/source/execution_environment/execution_environment.cpp b/shared/source/execution_environment/execution_environment.cpp index d6ddae1bd4..e32d4e1463 100644 --- a/shared/source/execution_environment/execution_environment.cpp +++ b/shared/source/execution_environment/execution_environment.cpp @@ -9,6 +9,7 @@ #include "shared/source/built_ins/built_ins.h" #include "shared/source/built_ins/sip.h" +#include "shared/source/debug_settings/debug_settings_manager.h" #include "shared/source/direct_submission/direct_submission_controller.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/helpers/affinity_mask.h" diff --git a/shared/source/helpers/blit_commands_helper.h b/shared/source/helpers/blit_commands_helper.h index 97b94196a2..4c10871b80 100644 --- a/shared/source/helpers/blit_commands_helper.h +++ b/shared/source/helpers/blit_commands_helper.h @@ -7,7 +7,7 @@ #pragma once #include "shared/source/command_stream/csr_deps.h" -#include "shared/source/gmm_helper/gmm.h" +#include "shared/source/gmm_helper/gmm_lib.h" #include "shared/source/helpers/aux_translation.h" #include "shared/source/helpers/common_types.h" #include "shared/source/helpers/constants.h" diff --git a/shared/source/helpers/heap_assigner.cpp b/shared/source/helpers/heap_assigner.cpp index dc2ab0decb..c1e8893046 100644 --- a/shared/source/helpers/heap_assigner.cpp +++ b/shared/source/helpers/heap_assigner.cpp @@ -9,6 +9,7 @@ #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/hw_info.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/memory_manager.h" namespace NEO { diff --git a/shared/source/helpers/heap_assigner.h b/shared/source/helpers/heap_assigner.h index 73d8e23e2c..f95135e3e3 100644 --- a/shared/source/helpers/heap_assigner.h +++ b/shared/source/helpers/heap_assigner.h @@ -6,11 +6,10 @@ */ #pragma once -#include "shared/source/memory_manager/allocation_type.h" - #include namespace NEO { +enum class AllocationType; enum class HeapIndex : uint32_t; struct HardwareInfo; diff --git a/shared/source/memory_manager/alignment_selector.cpp b/shared/source/memory_manager/alignment_selector.cpp index ef2429759c..ed933d6dfd 100644 --- a/shared/source/memory_manager/alignment_selector.cpp +++ b/shared/source/memory_manager/alignment_selector.cpp @@ -9,6 +9,7 @@ #include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/debug_helpers.h" +#include "shared/source/memory_manager/gfx_partition.h" #include diff --git a/shared/source/memory_manager/alignment_selector.h b/shared/source/memory_manager/alignment_selector.h index 694a83f16e..c58e1b6d32 100644 --- a/shared/source/memory_manager/alignment_selector.h +++ b/shared/source/memory_manager/alignment_selector.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Intel Corporation + * Copyright (C) 2021-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,11 +7,12 @@ #pragma once -#include "shared/source/memory_manager/gfx_partition.h" - +#include +#include #include namespace NEO { +enum class HeapIndex : uint32_t; class AlignmentSelector { public: diff --git a/shared/source/memory_manager/definitions/storage_info.cpp b/shared/source/memory_manager/definitions/storage_info.cpp index 70e5bce398..bcf7bc8124 100644 --- a/shared/source/memory_manager/definitions/storage_info.cpp +++ b/shared/source/memory_manager/definitions/storage_info.cpp @@ -5,6 +5,8 @@ * */ +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/execution_environment/execution_environment.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/helpers/app_resource_helper.h" #include "shared/source/helpers/hw_helper.h" diff --git a/shared/source/memory_manager/gfx_partition.cpp b/shared/source/memory_manager/gfx_partition.cpp index 6b43e79662..741abc7544 100644 --- a/shared/source/memory_manager/gfx_partition.cpp +++ b/shared/source/memory_manager/gfx_partition.cpp @@ -144,6 +144,28 @@ void GfxPartition::freeGpuAddressRange(uint64_t ptr, size_t size) { } } +uint64_t GfxPartition::getHeapMinimalAddress(HeapIndex heapIndex) { + if (heapIndex == HeapIndex::HEAP_SVM || + heapIndex == HeapIndex::HEAP_EXTERNAL_DEVICE_FRONT_WINDOW || + heapIndex == HeapIndex::HEAP_EXTERNAL_FRONT_WINDOW || + heapIndex == HeapIndex::HEAP_INTERNAL_DEVICE_FRONT_WINDOW || + heapIndex == HeapIndex::HEAP_INTERNAL_FRONT_WINDOW) { + return getHeapBase(heapIndex); + } else { + if ((heapIndex == HeapIndex::HEAP_EXTERNAL || + heapIndex == HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY) && + (getHeapLimit(HeapAssigner::mapExternalWindowIndex(heapIndex)) != 0)) { + return getHeapBase(heapIndex) + GfxPartition::externalFrontWindowPoolSize; + } else if (heapIndex == HeapIndex::HEAP_INTERNAL || + heapIndex == HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY) { + return getHeapBase(heapIndex) + GfxPartition::internalFrontWindowPoolSize; + } else if (heapIndex == HeapIndex::HEAP_STANDARD2MB) { + return getHeapBase(heapIndex) + GfxPartition::heapGranularity2MB; + } + return getHeapBase(heapIndex) + GfxPartition::heapGranularity; + } +} + bool GfxPartition::init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToReserve, uint32_t rootDeviceIndex, size_t numRootDevices, bool useExternalFrontWindowPool) { /* diff --git a/shared/source/memory_manager/gfx_partition.h b/shared/source/memory_manager/gfx_partition.h index d88dbd0576..36462d83cc 100644 --- a/shared/source/memory_manager/gfx_partition.h +++ b/shared/source/memory_manager/gfx_partition.h @@ -7,7 +7,6 @@ #pragma once #include "shared/source/helpers/constants.h" -#include "shared/source/helpers/heap_assigner.h" #include "shared/source/os_interface/os_memory.h" #include "shared/source/utilities/heap_allocator.h" @@ -86,27 +85,7 @@ class GfxPartition { return getHeap(heapIndex).getLimit(); } - uint64_t getHeapMinimalAddress(HeapIndex heapIndex) { - if (heapIndex == HeapIndex::HEAP_SVM || - heapIndex == HeapIndex::HEAP_EXTERNAL_DEVICE_FRONT_WINDOW || - heapIndex == HeapIndex::HEAP_EXTERNAL_FRONT_WINDOW || - heapIndex == HeapIndex::HEAP_INTERNAL_DEVICE_FRONT_WINDOW || - heapIndex == HeapIndex::HEAP_INTERNAL_FRONT_WINDOW) { - return getHeapBase(heapIndex); - } else { - if ((heapIndex == HeapIndex::HEAP_EXTERNAL || - heapIndex == HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY) && - (getHeapLimit(HeapAssigner::mapExternalWindowIndex(heapIndex)) != 0)) { - return getHeapBase(heapIndex) + GfxPartition::externalFrontWindowPoolSize; - } else if (heapIndex == HeapIndex::HEAP_INTERNAL || - heapIndex == HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY) { - return getHeapBase(heapIndex) + GfxPartition::internalFrontWindowPoolSize; - } else if (heapIndex == HeapIndex::HEAP_STANDARD2MB) { - return getHeapBase(heapIndex) + GfxPartition::heapGranularity2MB; - } - return getHeapBase(heapIndex) + GfxPartition::heapGranularity; - } - } + uint64_t getHeapMinimalAddress(HeapIndex heapIndex); bool isLimitedRange() { return getHeap(HeapIndex::HEAP_SVM).getSize() == 0ull; } diff --git a/shared/source/memory_manager/host_ptr_manager.cpp b/shared/source/memory_manager/host_ptr_manager.cpp index 659d707617..f73806f491 100644 --- a/shared/source/memory_manager/host_ptr_manager.cpp +++ b/shared/source/memory_manager/host_ptr_manager.cpp @@ -8,6 +8,7 @@ #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/helpers/abort.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/memory_manager/memory_manager.h" using namespace NEO; diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 8faff533b0..9a269df761 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -25,6 +25,7 @@ #include "shared/source/memory_manager/compression_selector.h" #include "shared/source/memory_manager/deferrable_allocation_deletion.h" #include "shared/source/memory_manager/deferred_deleter.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/memory_manager/internal_allocation_storage.h" #include "shared/source/memory_manager/local_memory_usage.h" @@ -93,10 +94,25 @@ bool MemoryManager::isLimitedGPUOnType(uint32_t rootDeviceIndex, AllocationType (type != AllocationType::IMAGE); } +void *MemoryManager::alignedMallocWrapper(size_t bytes, size_t alignment) { + return ::alignedMalloc(bytes, alignment); +} + +void MemoryManager::alignedFreeWrapper(void *ptr) { + ::alignedFree(ptr); +} + GmmHelper *MemoryManager::getGmmHelper(uint32_t rootDeviceIndex) { return executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->getGmmHelper(); } +HeapIndex MemoryManager::selectInternalHeap(bool useLocalMemory) { + return useLocalMemory ? HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_INTERNAL; +} +HeapIndex MemoryManager::selectExternalHeap(bool useLocalMemory) { + return useLocalMemory ? HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_EXTERNAL; +} + void MemoryManager::zeroCpuMemoryIfRequested(const AllocationData &allocationData, void *cpuPtr, size_t size) { if (allocationData.flags.zeroMemory) { memset(cpuPtr, 0, size); @@ -259,6 +275,17 @@ void MemoryManager::checkGpuUsageAndDestroyGraphicsAllocations(GraphicsAllocatio freeGraphicsMemory(gfxAllocation); } +uint64_t MemoryManager::getInternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory) { + return getGfxPartition(rootDeviceIndex)->getHeapBase(selectInternalHeap(useLocalMemory)); +} +uint64_t MemoryManager::getExternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory) { + return getGfxPartition(rootDeviceIndex)->getHeapBase(selectExternalHeap(useLocalMemory)); +} + +bool MemoryManager::isLimitedRange(uint32_t rootDeviceIndex) { + return getGfxPartition(rootDeviceIndex)->isLimitedRange(); +} + void MemoryManager::waitForDeletions() { if (deferredDeleter) { deferredDeleter->drain(false); @@ -906,6 +933,16 @@ bool MemoryManager::isLocalMemoryUsedForIsa(uint32_t rootDeviceIndex) { return isaInLocalMemory[rootDeviceIndex]; } +bool MemoryManager::isKernelBinaryReuseEnabled() { + auto reuseBinaries = false; + + if (DebugManager.flags.ReuseKernelBinaries.get() != -1) { + reuseBinaries = DebugManager.flags.ReuseKernelBinaries.get(); + } + + return reuseBinaries; +} + OsContext *MemoryManager::getDefaultEngineContext(uint32_t rootDeviceIndex, DeviceBitfield subdevicesBitfield) { OsContext *defaultContext = nullptr; for (auto engineIndex = 0u; engineIndex < this->getRegisteredEnginesCount(); engineIndex++) { diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 6422876421..ca2b352163 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -25,6 +25,7 @@ #include namespace NEO { +class GfxPartition; struct ImageInfo; struct AllocationData; class GmmHelper; @@ -141,10 +142,10 @@ class MemoryManager { virtual double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) = 0; uint64_t getMaxApplicationAddress() { return is64bit ? MemoryConstants::max64BitAppAddress : MemoryConstants::max32BitAppAddress; }; - MOCKABLE_VIRTUAL uint64_t getInternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory) { return getGfxPartition(rootDeviceIndex)->getHeapBase(selectInternalHeap(useLocalMemory)); } - uint64_t getExternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory) { return getGfxPartition(rootDeviceIndex)->getHeapBase(selectExternalHeap(useLocalMemory)); } + MOCKABLE_VIRTUAL uint64_t getInternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory); + uint64_t getExternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory); - bool isLimitedRange(uint32_t rootDeviceIndex) { return getGfxPartition(rootDeviceIndex)->isLimitedRange(); } + bool isLimitedRange(uint32_t rootDeviceIndex); bool peek64kbPagesEnabled(uint32_t rootDeviceIndex) const; bool peekForce32BitAllocations() const { return force32bitAllocations; } @@ -176,13 +177,9 @@ class MemoryManager { return nullptr; } - MOCKABLE_VIRTUAL void *alignedMallocWrapper(size_t bytes, size_t alignment) { - return ::alignedMalloc(bytes, alignment); - } + MOCKABLE_VIRTUAL void *alignedMallocWrapper(size_t bytes, size_t alignment); - MOCKABLE_VIRTUAL void alignedFreeWrapper(void *ptr) { - ::alignedFree(ptr); - } + MOCKABLE_VIRTUAL void alignedFreeWrapper(void *ptr); MOCKABLE_VIRTUAL bool isHostPointerTrackingEnabled(uint32_t rootDeviceIndex); @@ -213,8 +210,8 @@ class MemoryManager { GmmHelper *getGmmHelper(uint32_t rootDeviceIndex); virtual AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) = 0; virtual void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) = 0; - static HeapIndex selectInternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_INTERNAL; } - static HeapIndex selectExternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_EXTERNAL; } + static HeapIndex selectInternalHeap(bool useLocalMemory); + static HeapIndex selectExternalHeap(bool useLocalMemory); static uint32_t maxOsContextCount; virtual void commonCleanup(){}; @@ -245,15 +242,7 @@ class MemoryManager { return false; } - bool isKernelBinaryReuseEnabled() { - auto reuseBinaries = false; - - if (DebugManager.flags.ReuseKernelBinaries.get() != -1) { - reuseBinaries = DebugManager.flags.ReuseKernelBinaries.get(); - } - - return reuseBinaries; - } + bool isKernelBinaryReuseEnabled(); struct KernelAllocationInfo { KernelAllocationInfo(GraphicsAllocation *allocation, uint32_t reuseCounter) : kernelAllocation(allocation), reuseCounter(reuseCounter) {} diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.cpp b/shared/source/memory_manager/os_agnostic_memory_manager.cpp index 6134c919fb..486bc6e769 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.cpp +++ b/shared/source/memory_manager/os_agnostic_memory_manager.cpp @@ -23,6 +23,7 @@ #include "shared/source/helpers/ptr_math.h" #include "shared/source/helpers/surface_format_info.h" #include "shared/source/memory_manager/allocation_properties.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/memory_manager/residency.h" #include "shared/source/os_interface/os_memory.h" diff --git a/shared/source/os_interface/linux/drm_allocation.cpp b/shared/source/os_interface/linux/drm_allocation.cpp index 07185012ac..d3e007c0d1 100644 --- a/shared/source/os_interface/linux/drm_allocation.cpp +++ b/shared/source/os_interface/linux/drm_allocation.cpp @@ -7,6 +7,7 @@ #include "shared/source/os_interface/linux/drm_allocation.h" +#include "shared/source/debug_settings/debug_settings_manager.h" #include "shared/source/memory_manager/residency.h" #include "shared/source/os_interface/linux/cache_info.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index dec4c831bd..1d67421453 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -22,6 +22,7 @@ #include "shared/source/helpers/string.h" #include "shared/source/helpers/surface_format_info.h" #include "shared/source/memory_manager/allocation_properties.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/memory_manager/memory_banks.h" #include "shared/source/memory_manager/memory_pool.h" diff --git a/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp b/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp index 248dd97891..9579e34fa2 100644 --- a/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp @@ -7,7 +7,9 @@ #include "shared/source/gmm_helper/gmm.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/memory_manager/allocation_properties.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/memory_pool.h" #include "shared/source/os_interface/linux/drm_allocation.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" diff --git a/shared/source/os_interface/linux/drm_neo.cpp b/shared/source/os_interface/linux/drm_neo.cpp index 16280c587a..e3ba9589ef 100644 --- a/shared/source/os_interface/linux/drm_neo.cpp +++ b/shared/source/os_interface/linux/drm_neo.cpp @@ -46,6 +46,7 @@ #include #include #include +#include namespace NEO { diff --git a/shared/source/os_interface/os_memory.h b/shared/source/os_interface/os_memory.h index df08da843e..7c521b004d 100644 --- a/shared/source/os_interface/os_memory.h +++ b/shared/source/os_interface/os_memory.h @@ -1,12 +1,11 @@ /* - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation * * SPDX-License-Identifier: MIT * */ #pragma once -#include #include #include diff --git a/shared/source/os_interface/windows/wddm/configure_device_address_space_drm_or_wddm.cpp b/shared/source/os_interface/windows/wddm/configure_device_address_space_drm_or_wddm.cpp index 6a94852da6..373225c6c5 100644 --- a/shared/source/os_interface/windows/wddm/configure_device_address_space_drm_or_wddm.cpp +++ b/shared/source/os_interface/windows/wddm/configure_device_address_space_drm_or_wddm.cpp @@ -12,6 +12,7 @@ #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/gmm_helper/page_table_mngr.h" #include "shared/source/gmm_helper/resource_info.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/helpers/windows/gmm_callbacks.h" #include "shared/source/memory_manager/gfx_partition.h" diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index f4b2bb9948..237778a15d 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -22,6 +22,7 @@ #include "shared/source/helpers/mt_helpers.h" #include "shared/source/helpers/string.h" #include "shared/source/helpers/windows/gmm_callbacks.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/os_interface/hw_info_config.h" #include "shared/source/os_interface/sys_calls_common.h" #include "shared/source/os_interface/windows/driver_info_windows.h" diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index 51bcbb7b0d..06483c39fa 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -26,6 +26,7 @@ #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/deferrable_deletion.h" #include "shared/source/memory_manager/deferred_deleter.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/memory_manager/memory_operations_handler.h" #include "shared/source/os_interface/hw_info_config.h" diff --git a/shared/source/utilities/heap_allocator.cpp b/shared/source/utilities/heap_allocator.cpp index e7a212452a..b076efe53d 100644 --- a/shared/source/utilities/heap_allocator.cpp +++ b/shared/source/utilities/heap_allocator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,9 +7,125 @@ #include "shared/source/utilities/heap_allocator.h" +#include "shared/source/helpers/aligned_memory.h" + namespace NEO { bool operator<(const HeapChunk &hc1, const HeapChunk &hc2) { return hc1.ptr < hc2.ptr; } + +uint64_t HeapAllocator::allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment) { + if (alignment == 0) { + alignment = this->allocationAlignment; + } + + UNRECOVERABLE_IF(alignment % allocationAlignment != 0); // custom alignment have to be a multiple of allocator alignment + sizeToAllocate = alignUp(sizeToAllocate, allocationAlignment); + + std::lock_guard lock(mtx); + DBG_LOG(LogAllocationMemoryPool, __FUNCTION__, "Allocator usage == ", this->getUsage()); + if (availableSize < sizeToAllocate) { + return 0llu; + } + + std::vector &freedChunks = (sizeToAllocate > sizeThreshold) ? freedChunksBig : freedChunksSmall; + uint32_t defragmentCount = 0; + + for (;;) { + size_t sizeOfFreedChunk = 0; + uint64_t ptrReturn = getFromFreedChunks(sizeToAllocate, freedChunks, sizeOfFreedChunk, alignment); + + if (ptrReturn == 0llu) { + if (sizeToAllocate > sizeThreshold) { + const uint64_t misalignment = alignUp(pLeftBound, alignment) - pLeftBound; + if (pLeftBound + misalignment + sizeToAllocate <= pRightBound) { + if (misalignment) { + storeInFreedChunks(pLeftBound, static_cast(misalignment), freedChunks); + pLeftBound += misalignment; + } + ptrReturn = pLeftBound; + pLeftBound += sizeToAllocate; + } + } else { + const uint64_t pStart = pRightBound - sizeToAllocate; + const uint64_t misalignment = pStart - alignDown(pStart, alignment); + if (pLeftBound + sizeToAllocate + misalignment <= pRightBound) { + if (misalignment) { + pRightBound -= misalignment; + storeInFreedChunks(pRightBound, static_cast(misalignment), freedChunks); + } + pRightBound -= sizeToAllocate; + ptrReturn = pRightBound; + } + } + } + + if (ptrReturn != 0llu) { + if (sizeOfFreedChunk > 0) { + availableSize -= sizeOfFreedChunk; + sizeToAllocate = sizeOfFreedChunk; + } else { + availableSize -= sizeToAllocate; + } + DEBUG_BREAK_IF(!isAligned(ptrReturn, alignment)); + return ptrReturn; + } + + if (defragmentCount == 1) + return 0llu; + defragment(); + defragmentCount++; + } +} + +uint64_t HeapAllocator::getFromFreedChunks(size_t size, std::vector &freedChunks, size_t &sizeOfFreedChunk, size_t requiredAlignment) { + size_t elements = freedChunks.size(); + size_t bestFitIndex = -1; + size_t bestFitSize = 0; + sizeOfFreedChunk = 0; + + for (size_t i = 0; i < elements; i++) { + const bool chunkAligned = isAligned(freedChunks[i].ptr, requiredAlignment); + if (!chunkAligned) { + continue; + } + + if (freedChunks[i].size == size) { + auto ptr = freedChunks[i].ptr; + freedChunks.erase(freedChunks.begin() + i); + return ptr; + } + + if (freedChunks[i].size > size) { + if (freedChunks[i].size < bestFitSize || bestFitSize == 0) { + bestFitIndex = i; + bestFitSize = freedChunks[i].size; + } + } + } + + if (bestFitSize != 0) { + if (bestFitSize < (size << 1)) { + auto ptr = freedChunks[bestFitIndex].ptr; + sizeOfFreedChunk = freedChunks[bestFitIndex].size; + freedChunks.erase(freedChunks.begin() + bestFitIndex); + return ptr; + } else { + size_t sizeDelta = freedChunks[bestFitIndex].size - size; + + DEBUG_BREAK_IF(!(size <= sizeThreshold || (size > sizeThreshold && sizeDelta > sizeThreshold))); + + auto ptr = freedChunks[bestFitIndex].ptr + sizeDelta; + freedChunks[bestFitIndex].size = sizeDelta; + + if (!isAligned(ptr, requiredAlignment)) { + return 0llu; + } + return ptr; + } + } + return 0llu; +} + } // namespace NEO diff --git a/shared/source/utilities/heap_allocator.h b/shared/source/utilities/heap_allocator.h index 8e8a5b4d0f..56879047b6 100644 --- a/shared/source/utilities/heap_allocator.h +++ b/shared/source/utilities/heap_allocator.h @@ -6,10 +6,11 @@ */ #pragma once -#include "shared/source/helpers/aligned_memory.h" -#include "shared/source/helpers/debug_helpers.h" -#include +#include "shared/source/helpers/constants.h" +#include "shared/source/helpers/debug_helpers.h" +#include "shared/source/utilities/logger.h" + #include #include @@ -42,69 +43,7 @@ class HeapAllocator { return allocateWithCustomAlignment(sizeToAllocate, 0u); } - uint64_t allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment) { - if (alignment == 0) { - alignment = this->allocationAlignment; - } - - UNRECOVERABLE_IF(alignment % allocationAlignment != 0); // custom alignment have to be a multiple of allocator alignment - sizeToAllocate = alignUp(sizeToAllocate, allocationAlignment); - - std::lock_guard lock(mtx); - DBG_LOG(LogAllocationMemoryPool, __FUNCTION__, "Allocator usage == ", this->getUsage()); - if (availableSize < sizeToAllocate) { - return 0llu; - } - - std::vector &freedChunks = (sizeToAllocate > sizeThreshold) ? freedChunksBig : freedChunksSmall; - uint32_t defragmentCount = 0; - - for (;;) { - size_t sizeOfFreedChunk = 0; - uint64_t ptrReturn = getFromFreedChunks(sizeToAllocate, freedChunks, sizeOfFreedChunk, alignment); - - if (ptrReturn == 0llu) { - if (sizeToAllocate > sizeThreshold) { - const uint64_t misalignment = alignUp(pLeftBound, alignment) - pLeftBound; - if (pLeftBound + misalignment + sizeToAllocate <= pRightBound) { - if (misalignment) { - storeInFreedChunks(pLeftBound, static_cast(misalignment), freedChunks); - pLeftBound += misalignment; - } - ptrReturn = pLeftBound; - pLeftBound += sizeToAllocate; - } - } else { - const uint64_t pStart = pRightBound - sizeToAllocate; - const uint64_t misalignment = pStart - alignDown(pStart, alignment); - if (pLeftBound + sizeToAllocate + misalignment <= pRightBound) { - if (misalignment) { - pRightBound -= misalignment; - storeInFreedChunks(pRightBound, static_cast(misalignment), freedChunks); - } - pRightBound -= sizeToAllocate; - ptrReturn = pRightBound; - } - } - } - - if (ptrReturn != 0llu) { - if (sizeOfFreedChunk > 0) { - availableSize -= sizeOfFreedChunk; - sizeToAllocate = sizeOfFreedChunk; - } else { - availableSize -= sizeToAllocate; - } - DEBUG_BREAK_IF(!isAligned(ptrReturn, alignment)); - return ptrReturn; - } - - if (defragmentCount == 1) - return 0llu; - defragment(); - defragmentCount++; - } - } + uint64_t allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment); void free(uint64_t ptr, size_t size) { if (ptr == 0llu) @@ -153,54 +92,7 @@ class HeapAllocator { std::vector freedChunksBig; std::mutex mtx; - uint64_t getFromFreedChunks(size_t size, std::vector &freedChunks, size_t &sizeOfFreedChunk, size_t requiredAlignment) { - size_t elements = freedChunks.size(); - size_t bestFitIndex = -1; - size_t bestFitSize = 0; - sizeOfFreedChunk = 0; - - for (size_t i = 0; i < elements; i++) { - const bool chunkAligned = isAligned(freedChunks[i].ptr, requiredAlignment); - if (!chunkAligned) { - continue; - } - - if (freedChunks[i].size == size) { - auto ptr = freedChunks[i].ptr; - freedChunks.erase(freedChunks.begin() + i); - return ptr; - } - - if (freedChunks[i].size > size) { - if (freedChunks[i].size < bestFitSize || bestFitSize == 0) { - bestFitIndex = i; - bestFitSize = freedChunks[i].size; - } - } - } - - if (bestFitSize != 0) { - if (bestFitSize < (size << 1)) { - auto ptr = freedChunks[bestFitIndex].ptr; - sizeOfFreedChunk = freedChunks[bestFitIndex].size; - freedChunks.erase(freedChunks.begin() + bestFitIndex); - return ptr; - } else { - size_t sizeDelta = freedChunks[bestFitIndex].size - size; - - DEBUG_BREAK_IF(!(size <= sizeThreshold || (size > sizeThreshold && sizeDelta > sizeThreshold))); - - auto ptr = freedChunks[bestFitIndex].ptr + sizeDelta; - freedChunks[bestFitIndex].size = sizeDelta; - - if (!isAligned(ptr, requiredAlignment)) { - return 0llu; - } - return ptr; - } - } - return 0llu; - } + uint64_t getFromFreedChunks(size_t size, std::vector &freedChunks, size_t &sizeOfFreedChunk, size_t requiredAlignment); void storeInFreedChunks(uint64_t ptr, size_t size, std::vector &freedChunks) { for (auto &freedChunk : freedChunks) { diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index d546401526..54ab9b15da 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -9,6 +9,7 @@ #include "shared/source/command_stream/command_stream_receiver.h" #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/memory_manager/allocation_properties.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/os_agnostic_memory_manager.h" #include "shared/test/common/helpers/default_hw_info.h" #include "shared/test/common/mocks/mock_execution_environment.h" diff --git a/shared/test/unit_test/command_stream/stream_properties_tests_common.cpp b/shared/test/unit_test/command_stream/stream_properties_tests_common.cpp index 7488a35230..bbcbb98951 100644 --- a/shared/test/unit_test/command_stream/stream_properties_tests_common.cpp +++ b/shared/test/unit_test/command_stream/stream_properties_tests_common.cpp @@ -7,6 +7,7 @@ #include "shared/test/unit_test/command_stream/stream_properties_tests_common.h" +#include "shared/source/command_stream/preemption_mode.h" #include "shared/source/command_stream/stream_properties.h" #include "shared/source/command_stream/thread_arbitration_policy.h" #include "shared/source/helpers/hw_helper.h" diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp index 72a361bd76..ff528e6c81 100644 --- a/shared/test/unit_test/device/neo_device_tests.cpp +++ b/shared/test/unit_test/device/neo_device_tests.cpp @@ -6,6 +6,7 @@ */ #include "shared/source/device/device.h" +#include "shared/source/gmm_helper/gmm.h" #include "shared/source/os_interface/device_factory.h" #include "shared/source/os_interface/os_interface.h" #include "shared/test/common/fixtures/device_fixture.h" diff --git a/shared/test/unit_test/kernel/local_ids_cache_tests.cpp b/shared/test/unit_test/kernel/local_ids_cache_tests.cpp index e61d288e3e..4a95f192f4 100644 --- a/shared/test/unit_test/kernel/local_ids_cache_tests.cpp +++ b/shared/test/unit_test/kernel/local_ids_cache_tests.cpp @@ -6,6 +6,7 @@ */ #include "shared/source/command_stream/linear_stream.h" +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/per_thread_data.h" #include "shared/source/kernel/local_ids_cache.h" #include "shared/test/common/mocks/mock_graphics_allocation.h" diff --git a/shared/test/unit_test/memory_manager/alignment_selector_tests.cpp b/shared/test/unit_test/memory_manager/alignment_selector_tests.cpp index 1e402ac2b9..d5813e1e07 100644 --- a/shared/test/unit_test/memory_manager/alignment_selector_tests.cpp +++ b/shared/test/unit_test/memory_manager/alignment_selector_tests.cpp @@ -1,11 +1,12 @@ /* - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation * * SPDX-License-Identifier: MIT * */ #include "shared/source/memory_manager/alignment_selector.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/test/common/test_macros/test.h" namespace NEO { diff --git a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp index 4f1e308a93..c896ea339b 100644 --- a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp +++ b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp @@ -7,6 +7,7 @@ #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/surface_format_info.h" +#include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/os_interface/hw_info_config.h" #include "shared/source/os_interface/linux/os_time_linux.h" #include "shared/source/os_interface/os_interface.h" diff --git a/shared/test/unit_test/utilities/heap_allocator_tests.cpp b/shared/test/unit_test/utilities/heap_allocator_tests.cpp index 43dc12daf5..bdd28df867 100644 --- a/shared/test/unit_test/utilities/heap_allocator_tests.cpp +++ b/shared/test/unit_test/utilities/heap_allocator_tests.cpp @@ -5,6 +5,7 @@ * */ +#include "shared/source/helpers/aligned_memory.h" #include "shared/source/utilities/heap_allocator.h" #include "shared/test/common/test_macros/test.h"