From 2ac968e6c21b830957bd7a7abdfb898b6042c8d2 Mon Sep 17 00:00:00 2001 From: Igor Venevtsev Date: Sun, 29 Mar 2020 19:40:34 +0200 Subject: [PATCH] Add alignment capability do OSMemory::reserveCpuAddressRange Resolves: NEO-4510 Change-Id: Iffcb33d1c06ca930345df0216bc5d3d1ce12c313 Signed-off-by: Igor Venevtsev --- .../test/unit_test/mocks/mock_gfx_partition.h | 4 +-- .../linux/drm_os_memory_tests.cpp | 26 ++++++++++++--- .../os_interface/os_memory_tests.cpp | 9 +++-- .../source/memory_manager/gfx_partition.cpp | 17 ++++------ shared/source/memory_manager/gfx_partition.h | 3 +- shared/source/os_interface/CMakeLists.txt | 1 + .../os_interface/linux/os_memory_linux.cpp | 6 ++-- .../os_interface/linux/os_memory_linux.h | 5 +-- shared/source/os_interface/os_memory.cpp | 33 +++++++++++++++++++ shared/source/os_interface/os_memory.h | 16 +++++++-- .../os_interface/windows/os_memory_win.cpp | 4 +-- .../os_interface/windows/os_memory_win.h | 5 +-- 12 files changed, 97 insertions(+), 32 deletions(-) create mode 100644 shared/source/os_interface/os_memory.cpp diff --git a/opencl/test/unit_test/mocks/mock_gfx_partition.h b/opencl/test/unit_test/mocks/mock_gfx_partition.h index 46421a556d..100901e63d 100644 --- a/opencl/test/unit_test/mocks/mock_gfx_partition.h +++ b/opencl/test/unit_test/mocks/mock_gfx_partition.h @@ -24,11 +24,11 @@ class MockGfxPartition : public GfxPartition { } void *getReservedCpuAddressRange() { - return reservedCpuAddressRange; + return reservedCpuAddressRange.alignedPtr; } size_t getReservedCpuAddressRangeSize() { - return reservedCpuAddressRangeSize; + return reservedCpuAddressRange.actualReservedSize - GfxPartition::heapGranularity; } MOCK_METHOD2(freeGpuAddressRange, void(uint64_t gpuAddress, size_t size)); diff --git a/opencl/test/unit_test/os_interface/linux/drm_os_memory_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_os_memory_tests.cpp index ed4532f8a7..9190a3c819 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_os_memory_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_os_memory_tests.cpp @@ -5,6 +5,7 @@ * */ +#include "shared/source/memory_manager/memory_constants.h" #include "shared/source/os_interface/linux/os_memory_linux.h" #include "gmock/gmock.h" @@ -20,8 +21,26 @@ class MockOSMemoryLinux : public OSMemoryLinux { return std::make_unique(); } + MockOSMemoryLinux() { + ON_CALL(*this, mmapWrapper).WillByDefault([this](void *addr, size_t size, int prot, int flags, int fd, off_t off) { + return this->baseMmapWrapper(addr, size, prot, flags, fd, off); + }); + + ON_CALL(*this, munmapWrapper).WillByDefault([this](void *addr, size_t size) { + return this->baseMunmapWrapper(addr, size); + }); + } + MOCK_METHOD6(mmapWrapper, void *(void *, size_t, int, int, int, off_t)); MOCK_METHOD2(munmapWrapper, int(void *, size_t)); + + void *baseMmapWrapper(void *addr, size_t size, int prot, int flags, int fd, off_t off) { + return OSMemoryLinux::mmapWrapper(addr, size, prot, flags, fd, off); + } + + int baseMunmapWrapper(void *addr, size_t size) { + return OSMemoryLinux::munmapWrapper(addr, size); + } }; TEST(OSMemoryLinux, givenOSMemoryLinuxWhenReserveCpuAddressRangeIsCalledThenMinusOneIsPassedToMmapAsFdParam) { @@ -29,12 +48,11 @@ TEST(OSMemoryLinux, givenOSMemoryLinuxWhenReserveCpuAddressRangeIsCalledThenMinu EXPECT_CALL(*mockOSMemoryLinux, mmapWrapper(_, _, _, _, -1, _)); - size_t size = 0x1024; - auto reservedCpuAddr = mockOSMemoryLinux->reserveCpuAddressRange(size); + auto reservedCpuRange = mockOSMemoryLinux->reserveCpuAddressRange(MemoryConstants::pageSize, MemoryConstants::pageSize64k); - EXPECT_CALL(*mockOSMemoryLinux, munmapWrapper(reservedCpuAddr, size)); + EXPECT_CALL(*mockOSMemoryLinux, munmapWrapper(reservedCpuRange.originalPtr, reservedCpuRange.actualReservedSize)); - mockOSMemoryLinux->releaseCpuAddressRange(reservedCpuAddr, size); + mockOSMemoryLinux->releaseCpuAddressRange(reservedCpuRange); } }; // namespace NEO diff --git a/opencl/test/unit_test/os_interface/os_memory_tests.cpp b/opencl/test/unit_test/os_interface/os_memory_tests.cpp index c1274120d9..f861d41a28 100644 --- a/opencl/test/unit_test/os_interface/os_memory_tests.cpp +++ b/opencl/test/unit_test/os_interface/os_memory_tests.cpp @@ -5,6 +5,8 @@ * */ +#include "shared/source/helpers/aligned_memory.h" +#include "shared/source/memory_manager/memory_constants.h" #include "shared/source/os_interface/os_memory.h" #include "gtest/gtest.h" @@ -14,7 +16,8 @@ using namespace NEO; TEST(OSMemory, reserveCpuAddressRange) { auto osMemory = OSMemory::create(); size_t reservedCpuAddressRangeSize = 1024; - auto reservedCpuAddressRange = osMemory->reserveCpuAddressRange(reservedCpuAddressRangeSize); - EXPECT_NE(reservedCpuAddressRange, nullptr); - osMemory->releaseCpuAddressRange(reservedCpuAddressRange, reservedCpuAddressRangeSize); + auto reservedCpuAddressRange = osMemory->reserveCpuAddressRange(reservedCpuAddressRangeSize, MemoryConstants::pageSize64k); + EXPECT_NE(reservedCpuAddressRange.originalPtr, nullptr); + EXPECT_TRUE(isAligned(reservedCpuAddressRange.alignedPtr)); + osMemory->releaseCpuAddressRange(reservedCpuAddressRange); } diff --git a/shared/source/memory_manager/gfx_partition.cpp b/shared/source/memory_manager/gfx_partition.cpp index ab25fe5e15..e929584d2d 100644 --- a/shared/source/memory_manager/gfx_partition.cpp +++ b/shared/source/memory_manager/gfx_partition.cpp @@ -27,9 +27,7 @@ const std::array GfxPartition::heapNonSvmNames{{HeapIndex::HEAP_IN GfxPartition::GfxPartition() : osMemory(OSMemory::create()) {} GfxPartition::~GfxPartition() { - if (reservedCpuAddressRange) { - osMemory->releaseCpuAddressRange(reservedCpuAddressRange, reservedCpuAddressRangeSize); - } + osMemory->releaseCpuAddressRange(reservedCpuAddressRange); } void GfxPartition::Heap::init(uint64_t base, uint64_t size) { @@ -114,13 +112,12 @@ void GfxPartition::init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToRe gfxBase = maxNBitValue(48 - 1) + 1; heapInit(HeapIndex::HEAP_SVM, 0ull, gfxBase); } else if (gpuAddressSpace == maxNBitValue(47)) { - reservedCpuAddressRangeSize = cpuAddressRangeSizeToReserve; - UNRECOVERABLE_IF(reservedCpuAddressRangeSize == 0); - reservedCpuAddressRange = osMemory->reserveCpuAddressRange(reservedCpuAddressRangeSize); - UNRECOVERABLE_IF(reservedCpuAddressRange == nullptr); - UNRECOVERABLE_IF(!isAligned(reservedCpuAddressRange)); - gfxBase = reinterpret_cast(reservedCpuAddressRange); - gfxTop = gfxBase + reservedCpuAddressRangeSize; + UNRECOVERABLE_IF(cpuAddressRangeSizeToReserve == 0); + reservedCpuAddressRange = osMemory->reserveCpuAddressRange(cpuAddressRangeSizeToReserve, GfxPartition::heapGranularity); + UNRECOVERABLE_IF(reservedCpuAddressRange.originalPtr == nullptr); + UNRECOVERABLE_IF(!isAligned(reservedCpuAddressRange.alignedPtr)); + gfxBase = reinterpret_cast(reservedCpuAddressRange.alignedPtr); + gfxTop = gfxBase + cpuAddressRangeSizeToReserve; heapInit(HeapIndex::HEAP_SVM, 0ull, gpuAddressSpace + 1); } else if (gpuAddressSpace < maxNBitValue(47)) { gfxBase = 0ull; diff --git a/shared/source/memory_manager/gfx_partition.h b/shared/source/memory_manager/gfx_partition.h index 0b9fe60c24..088576b688 100644 --- a/shared/source/memory_manager/gfx_partition.h +++ b/shared/source/memory_manager/gfx_partition.h @@ -92,8 +92,7 @@ class GfxPartition { std::array(HeapIndex::TOTAL_HEAPS)> heaps; - void *reservedCpuAddressRange = nullptr; - size_t reservedCpuAddressRangeSize = 0; + OSMemory::ReservedCpuAddressRange reservedCpuAddressRange; std::unique_ptr osMemory; }; diff --git a/shared/source/os_interface/CMakeLists.txt b/shared/source/os_interface/CMakeLists.txt index 0576412acc..4e6e41d43f 100644 --- a/shared/source/os_interface/CMakeLists.txt +++ b/shared/source/os_interface/CMakeLists.txt @@ -18,6 +18,7 @@ set(NEO_CORE_OS_INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/os_environment.h ${CMAKE_CURRENT_SOURCE_DIR}/os_interface.h ${CMAKE_CURRENT_SOURCE_DIR}/os_library.h + ${CMAKE_CURRENT_SOURCE_DIR}/os_memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/os_memory.h ${CMAKE_CURRENT_SOURCE_DIR}/os_thread.h ${CMAKE_CURRENT_SOURCE_DIR}/os_time.cpp diff --git a/shared/source/os_interface/linux/os_memory_linux.cpp b/shared/source/os_interface/linux/os_memory_linux.cpp index 78c6a4033a..1c8f5957cd 100644 --- a/shared/source/os_interface/linux/os_memory_linux.cpp +++ b/shared/source/os_interface/linux/os_memory_linux.cpp @@ -13,11 +13,11 @@ std::unique_ptr OSMemory::create() { return std::make_unique(); } -void *OSMemoryLinux::reserveCpuAddressRange(size_t sizeToReserve) { - return mmapWrapper(0, sizeToReserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_HUGETLB, -1, 0); +void *OSMemoryLinux::osReserveCpuAddressRange(size_t sizeToReserve) { + return mmapWrapper(0, sizeToReserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); } -void OSMemoryLinux::releaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) { +void OSMemoryLinux::osReleaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) { munmapWrapper(reservedCpuAddressRange, reservedSize); } diff --git a/shared/source/os_interface/linux/os_memory_linux.h b/shared/source/os_interface/linux/os_memory_linux.h index 432fbad9e4..a8abcdc334 100644 --- a/shared/source/os_interface/linux/os_memory_linux.h +++ b/shared/source/os_interface/linux/os_memory_linux.h @@ -15,10 +15,11 @@ namespace NEO { class OSMemoryLinux : public OSMemory { public: OSMemoryLinux() = default; - void *reserveCpuAddressRange(size_t sizeToReserve) override; - void releaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) override; protected: + void *osReserveCpuAddressRange(size_t sizeToReserve) override; + void osReleaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) override; + MOCKABLE_VIRTUAL void *mmapWrapper(void *, size_t, int, int, int, off_t); MOCKABLE_VIRTUAL int munmapWrapper(void *, size_t); }; diff --git a/shared/source/os_interface/os_memory.cpp b/shared/source/os_interface/os_memory.cpp new file mode 100644 index 0000000000..35cec82050 --- /dev/null +++ b/shared/source/os_interface/os_memory.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/os_memory.h" + +#include "shared/source/helpers/aligned_memory.h" + +namespace NEO { + +OSMemory::ReservedCpuAddressRange OSMemory::reserveCpuAddressRange(size_t sizeToReserve, size_t alignment) { + UNRECOVERABLE_IF(0 == alignment); + UNRECOVERABLE_IF(0 != (alignment & (alignment - 1))); + + ReservedCpuAddressRange reservedCpuAddressRange; + + reservedCpuAddressRange.actualReservedSize = sizeToReserve + alignment; + reservedCpuAddressRange.originalPtr = this->osReserveCpuAddressRange(reservedCpuAddressRange.actualReservedSize); + reservedCpuAddressRange.alignedPtr = alignUp(reservedCpuAddressRange.originalPtr, alignment); + + return reservedCpuAddressRange; +} + +void OSMemory::releaseCpuAddressRange(const ReservedCpuAddressRange &reservedCpuAddressRange) { + if (reservedCpuAddressRange.originalPtr != nullptr) { + this->osReleaseCpuAddressRange(reservedCpuAddressRange.originalPtr, reservedCpuAddressRange.actualReservedSize); + } +} + +} // namespace NEO diff --git a/shared/source/os_interface/os_memory.h b/shared/source/os_interface/os_memory.h index 40321e0e03..b7b1a5e59d 100644 --- a/shared/source/os_interface/os_memory.h +++ b/shared/source/os_interface/os_memory.h @@ -12,12 +12,24 @@ namespace NEO { struct OSMemory { + public: + struct ReservedCpuAddressRange { + void *originalPtr = nullptr; + void *alignedPtr = nullptr; + size_t actualReservedSize = 0; + }; + public: static std::unique_ptr create(); virtual ~OSMemory() = default; - virtual void *reserveCpuAddressRange(size_t sizeToReserve) = 0; - virtual void releaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) = 0; + + MOCKABLE_VIRTUAL ReservedCpuAddressRange reserveCpuAddressRange(size_t sizeToReserve, size_t alignment); + MOCKABLE_VIRTUAL void releaseCpuAddressRange(const ReservedCpuAddressRange &reservedCpuAddressRange); + + protected: + virtual void *osReserveCpuAddressRange(size_t sizeToReserve) = 0; + virtual void osReleaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) = 0; }; } // namespace NEO diff --git a/shared/source/os_interface/windows/os_memory_win.cpp b/shared/source/os_interface/windows/os_memory_win.cpp index 27d6c2dddd..405c4b0bc0 100644 --- a/shared/source/os_interface/windows/os_memory_win.cpp +++ b/shared/source/os_interface/windows/os_memory_win.cpp @@ -13,11 +13,11 @@ std::unique_ptr OSMemory::create() { return std::make_unique(); } -void *OSMemoryWindows::reserveCpuAddressRange(size_t sizeToReserve) { +void *OSMemoryWindows::osReserveCpuAddressRange(size_t sizeToReserve) { return virtualAllocWrapper(0, sizeToReserve, MEM_RESERVE, PAGE_READWRITE); } -void OSMemoryWindows::releaseCpuAddressRange(void *reservedCpuAddressRange, size_t /* reservedSize */) { +void OSMemoryWindows::osReleaseCpuAddressRange(void *reservedCpuAddressRange, size_t /* reservedSize */) { virtualFreeWrapper(reservedCpuAddressRange, 0, MEM_RELEASE); } diff --git a/shared/source/os_interface/windows/os_memory_win.h b/shared/source/os_interface/windows/os_memory_win.h index bc60be96a2..8f688e8a20 100644 --- a/shared/source/os_interface/windows/os_memory_win.h +++ b/shared/source/os_interface/windows/os_memory_win.h @@ -14,10 +14,11 @@ namespace NEO { class OSMemoryWindows : public OSMemory { public: OSMemoryWindows() = default; - void *reserveCpuAddressRange(size_t sizeToReserve) override; - void releaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) override; protected: + void *osReserveCpuAddressRange(size_t sizeToReserve) override; + void osReleaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) override; + MOCKABLE_VIRTUAL LPVOID virtualAllocWrapper(LPVOID, SIZE_T, DWORD, DWORD); MOCKABLE_VIRTUAL BOOL virtualFreeWrapper(LPVOID, SIZE_T, DWORD); };