diff --git a/runtime/memory_manager/CMakeLists.txt b/runtime/memory_manager/CMakeLists.txt index 96cdd80b9d..699ead3052 100644 --- a/runtime/memory_manager/CMakeLists.txt +++ b/runtime/memory_manager/CMakeLists.txt @@ -39,6 +39,7 @@ set(RUNTIME_SRCS_MEMORY_MANAGER ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/os_agnostic_memory_manager_allocate_in_device_pool.cpp ${CMAKE_CURRENT_SOURCE_DIR}/page_table.cpp ${CMAKE_CURRENT_SOURCE_DIR}/page_table.h + ${CMAKE_CURRENT_SOURCE_DIR}/page_table.inl ${CMAKE_CURRENT_SOURCE_DIR}/residency.h ${CMAKE_CURRENT_SOURCE_DIR}/residency.cpp ${CMAKE_CURRENT_SOURCE_DIR}/residency_container.h diff --git a/runtime/memory_manager/page_table.cpp b/runtime/memory_manager/page_table.cpp index 573d92738d..493c84c5f7 100644 --- a/runtime/memory_manager/page_table.cpp +++ b/runtime/memory_manager/page_table.cpp @@ -21,25 +21,13 @@ */ #include "runtime/memory_manager/page_table.h" - -#include -#include +#include "runtime/memory_manager/page_table.inl" namespace OCLRT { const uint32_t PTE::initialPage = 1; std::atomic PTE::nextPage(PTE::initialPage); -template <> -uintptr_t PageTable::map(uintptr_t vm, size_t size) { - return 0; -} - -template <> -size_t PageTable::getBits() { - return 9; -} - uintptr_t PTE::map(uintptr_t vm, size_t size) { const size_t shift = 12; const uint32_t mask = (1 << bits) - 1; @@ -57,30 +45,6 @@ uintptr_t PTE::map(uintptr_t vm, size_t size) { return (res & ~0x1) + (vm & (pageSize - 1)); } -template -uintptr_t PageTable::map(uintptr_t vm, size_t size) { - const size_t shift = T::getBits() + 12; - const uintptr_t mask = (1 << bits) - 1; - size_t indexStart = (vm >> shift) & mask; - size_t indexEnd = ((vm + size - 1) >> shift) & mask; - uintptr_t res = -1; - uintptr_t vmMask = (uintptr_t(-1) >> (sizeof(void *) * 8 - shift - bits)); - auto maskedVm = vm & vmMask; - - for (size_t index = indexStart; index <= indexEnd; index++) { - uintptr_t vmStart = (uintptr_t(1) << shift) * index; - vmStart = std::max(vmStart, maskedVm); - uintptr_t vmEnd = (uintptr_t(1) << shift) * (index + 1) - 1; - vmEnd = std::min(vmEnd, maskedVm + size - 1); - - if (entries[index] == nullptr) { - entries[index] = new T; - } - res = std::min((entries[index])->map(vmStart, vmEnd - vmStart + 1), res); - } - return res; -} - void PTE::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker) { static const uint32_t bits = 9; const size_t shift = 12; @@ -106,44 +70,6 @@ void PTE::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWal } } -template <> -void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker) { -} - -template -void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker) { - const size_t shift = T::getBits() + 12; - const uintptr_t mask = (1 << bits) - 1; - size_t indexStart = (vm >> shift) & mask; - size_t indexEnd = ((vm + size - 1) >> shift) & mask; - uintptr_t vmMask = (uintptr_t(-1) >> (sizeof(void *) * 8 - shift - bits)); - auto maskedVm = vm & vmMask; - - for (size_t index = indexStart; index <= indexEnd; index++) { - uintptr_t vmStart = (uintptr_t(1) << shift) * index; - vmStart = std::max(vmStart, maskedVm); - uintptr_t vmEnd = (uintptr_t(1) << shift) * (index + 1) - 1; - vmEnd = std::min(vmEnd, maskedVm + size - 1); - - if (entries[index] == nullptr) { - entries[index] = new T; - } - entries[index]->pageWalk(vmStart, vmEnd - vmStart + 1, offset, pageWalker); - - offset += (vmEnd - vmStart + 1); - } -} - -template <> -PageTable::~PageTable() { -} - -template -PageTable::~PageTable() { - for (auto &e : entries) - delete e; -} - template class PageTable; template class PageTable; } // namespace OCLRT diff --git a/runtime/memory_manager/page_table.h b/runtime/memory_manager/page_table.h index 15d72719ae..25552d24a7 100644 --- a/runtime/memory_manager/page_table.h +++ b/runtime/memory_manager/page_table.h @@ -23,9 +23,10 @@ #pragma once #include "runtime/helpers/basic_math.h" -#include -#include #include +#include +#include +#include #include #include diff --git a/runtime/memory_manager/page_table.inl b/runtime/memory_manager/page_table.inl new file mode 100644 index 0000000000..21a26c874f --- /dev/null +++ b/runtime/memory_manager/page_table.inl @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018, Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +namespace OCLRT { + +template <> +inline uintptr_t PageTable::map(uintptr_t vm, size_t size) { + return 0; +} + +template <> +inline size_t PageTable::getBits() { + return 9; +} + +template <> +inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker) { +} + +template <> +inline PageTable::~PageTable() { +} + +template +inline uintptr_t PageTable::map(uintptr_t vm, size_t size) { + const size_t shift = T::getBits() + 12; + const uintptr_t mask = (1 << bits) - 1; + size_t indexStart = (vm >> shift) & mask; + size_t indexEnd = ((vm + size - 1) >> shift) & mask; + uintptr_t res = -1; + uintptr_t vmMask = (uintptr_t(-1) >> (sizeof(void *) * 8 - shift - bits)); + auto maskedVm = vm & vmMask; + + for (size_t index = indexStart; index <= indexEnd; index++) { + uintptr_t vmStart = (uintptr_t(1) << shift) * index; + vmStart = std::max(vmStart, maskedVm); + uintptr_t vmEnd = (uintptr_t(1) << shift) * (index + 1) - 1; + vmEnd = std::min(vmEnd, maskedVm + size - 1); + + if (entries[index] == nullptr) { + entries[index] = new T; + } + res = std::min((entries[index])->map(vmStart, vmEnd - vmStart + 1), res); + } + return res; +} + +template +inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker) { + const size_t shift = T::getBits() + 12; + const uintptr_t mask = (1 << bits) - 1; + size_t indexStart = (vm >> shift) & mask; + size_t indexEnd = ((vm + size - 1) >> shift) & mask; + uintptr_t vmMask = (uintptr_t(-1) >> (sizeof(void *) * 8 - shift - bits)); + auto maskedVm = vm & vmMask; + + for (size_t index = indexStart; index <= indexEnd; index++) { + uintptr_t vmStart = (uintptr_t(1) << shift) * index; + vmStart = std::max(vmStart, maskedVm); + uintptr_t vmEnd = (uintptr_t(1) << shift) * (index + 1) - 1; + vmEnd = std::min(vmEnd, maskedVm + size - 1); + + if (entries[index] == nullptr) { + entries[index] = new T; + } + entries[index]->pageWalk(vmStart, vmEnd - vmStart + 1, offset, pageWalker); + + offset += (vmEnd - vmStart + 1); + } +} + +template +inline PageTable::~PageTable() { + for (auto &e : entries) + delete e; +} + +} // namespace OCLRT diff --git a/unit_tests/memory_manager/page_table_tests.cpp b/unit_tests/memory_manager/page_table_tests.cpp index 2992084e0c..c57393a478 100644 --- a/unit_tests/memory_manager/page_table_tests.cpp +++ b/unit_tests/memory_manager/page_table_tests.cpp @@ -22,6 +22,7 @@ #include "runtime/helpers/selectors.h" #include "runtime/memory_manager/page_table.h" +#include "runtime/memory_manager/page_table.inl" #include "test.h" #include "gtest/gtest.h" #include "unit_tests/helpers/memory_management.h" @@ -58,6 +59,40 @@ TEST_F(PTETest, physicalAddressesInAUBCantStartAt0) { EXPECT_NE(0u, physAddress); } +template +class MockPageTable : public PageTable { + public: + using PageTable::PageTable; + using PageTable::entries; +}; + +class MockPTE : public PTE { + public: + using PTE::entries; + + uintptr_t map(uintptr_t vm, size_t size) override { + return PTE::map(vm, size); + } + void pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker) override { + return PTE::pageWalk(vm, size, offset, pageWalker); + } +}; + +class MockPDE : public MockPageTable { + public: + using MockPageTable::entries; +}; + +class MockPDP : public MockPageTable { + public: + using MockPageTable::entries; +}; + +class MockPML4 : public MockPageTable { + public: + using MockPageTable::entries; +}; + class PPGTTPageTable : public TypeSelector::type { public: const size_t ppgttEntries = IntSelector<512u, 4u, sizeof(void *) == 8>::value; @@ -128,6 +163,38 @@ TEST_F(PageTableTests48, DISABLED_mapSizeZero) { std::cerr << phys1 << std::endl; } +TEST_F(PageTableTests48, givenReservedPhysicalAddressWhenPageWalkIsCalledThenPageTablesAreFilledWithProperAddresses) { + if (is64Bit) { + std::unique_ptr pageTable(std::make_unique()); + + int shiftPML4 = is64Bit ? (9 + 9 + 9 + 12) : 0; + int shiftPDP = is64Bit ? (9 + 9 + 12) : 0; + + uintptr_t gpuVa = (uintptr_t(0x1) << (shiftPML4)) | (uintptr_t(0x1) << (shiftPDP)) | (uintptr_t(0x1) << (9 + 12)) | 0x100; + + size_t size = 10 * pageSize; + + size_t walked = 0u; + auto address = this->getNextPage() * pageSize; + + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + walked += size; + }; + pageTable->pageWalk(gpuVa, size, 0, walker); + + EXPECT_EQ(size, walked); + + ASSERT_NE(nullptr, pageTable->entries[1]); + ASSERT_NE(nullptr, pageTable->entries[1]->entries[1]); + ASSERT_NE(nullptr, pageTable->entries[1]->entries[1]->entries[1]); + + for (uint32_t i = 0; i < 10; i++) { + EXPECT_EQ(reinterpret_cast(address | 0x1), pageTable->entries[1]->entries[1]->entries[1]->entries[i]); + address += pageSize; + } + } +} + TEST_F(PageTableTests48, pageWalkSimple) { std::unique_ptr pageTable(new PPGTTPageTable); uintptr_t addr1 = refAddr + (510 * pageSize) + 0x10;