From 4af432ae1083d99716fed4395f31c83f3cae1d2e Mon Sep 17 00:00:00 2001 From: "Hoppe, Mateusz" Date: Tue, 18 Sep 2018 13:17:55 -0700 Subject: [PATCH] Store page entry bits in PageTable entries - set Present bit when entry is allocated regardless entry bits passed. Change-Id: Ib1393927f66c4ed0b577a4df58d2760fbff86df7 --- .../aub_command_stream_receiver_hw.h | 20 +- .../aub_command_stream_receiver_hw.inl | 55 +++-- .../tbx_command_stream_receiver_hw.h | 20 +- .../tbx_command_stream_receiver_hw.inl | 48 ++-- runtime/memory_manager/page_table.cpp | 43 ++-- runtime/memory_manager/page_table.h | 29 +-- runtime/memory_manager/page_table.inl | 31 +-- .../aub_command_stream_fixture.h | 4 +- .../aub_command_stream_receiver_tests.cpp | 25 +- .../tbx_command_stream_tests.cpp | 23 +- .../memory_manager/page_table_tests.cpp | 221 +++++++++++++----- 11 files changed, 263 insertions(+), 256 deletions(-) diff --git a/runtime/command_stream/aub_command_stream_receiver_hw.h b/runtime/command_stream/aub_command_stream_receiver_hw.h index 036d9bf834..2d6a06f2a8 100644 --- a/runtime/command_stream/aub_command_stream_receiver_hw.h +++ b/runtime/command_stream/aub_command_stream_receiver_hw.h @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #pragma once @@ -112,6 +97,7 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverHw { uint32_t getGUCWorkQueueItemHeader(EngineType engineType); uint64_t getPPGTTAdditionalBits(GraphicsAllocation *gfxAllocation); void getGTTData(void *memory, AubGTTData &data); + uint64_t getGTTBits() const; CommandStreamReceiverType getType() override { return CommandStreamReceiverType::CSR_AUB; diff --git a/runtime/command_stream/aub_command_stream_receiver_hw.inl b/runtime/command_stream/aub_command_stream_receiver_hw.inl index 17ac52fbdc..682b1fbc4b 100644 --- a/runtime/command_stream/aub_command_stream_receiver_hw.inl +++ b/runtime/command_stream/aub_command_stream_receiver_hw.inl @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include "hw_cmds.h" @@ -140,7 +125,8 @@ void AUBCommandStreamReceiverHw::initializeEngine(EngineType engineTy const size_t alignHWSP = 0x1000; engineInfo.pGlobalHWStatusPage = alignedMalloc(sizeHWSP, alignHWSP); engineInfo.ggttHWSP = gttRemap.map(engineInfo.pGlobalHWStatusPage, sizeHWSP); - auto physHWSP = ggtt->map(engineInfo.ggttHWSP, sizeHWSP, PageTableHelper::memoryBankNotSpecified); + + auto physHWSP = ggtt->map(engineInfo.ggttHWSP, sizeHWSP, getGTTBits(), PageTableHelper::memoryBankNotSpecified); // Write our GHWSP { @@ -171,7 +157,7 @@ void AUBCommandStreamReceiverHw::initializeEngine(EngineType engineTy const size_t alignRingBuffer = 0x1000; engineInfo.pRingBuffer = alignedMalloc(engineInfo.sizeRingBuffer, alignRingBuffer); engineInfo.ggttRingBuffer = gttRemap.map(engineInfo.pRingBuffer, engineInfo.sizeRingBuffer); - auto physRingBuffer = ggtt->map(engineInfo.ggttRingBuffer, engineInfo.sizeRingBuffer, PageTableHelper::memoryBankNotSpecified); + auto physRingBuffer = ggtt->map(engineInfo.ggttRingBuffer, engineInfo.sizeRingBuffer, getGTTBits(), PageTableHelper::memoryBankNotSpecified); { std::ostringstream str; @@ -199,7 +185,7 @@ void AUBCommandStreamReceiverHw::initializeEngine(EngineType engineTy // Write our LRCA { engineInfo.ggttLRCA = gttRemap.map(engineInfo.pLRCA, sizeLRCA); - auto lrcAddressPhys = ggtt->map(engineInfo.ggttLRCA, sizeLRCA, PageTableHelper::memoryBankNotSpecified); + auto lrcAddressPhys = ggtt->map(engineInfo.ggttLRCA, sizeLRCA, getGTTBits(), PageTableHelper::memoryBankNotSpecified); { std::ostringstream str; @@ -294,7 +280,9 @@ FlushStamp AUBCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer stream->addComment(str.str().c_str()); } - auto physBatchBuffer = ppgtt->map(static_cast(batchBufferGpuAddress), sizeBatchBuffer, PageTableHelper::getMemoryBankIndex(*batchBuffer.commandBufferAllocation)); + auto physBatchBuffer = ppgtt->map(static_cast(batchBufferGpuAddress), sizeBatchBuffer, + getPPGTTAdditionalBits(batchBuffer.commandBufferAllocation), + PageTableHelper::getMemoryBankIndex(*batchBuffer.commandBufferAllocation)); AubHelperHw aubHelperHw(this->localMemoryEnabled); AUB::reserveAddressPPGTT(*stream, static_cast(batchBufferGpuAddress), sizeBatchBuffer, physBatchBuffer, getPPGTTAdditionalBits(batchBuffer.commandBufferAllocation), @@ -348,7 +336,8 @@ FlushStamp AUBCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer auto sizeToWrap = engineInfo.sizeRingBuffer - engineInfo.tailRingBuffer; memset(pTail, 0, sizeToWrap); // write remaining ring - auto physDumpStart = ggtt->map(ggttTail, sizeToWrap, PageTableHelper::memoryBankNotSpecified); + + auto physDumpStart = ggtt->map(ggttTail, sizeToWrap, getGTTBits(), PageTableHelper::memoryBankNotSpecified); AUB::addMemoryWrite( *stream, physDumpStart, @@ -398,7 +387,7 @@ FlushStamp AUBCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer stream->addComment(str.str().c_str()); } - auto physDumpStart = ggtt->map(ggttDumpStart, dumpLength, PageTableHelper::memoryBankNotSpecified); + auto physDumpStart = ggtt->map(ggttDumpStart, dumpLength, getGTTBits(), PageTableHelper::memoryBankNotSpecified); AUB::addMemoryWrite( *stream, physDumpStart, @@ -414,7 +403,7 @@ FlushStamp AUBCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer stream->addComment(str.str().c_str()); } - auto physLRCA = ggtt->map(engineInfo.ggttLRCA, sizeof(engineInfo.tailRingBuffer), PageTableHelper::memoryBankNotSpecified); + auto physLRCA = ggtt->map(engineInfo.ggttLRCA, sizeof(engineInfo.tailRingBuffer), getGTTBits(), PageTableHelper::memoryBankNotSpecified); AUB::addMemoryWrite( *stream, physLRCA + 0x101c, @@ -474,12 +463,12 @@ bool AUBCommandStreamReceiverHw::addPatchInfoComments() { if (patchInfoData.sourceAllocation) { allocationsMap.insert(std::pair(patchInfoData.sourceAllocation, - ppgtt->map(static_cast(patchInfoData.sourceAllocation), 1, PageTableHelper::memoryBankNotSpecified))); + ppgtt->map(static_cast(patchInfoData.sourceAllocation), 1, 0, PageTableHelper::memoryBankNotSpecified))); } if (patchInfoData.targetAllocation) { allocationsMap.insert(std::pair(patchInfoData.targetAllocation, - ppgtt->map(static_cast(patchInfoData.targetAllocation), 1, PageTableHelper::memoryBankNotSpecified))); + ppgtt->map(static_cast(patchInfoData.targetAllocation), 1, 0, PageTableHelper::memoryBankNotSpecified))); } } bool result = stream->addComment(str.str().c_str()); @@ -564,12 +553,12 @@ bool AUBCommandStreamReceiverHw::writeMemory(GraphicsAllocation &gfxA AubHelperHw aubHelperHw(this->localMemoryEnabled); - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { AUB::reserveAddressGGTTAndWriteMmeory(*stream, static_cast(gpuAddress), cpuAddress, physAddress, size, offset, getPPGTTAdditionalBits(&gfxAllocation), aubHelperHw); }; - ppgtt->pageWalk(static_cast(gpuAddress), size, 0, walker, PageTableHelper::getMemoryBankIndex(gfxAllocation)); + ppgtt->pageWalk(static_cast(gpuAddress), size, 0, getPPGTTAdditionalBits(&gfxAllocation), walker, PageTableHelper::getMemoryBankIndex(gfxAllocation)); if (gfxAllocation.isLocked()) { this->getMemoryManager()->unlockResource(&gfxAllocation); @@ -674,7 +663,10 @@ void AUBCommandStreamReceiverHw::addGUCStartMessage(uint64_t batchBuf miBatchBufferStart->setBatchBufferStartAddressGraphicsaddress472(AUB::ptrToPPGTT(buffer.get())); miBatchBufferStart->setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT); - auto physBufferAddres = ppgtt->map(reinterpret_cast(buffer.get()), bufferSize, PageTableHelper::memoryBankNotSpecified); + auto physBufferAddres = ppgtt->map(reinterpret_cast(buffer.get()), bufferSize, + getPPGTTAdditionalBits(linearStream.getGraphicsAllocation()), + PageTableHelper::memoryBankNotSpecified); + AUB::reserveAddressPPGTT(*stream, reinterpret_cast(buffer.get()), bufferSize, physBufferAddres, getPPGTTAdditionalBits(linearStream.getGraphicsAllocation()), aubHelperHw); @@ -712,4 +704,9 @@ int AUBCommandStreamReceiverHw::getAddressSpace(int hint) { return AubMemDump::AddressSpaceValues::TraceNonlocal; } +template +uint64_t AUBCommandStreamReceiverHw::getGTTBits() const { + return 0; +} + } // namespace OCLRT diff --git a/runtime/command_stream/tbx_command_stream_receiver_hw.h b/runtime/command_stream/tbx_command_stream_receiver_hw.h index e10179b2d3..109672279b 100644 --- a/runtime/command_stream/tbx_command_stream_receiver_hw.h +++ b/runtime/command_stream/tbx_command_stream_receiver_hw.h @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #pragma once @@ -88,6 +73,7 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverHw { } uint64_t getPPGTTAdditionalBits(GraphicsAllocation *gfxAllocation); void getGTTData(void *memory, AubGTTData &data); + uint64_t getGTTBits() const; TbxCommandStreamReceiver::TbxStream stream; uint32_t aubDeviceId; diff --git a/runtime/command_stream/tbx_command_stream_receiver_hw.inl b/runtime/command_stream/tbx_command_stream_receiver_hw.inl index 129c1a0126..6c31deeb87 100644 --- a/runtime/command_stream/tbx_command_stream_receiver_hw.inl +++ b/runtime/command_stream/tbx_command_stream_receiver_hw.inl @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include "hw_cmds.h" @@ -113,7 +98,7 @@ void TbxCommandStreamReceiverHw::initializeEngine(EngineType engineTy const size_t alignHWSP = 0x1000; engineInfo.pGlobalHWStatusPage = alignedMalloc(sizeHWSP, alignHWSP); engineInfo.ggttHWSP = gttRemap.map(engineInfo.pGlobalHWStatusPage, sizeHWSP); - auto physHWSP = ggtt->map(engineInfo.ggttHWSP, sizeHWSP, PageTableHelper::memoryBankNotSpecified); + auto physHWSP = ggtt->map(engineInfo.ggttHWSP, sizeHWSP, getGTTBits(), PageTableHelper::memoryBankNotSpecified); // Write our GHWSP AubGTTData data = {0}; @@ -138,7 +123,7 @@ void TbxCommandStreamReceiverHw::initializeEngine(EngineType engineTy const size_t alignRCS = 0x1000; engineInfo.pRCS = alignedMalloc(engineInfo.sizeRCS, alignRCS); engineInfo.ggttRCS = gttRemap.map(engineInfo.pRCS, engineInfo.sizeRCS); - auto physRCS = ggtt->map(engineInfo.ggttRCS, engineInfo.sizeRCS, PageTableHelper::memoryBankNotSpecified); + auto physRCS = ggtt->map(engineInfo.ggttRCS, engineInfo.sizeRCS, getGTTBits(), PageTableHelper::memoryBankNotSpecified); AubGTTData data = {0}; getGTTData(reinterpret_cast(physRCS), data); @@ -160,7 +145,7 @@ void TbxCommandStreamReceiverHw::initializeEngine(EngineType engineTy // Write our LRCA { engineInfo.ggttLRCA = gttRemap.map(engineInfo.pLRCA, sizeLRCA); - auto lrcAddressPhys = ggtt->map(engineInfo.ggttLRCA, sizeLRCA, PageTableHelper::memoryBankNotSpecified); + auto lrcAddressPhys = ggtt->map(engineInfo.ggttLRCA, sizeLRCA, getGTTBits(), PageTableHelper::memoryBankNotSpecified); AubGTTData data = {0}; getGTTData(reinterpret_cast(lrcAddressPhys), data); @@ -210,7 +195,9 @@ FlushStamp TbxCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer DEBUG_BREAK_IF(currentOffset < batchBuffer.startOffset); auto sizeBatchBuffer = currentOffset - batchBuffer.startOffset; { - auto physBatchBuffer = ppgtt->map(reinterpret_cast(pBatchBuffer), sizeBatchBuffer, PageTableHelper::getMemoryBankIndex(*batchBuffer.commandBufferAllocation)); + auto physBatchBuffer = ppgtt->map(reinterpret_cast(pBatchBuffer), sizeBatchBuffer, + getPPGTTAdditionalBits(batchBuffer.commandBufferAllocation), + PageTableHelper::getMemoryBankIndex(*batchBuffer.commandBufferAllocation)); AubHelperHw aubHelperHw(this->localMemoryEnabled); AUB::reserveAddressPPGTT(stream, reinterpret_cast(pBatchBuffer), sizeBatchBuffer, physBatchBuffer, @@ -249,7 +236,7 @@ FlushStamp TbxCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer auto sizeToWrap = engineInfo.sizeRCS - engineInfo.tailRCS; memset(pTail, 0, sizeToWrap); // write remaining ring - auto physDumpStart = ggtt->map(ggttTail, sizeToWrap, PageTableHelper::memoryBankNotSpecified); + auto physDumpStart = ggtt->map(ggttTail, sizeToWrap, getGTTBits(), PageTableHelper::memoryBankNotSpecified); AUB::addMemoryWrite( stream, physDumpStart, @@ -289,7 +276,7 @@ FlushStamp TbxCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer auto dumpLength = engineInfo.tailRCS - previousTail; // write RCS - auto physDumpStart = ggtt->map(ggttDumpStart, dumpLength, PageTableHelper::memoryBankNotSpecified); + auto physDumpStart = ggtt->map(ggttDumpStart, dumpLength, getGTTBits(), PageTableHelper::memoryBankNotSpecified); AUB::addMemoryWrite( stream, physDumpStart, @@ -299,7 +286,7 @@ FlushStamp TbxCommandStreamReceiverHw::flush(BatchBuffer &batchBuffer AubMemDump::DataTypeHintValues::TraceCommandBuffer); // update the RCS mmio tail in the LRCA - auto physLRCA = ggtt->map(engineInfo.ggttLRCA, sizeof(engineInfo.tailRCS), PageTableHelper::memoryBankNotSpecified); + auto physLRCA = ggtt->map(engineInfo.ggttLRCA, sizeof(engineInfo.tailRCS), getGTTBits(), PageTableHelper::memoryBankNotSpecified); AUB::addMemoryWrite( stream, physLRCA + 0x101c, @@ -367,12 +354,12 @@ bool TbxCommandStreamReceiverHw::writeMemory(GraphicsAllocation &gfxA AubHelperHw aubHelperHw(this->localMemoryEnabled); - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { AUB::reserveAddressGGTTAndWriteMmeory(stream, static_cast(gpuAddress), cpuAddress, physAddress, size, offset, getPPGTTAdditionalBits(&gfxAllocation), aubHelperHw); }; - ppgtt->pageWalk(static_cast(gpuAddress), size, 0, walker, PageTableHelper::getMemoryBankIndex(gfxAllocation)); + ppgtt->pageWalk(static_cast(gpuAddress), size, 0, getPPGTTAdditionalBits(&gfxAllocation), walker, PageTableHelper::getMemoryBankIndex(gfxAllocation)); return true; } @@ -393,11 +380,11 @@ void TbxCommandStreamReceiverHw::makeCoherent(GraphicsAllocation &gfx auto length = gfxAllocation.getUnderlyingBufferSize(); if (length) { - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { DEBUG_BREAK_IF(offset > length); stream.readMemory(physAddress, ptrOffset(cpuAddress, offset), size); }; - ppgtt->pageWalk(static_cast(gpuAddress), length, 0, walker, PageTableHelper::getMemoryBankIndex(gfxAllocation)); + ppgtt->pageWalk(static_cast(gpuAddress), length, 0, 0, walker, PageTableHelper::getMemoryBankIndex(gfxAllocation)); } } @@ -427,4 +414,9 @@ int TbxCommandStreamReceiverHw::getAddressSpace(int hint) { return AubMemDump::AddressSpaceValues::TraceNonlocal; } +template +uint64_t TbxCommandStreamReceiverHw::getGTTBits() const { + return 0; +} + } // namespace OCLRT diff --git a/runtime/memory_manager/page_table.cpp b/runtime/memory_manager/page_table.cpp index 2ec6bc7899..7c0988eb0e 100644 --- a/runtime/memory_manager/page_table.cpp +++ b/runtime/memory_manager/page_table.cpp @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include "runtime/memory_manager/page_table.h" @@ -25,24 +10,28 @@ namespace OCLRT { -uintptr_t PTE::map(uintptr_t vm, size_t size, uint32_t memoryBank) { +uintptr_t PTE::map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memoryBank) { const size_t shift = 12; const uint32_t mask = (1 << bits) - 1; size_t indexStart = (vm >> shift) & mask; size_t indexEnd = ((vm + size - 1) >> shift) & mask; uintptr_t res = -1; + entryBits &= 0xfff; + entryBits |= 0x1; for (size_t index = indexStart; index <= indexEnd; index++) { if (entries[index] == 0x0) { uint64_t tmp = allocator->reservePage(memoryBank); - entries[index] = reinterpret_cast(tmp | 0x1); + entries[index] = reinterpret_cast(tmp | entryBits); + } else { + entries[index] = reinterpret_cast((reinterpret_cast(entries[index]) & 0xfffff000u) | entryBits); } - res = std::min(reinterpret_cast(entries[index]) & 0xfffffffeu, res); + res = std::min(reinterpret_cast(entries[index]) & 0xfffff000u, res); } - return (res & ~0x1) + (vm & (pageSize - 1)); + return (res & ~entryBits) + (vm & (pageSize - 1)); } -void PTE::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank) { +void PTE::pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank) { static const uint32_t bits = 9; const size_t shift = 12; const uint32_t mask = (1 << bits) - 1; @@ -50,16 +39,20 @@ void PTE::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWal size_t indexEnd = ((vm + size - 1) >> shift) & mask; uint64_t res = -1; uintptr_t rem = vm & (pageSize - 1); + entryBits &= 0xfff; + entryBits |= 0x1; for (size_t index = indexStart; index <= indexEnd; index++) { if (entries[index] == 0x0) { uint64_t tmp = allocator->reservePage(memoryBank); - entries[index] = reinterpret_cast(tmp | 0x1); + entries[index] = reinterpret_cast(tmp | entryBits); + } else { + entries[index] = reinterpret_cast((reinterpret_cast(entries[index]) & 0xfffff000u) | entryBits); } - res = reinterpret_cast(entries[index]) & 0xfffffffeu; + res = reinterpret_cast(entries[index]) & 0xfffff000u; size_t lSize = std::min(pageSize - rem, size); - pageWalker((res & ~0x1) + rem, lSize, offset); + pageWalker((res & ~0x1) + rem, lSize, offset, reinterpret_cast(entries[index]) & 0xfffu); size -= lSize; offset += lSize; diff --git a/runtime/memory_manager/page_table.h b/runtime/memory_manager/page_table.h index f221c5a966..6a93c23926 100644 --- a/runtime/memory_manager/page_table.h +++ b/runtime/memory_manager/page_table.h @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #pragma once @@ -44,7 +29,7 @@ class PageTableHelper { static const uint32_t memoryBankNotSpecified = 0; }; -typedef std::function PageWalker; +typedef std::function PageWalker; template class PageTable { public: @@ -57,8 +42,8 @@ class PageTable { delete e; } - virtual uintptr_t map(uintptr_t vm, size_t size, uint32_t memoryBank); - virtual void pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank); + virtual uintptr_t map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memoryBank); + virtual void pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank); static const size_t pageSize = 1 << 12; static size_t getBits() { @@ -78,8 +63,8 @@ class PTE : public PageTable { public: PTE(PhysicalAddressAllocator *physicalAddressAllocator) : PageTable(physicalAddressAllocator) {} - uintptr_t map(uintptr_t vm, size_t size, uint32_t memoryBank) override; - void pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank) override; + uintptr_t map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memoryBank) override; + void pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank) override; static const uint32_t level = 0; static const uint32_t bits = 9; diff --git a/runtime/memory_manager/page_table.inl b/runtime/memory_manager/page_table.inl index 1c53954565..bc9f12b3ad 100644 --- a/runtime/memory_manager/page_table.inl +++ b/runtime/memory_manager/page_table.inl @@ -1,29 +1,14 @@ /* - * Copyright (c) 2018, Intel Corporation + * Copyright (C) 2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ namespace OCLRT { template <> -inline uintptr_t PageTable::map(uintptr_t vm, size_t size, uint32_t memoryBank) { +inline uintptr_t PageTable::map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memoryBank) { return 0; } @@ -33,11 +18,11 @@ inline size_t PageTable::getBits() { } template <> -inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank) { +inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank) { } template -inline uintptr_t PageTable::map(uintptr_t vm, size_t size, uint32_t memoryBank) { +inline uintptr_t PageTable::map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memoryBank) { const size_t shift = T::getBits() + 12; const uintptr_t mask = (1 << bits) - 1; size_t indexStart = (vm >> shift) & mask; @@ -55,13 +40,13 @@ inline uintptr_t PageTable::map(uintptr_t vm, size_t size, uint3 if (entries[index] == nullptr) { entries[index] = new T(allocator); } - res = std::min((entries[index])->map(vmStart, vmEnd - vmStart + 1, memoryBank), res); + res = std::min((entries[index])->map(vmStart, vmEnd - vmStart + 1, entryBits, memoryBank), res); } return res; } template -inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank) { +inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank) { const size_t shift = T::getBits() + 12; const uintptr_t mask = (1 << bits) - 1; size_t indexStart = (vm >> shift) & mask; @@ -78,7 +63,7 @@ inline void PageTable::pageWalk(uintptr_t vm, size_t size, size_ if (entries[index] == nullptr) { entries[index] = new T(allocator); } - entries[index]->pageWalk(vmStart, vmEnd - vmStart + 1, offset, pageWalker, memoryBank); + entries[index]->pageWalk(vmStart, vmEnd - vmStart + 1, offset, entryBits, pageWalker, memoryBank); offset += (vmEnd - vmStart + 1); } diff --git a/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h b/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h index cd8556f263..84171856fb 100644 --- a/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h +++ b/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h @@ -56,7 +56,7 @@ class AUBCommandStreamFixture : public CommandStreamFixture { } auto aubCsr = reinterpret_cast *>(csr); - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { if (offset > length) abort(); @@ -65,7 +65,7 @@ class AUBCommandStreamFixture : public CommandStreamFixture { size); }; - aubCsr->ppgtt->pageWalk(reinterpret_cast(gfxAddress), length, 0, walker, PageTableHelper::memoryBankNotSpecified); + aubCsr->ppgtt->pageWalk(reinterpret_cast(gfxAddress), length, 0, 0, walker, PageTableHelper::memoryBankNotSpecified); } CommandStreamReceiver *pCommandStreamReceiver = nullptr; diff --git a/unit_tests/command_stream/aub_command_stream_receiver_tests.cpp b/unit_tests/command_stream/aub_command_stream_receiver_tests.cpp index 92263ac6c8..3df9765b61 100644 --- a/unit_tests/command_stream/aub_command_stream_receiver_tests.cpp +++ b/unit_tests/command_stream/aub_command_stream_receiver_tests.cpp @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include "runtime/aub_mem_dump/aub_mem_dump.h" @@ -1920,7 +1905,7 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverWhenWriteMe struct PpgttMock : TypeSelector::type { PpgttMock(PhysicalAddressAllocator *allocator) : TypeSelector::type(allocator) {} - void pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank) override { + void pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank) override { receivedSize = size; } size_t receivedSize = 0; @@ -1956,10 +1941,10 @@ HWTEST_F(AubCommandStreamReceiverTests, whenAubCommandStreamReceiverIsCreatedThe ASSERT_NE(nullptr, aubCsr->ggtt.get()); uintptr_t address = 0x20000; - auto physicalAddress = aubCsr->ppgtt->map(address, MemoryConstants::pageSize, PageTableHelper::memoryBankNotSpecified); + auto physicalAddress = aubCsr->ppgtt->map(address, MemoryConstants::pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(0u, physicalAddress); - physicalAddress = aubCsr->ggtt->map(address, MemoryConstants::pageSize, PageTableHelper::memoryBankNotSpecified); + physicalAddress = aubCsr->ggtt->map(address, MemoryConstants::pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(0u, physicalAddress); } diff --git a/unit_tests/command_stream/tbx_command_stream_tests.cpp b/unit_tests/command_stream/tbx_command_stream_tests.cpp index 17a0b5e152..58ba21d649 100644 --- a/unit_tests/command_stream/tbx_command_stream_tests.cpp +++ b/unit_tests/command_stream/tbx_command_stream_tests.cpp @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include "runtime/mem_obj/mem_obj.h" @@ -338,9 +323,9 @@ HWTEST_F(TbxCommandSteamSimpleTest, whenTbxCommandStreamReceiverIsCreatedThenPPG MockTbxCsr tbxCsr(*platformDevices[0], *pDevice->executionEnvironment); uintptr_t address = 0x20000; - auto physicalAddress = tbxCsr.ppgtt->map(address, MemoryConstants::pageSize, PageTableHelper::memoryBankNotSpecified); + auto physicalAddress = tbxCsr.ppgtt->map(address, MemoryConstants::pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(0u, physicalAddress); - physicalAddress = tbxCsr.ggtt->map(address, MemoryConstants::pageSize, PageTableHelper::memoryBankNotSpecified); + physicalAddress = tbxCsr.ggtt->map(address, MemoryConstants::pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(0u, physicalAddress); } diff --git a/unit_tests/memory_manager/page_table_tests.cpp b/unit_tests/memory_manager/page_table_tests.cpp index 91b3727589..472dea0a30 100644 --- a/unit_tests/memory_manager/page_table_tests.cpp +++ b/unit_tests/memory_manager/page_table_tests.cpp @@ -1,23 +1,8 @@ /* - * Copyright (c) 2017 - 2018, Intel Corporation + * Copyright (C) 2017-2018 Intel Corporation * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: + * SPDX-License-Identifier: MIT * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include "runtime/helpers/ptr_math.h" @@ -48,11 +33,11 @@ class MockPTE : public PTE { MockPTE(PhysicalAddressAllocator *physicalAddressAllocator) : PTE(physicalAddressAllocator) {} - uintptr_t map(uintptr_t vm, size_t size, uint32_t memoryBank) override { - return PTE::map(vm, size, memoryBank); + uintptr_t map(uintptr_t vm, size_t size, uint64_t entryBits, uint32_t memoryBank) override { + return PTE::map(vm, size, entryBits, memoryBank); } - void pageWalk(uintptr_t vm, size_t size, size_t offset, PageWalker &pageWalker, uint32_t memoryBank) override { - return PTE::pageWalk(vm, size, offset, pageWalker, memoryBank); + void pageWalk(uintptr_t vm, size_t size, size_t offset, uint64_t entryBits, PageWalker &pageWalker, uint32_t memoryBank) override { + return PTE::pageWalk(vm, size, offset, entryBits, pageWalker, memoryBank); } }; @@ -79,6 +64,15 @@ class MockPML4 : public MockPageTable { } }; +class MockPDPE : public MockPageTable { + public: + using MockPageTable::entries; + using PageTable::allocator; + + MockPDPE(PhysicalAddressAllocator *physicalAddressAllocator) : MockPageTable(physicalAddressAllocator) { + } +}; + class PPGTTPageTable : public TypeSelector::type { public: const size_t ppgttEntries = IntSelector<512u, 4u, sizeof(void *) == 8>::value; @@ -122,18 +116,38 @@ class PageTableFixture { } }; -typedef Test PageTableTests32; +class PageTableEntryChecker { + public: + template + static void testEntry(T *pageTable, uint32_t pteIndex, uintptr_t expectedValue) { + } +}; +template <> +void PageTableEntryChecker::testEntry(MockPML4 *pageTable, uint32_t pteIndex, uintptr_t expectedValue) { + ASSERT_NE(nullptr, pageTable->entries[0]); + ASSERT_NE(nullptr, pageTable->entries[0]->entries[0]); + ASSERT_NE(nullptr, pageTable->entries[0]->entries[0]->entries[0]); + EXPECT_EQ(reinterpret_cast(expectedValue), pageTable->entries[0]->entries[0]->entries[0]->entries[pteIndex]); +} + +template <> +void PageTableEntryChecker::testEntry(MockPDPE *pageTable, uint32_t pteIndex, uintptr_t expectedValue) { + ASSERT_NE(nullptr, pageTable->entries[0]); + EXPECT_EQ(reinterpret_cast(expectedValue), pageTable->entries[0]->entries[0]->entries[pteIndex]); +} + +typedef Test PageTableTests32; typedef Test PageTableTests48; typedef Test PageTableTestsGPU; TEST_F(PageTableTests48, dummy) { PageTable pt(&allocator); - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { }; - pt.pageWalk(0, pageSize, 0, walker, PageTableHelper::memoryBankNotSpecified); + pt.pageWalk(0, pageSize, 0, 0, walker, PageTableHelper::memoryBankNotSpecified); } TEST_F(PageTableTests48, newIsEmpty) { @@ -147,7 +161,7 @@ TEST_F(PageTableTests48, DISABLED_mapSizeZero) { EXPECT_TRUE(pageTable->isEmpty()); EXPECT_EQ(allocator.initialPageAddress, allocator.nextPageAddress); - auto phys1 = pageTable->map(0x0, 0x0, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(0x0, 0x0, 0, PageTableHelper::memoryBankNotSpecified); std::cerr << phys1 << std::endl; } @@ -158,14 +172,14 @@ TEST_F(PageTableTests48, pageWalkSimple) { size_t walked = 0u; size_t lastOffset = 0; - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { EXPECT_EQ(lastOffset, offset); EXPECT_GE(pageSize, size); walked += size; lastOffset += size; }; - pageTable->pageWalk(addr1, lSize, 0, walker, PageTableHelper::memoryBankNotSpecified); + pageTable->pageWalk(addr1, lSize, 0, 0, walker, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(lSize, walked); } @@ -183,10 +197,10 @@ TEST_F(PageTableTests48, givenReservedPhysicalAddressWhenPageWalkIsCalledThenPag size_t walked = 0u; auto address = allocator.nextPageAddress.load(); - PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset) { + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { walked += size; }; - pageTable->pageWalk(gpuVa, size, 0, walker, PageTableHelper::memoryBankNotSpecified); + pageTable->pageWalk(gpuVa, size, 0, 0, walker, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(size, walked); @@ -201,27 +215,126 @@ TEST_F(PageTableTests48, givenReservedPhysicalAddressWhenPageWalkIsCalledThenPag } } +TEST_F(PageTableTests48, givenZeroEntryBitsWhenPageWalkIsCalledThenPageTableEntryHasPresentBitSet) { + std::unique_ptr::type> + pageTable(std::make_unique::type>(&allocator)); + + uintptr_t gpuVa = 0x1000; + + size_t size = pageSize; + + size_t walked = 0u; + auto address = allocator.nextPageAddress.load(); + + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { + walked += size; + }; + pageTable->pageWalk(gpuVa, size, 0, 0, walker, PageTableHelper::memoryBankNotSpecified); + + EXPECT_EQ(size, walked); + ASSERT_NE(nullptr, pageTable->entries[0]); + + PageTableEntryChecker::testEntry::type>(pageTable.get(), 1, static_cast(address | 0x1)); +} + +TEST_F(PageTableTests48, givenZeroEntryBitsWhenMapIsCalledThenPageTableEntryHasPresentBitSet) { + std::unique_ptr::type> + pageTable(std::make_unique::type>(&allocator)); + uintptr_t gpuVa = 0x1000; + size_t size = pageSize; + auto address = allocator.nextPageAddress.load(); + + pageTable->map(gpuVa, size, 0, 0); + ASSERT_NE(nullptr, pageTable->entries[0]); + + PageTableEntryChecker::testEntry::type>(pageTable.get(), 1, static_cast(address | 0x1)); +} + +TEST_F(PageTableTests48, givenEntryBitsWhenPageWalkIsCalledThenEntryBitsArePassedToPageWalker) { + std::unique_ptr::type> + pageTable(std::make_unique::type>(&allocator)); + + uintptr_t gpuVa = 0x1000; + size_t size = pageSize; + size_t walked = 0u; + uint64_t ppgttBits = 0xabc; + uint64_t entryBitsPassed = 0u; + + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { + walked += size; + entryBitsPassed = entryBits; + }; + pageTable->pageWalk(gpuVa, size, 0, ppgttBits, walker, PageTableHelper::memoryBankNotSpecified); + + ppgttBits |= 0x1; + EXPECT_EQ(ppgttBits, entryBitsPassed); +} + +TEST_F(PageTableTests48, givenTwoPageWalksWhenSecondWalkHasDifferenEntryBitsThenEntryIsUpdated) { + std::unique_ptr::type> + pageTable(std::make_unique::type>(&allocator)); + + uintptr_t gpuVa = 0x1000; + size_t size = pageSize; + size_t walked = 0u; + uint64_t ppgttBits = 0xabc; + uint64_t entryBitsPassed = 0u; + + PageWalker walker = [&](uint64_t physAddress, size_t size, size_t offset, uint64_t entryBits) { + walked += size; + entryBitsPassed = entryBits; + }; + pageTable->pageWalk(gpuVa, size, 0, ppgttBits, walker, PageTableHelper::memoryBankNotSpecified); + + ppgttBits |= 0x1; + EXPECT_EQ(ppgttBits, entryBitsPassed); + + ppgttBits = 0x345; + pageTable->pageWalk(gpuVa, size, 0, ppgttBits, walker, PageTableHelper::memoryBankNotSpecified); + + EXPECT_EQ(ppgttBits, entryBitsPassed); +} + +TEST_F(PageTableTests48, givenTwoMapsWhenSecondMapHasDifferentEntryBitsThenEntryIsUpdated) { + std::unique_ptr::type> + pageTable(std::make_unique::type>(&allocator)); + uintptr_t gpuVa = 0x1000; + size_t size = pageSize; + uint64_t ppgttBits = 0xabc; + auto address = allocator.nextPageAddress.load(); + + pageTable->map(gpuVa, size, ppgttBits, 0); + ASSERT_NE(nullptr, pageTable->entries[0]); + + PageTableEntryChecker::testEntry::type>(pageTable.get(), 1, static_cast(address | ppgttBits | 0x1)); + + ppgttBits = 0x345; + pageTable->map(gpuVa, size, ppgttBits, 0); + + PageTableEntryChecker::testEntry::type>(pageTable.get(), 1, static_cast(address | ppgttBits | 0x1)); +} + TEST_F(PageTableTests48, givenPageTableWhenMappingTheSameAddressMultipleTimesThenNumberOfPagesReservedInAllocatorMatchPagesMapped) { std::unique_ptr pageTable(new PPGTTPageTable(&allocator)); uintptr_t address = refAddr; auto initialAddress = allocator.initialPageAddress; - auto phys1 = pageTable->map(address, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(address, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); - auto phys1_1 = pageTable->map(address, 1, PageTableHelper::memoryBankNotSpecified); + auto phys1_1 = pageTable->map(address, 1, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1_1); - auto phys2 = pageTable->map(address, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys2 = pageTable->map(address, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(phys1, phys2); address = ptrOffset(address, pageSize); - auto phys3 = pageTable->map(address, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys3 = pageTable->map(address, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(phys1, phys3); address = ptrOffset(address, pageSize); - auto phys4 = pageTable->map(address, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys4 = pageTable->map(address, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(phys3, phys4); auto nextFreeAddress = initialAddress + ptrDiff(phys4 + pageSize, initialAddress); @@ -233,7 +346,7 @@ TEST_F(PageTableTests48, physicalAddressesInAUBCantStartAt0) { std::unique_ptr pageTable(new PPGTTPageTable(&allocator)); uintptr_t addr1 = refAddr; - auto phys1 = pageTable->map(addr1, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(addr1, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(0u, phys1); } @@ -241,11 +354,11 @@ TEST_F(PageTableTests48, mapPageMapByteInMapped) { std::unique_ptr pageTable(new PPGTTPageTable(&allocator)); uintptr_t addr1 = refAddr; - auto phys1 = pageTable->map(addr1, pageSize, 0); + auto phys1 = pageTable->map(addr1, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); EXPECT_EQ(allocator.initialPageAddress + pageSize, allocator.nextPageAddress); - auto phys1_1 = pageTable->map(addr1, 1, PageTableHelper::memoryBankNotSpecified); + auto phys1_1 = pageTable->map(addr1, 1, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1_1); EXPECT_EQ(allocator.initialPageAddress + pageSize, allocator.nextPageAddress); } @@ -254,37 +367,37 @@ TEST_F(PageTableTests48, mapsCorrectlyEvenMultipleCalls) { std::unique_ptr pageTable(new PPGTTPageTable(&allocator)); uintptr_t addr1 = refAddr; - auto phys1 = pageTable->map(addr1, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(addr1, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); EXPECT_EQ(allocator.initialPageAddress + pageSize, allocator.nextPageAddress); - auto phys1_1 = pageTable->map(addr1, 1, PageTableHelper::memoryBankNotSpecified); + auto phys1_1 = pageTable->map(addr1, 1, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1_1); EXPECT_EQ(allocator.initialPageAddress + pageSize, allocator.nextPageAddress); - auto phys2 = pageTable->map(addr1, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys2 = pageTable->map(addr1, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(phys1, phys2); EXPECT_EQ(allocator.initialPageAddress + pageSize, allocator.nextPageAddress); - auto phys3 = pageTable->map(addr1 + pageSize, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys3 = pageTable->map(addr1 + pageSize, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(phys1, phys3); EXPECT_EQ(allocator.initialPageAddress + 2 * pageSize, allocator.nextPageAddress); - auto phys4 = pageTable->map(addr1 + pageSize, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys4 = pageTable->map(addr1 + pageSize, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(phys1, phys3); EXPECT_EQ(phys3, phys4); auto addr2 = addr1 + pageSize + pageSize; - auto phys5 = pageTable->map(addr2, 2 * pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys5 = pageTable->map(addr2, 2 * pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(phys1, phys5); EXPECT_NE(phys3, phys5); - auto phys6 = pageTable->map(addr2, 2 * pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys6 = pageTable->map(addr2, 2 * pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(phys1, phys6); EXPECT_NE(phys3, phys6); EXPECT_EQ(phys5, phys6); - auto phys7 = pageTable->map(addr2 + pageSize, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys7 = pageTable->map(addr2 + pageSize, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_NE(0u, phys7); EXPECT_NE(phys1, phys7); EXPECT_NE(phys3, phys7); @@ -299,7 +412,7 @@ TEST_F(PageTableTests48, mapsPagesOnTableBoundary) { size_t pages = (1 << 9) * 2; size_t size = pages * pageSize; - auto phys1 = pageTable->map(addr1, size, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(addr1, size, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); } @@ -309,10 +422,10 @@ TEST_F(PageTableTests48, mapsPagesOnTableBoundary2ndAllocation) { size_t pages = (1 << 9) * 2; size_t size = pages * pageSize; - auto phys1 = pageTable->map(0x0, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(0x0, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); - auto phys2 = pageTable->map(addr1, size, PageTableHelper::memoryBankNotSpecified); + auto phys2 = pageTable->map(addr1, size, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress + pageSize, phys2); } @@ -325,10 +438,10 @@ TEST_F(PageTableTestsGPU, mapsPagesOnTableBoundary) { size_t pages = (1 << 9) * 2; size_t size = pages * pageSize; - auto phys32 = ggtt->map(addrGGTT, size, PageTableHelper::memoryBankNotSpecified); + auto phys32 = ggtt->map(addrGGTT, size, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys32); - auto phys48 = ppgtt->map(addrPPGTT, size, PageTableHelper::memoryBankNotSpecified); + auto phys48 = ppgtt->map(addrPPGTT, size, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress + size, phys48); } @@ -342,7 +455,7 @@ TEST_F(PageTableTestsGPU, newIsEmpty) { TEST_F(PageTableTests32, level0) { std::unique_ptr> pt(new PageTable(&allocator)); - auto phys = pt->map(0x10000, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys = pt->map(0x10000, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(0u, phys); } @@ -357,7 +470,7 @@ TEST_F(PageTableTests32, mapsPagesOnTableBoundary) { size_t pages = (1 << 9) * 2; size_t size = pages * pageSize; - auto phys1 = pageTable->map(addr1, size, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(addr1, size, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); } @@ -367,9 +480,9 @@ TEST_F(PageTableTests32, mapsPagesOnTableBoundary2ndAllocation) { size_t pages = (1 << 9) * 2; size_t size = pages * pageSize; - auto phys1 = pageTable->map(0x0, pageSize, PageTableHelper::memoryBankNotSpecified); + auto phys1 = pageTable->map(0x0, pageSize, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress, phys1); - auto phys2 = pageTable->map(addr1, size, PageTableHelper::memoryBankNotSpecified); + auto phys2 = pageTable->map(addr1, size, 0, PageTableHelper::memoryBankNotSpecified); EXPECT_EQ(startAddress + pageSize, phys2); }