From f157943610c60273294165008385abbb23c3dd0c Mon Sep 17 00:00:00 2001 From: Mateusz Jablonski Date: Fri, 21 Dec 2018 10:16:27 +0100 Subject: [PATCH] Allocate internal allocations through preferred pool Change-Id: Ib17431ceefc1eb72f86625e0998f679baaa7cb0d Signed-off-by: Mateusz Jablonski --- .../command_stream/command_stream_receiver.cpp | 6 +++--- runtime/memory_manager/graphics_allocation.h | 2 ++ runtime/memory_manager/memory_manager.cpp | 16 ++++++++++++++-- runtime/memory_manager/memory_manager.h | 6 +++--- .../os_agnostic_memory_manager.cpp | 16 ++++++++-------- .../os_agnostic_memory_manager.h | 4 ++-- .../os_interface/linux/drm_memory_manager.cpp | 18 +++++++++--------- .../os_interface/linux/drm_memory_manager.h | 2 +- .../windows/wddm_memory_manager.cpp | 18 +++++++++--------- .../os_interface/windows/wddm_memory_manager.h | 4 ++-- runtime/program/kernel_info.cpp | 4 ++-- .../command_queue/command_queue_tests.cpp | 7 +++---- .../command_queue/dispatch_walker_tests.cpp | 4 ++-- unit_tests/fixtures/memory_allocator_fixture.h | 5 +++-- unit_tests/helpers/kernel_commands_tests.cpp | 2 +- ...y_manager_allocate_in_device_pool_tests.cpp | 4 ++-- ...y_manager_allocate_in_device_pool_tests.inl | 4 ++-- .../memory_manager/memory_manager_tests.cpp | 10 +++------- .../mocks/linux/mock_drm_memory_manager.h | 11 +++++++++++ unit_tests/mocks/mock_allocation_properties.h | 9 ++++++++- unit_tests/mocks/mock_memory_manager.cpp | 11 +++++++++++ unit_tests/mocks/mock_memory_manager.h | 3 ++- .../linux/drm_command_stream_tests.cpp | 4 ++-- .../windows/mock_wddm_memory_manager.h | 13 ++++++++++++- unit_tests/program/kernel_info_tests.cpp | 4 ++-- 25 files changed, 119 insertions(+), 68 deletions(-) diff --git a/runtime/command_stream/command_stream_receiver.cpp b/runtime/command_stream/command_stream_receiver.cpp index fcd27b0bcf..592fcf6c58 100644 --- a/runtime/command_stream/command_stream_receiver.cpp +++ b/runtime/command_stream/command_stream_receiver.cpp @@ -287,11 +287,11 @@ void CommandStreamReceiver::allocateHeapMemory(IndirectHeap::Type heapType, auto heapMemory = internalAllocationStorage->obtainReusableAllocation(finalHeapSize, requireInternalHeap).release(); if (!heapMemory) { + auto allocationType = GraphicsAllocation::AllocationType::LINEAR_STREAM; if (requireInternalHeap) { - heapMemory = getMemoryManager()->allocate32BitGraphicsMemory(finalHeapSize, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); - } else { - heapMemory = getMemoryManager()->allocateGraphicsMemoryWithProperties({finalHeapSize, GraphicsAllocation::AllocationType::LINEAR_STREAM}); + allocationType = GraphicsAllocation::AllocationType::INTERNAL_HEAP; } + heapMemory = getMemoryManager()->allocateGraphicsMemoryWithProperties({finalHeapSize, allocationType}); } else { finalHeapSize = std::max(heapMemory->getUnderlyingBufferSize(), finalHeapSize); } diff --git a/runtime/memory_manager/graphics_allocation.h b/runtime/memory_manager/graphics_allocation.h index b4998d9a47..478cfef92c 100644 --- a/runtime/memory_manager/graphics_allocation.h +++ b/runtime/memory_manager/graphics_allocation.h @@ -64,6 +64,8 @@ class GraphicsAllocation : public IDNode { DYNAMIC_STATE_HEAP, SHARED_RESOURCE_COPY, SVM, + KERNEL_ISA, + INTERNAL_HEAP, UNDECIDED, }; diff --git a/runtime/memory_manager/memory_manager.cpp b/runtime/memory_manager/memory_manager.cpp index ae105b779d..821536f329 100644 --- a/runtime/memory_manager/memory_manager.cpp +++ b/runtime/memory_manager/memory_manager.cpp @@ -260,12 +260,23 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo case GraphicsAllocation::AllocationType::LINEAR_STREAM: case GraphicsAllocation::AllocationType::FILL_PATTERN: case GraphicsAllocation::AllocationType::TIMESTAMP_TAG_BUFFER: + case GraphicsAllocation::AllocationType::KERNEL_ISA: + case GraphicsAllocation::AllocationType::INTERNAL_HEAP: allocationData.flags.useSystemMemory = true; break; default: break; } + switch (properties.allocationType) { + case GraphicsAllocation::AllocationType::KERNEL_ISA: + case GraphicsAllocation::AllocationType::INTERNAL_HEAP: + allocationData.allocationOrigin = AllocationOrigin::INTERNAL_ALLOCATION; + break; + default: + break; + } + allocationData.flags.mustBeZeroCopy = mustBeZeroCopy; allocationData.flags.allocateMemory = properties.flags.allocateMemory; allocationData.flags.allow32Bit = allow32Bit; @@ -315,8 +326,9 @@ GraphicsAllocation *MemoryManager::allocateGraphicsMemory(const AllocationData & return allocateGraphicsMemoryForImage(allocationData); } - if (force32bitAllocations && allocationData.flags.allow32Bit && is64bit) { - return allocate32BitGraphicsMemory(allocationData.size, allocationData.hostPtr, AllocationOrigin::EXTERNAL_ALLOCATION); + if (allocationData.allocationOrigin == AllocationOrigin::INTERNAL_ALLOCATION || + (force32bitAllocations && allocationData.flags.allow32Bit && is64bit)) { + return allocate32BitGraphicsMemoryImpl(allocationData); } if (allocationData.hostPtr) { return allocateGraphicsMemoryWithHostPtr(allocationData); diff --git a/runtime/memory_manager/memory_manager.h b/runtime/memory_manager/memory_manager.h index 3e2113017b..094a6c4db4 100644 --- a/runtime/memory_manager/memory_manager.h +++ b/runtime/memory_manager/memory_manager.h @@ -36,7 +36,7 @@ enum AllocationUsage { REUSABLE_ALLOCATION }; -enum AllocationOrigin { +enum class AllocationOrigin { EXTERNAL_ALLOCATION, INTERNAL_ALLOCATION }; @@ -117,8 +117,6 @@ class MemoryManager { } } - virtual GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) = 0; - GraphicsAllocation *allocateGraphicsMemoryInPreferredPool(AllocationProperties properties, DevicesBitfield devicesBitfield, const void *hostPtr); virtual GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness) = 0; @@ -212,6 +210,7 @@ class MemoryManager { }; static_assert(sizeof(AllocationData::flags) == sizeof(AllocationData::allFlags), ""); GraphicsAllocation::AllocationType type = GraphicsAllocation::AllocationType::UNKNOWN; + AllocationOrigin allocationOrigin = AllocationOrigin::EXTERNAL_ALLOCATION; const void *hostPtr = nullptr; size_t size = 0; size_t alignment = 0; @@ -227,6 +226,7 @@ class MemoryManager { virtual GraphicsAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData); virtual GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) = 0; virtual GraphicsAllocation *allocateGraphicsMemory64kb(AllocationData allocationData) = 0; + virtual GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) = 0; virtual GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::Error; switch (allocationData.type) { diff --git a/runtime/memory_manager/os_agnostic_memory_manager.cpp b/runtime/memory_manager/os_agnostic_memory_manager.cpp index 4879fcbbec..21670591af 100644 --- a/runtime/memory_manager/os_agnostic_memory_manager.cpp +++ b/runtime/memory_manager/os_agnostic_memory_manager.cpp @@ -78,15 +78,15 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemory64kb(Allocati return memoryAllocation; } -GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) { - if (ptr) { - auto allocationSize = alignSizeWholePage(ptr, size); +GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) { + if (allocationData.hostPtr) { + auto allocationSize = alignSizeWholePage(allocationData.hostPtr, allocationData.size); auto gpuVirtualAddress = allocator32Bit->allocate(allocationSize); if (!gpuVirtualAddress) { return nullptr; } - uint64_t offset = static_cast(reinterpret_cast(ptr) & MemoryConstants::pageMask); - MemoryAllocation *memAlloc = new MemoryAllocation(nullptr, const_cast(ptr), GmmHelper::canonize(gpuVirtualAddress + offset), size, + uint64_t offset = static_cast(reinterpret_cast(allocationData.hostPtr) & MemoryConstants::pageMask); + MemoryAllocation *memAlloc = new MemoryAllocation(nullptr, const_cast(allocationData.hostPtr), GmmHelper::canonize(gpuVirtualAddress + offset), allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, this->getOsContextCount(), false); memAlloc->is32BitAllocation = true; memAlloc->gpuBaseAddress = GmmHelper::canonize(allocator32Bit->getBase()); @@ -96,17 +96,17 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemory(size_t return memAlloc; } - auto allocationSize = alignUp(size, MemoryConstants::pageSize); + auto allocationSize = alignUp(allocationData.size, MemoryConstants::pageSize); void *ptrAlloc = nullptr; auto gpuAddress = allocator32Bit->allocate(allocationSize); - if (size < 0xfffff000) { + if (allocationData.size < 0xfffff000) { ptrAlloc = alignedMallocWrapper(allocationSize, MemoryConstants::allocationAlignment); } MemoryAllocation *memoryAllocation = nullptr; if (ptrAlloc != nullptr) { - memoryAllocation = new MemoryAllocation(ptrAlloc, ptrAlloc, GmmHelper::canonize(gpuAddress), size, counter, + memoryAllocation = new MemoryAllocation(ptrAlloc, ptrAlloc, GmmHelper::canonize(gpuAddress), allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, this->getOsContextCount(), false); memoryAllocation->is32BitAllocation = true; memoryAllocation->gpuBaseAddress = GmmHelper::canonize(allocator32Bit->getBase()); diff --git a/runtime/memory_manager/os_agnostic_memory_manager.h b/runtime/memory_manager/os_agnostic_memory_manager.h index e4b9cf7ed2..a9f9277c8c 100644 --- a/runtime/memory_manager/os_agnostic_memory_manager.h +++ b/runtime/memory_manager/os_agnostic_memory_manager.h @@ -45,10 +45,8 @@ class OsAgnosticMemoryManager : public MemoryManager { ~OsAgnosticMemoryManager() override; GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(size_t size, void *cpuPtr) override; - GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) override; GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness) override; GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) override { return nullptr; } - GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; void addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) override; void removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) override; @@ -74,6 +72,8 @@ class OsAgnosticMemoryManager : public MemoryManager { void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override { return ptrOffset(graphicsAllocation.getUnderlyingBuffer(), static_cast(graphicsAllocation.allocationOffset)); }; void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override{}; + GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override; + GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; private: unsigned long long counter = 0; diff --git a/runtime/os_interface/linux/drm_memory_manager.cpp b/runtime/os_interface/linux/drm_memory_manager.cpp index 7cfa2474e3..cde443fd05 100644 --- a/runtime/os_interface/linux/drm_memory_manager.cpp +++ b/runtime/os_interface/linux/drm_memory_manager.cpp @@ -324,19 +324,19 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryForImageImpl(const A return allocation; } -DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) { - auto allocatorToUse = allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION ? allocator32Bit.get() : internal32bitAllocator.get(); - auto allocatorType = allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION ? BIT32_ALLOCATOR_EXTERNAL : BIT32_ALLOCATOR_INTERNAL; +DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) { + auto allocatorToUse = allocationData.allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION ? allocator32Bit.get() : internal32bitAllocator.get(); + auto allocatorType = allocationData.allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION ? BIT32_ALLOCATOR_EXTERNAL : BIT32_ALLOCATOR_INTERNAL; - if (ptr) { - uintptr_t inputPtr = (uintptr_t)ptr; - auto allocationSize = alignSizeWholePage((void *)ptr, size); + if (allocationData.hostPtr) { + uintptr_t inputPtr = reinterpret_cast(allocationData.hostPtr); + auto allocationSize = alignSizeWholePage(allocationData.hostPtr, allocationData.size); auto realAllocationSize = allocationSize; auto gpuVirtualAddress = allocatorToUse->allocate(realAllocationSize); if (!gpuVirtualAddress) { return nullptr; } - auto alignedUserPointer = (uintptr_t)alignDown(ptr, MemoryConstants::pageSize); + auto alignedUserPointer = reinterpret_cast(alignDown(allocationData.hostPtr, MemoryConstants::pageSize)); auto inputPointerOffset = inputPtr - alignedUserPointer; BufferObject *bo = allocUserptr(alignedUserPointer, allocationSize, 0, true); @@ -351,14 +351,14 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemory(size_t size, const uintptr_t offset = (uintptr_t)bo->address; bo->softPin((uint64_t)offset); bo->setAllocationType(allocatorType); - auto drmAllocation = new DrmAllocation(bo, (void *)ptr, (uint64_t)ptrOffset(gpuVirtualAddress, inputPointerOffset), + auto drmAllocation = new DrmAllocation(bo, const_cast(allocationData.hostPtr), static_cast(ptrOffset(gpuVirtualAddress, inputPointerOffset)), allocationSize, MemoryPool::System4KBPagesWith32BitGpuAddressing, getOsContextCount(), false); drmAllocation->is32BitAllocation = true; drmAllocation->gpuBaseAddress = allocatorToUse->getBase(); return drmAllocation; } - size_t alignedAllocationSize = alignUp(size, MemoryConstants::pageSize); + size_t alignedAllocationSize = alignUp(allocationData.size, MemoryConstants::pageSize); auto allocationSize = alignedAllocationSize; auto res = allocatorToUse->allocate(allocationSize); diff --git a/runtime/os_interface/linux/drm_memory_manager.h b/runtime/os_interface/linux/drm_memory_manager.h index e165f300ce..f63f5880b4 100644 --- a/runtime/os_interface/linux/drm_memory_manager.h +++ b/runtime/os_interface/linux/drm_memory_manager.h @@ -32,7 +32,6 @@ class DrmMemoryManager : public MemoryManager { void removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) override; void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override; DrmAllocation *allocateGraphicsMemoryForNonSvmHostPtr(size_t size, void *cpuPtr) override; - DrmAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) override; GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness) override; GraphicsAllocation *createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding) override; GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) override { return nullptr; } @@ -72,6 +71,7 @@ class DrmMemoryManager : public MemoryManager { void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override; void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override; + DrmAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override; Drm *drm; BufferObject *pinBB; diff --git a/runtime/os_interface/windows/wddm_memory_manager.cpp b/runtime/os_interface/windows/wddm_memory_manager.cpp index 4fba4bc98b..97b280b40f 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.cpp +++ b/runtime/os_interface/windows/wddm_memory_manager.cpp @@ -167,19 +167,19 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory(const AllocationPr return allocation; } -GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) { +GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) { Gmm *gmm = nullptr; const void *ptrAligned = nullptr; - size_t sizeAligned = size; + size_t sizeAligned = allocationData.size; void *pSysMem = nullptr; size_t offset = 0; - if (ptr) { - ptrAligned = alignDown(ptr, MemoryConstants::allocationAlignment); - sizeAligned = alignSizeWholePage(ptr, size); - offset = ptrDiff(ptr, ptrAligned); + if (allocationData.hostPtr) { + ptrAligned = alignDown(allocationData.hostPtr, MemoryConstants::allocationAlignment); + sizeAligned = alignSizeWholePage(allocationData.hostPtr, sizeAligned); + offset = ptrDiff(allocationData.hostPtr, ptrAligned); } else { - sizeAligned = alignUp(size, MemoryConstants::allocationAlignment); + sizeAligned = alignUp(sizeAligned, MemoryConstants::allocationAlignment); pSysMem = allocateSystemMemory(sizeAligned, MemoryConstants::allocationAlignment); if (pSysMem == nullptr) { return nullptr; @@ -196,14 +196,14 @@ GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemory(size_t size, gmm = new Gmm(ptrAligned, sizeAligned, false); wddmAllocation->gmm = gmm; - if (!createWddmAllocation(wddmAllocation.get(), allocationOrigin)) { + if (!createWddmAllocation(wddmAllocation.get(), allocationData.allocationOrigin)) { delete gmm; freeSystemMemory(pSysMem); return nullptr; } wddmAllocation->is32BitAllocation = true; - auto baseAddress = allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION ? allocator32Bit->getBase() : this->wddm->getGfxPartition().Heap32[1].Base; + auto baseAddress = allocationData.allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION ? allocator32Bit->getBase() : this->wddm->getGfxPartition().Heap32[1].Base; wddmAllocation->gpuBaseAddress = GmmHelper::canonize(baseAddress); DebugManager.logAllocation(wddmAllocation.get()); diff --git a/runtime/os_interface/windows/wddm_memory_manager.h b/runtime/os_interface/windows/wddm_memory_manager.h index 5297703c1b..97cdc694f1 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.h +++ b/runtime/os_interface/windows/wddm_memory_manager.h @@ -36,10 +36,8 @@ class WddmMemoryManager : public MemoryManager { void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override; GraphicsAllocation *allocateGraphicsMemory(const AllocationProperties &properties, const void *ptr) override; GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(size_t size, void *cpuPtr) override; - GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) override; GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness) override; GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) override; - GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; void addAllocationToHostPtrManager(GraphicsAllocation *memory) override; void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) override; @@ -75,6 +73,8 @@ class WddmMemoryManager : public MemoryManager { void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override; void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override; + GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override; + GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle); static bool validateAllocation(WddmAllocation *alloc); diff --git a/runtime/program/kernel_info.cpp b/runtime/program/kernel_info.cpp index 6d064a58f0..1e87403246 100644 --- a/runtime/program/kernel_info.cpp +++ b/runtime/program/kernel_info.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017-2018 Intel Corporation + * Copyright (C) 2017-2019 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -485,7 +485,7 @@ uint32_t KernelInfo::getConstantBufferSize() const { bool KernelInfo::createKernelAllocation(MemoryManager *memoryManager) { UNRECOVERABLE_IF(kernelAllocation); auto kernelIsaSize = heapInfo.pKernelHeader->KernelHeapSize; - kernelAllocation = memoryManager->allocate32BitGraphicsMemory(kernelIsaSize, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); + kernelAllocation = memoryManager->allocateGraphicsMemoryWithProperties({kernelIsaSize, GraphicsAllocation::AllocationType::KERNEL_ISA}); if (kernelAllocation) { memcpy_s(kernelAllocation->getUnderlyingBuffer(), kernelIsaSize, heapInfo.pKernelHeap, kernelIsaSize); } else { diff --git a/unit_tests/command_queue/command_queue_tests.cpp b/unit_tests/command_queue/command_queue_tests.cpp index 252bb82629..6aebe47a1f 100644 --- a/unit_tests/command_queue/command_queue_tests.cpp +++ b/unit_tests/command_queue/command_queue_tests.cpp @@ -489,12 +489,11 @@ TEST_P(CommandQueueIndirectHeapTest, givenCommandStreamReceiverWithReusableAlloc GraphicsAllocation *allocation = nullptr; auto &commandStreamReceiver = cmdQ.getCommandStreamReceiver(); - + auto allocationType = GraphicsAllocation::AllocationType::LINEAR_STREAM; if (this->GetParam() == IndirectHeap::INDIRECT_OBJECT) { - allocation = memoryManager->allocate32BitGraphicsMemory(allocationSize, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); - } else { - allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{allocationSize}); + allocationType = GraphicsAllocation::AllocationType::INTERNAL_HEAP; } + allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{allocationSize, allocationType}); if (this->GetParam() == IndirectHeap::SURFACE_STATE) { allocation->setSize(commandStreamReceiver.defaultSshSize * 2); } diff --git a/unit_tests/command_queue/dispatch_walker_tests.cpp b/unit_tests/command_queue/dispatch_walker_tests.cpp index b72112cd88..5d04088caa 100644 --- a/unit_tests/command_queue/dispatch_walker_tests.cpp +++ b/unit_tests/command_queue/dispatch_walker_tests.cpp @@ -813,8 +813,8 @@ HWCMDTEST_F(IGFX_GEN8_CORE, DispatchWalkerTest, dispatchWalkerWithMultipleDispat using INTERFACE_DESCRIPTOR_DATA = typename FamilyType::INTERFACE_DESCRIPTOR_DATA; auto memoryManager = this->pDevice->getMemoryManager(); - auto kernelIsaAllocation = memoryManager->allocate32BitGraphicsMemory(MemoryConstants::pageSize, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); - auto kernelIsaWithSamplerAllocation = memoryManager->allocate32BitGraphicsMemory(MemoryConstants::pageSize, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); + auto kernelIsaAllocation = memoryManager->allocateGraphicsMemoryWithProperties({MemoryConstants::pageSize, GraphicsAllocation::AllocationType::KERNEL_ISA}); + auto kernelIsaWithSamplerAllocation = memoryManager->allocateGraphicsMemoryWithProperties({MemoryConstants::pageSize, GraphicsAllocation::AllocationType::KERNEL_ISA}); kernelInfo.kernelAllocation = kernelIsaAllocation; kernelInfoWithSampler.kernelAllocation = kernelIsaWithSamplerAllocation; auto gpuAddress1 = kernelIsaAllocation->getGpuAddressToPatch(); diff --git a/unit_tests/fixtures/memory_allocator_fixture.h b/unit_tests/fixtures/memory_allocator_fixture.h index 4f03c90a94..a0ab56c68b 100644 --- a/unit_tests/fixtures/memory_allocator_fixture.h +++ b/unit_tests/fixtures/memory_allocator_fixture.h @@ -12,6 +12,7 @@ #include "runtime/memory_manager/os_agnostic_memory_manager.h" #include "unit_tests/fixtures/memory_management_fixture.h" #include "unit_tests/libult/create_command_stream.h" +#include "unit_tests/mocks/mock_memory_manager.h" using namespace OCLRT; @@ -21,7 +22,7 @@ class MemoryAllocatorFixture : public MemoryManagementFixture { MemoryManagementFixture::SetUp(); executionEnvironment = std::make_unique(); executionEnvironment->initializeCommandStreamReceiver(*platformDevices, 0, 0); - memoryManager = new OsAgnosticMemoryManager(false, false, *executionEnvironment); + memoryManager = new MockMemoryManager(false, false, *executionEnvironment); executionEnvironment->memoryManager.reset(memoryManager); csr = memoryManager->getDefaultCommandStreamReceiver(0); csr->setupContext(*memoryManager->createAndRegisterOsContext(HwHelper::get(platformDevices[0]->pPlatform->eRenderCoreFamily).getGpgpuEngineInstances()[0], PreemptionHelper::getDefaultPreemptionMode(*platformDevices[0]))); @@ -34,6 +35,6 @@ class MemoryAllocatorFixture : public MemoryManagementFixture { protected: std::unique_ptr executionEnvironment; - MemoryManager *memoryManager; + MockMemoryManager *memoryManager; CommandStreamReceiver *csr; }; diff --git a/unit_tests/helpers/kernel_commands_tests.cpp b/unit_tests/helpers/kernel_commands_tests.cpp index c3e66e88f0..0504b7ab5f 100644 --- a/unit_tests/helpers/kernel_commands_tests.cpp +++ b/unit_tests/helpers/kernel_commands_tests.cpp @@ -210,7 +210,7 @@ HWTEST_F(KernelCommandsTest, givenIndirectHeapNotAllocatedFromInternalPoolWhenSe } HWTEST_F(KernelCommandsTest, givenIndirectHeapAllocatedFromInternalPoolWhenSendCrossThreadDataIsCalledThenHeapBaseOffsetIsReturned) { - auto internalAllocation = pDevice->getMemoryManager()->allocate32BitGraphicsMemory(MemoryConstants::pageSize, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); + auto internalAllocation = pDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties(MockAllocationProperties::getPropertiesFor32BitInternalAllocation(MemoryConstants::pageSize, true)); IndirectHeap indirectHeap(internalAllocation, true); auto expectedOffset = internalAllocation->getGpuAddressToPatch(); diff --git a/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp b/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp index fa9fe89eea..301bba93d4 100644 --- a/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp +++ b/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp @@ -10,7 +10,7 @@ TEST(MemoryManagerTest, givenNotSetUseSystemMemoryWhenGraphicsAllocationInDevicePoolIsAllocatedThenAllocationIsReturned) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; AllocationData allocData; @@ -26,7 +26,7 @@ TEST(MemoryManagerTest, givenNotSetUseSystemMemoryWhenGraphicsAllocationInDevice TEST(MemoryManagerTest, givenImageOrSharedResourceCopyWhenGraphicsAllocationInDevicePoolIsAllocatedThenNullptrIsReturned) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; AllocationData allocData; diff --git a/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl b/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl index dfeccb9ad4..0d356f75ab 100644 --- a/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl +++ b/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl @@ -18,7 +18,7 @@ using namespace OCLRT; TEST(MemoryManagerTest, givenSetUseSytemMemoryWhenGraphicsAllocationInDevicePoolIsAllocatedThenNullptrIsReturned) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; AllocationData allocData; allocData.size = MemoryConstants::pageSize; @@ -32,7 +32,7 @@ TEST(MemoryManagerTest, givenSetUseSytemMemoryWhenGraphicsAllocationInDevicePool TEST(MemoryManagerTest, givenAllowed32BitAndFroce32BitWhenGraphicsAllocationInDevicePoolIsAllocatedThenNullptrIsReturned) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); memoryManager.setForce32BitAllocations(true); MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; diff --git a/unit_tests/memory_manager/memory_manager_tests.cpp b/unit_tests/memory_manager/memory_manager_tests.cpp index b88edcf737..363cd7956a 100644 --- a/unit_tests/memory_manager/memory_manager_tests.cpp +++ b/unit_tests/memory_manager/memory_manager_tests.cpp @@ -358,8 +358,6 @@ TEST_F(MemoryAllocatorTest, givenNotEnoughSpaceInAllocatorWhenAskedFor32bitAlloc auto allocationFirst = memoryManager->allocate32BitGraphicsMemory(0x5000, nullptr, AllocationOrigin::EXTERNAL_ALLOCATION); auto allocation = memoryManager->allocate32BitGraphicsMemory(size, nullptr, AllocationOrigin::EXTERNAL_ALLOCATION); EXPECT_EQ(nullptr, allocation); - if (allocation) - memoryManager->freeGraphicsMemory(allocation); memoryManager->freeGraphicsMemory(allocationFirst); } @@ -369,8 +367,6 @@ TEST_F(MemoryAllocatorTest, givenNotEnoughSpaceInAllocatorWhenAskedFor32bitAlloc auto allocationFirst = memoryManager->allocate32BitGraphicsMemory(0x5000, nullptr, AllocationOrigin::EXTERNAL_ALLOCATION); auto allocation = memoryManager->allocate32BitGraphicsMemory(size, ptr, AllocationOrigin::EXTERNAL_ALLOCATION); EXPECT_EQ(nullptr, allocation); - if (allocation) - memoryManager->freeGraphicsMemory(allocation); memoryManager->freeGraphicsMemory(allocationFirst); } @@ -718,7 +714,7 @@ TEST(OsAgnosticMemoryManager, givenMemoryManagerWhenAllocateGraphicsMemoryWithPt TEST(OsAgnosticMemoryManager, givenMemoryManagerWhenAllocate32BitGraphicsMemoryWithPtrIsCalledThenMemoryPoolIsSystem4KBPagesWith32BitGpuAddressing) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); void *ptr = reinterpret_cast(0x1001); auto size = MemoryConstants::pageSize; @@ -732,7 +728,7 @@ TEST(OsAgnosticMemoryManager, givenMemoryManagerWhenAllocate32BitGraphicsMemoryW TEST(OsAgnosticMemoryManager, givenMemoryManagerWhenAllocate32BitGraphicsMemoryWithoutPtrIsCalledThenMemoryPoolIsSystem4KBPagesWith32BitGpuAddressing) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); void *ptr = nullptr; auto size = MemoryConstants::pageSize; @@ -1020,7 +1016,7 @@ TEST(OsAgnosticMemoryManager, GivenEnabled64kbPagesWhenHostMemoryAllocationIsCre TEST(OsAgnosticMemoryManager, givenPointerAndSizeWhenCreateInternalAllocationIsCalledThenGraphicsAllocationIsReturned) { ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); + MockMemoryManager memoryManager(false, false, executionEnvironment); auto ptr = (void *)0x100000; size_t allocationSize = 4096; auto graphicsAllocation = memoryManager.allocate32BitGraphicsMemory(allocationSize, ptr, AllocationOrigin::INTERNAL_ALLOCATION); diff --git a/unit_tests/mocks/linux/mock_drm_memory_manager.h b/unit_tests/mocks/linux/mock_drm_memory_manager.h index c9a469084c..b53faa7ed6 100644 --- a/unit_tests/mocks/linux/mock_drm_memory_manager.h +++ b/unit_tests/mocks/linux/mock_drm_memory_manager.h @@ -6,6 +6,7 @@ */ #include "runtime/os_interface/linux/drm_memory_manager.h" +#include "unit_tests/mocks/mock_allocation_properties.h" #include "unit_tests/mocks/mock_host_ptr_manager.h" #include @@ -85,5 +86,15 @@ class TestedDrmMemoryManager : public DrmMemoryManager { Allocator32bit *getDrmInternal32BitAllocator() const { return internal32bitAllocator.get(); } AllocatorLimitedRange *getDrmLimitedRangeAllocator() const { return limitedGpuAddressRangeAllocator.get(); } + DrmAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) { + bool allocateMemory = ptr == nullptr; + AllocationData allocationData; + if (allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION) { + getAllocationData(allocationData, MockAllocationProperties::getPropertiesFor32BitExternalAllocation(size, allocateMemory), 0u, ptr); + } else { + getAllocationData(allocationData, MockAllocationProperties::getPropertiesFor32BitInternalAllocation(size, allocateMemory), 0u, ptr); + } + return allocate32BitGraphicsMemoryImpl(allocationData); + } }; } // namespace OCLRT diff --git a/unit_tests/mocks/mock_allocation_properties.h b/unit_tests/mocks/mock_allocation_properties.h index 7d10ddf63a..5ba915cc81 100644 --- a/unit_tests/mocks/mock_allocation_properties.h +++ b/unit_tests/mocks/mock_allocation_properties.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Intel Corporation + * Copyright (C) 2018-2019 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -14,5 +14,12 @@ struct MockAllocationProperties : public AllocationProperties { MockAllocationProperties(size_t size, GraphicsAllocation::AllocationType allocationType) : AllocationProperties(size, allocationType) {} MockAllocationProperties(size_t size) : AllocationProperties(size, GraphicsAllocation::AllocationType::UNDECIDED) {} MockAllocationProperties(bool allocateMemory, size_t size) : AllocationProperties(allocateMemory, size, GraphicsAllocation::AllocationType::UNDECIDED) {} + + static AllocationProperties getPropertiesFor32BitInternalAllocation(size_t size, bool allocateMemory) { + return AllocationProperties{allocateMemory, size, GraphicsAllocation::AllocationType::INTERNAL_HEAP}; + } + static AllocationProperties getPropertiesFor32BitExternalAllocation(size_t size, bool allocateMemory) { + return AllocationProperties{allocateMemory, size, GraphicsAllocation::AllocationType::BUFFER}; + } }; } // namespace OCLRT diff --git a/unit_tests/mocks/mock_memory_manager.cpp b/unit_tests/mocks/mock_memory_manager.cpp index d52724012d..66b2229ede 100644 --- a/unit_tests/mocks/mock_memory_manager.cpp +++ b/unit_tests/mocks/mock_memory_manager.cpp @@ -9,6 +9,7 @@ #include "runtime/memory_manager/deferred_deleter.h" #include "runtime/gmm_helper/gmm.h" #include "runtime/helpers/surface_formats.h" +#include "unit_tests/mocks/mock_allocation_properties.h" #include "unit_tests/mocks/mock_memory_manager.h" #include @@ -80,6 +81,16 @@ GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryWithAlignment(const allocationCreated = true; return OsAgnosticMemoryManager::allocateGraphicsMemoryWithAlignment(allocationData); } +GraphicsAllocation *MockMemoryManager::allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) { + bool allocateMemory = ptr == nullptr; + AllocationData allocationData; + if (allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION) { + getAllocationData(allocationData, MockAllocationProperties::getPropertiesFor32BitExternalAllocation(size, allocateMemory), 0u, ptr); + } else { + getAllocationData(allocationData, MockAllocationProperties::getPropertiesFor32BitInternalAllocation(size, allocateMemory), 0u, ptr); + } + return allocate32BitGraphicsMemoryImpl(allocationData); +} FailMemoryManager::FailMemoryManager(int32_t failedAllocationsCount) { this->failedAllocationsCount = failedAllocationsCount; diff --git a/unit_tests/mocks/mock_memory_manager.h b/unit_tests/mocks/mock_memory_manager.h index b96661c1d3..96e5e8f81f 100644 --- a/unit_tests/mocks/mock_memory_manager.h +++ b/unit_tests/mocks/mock_memory_manager.h @@ -64,6 +64,7 @@ class MockMemoryManager : public OsAgnosticMemoryManager { unlockResourceCalled++; OsAgnosticMemoryManager::unlockResourceImpl(gfxAllocation); } + GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin); uint32_t freeGraphicsMemoryCalled = 0u; uint32_t unlockResourceCalled = 0u; @@ -131,7 +132,7 @@ class FailMemoryManager : public MockMemoryManager { GraphicsAllocation *allocateGraphicsMemory(const AllocationProperties &properties, const void *ptr) override { return nullptr; }; - GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) override { + GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override { return nullptr; }; GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness) override { diff --git a/unit_tests/os_interface/linux/drm_command_stream_tests.cpp b/unit_tests/os_interface/linux/drm_command_stream_tests.cpp index 8add89cba6..945c73e477 100644 --- a/unit_tests/os_interface/linux/drm_command_stream_tests.cpp +++ b/unit_tests/os_interface/linux/drm_command_stream_tests.cpp @@ -236,8 +236,8 @@ TEST_F(DrmCommandStreamTest, givenDrmContextIdWhenFlushingThenSetIdToAllExecBuff return 0; }; - auto allocation1 = memoryManager->allocate32BitGraphicsMemory(1, reinterpret_cast(1), AllocationOrigin::INTERNAL_ALLOCATION); - auto allocation2 = memoryManager->allocate32BitGraphicsMemory(1, reinterpret_cast(2), AllocationOrigin::INTERNAL_ALLOCATION); + auto allocation1 = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{MemoryConstants::pageSize}); + auto allocation2 = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{MemoryConstants::pageSize}); csr->makeResident(*allocation1); csr->makeResident(*allocation2); diff --git a/unit_tests/os_interface/windows/mock_wddm_memory_manager.h b/unit_tests/os_interface/windows/mock_wddm_memory_manager.h index 4fe5807c69..7beb18554f 100644 --- a/unit_tests/os_interface/windows/mock_wddm_memory_manager.h +++ b/unit_tests/os_interface/windows/mock_wddm_memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017-2018 Intel Corporation + * Copyright (C) 2017-2019 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -17,6 +17,7 @@ class MockWddmMemoryManager : public WddmMemoryManager { public: using BaseClass::allocateGraphicsMemory; using BaseClass::allocateGraphicsMemory64kb; + using BaseClass::allocateGraphicsMemoryInDevicePool; using BaseClass::createWddmAllocation; using BaseClass::WddmMemoryManager; @@ -32,5 +33,15 @@ class MockWddmMemoryManager : public WddmMemoryManager { bool validateAllocationMock(WddmAllocation *graphicsAllocation) { return this->validateAllocation(graphicsAllocation); } + GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) { + bool allocateMemory = ptr == nullptr; + AllocationData allocationData; + if (allocationOrigin == AllocationOrigin::EXTERNAL_ALLOCATION) { + getAllocationData(allocationData, MockAllocationProperties::getPropertiesFor32BitExternalAllocation(size, allocateMemory), 0u, ptr); + } else { + getAllocationData(allocationData, MockAllocationProperties::getPropertiesFor32BitInternalAllocation(size, allocateMemory), 0u, ptr); + } + return allocate32BitGraphicsMemoryImpl(allocationData); + } }; } // namespace OCLRT diff --git a/unit_tests/program/kernel_info_tests.cpp b/unit_tests/program/kernel_info_tests.cpp index e464879c00..eb890cd8c9 100644 --- a/unit_tests/program/kernel_info_tests.cpp +++ b/unit_tests/program/kernel_info_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017-2018 Intel Corporation + * Copyright (C) 2017-2019 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -130,7 +130,7 @@ TEST(KernelInfoTest, givenKernelInfoWhenCreateKernelAllocationThenCopyWholeKerne class MyMemoryManager : public OsAgnosticMemoryManager { public: using OsAgnosticMemoryManager::OsAgnosticMemoryManager; - GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) override { return nullptr; } + GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override { return nullptr; } }; TEST(KernelInfoTest, givenKernelInfoWhenCreateKernelAllocationAndCannotAllocateMemoryThenReturnsFalse) {