From cdb52400c4f6b82aea1c0fea693ef212012ce5a9 Mon Sep 17 00:00:00 2001 From: Zbigniew Zdanowicz Date: Wed, 13 Mar 2019 15:31:46 +0100 Subject: [PATCH] Add new functions reserving address range to Memory Manager Change-Id: I947203c24495c9e5a206b95bb0c69440824586b6 Signed-off-by: Zbigniew Zdanowicz --- .../memory_manager/graphics_allocation.cpp | 10 ++--- runtime/memory_manager/graphics_allocation.h | 37 +++++++++++++------ runtime/memory_manager/memory_manager.h | 2 + .../os_agnostic_memory_manager.cpp | 12 ++++++ .../os_agnostic_memory_manager.h | 3 ++ .../os_interface/linux/drm_memory_manager.cpp | 19 ++++++++-- .../os_interface/linux/drm_memory_manager.h | 2 + runtime/os_interface/windows/wddm/wddm.cpp | 2 +- .../os_interface/windows/wddm_allocation.h | 12 ++---- .../windows/wddm_memory_manager.cpp | 16 +++++++- .../windows/wddm_memory_manager.h | 2 + .../memory_manager/memory_manager_tests.cpp | 12 ++++++ unit_tests/mocks/mock_wddm.cpp | 2 +- .../linux/drm_memory_manager_tests.cpp | 12 ++++++ .../windows/device_command_stream_tests.cpp | 2 +- .../windows/wddm_memory_manager_tests.cpp | 2 +- 16 files changed, 112 insertions(+), 35 deletions(-) diff --git a/runtime/memory_manager/graphics_allocation.cpp b/runtime/memory_manager/graphics_allocation.cpp index cc9e03dcd9..3bb5ebb91f 100644 --- a/runtime/memory_manager/graphics_allocation.cpp +++ b/runtime/memory_manager/graphics_allocation.cpp @@ -19,10 +19,10 @@ void GraphicsAllocation::setAllocationType(AllocationType allocationType) { GraphicsAllocation::GraphicsAllocation(AllocationType allocationType, void *cpuPtrIn, uint64_t gpuAddress, uint64_t baseAddress, size_t sizeIn, MemoryPool::Type pool, bool multiOsContextCapable) - : size(sizeIn), - cpuPtr(cpuPtrIn), - gpuBaseAddress(baseAddress), + : gpuBaseAddress(baseAddress), gpuAddress(gpuAddress), + size(sizeIn), + cpuPtr(cpuPtrIn), memoryPool(pool), allocationType(allocationType) { allocationInfo.flags.multiOsContextCapable = multiOsContextCapable; @@ -30,9 +30,9 @@ GraphicsAllocation::GraphicsAllocation(AllocationType allocationType, void *cpuP GraphicsAllocation::GraphicsAllocation(AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandleIn, MemoryPool::Type pool, bool multiOsContextCapable) - : size(sizeIn), + : gpuAddress(castToUint64(cpuPtrIn)), + size(sizeIn), cpuPtr(cpuPtrIn), - gpuAddress(castToUint64(cpuPtrIn)), memoryPool(pool), allocationType(allocationType) { sharingInfo.sharedHandle = sharedHandleIn; diff --git a/runtime/memory_manager/graphics_allocation.h b/runtime/memory_manager/graphics_allocation.h index ddbad7ca81..e433fb2bf5 100644 --- a/runtime/memory_manager/graphics_allocation.h +++ b/runtime/memory_manager/graphics_allocation.h @@ -165,6 +165,16 @@ class GraphicsAllocation : public IDNode { allocationType == AllocationType::COMMAND_BUFFER; } static StorageInfo createStorageInfoFromProperties(const AllocationProperties &properties); + void *getReservedAddressPtr() const { + return this->reservedAddressRangeInfo.addressPtr; + } + size_t getReservedAddressSize() const { + return this->reservedAddressRangeInfo.rangeSize; + } + void setReservedAddressRange(void *reserveAddress, size_t size) { + this->reservedAddressRangeInfo.addressPtr = reserveAddress; + this->reservedAddressRangeInfo.rangeSize = size; + } Gmm *getDefaultGmm() const { return getGmm(0u); @@ -222,23 +232,28 @@ class GraphicsAllocation : public IDNode { } }; - uint64_t allocationOffset = 0u; - void *driverAllocatedCpuPointer = nullptr; + struct ReservedAddressRange { + void *addressPtr = nullptr; + size_t rangeSize = 0; + }; - //this variable can only be modified from SubmissionAggregator friend class SubmissionAggregator; - size_t size = 0; - void *cpuPtr = nullptr; - uint64_t gpuBaseAddress = 0; - uint64_t gpuAddress = 0; - void *lockedPtr = nullptr; - - MemoryPool::Type memoryPool = MemoryPool::MemoryNull; - AllocationType allocationType = AllocationType::UNKNOWN; AllocationInfo allocationInfo; AubInfo aubInfo; SharingInfo sharingInfo; + ReservedAddressRange reservedAddressRangeInfo; + + uint64_t allocationOffset = 0u; + uint64_t gpuBaseAddress = 0; + uint64_t gpuAddress = 0; + void *driverAllocatedCpuPointer = nullptr; + size_t size = 0; + void *cpuPtr = nullptr; + void *lockedPtr = nullptr; + + MemoryPool::Type memoryPool = MemoryPool::MemoryNull; + AllocationType allocationType = AllocationType::UNKNOWN; std::array usageInfos; std::atomic registeredContextsNum{0}; diff --git a/runtime/memory_manager/memory_manager.h b/runtime/memory_manager/memory_manager.h index 3291aa44a9..c77004f349 100644 --- a/runtime/memory_manager/memory_manager.h +++ b/runtime/memory_manager/memory_manager.h @@ -185,6 +185,8 @@ class MemoryManager { virtual bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, const void *memoryToCopy, uint32_t sizeToCopy) const; static HeapIndex selectHeap(const GraphicsAllocation *allocation, const void *ptr, const HardwareInfo &hwInfo); static std::unique_ptr createMemoryManager(bool enable64KBpages, bool enableLocalMemory, ExecutionEnvironment &exeEnv); + virtual void *reserveCpuAddressRange(size_t size) = 0; + virtual void releaseReservedCpuAddressRange(void *reserved, size_t size) = 0; protected: struct AllocationData { diff --git a/runtime/memory_manager/os_agnostic_memory_manager.cpp b/runtime/memory_manager/os_agnostic_memory_manager.cpp index d667774cbf..4b655912de 100644 --- a/runtime/memory_manager/os_agnostic_memory_manager.cpp +++ b/runtime/memory_manager/os_agnostic_memory_manager.cpp @@ -169,6 +169,9 @@ void OsAgnosticMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllo } alignedFreeWrapper(gfxAllocation->getDriverAllocatedCpuPtr()); + if (gfxAllocation->getReservedAddressPtr()) { + releaseReservedCpuAddressRange(gfxAllocation->getReservedAddressPtr(), gfxAllocation->getReservedAddressSize()); + } delete gfxAllocation; } @@ -256,4 +259,13 @@ Allocator32bit *OsAgnosticMemoryManager::create32BitAllocator(bool aubUsage) { return new Allocator32bit(heap32Base, allocatorSize); } + +void *OsAgnosticMemoryManager::reserveCpuAddressRange(size_t size) { + void *reservePtr = allocateSystemMemory(size, MemoryConstants::preferredAlignment); + return reservePtr; +} + +void OsAgnosticMemoryManager::releaseReservedCpuAddressRange(void *reserved, size_t size) { + alignedFreeWrapper(reserved); +} } // namespace OCLRT diff --git a/runtime/memory_manager/os_agnostic_memory_manager.h b/runtime/memory_manager/os_agnostic_memory_manager.h index cb0853de6c..9e71ee5498 100644 --- a/runtime/memory_manager/os_agnostic_memory_manager.h +++ b/runtime/memory_manager/os_agnostic_memory_manager.h @@ -63,6 +63,9 @@ class OsAgnosticMemoryManager : public MemoryManager { Allocator32bit *create32BitAllocator(bool enableLocalMemory); + void *reserveCpuAddressRange(size_t size) override; + void releaseReservedCpuAddressRange(void *reserved, size_t size) override; + protected: GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override; GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) override; diff --git a/runtime/os_interface/linux/drm_memory_manager.cpp b/runtime/os_interface/linux/drm_memory_manager.cpp index b64220f161..f095a004ec 100644 --- a/runtime/os_interface/linux/drm_memory_manager.cpp +++ b/runtime/os_interface/linux/drm_memory_manager.cpp @@ -170,12 +170,12 @@ uint64_t DrmMemoryManager::acquireGpuRange(size_t &size, StorageAllocatorType &s } storageType = MMAP_ALLOCATOR; - return reinterpret_cast(mmapFunction(nullptr, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0)); + return reinterpret_cast(reserveCpuAddressRange(size)); } void DrmMemoryManager::releaseGpuRange(void *address, size_t unmapSize, StorageAllocatorType allocatorType) { if (allocatorType == MMAP_ALLOCATOR) { - munmapFunction(address, unmapSize); + releaseReservedCpuAddressRange(address, unmapSize); return; } @@ -579,7 +579,10 @@ void DrmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) if (gfxAllocation->peekSharedHandle() != Sharing::nonSharedResource) { closeFunction(gfxAllocation->peekSharedHandle()); } - + void *reserveAddress = gfxAllocation->getReservedAddressPtr(); + if (reserveAddress) { + releaseReservedCpuAddressRange(reserveAddress, gfxAllocation->getReservedAddressSize()); + } delete gfxAllocation; unreference(search); @@ -733,8 +736,16 @@ void DrmMemoryManager::unlockResourceImpl(GraphicsAllocation &graphicsAllocation if (bo == nullptr) return; - munmapFunction(bo->peekLockedAddress(), bo->peekSize()); + releaseReservedCpuAddressRange(bo->peekLockedAddress(), bo->peekSize()); bo->setLockedAddress(nullptr); } +void *DrmMemoryManager::reserveCpuAddressRange(size_t size) { + void *reservePtr = mmapFunction(nullptr, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + return reservePtr; +} + +void DrmMemoryManager::releaseReservedCpuAddressRange(void *reserved, size_t size) { + munmapFunction(reserved, size); +} } // namespace OCLRT diff --git a/runtime/os_interface/linux/drm_memory_manager.h b/runtime/os_interface/linux/drm_memory_manager.h index b27eb9af00..d92330aa6b 100644 --- a/runtime/os_interface/linux/drm_memory_manager.h +++ b/runtime/os_interface/linux/drm_memory_manager.h @@ -55,6 +55,8 @@ class DrmMemoryManager : public MemoryManager { } DrmGemCloseWorker *peekGemCloseWorker() const { return this->gemCloseWorker.get(); } + void *reserveCpuAddressRange(size_t size) override; + void releaseReservedCpuAddressRange(void *reserved, size_t size) override; protected: BufferObject *findAndReferenceSharedBufferObject(int boHandle); diff --git a/runtime/os_interface/windows/wddm/wddm.cpp b/runtime/os_interface/windows/wddm/wddm.cpp index 448b1a789f..666a0aa9f4 100644 --- a/runtime/os_interface/windows/wddm/wddm.cpp +++ b/runtime/os_interface/windows/wddm/wddm.cpp @@ -289,7 +289,7 @@ bool Wddm::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantT } bool Wddm::mapGpuVirtualAddress(WddmAllocation *allocation, void *cpuPtr) { - void *mapPtr = allocation->getReservedAddress() != nullptr ? allocation->getReservedAddress() : cpuPtr; + void *mapPtr = allocation->getReservedAddressPtr() != nullptr ? allocation->getReservedAddressPtr() : cpuPtr; return mapGpuVirtualAddressImpl(allocation->getDefaultGmm(), allocation->getDefaultHandle(), mapPtr, allocation->getGpuAddressToModify(), MemoryManager::selectHeap(allocation, mapPtr, *hardwareInfoTable[gfxPlatform->eProductFamily])); } diff --git a/runtime/os_interface/windows/wddm_allocation.h b/runtime/os_interface/windows/wddm_allocation.h index 24557b365c..5327f54c00 100644 --- a/runtime/os_interface/windows/wddm_allocation.h +++ b/runtime/os_interface/windows/wddm_allocation.h @@ -26,8 +26,10 @@ constexpr size_t trimListUnusedPosition = std::numeric_limits::max(); class WddmAllocation : public GraphicsAllocation { public: WddmAllocation(AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool, bool multiOsContextCapable) - : GraphicsAllocation(allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool, multiOsContextCapable), reservedAddressSpace(reservedAddr) { + : GraphicsAllocation(allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool, multiOsContextCapable) { trimCandidateListPositions.fill(trimListUnusedPosition); + reservedAddressRangeInfo.addressPtr = reservedAddr; + reservedAddressRangeInfo.rangeSize = sizeIn; } WddmAllocation(AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandle, MemoryPool::Type pool, bool multiOsContextCapable) @@ -65,13 +67,6 @@ class WddmAllocation : public GraphicsAllocation { return trimListUnusedPosition; } - void *getReservedAddress() const { - return this->reservedAddressSpace; - } - - void setReservedAddress(void *reserveMem) { - this->reservedAddressSpace = reserveMem; - } void setGpuAddress(uint64_t graphicsAddress) { this->gpuAddress = graphicsAddress; } void setCpuAddress(void *cpuPtr) { this->cpuPtr = cpuPtr; } @@ -93,6 +88,5 @@ class WddmAllocation : public GraphicsAllocation { std::array handles{}; ResidencyData residency; std::array trimCandidateListPositions; - void *reservedAddressSpace = nullptr; }; } // namespace OCLRT diff --git a/runtime/os_interface/windows/wddm_memory_manager.cpp b/runtime/os_interface/windows/wddm_memory_manager.cpp index 0e19f3d165..671d0be03f 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.cpp +++ b/runtime/os_interface/windows/wddm_memory_manager.cpp @@ -230,7 +230,7 @@ GraphicsAllocation *WddmMemoryManager::createAllocationFromHandle(osHandle handl if (!wddm->reserveValidAddressRange(size, ptr)) { return nullptr; } - allocation->setReservedAddress(ptr); + allocation->setReservedAddressRange(ptr, size); } else if (requireSpecificBitness && this->force32bitAllocations) { allocation->set32BitAllocation(true); allocation->setGpuBaseAddress(GmmHelper::canonize(allocator32Bit->getBase())); @@ -339,7 +339,9 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation DEBUG_BREAK_IF(!status); alignedFreeWrapper(input->getDriverAllocatedCpuPtr()); } - wddm->releaseReservedAddress(input->getReservedAddress()); + if (input->getReservedAddressPtr()) { + releaseReservedCpuAddressRange(input->getReservedAddressPtr(), input->getReservedAddressSize()); + } delete gfxAllocation; } @@ -502,4 +504,14 @@ bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation) { return (wddmSuccess == STATUS_SUCCESS); } +void *WddmMemoryManager::reserveCpuAddressRange(size_t size) { + void *reservePtr = nullptr; + wddm->reserveValidAddressRange(size, reservePtr); + return reservePtr; +} + +void WddmMemoryManager::releaseReservedCpuAddressRange(void *reserved, size_t size) { + wddm->releaseReservedAddress(reserved); +} + } // namespace OCLRT diff --git a/runtime/os_interface/windows/wddm_memory_manager.h b/runtime/os_interface/windows/wddm_memory_manager.h index c5305db79d..7c6ee5cfa8 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.h +++ b/runtime/os_interface/windows/wddm_memory_manager.h @@ -64,6 +64,8 @@ class WddmMemoryManager : public MemoryManager { AlignedMallocRestrictions *getAlignedMallocRestrictions() override; bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, const void *memoryToCopy, uint32_t sizeToCopy) const override; + void *reserveCpuAddressRange(size_t size) override; + void releaseReservedCpuAddressRange(void *reserved, size_t size) override; protected: GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override; diff --git a/unit_tests/memory_manager/memory_manager_tests.cpp b/unit_tests/memory_manager/memory_manager_tests.cpp index ec8e13b82c..6dda816d5f 100644 --- a/unit_tests/memory_manager/memory_manager_tests.cpp +++ b/unit_tests/memory_manager/memory_manager_tests.cpp @@ -1758,3 +1758,15 @@ TEST(MemoryManagerCopyMemoryTest, givenValidAllocationAndMemoryWhenCopyMemoryToA EXPECT_TRUE(memoryManager.copyMemoryToAllocation(&allocation, &memory, sizeof(memory))); EXPECT_EQ(memory, allocationStorage[0]); } + +TEST_F(MemoryAllocatorTest, whenReservingAddressRangeThenExpectProperAddressAndReleaseWhenFreeing) { + size_t size = 0x1000; + auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{size}); + ASSERT_NE(nullptr, allocation); + void *reserve = memoryManager->reserveCpuAddressRange(size); + EXPECT_NE(nullptr, reserve); + allocation->setReservedAddressRange(reserve, size); + EXPECT_EQ(reserve, allocation->getReservedAddressPtr()); + EXPECT_EQ(size, allocation->getReservedAddressSize()); + memoryManager->freeGraphicsMemory(allocation); +} diff --git a/unit_tests/mocks/mock_wddm.cpp b/unit_tests/mocks/mock_wddm.cpp index e9c67c7bb5..47cf920ade 100644 --- a/unit_tests/mocks/mock_wddm.cpp +++ b/unit_tests/mocks/mock_wddm.cpp @@ -93,7 +93,7 @@ bool WddmMock::destroyAllocation(WddmAllocation *alloc, OsContextWin *osContext) const D3DKMT_HANDLE *allocationHandles = nullptr; uint32_t allocationCount = 0; D3DKMT_HANDLE resourceHandle = 0; - void *reserveAddress = alloc->getReservedAddress(); + void *reserveAddress = alloc->getReservedAddressPtr(); if (alloc->peekSharedHandle()) { resourceHandle = alloc->resourceHandle; } else { diff --git a/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp b/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp index 126a08aba4..372e21d45a 100644 --- a/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp @@ -3025,3 +3025,15 @@ TEST(DrmAllocationTest, givenMemoryPoolWhenPassedToDrmAllocationConstructorThenM DrmAllocation allocation2{GraphicsAllocation::AllocationType::UNDECIDED, nullptr, nullptr, 0ULL, static_cast(0), MemoryPool::SystemCpuInaccessible, false}; EXPECT_EQ(MemoryPool::SystemCpuInaccessible, allocation2.getMemoryPool()); } + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, whenReservingAddressRangeThenExpectProperAddressAndReleaseWhenFreeing) { + constexpr size_t size = 0x1000; + auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{size}); + ASSERT_NE(nullptr, allocation); + void *reserve = memoryManager->reserveCpuAddressRange(size); + EXPECT_NE(nullptr, reserve); + allocation->setReservedAddressRange(reserve, size); + EXPECT_EQ(reserve, allocation->getReservedAddressPtr()); + EXPECT_EQ(size, allocation->getReservedAddressSize()); + memoryManager->freeGraphicsMemory(allocation); +} diff --git a/unit_tests/os_interface/windows/device_command_stream_tests.cpp b/unit_tests/os_interface/windows/device_command_stream_tests.cpp index 6db4a05963..1a69ea7dd8 100644 --- a/unit_tests/os_interface/windows/device_command_stream_tests.cpp +++ b/unit_tests/os_interface/windows/device_command_stream_tests.cpp @@ -613,7 +613,7 @@ TEST_F(WddmCommandStreamTest, givenHostPtrWhenPtrBelowRestrictionThenCreateAlloc EXPECT_EQ(1u, csr->getResidencyAllocations().size()); EXPECT_EQ(hostPtr, gfxAllocation->getUnderlyingBuffer()); - EXPECT_EQ(expectedReserve, gfxAllocation->getReservedAddress()); + EXPECT_EQ(expectedReserve, gfxAllocation->getReservedAddressPtr()); memoryManager->freeGraphicsMemory(gfxAllocation); } diff --git a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp index a634312036..bb3f74c339 100644 --- a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp @@ -891,7 +891,7 @@ TEST_F(WddmMemoryManagerTest, givenWddmMemoryManagerWhenCpuMemNotMeetRestriction void *expectReserve = reinterpret_cast(wddm->virtualAllocAddress); ASSERT_NE(nullptr, allocation); - EXPECT_EQ(expectReserve, allocation->getReservedAddress()); + EXPECT_EQ(expectReserve, allocation->getReservedAddressPtr()); memoryManager->freeGraphicsMemory(allocation); }