From 477a06a4eba6dcdd37831881e963e6c040c9cbb6 Mon Sep 17 00:00:00 2001 From: Mateusz Jablonski Date: Tue, 23 Oct 2018 11:42:40 +0200 Subject: [PATCH] Move creation of os storage for host ptr allocation to host ptr manager Change-Id: If7b6c17e21c72c807031232a502265559dfa48b1 Signed-off-by: Mateusz Jablonski --- runtime/memory_manager/host_ptr_manager.cpp | 90 ++++++++++++++++--- runtime/memory_manager/host_ptr_manager.h | 6 +- runtime/memory_manager/memory_manager.cpp | 69 +------------- runtime/memory_manager/memory_manager.h | 2 - .../memory_manager/memory_manager_tests.cpp | 24 +---- 5 files changed, 88 insertions(+), 103 deletions(-) diff --git a/runtime/memory_manager/host_ptr_manager.cpp b/runtime/memory_manager/host_ptr_manager.cpp index 254cdf99e3..66bec33b64 100644 --- a/runtime/memory_manager/host_ptr_manager.cpp +++ b/runtime/memory_manager/host_ptr_manager.cpp @@ -5,13 +5,14 @@ * */ -#include "host_ptr_manager.h" +#include "runtime/command_stream/command_stream_receiver.h" #include "runtime/helpers/ptr_math.h" #include "runtime/helpers/abort.h" +#include "runtime/memory_manager/memory_manager.h" using namespace OCLRT; -std::map::iterator OCLRT::HostPtrManager::findElement(const void *ptr) { +std::map::iterator HostPtrManager::findElement(const void *ptr) { auto nextElement = partialAllocations.lower_bound(ptr); auto element = nextElement; if (element != partialAllocations.end()) { @@ -43,7 +44,7 @@ std::map::iterator OCLRT::HostPtrManager::findEle return partialAllocations.end(); } -AllocationRequirements OCLRT::HostPtrManager::getAllocationRequirements(const void *inputPtr, size_t size) { +AllocationRequirements HostPtrManager::getAllocationRequirements(const void *inputPtr, size_t size) { AllocationRequirements requiredAllocations; auto allocationCount = 0; @@ -89,7 +90,7 @@ AllocationRequirements OCLRT::HostPtrManager::getAllocationRequirements(const vo return requiredAllocations; } -OsHandleStorage OCLRT::HostPtrManager::populateAlreadyAllocatedFragments(AllocationRequirements &requirements, CheckedFragments *checkedFragments) { +OsHandleStorage HostPtrManager::populateAlreadyAllocatedFragments(AllocationRequirements &requirements, CheckedFragments *checkedFragments) { OsHandleStorage handleStorage; for (unsigned int i = 0; i < requirements.requiredFragmentsCount; i++) { OverlapStatus overlapStatus = OverlapStatus::FRAGMENT_NOT_CHECKED; @@ -133,8 +134,8 @@ OsHandleStorage OCLRT::HostPtrManager::populateAlreadyAllocatedFragments(Allocat return handleStorage; } -void OCLRT::HostPtrManager::storeFragment(FragmentStorage &fragment) { - std::lock_guard lock(allocationsMutex); +void HostPtrManager::storeFragment(FragmentStorage &fragment) { + std::lock_guard lock(allocationsMutex); auto element = findElement(fragment.fragmentCpuPointer); if (element != partialAllocations.end()) { element->second.refCount++; @@ -144,7 +145,7 @@ void OCLRT::HostPtrManager::storeFragment(FragmentStorage &fragment) { } } -void OCLRT::HostPtrManager::storeFragment(AllocationStorageData &storageData) { +void HostPtrManager::storeFragment(AllocationStorageData &storageData) { FragmentStorage fragment; fragment.fragmentCpuPointer = const_cast(storageData.cpuPtr); fragment.fragmentSize = storageData.fragmentSize; @@ -153,7 +154,7 @@ void OCLRT::HostPtrManager::storeFragment(AllocationStorageData &storageData) { storeFragment(fragment); } -void OCLRT::HostPtrManager::releaseHandleStorage(OsHandleStorage &fragments) { +void HostPtrManager::releaseHandleStorage(OsHandleStorage &fragments) { for (int i = 0; i < maxFragmentsCount; i++) { if (fragments.fragmentStorageData[i].fragmentSize || fragments.fragmentStorageData[i].cpuPtr) { fragments.fragmentStorageData[i].freeTheFragment = releaseHostPtr(fragments.fragmentStorageData[i].cpuPtr); @@ -161,8 +162,8 @@ void OCLRT::HostPtrManager::releaseHandleStorage(OsHandleStorage &fragments) { } } -bool OCLRT::HostPtrManager::releaseHostPtr(const void *ptr) { - std::lock_guard lock(allocationsMutex); +bool HostPtrManager::releaseHostPtr(const void *ptr) { + std::lock_guard lock(allocationsMutex); bool fragmentReadyToBeReleased = false; auto element = findElement(ptr); @@ -178,8 +179,8 @@ bool OCLRT::HostPtrManager::releaseHostPtr(const void *ptr) { return fragmentReadyToBeReleased; } -FragmentStorage *OCLRT::HostPtrManager::getFragment(const void *inputPtr) { - std::lock_guard lock(allocationsMutex); +FragmentStorage *HostPtrManager::getFragment(const void *inputPtr) { + std::lock_guard lock(allocationsMutex); auto element = findElement(inputPtr); if (element != partialAllocations.end()) { return &element->second; @@ -188,8 +189,8 @@ FragmentStorage *OCLRT::HostPtrManager::getFragment(const void *inputPtr) { } //for given inputs see if any allocation overlaps -FragmentStorage *OCLRT::HostPtrManager::getFragmentAndCheckForOverlaps(const void *inPtr, size_t size, OverlapStatus &overlappingStatus) { - std::lock_guard lock(allocationsMutex); +FragmentStorage *HostPtrManager::getFragmentAndCheckForOverlaps(const void *inPtr, size_t size, OverlapStatus &overlappingStatus) { + std::lock_guard lock(allocationsMutex); void *inputPtr = const_cast(inPtr); auto nextElement = partialAllocations.lower_bound(inputPtr); auto element = nextElement; @@ -246,3 +247,64 @@ FragmentStorage *OCLRT::HostPtrManager::getFragmentAndCheckForOverlaps(const voi } return nullptr; } + +OsHandleStorage HostPtrManager::prepareOsStorageForAllocation(MemoryManager &memoryManager, size_t size, const void *ptr) { + std::lock_guard lock(allocationsMutex); + auto requirements = HostPtrManager::getAllocationRequirements(ptr, size); + + CheckedFragments checkedFragments; + UNRECOVERABLE_IF(checkAllocationsForOverlapping(memoryManager, &requirements, &checkedFragments) == RequirementsStatus::FATAL); + + auto osStorage = populateAlreadyAllocatedFragments(requirements, &checkedFragments); + if (osStorage.fragmentCount > 0) { + if (memoryManager.populateOsHandles(osStorage) != MemoryManager::AllocationStatus::Success) { + memoryManager.cleanOsHandles(osStorage); + osStorage.fragmentCount = 0; + } + } + return osStorage; +} + +RequirementsStatus HostPtrManager::checkAllocationsForOverlapping(MemoryManager &memoryManager, AllocationRequirements *requirements, CheckedFragments *checkedFragments) { + DEBUG_BREAK_IF(requirements == nullptr); + DEBUG_BREAK_IF(checkedFragments == nullptr); + + RequirementsStatus status = RequirementsStatus::SUCCESS; + checkedFragments->count = 0; + + for (unsigned int i = 0; i < maxFragmentsCount; i++) { + checkedFragments->status[i] = OverlapStatus::FRAGMENT_NOT_CHECKED; + checkedFragments->fragments[i] = nullptr; + } + for (unsigned int i = 0; i < requirements->requiredFragmentsCount; i++) { + checkedFragments->count++; + checkedFragments->fragments[i] = getFragmentAndCheckForOverlaps(requirements->AllocationFragments[i].allocationPtr, requirements->AllocationFragments[i].allocationSize, checkedFragments->status[i]); + if (checkedFragments->status[i] == OverlapStatus::FRAGMENT_OVERLAPING_AND_BIGGER_THEN_STORED_FRAGMENT) { + // clean temporary allocations + + auto commandStreamReceiver = memoryManager.getCommandStreamReceiver(0); + uint32_t taskCount = *commandStreamReceiver->getTagAddress(); + memoryManager.cleanAllocationList(taskCount, TEMPORARY_ALLOCATION); + + // check overlapping again + checkedFragments->fragments[i] = getFragmentAndCheckForOverlaps(requirements->AllocationFragments[i].allocationPtr, requirements->AllocationFragments[i].allocationSize, checkedFragments->status[i]); + if (checkedFragments->status[i] == OverlapStatus::FRAGMENT_OVERLAPING_AND_BIGGER_THEN_STORED_FRAGMENT) { + + // Wait for completion + while (*commandStreamReceiver->getTagAddress() < commandStreamReceiver->peekLatestSentTaskCount()) + ; + + taskCount = *commandStreamReceiver->getTagAddress(); + memoryManager.cleanAllocationList(taskCount, TEMPORARY_ALLOCATION); + + // check overlapping last time + checkedFragments->fragments[i] = getFragmentAndCheckForOverlaps(requirements->AllocationFragments[i].allocationPtr, requirements->AllocationFragments[i].allocationSize, checkedFragments->status[i]); + if (checkedFragments->status[i] == OverlapStatus::FRAGMENT_OVERLAPING_AND_BIGGER_THEN_STORED_FRAGMENT) { + status = RequirementsStatus::FATAL; + break; + } + } + } + } + return status; +} \ No newline at end of file diff --git a/runtime/memory_manager/host_ptr_manager.h b/runtime/memory_manager/host_ptr_manager.h index f932d1b058..7afd0461a7 100644 --- a/runtime/memory_manager/host_ptr_manager.h +++ b/runtime/memory_manager/host_ptr_manager.h @@ -14,7 +14,7 @@ namespace OCLRT { typedef std::map HostPtrFragmentsContainer; - +class MemoryManager; class HostPtrManager { public: static AllocationRequirements getAllocationRequirements(const void *inputPtr, size_t size); @@ -28,11 +28,13 @@ class HostPtrManager { FragmentStorage *getFragment(const void *inputPtr); size_t getFragmentCount() { return partialAllocations.size(); } FragmentStorage *getFragmentAndCheckForOverlaps(const void *inputPtr, size_t size, OverlapStatus &overlappingStatus); + OsHandleStorage prepareOsStorageForAllocation(MemoryManager &memoryManager, size_t size, const void *ptr); + RequirementsStatus checkAllocationsForOverlapping(MemoryManager &memoryManager, AllocationRequirements *requirements, CheckedFragments *checkedFragments); private: std::map::iterator findElement(const void *ptr); HostPtrFragmentsContainer partialAllocations; - std::mutex allocationsMutex; + std::recursive_mutex allocationsMutex; }; } // namespace OCLRT diff --git a/runtime/memory_manager/memory_manager.cpp b/runtime/memory_manager/memory_manager.cpp index f86b3e5523..b1357d6b26 100644 --- a/runtime/memory_manager/memory_manager.cpp +++ b/runtime/memory_manager/memory_manager.cpp @@ -116,31 +116,14 @@ GraphicsAllocation *MemoryManager::allocateGraphicsMemoryForSVM(size_t size, boo GraphicsAllocation *MemoryManager::allocateGraphicsMemory(size_t size, const void *ptr, bool forcePin) { std::lock_guard lock(mtx); - auto requirements = HostPtrManager::getAllocationRequirements(ptr, size); - GraphicsAllocation *graphicsAllocation = nullptr; - if (deferredDeleter) { deferredDeleter->drain(true); } - - //check for overlaping - CheckedFragments checkedFragments; - if (checkAllocationsForOverlapping(&requirements, &checkedFragments) == RequirementsStatus::FATAL) { - //abort whole application instead of silently passing. - abortExecution(); + GraphicsAllocation *graphicsAllocation = nullptr; + auto osStorage = hostPtrManager.prepareOsStorageForAllocation(*this, size, ptr); + if (osStorage.fragmentCount > 0) { + graphicsAllocation = createGraphicsAllocation(osStorage, size, ptr); } - - auto osStorage = hostPtrManager.populateAlreadyAllocatedFragments(requirements, &checkedFragments); - if (osStorage.fragmentCount == 0) { - return nullptr; - } - auto result = populateOsHandles(osStorage); - if (result != AllocationStatus::Success) { - cleanOsHandles(osStorage); - return nullptr; - } - - graphicsAllocation = createGraphicsAllocation(osStorage, size, ptr); return graphicsAllocation; } @@ -261,50 +244,6 @@ bool MemoryManager::isMemoryBudgetExhausted() const { return false; } -RequirementsStatus MemoryManager::checkAllocationsForOverlapping(AllocationRequirements *requirements, CheckedFragments *checkedFragments) { - DEBUG_BREAK_IF(requirements == nullptr); - DEBUG_BREAK_IF(checkedFragments == nullptr); - - RequirementsStatus status = RequirementsStatus::SUCCESS; - checkedFragments->count = 0; - - for (unsigned int i = 0; i < maxFragmentsCount; i++) { - checkedFragments->status[i] = OverlapStatus::FRAGMENT_NOT_CHECKED; - checkedFragments->fragments[i] = nullptr; - } - - for (unsigned int i = 0; i < requirements->requiredFragmentsCount; i++) { - checkedFragments->count++; - checkedFragments->fragments[i] = hostPtrManager.getFragmentAndCheckForOverlaps(requirements->AllocationFragments[i].allocationPtr, requirements->AllocationFragments[i].allocationSize, checkedFragments->status[i]); - if (checkedFragments->status[i] == OverlapStatus::FRAGMENT_OVERLAPING_AND_BIGGER_THEN_STORED_FRAGMENT) { - // clean temporary allocations - - uint32_t taskCount = *getCommandStreamReceiver(0)->getTagAddress(); - cleanAllocationList(taskCount, TEMPORARY_ALLOCATION); - - // check overlapping again - checkedFragments->fragments[i] = hostPtrManager.getFragmentAndCheckForOverlaps(requirements->AllocationFragments[i].allocationPtr, requirements->AllocationFragments[i].allocationSize, checkedFragments->status[i]); - if (checkedFragments->status[i] == OverlapStatus::FRAGMENT_OVERLAPING_AND_BIGGER_THEN_STORED_FRAGMENT) { - - // Wait for completion - while (*getCommandStreamReceiver(0)->getTagAddress() < getCommandStreamReceiver(0)->peekLatestSentTaskCount()) { - } - - taskCount = *getCommandStreamReceiver(0)->getTagAddress(); - cleanAllocationList(taskCount, TEMPORARY_ALLOCATION); - - // check overlapping last time - checkedFragments->fragments[i] = hostPtrManager.getFragmentAndCheckForOverlaps(requirements->AllocationFragments[i].allocationPtr, requirements->AllocationFragments[i].allocationSize, checkedFragments->status[i]); - if (checkedFragments->status[i] == OverlapStatus::FRAGMENT_OVERLAPING_AND_BIGGER_THEN_STORED_FRAGMENT) { - status = RequirementsStatus::FATAL; - break; - } - } - } - } - return status; -} - void MemoryManager::registerOsContext(OsContext *contextToRegister) { auto contextId = contextToRegister->getContextId(); if (contextId + 1 > registeredOsContexts.size()) { diff --git a/runtime/memory_manager/memory_manager.h b/runtime/memory_manager/memory_manager.h index a73348de75..6eb8410799 100644 --- a/runtime/memory_manager/memory_manager.h +++ b/runtime/memory_manager/memory_manager.h @@ -207,8 +207,6 @@ class MemoryManager { void storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage); void storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage, uint32_t taskCount); - RequirementsStatus checkAllocationsForOverlapping(AllocationRequirements *requirements, CheckedFragments *checkedFragments); - TagAllocator *getEventTsAllocator(); TagAllocator *getEventPerfCountAllocator(); TagAllocator *getTimestampPacketAllocator(); diff --git a/unit_tests/memory_manager/memory_manager_tests.cpp b/unit_tests/memory_manager/memory_manager_tests.cpp index 0545be6c51..af1204dc50 100644 --- a/unit_tests/memory_manager/memory_manager_tests.cpp +++ b/unit_tests/memory_manager/memory_manager_tests.cpp @@ -1237,22 +1237,6 @@ TEST(OsAgnosticMemoryManager, GivenEnabled64kbPagesWhenHostMemoryAllocationIsCre memoryManager.freeGraphicsMemory(galloc); } -TEST(OsAgnosticMemoryManager, checkAllocationsForOverlappingWithNullCsrInMemoryManager) { - ExecutionEnvironment executionEnvironment; - OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); - - AllocationRequirements requirements; - CheckedFragments checkedFragments; - - requirements.requiredFragmentsCount = 1; - requirements.totalRequiredSize = MemoryConstants::pageSize * 10; - - RequirementsStatus status = memoryManager.checkAllocationsForOverlapping(&requirements, &checkedFragments); - - EXPECT_EQ(RequirementsStatus::SUCCESS, status); - EXPECT_EQ(1u, checkedFragments.count); -} - TEST(OsAgnosticMemoryManager, givenPointerAndSizeWhenCreateInternalAllocationIsCalledThenGraphicsAllocationIsReturned) { ExecutionEnvironment executionEnvironment; OsAgnosticMemoryManager memoryManager(false, false, executionEnvironment); @@ -1478,7 +1462,7 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithoutBiggerOver requirements.AllocationFragments[1].allocationSize = MemoryConstants::pageSize; requirements.AllocationFragments[1].fragmentPosition = FragmentPosition::TRAILING; - RequirementsStatus status = memoryManager->checkAllocationsForOverlapping(&requirements, &checkedFragments); + RequirementsStatus status = memoryManager->hostPtrManager.checkAllocationsForOverlapping(*memoryManager, &requirements, &checkedFragments); EXPECT_EQ(RequirementsStatus::SUCCESS, status); EXPECT_EQ(2u, checkedFragments.count); @@ -1528,7 +1512,7 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithBiggerOverlap requirements.AllocationFragments[0].allocationSize = MemoryConstants::pageSize * 10; requirements.AllocationFragments[0].fragmentPosition = FragmentPosition::NONE; - RequirementsStatus status = memoryManager->checkAllocationsForOverlapping(&requirements, &checkedFragments); + RequirementsStatus status = memoryManager->hostPtrManager.checkAllocationsForOverlapping(*memoryManager, &requirements, &checkedFragments); EXPECT_EQ(RequirementsStatus::SUCCESS, status); EXPECT_EQ(1u, checkedFragments.count); @@ -1584,7 +1568,7 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithBiggerOverlap EXPECT_CALL(*gmockMemoryManager, cleanAllocationList(::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Invoke(cleanAllocations)).WillOnce(::testing::Invoke(cleanAllocationsWithTaskCount)); - RequirementsStatus status = memoryManager->checkAllocationsForOverlapping(&requirements, &checkedFragments); + RequirementsStatus status = memoryManager->hostPtrManager.checkAllocationsForOverlapping(*memoryManager, &requirements, &checkedFragments); EXPECT_EQ(RequirementsStatus::SUCCESS, status); EXPECT_EQ(1u, checkedFragments.count); @@ -1636,7 +1620,7 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithBiggerOverlap EXPECT_CALL(*gmockMemoryManager, cleanAllocationList(::testing::_, ::testing::_)).Times(2).WillRepeatedly(::testing::Invoke(cleanAllocations)); - RequirementsStatus status = memoryManager->checkAllocationsForOverlapping(&requirements, &checkedFragments); + RequirementsStatus status = memoryManager->hostPtrManager.checkAllocationsForOverlapping(*memoryManager, &requirements, &checkedFragments); EXPECT_EQ(RequirementsStatus::FATAL, status); EXPECT_EQ(1u, checkedFragments.count);