performance: Ensure hostptrs removed before creating new one

Signed-off-by: Lukasz Jobczyk <lukasz.jobczyk@intel.com>

shared/source/memory_manager/deferrable_deletion.h

@@ -14,5 +14,8 @@ class DeferrableDeletion : public IDNode<DeferrableDeletion> {
     template <typename... Args>
     static DeferrableDeletion *create(Args... args);
     virtual bool apply() = 0;
+
+    bool isExternalHostptr() const { return externalHostptr; }
+    bool externalHostptr = false;
 };
 } // namespace NEO

shared/source/memory_manager/deferred_deleter.cpp

@@ -11,10 +11,7 @@
 #include "shared/source/os_interface/os_thread.h"
 
 namespace NEO {
-DeferredDeleter::DeferredDeleter() {
-    doWorkInBackground = false;
-    elementsToRelease = 0;
-}
+DeferredDeleter::DeferredDeleter() = default;
 
 void DeferredDeleter::stop() {
     // Called with threadMutex acquired
@@ -35,7 +32,7 @@ void DeferredDeleter::stop() {
         // Delete working thread
         worker.reset();
     }
-    drain(false);
+    drain(false, false);
 }
 
 void DeferredDeleter::safeStop() {
@@ -49,8 +46,14 @@ DeferredDeleter::~DeferredDeleter() {
 
 void DeferredDeleter::deferDeletion(DeferrableDeletion *deletion) {
     std::unique_lock<std::mutex> lock(queueMutex);
-    elementsToRelease++;
+
+    this->elementsToRelease++;
+    if (deletion->isExternalHostptr()) {
+        this->hostptrsToRelease++;
+    }
+
     queue.pushTailOne(*deletion);
+
     lock.unlock();
     condition.notify_one();
 }
@@ -76,8 +79,8 @@ void DeferredDeleter::ensureThread() {
     worker = Thread::createFunc(run, reinterpret_cast<void *>(this));
 }
 
-bool DeferredDeleter::areElementsReleased() {
-    return elementsToRelease == 0;
+bool DeferredDeleter::areElementsReleased(bool hostptrsOnly) {
+    return hostptrsOnly ? this->hostptrsToRelease == 0 : this->elementsToRelease == 0;
 }
 
 bool DeferredDeleter::shouldStop() {
@@ -96,7 +99,7 @@ void *DeferredDeleter::run(void *arg) {
         }
         lock.unlock();
         // Delete items placed into deferred delete queue
-        self->clearQueue();
+        self->clearQueue(false);
         lock.lock();
         // Check whether working thread should be stopped
     } while (!self->shouldStop());
@@ -104,24 +107,28 @@ void *DeferredDeleter::run(void *arg) {
     return nullptr;
 }
 
-void DeferredDeleter::drain(bool blocking) {
-    clearQueue();
+void DeferredDeleter::drain(bool blocking, bool hostptrsOnly) {
+    clearQueue(hostptrsOnly);
     if (blocking) {
-        while (!areElementsReleased())
+        while (!areElementsReleased(hostptrsOnly))
             ;
     }
 }
 
-void DeferredDeleter::clearQueue() {
+void DeferredDeleter::clearQueue(bool hostptrsOnly) {
     do {
         auto deletion = queue.removeFrontOne();
         if (deletion) {
-            if (deletion->apply()) {
-                elementsToRelease--;
+            bool isDeletionHostptr = deletion->isExternalHostptr();
+            if ((!hostptrsOnly || isDeletionHostptr) && deletion->apply()) {
+                this->elementsToRelease--;
+                if (isDeletionHostptr) {
+                    this->hostptrsToRelease--;
+                }
             } else {
                 queue.pushTailOne(*deletion.release());
             }
         }
-    } while (!queue.peekIsEmpty());
+    } while (hostptrsOnly ? !areElementsReleased(hostptrsOnly) : !queue.peekIsEmpty());
 }
 } // namespace NEO

shared/source/memory_manager/deferred_deleter.h

@@ -29,20 +29,21 @@ class DeferredDeleter {
 
     MOCKABLE_VIRTUAL void removeClient();
 
-    MOCKABLE_VIRTUAL void drain(bool blocking);
+    MOCKABLE_VIRTUAL void drain(bool blocking, bool hostptrsOnly);
 
   protected:
     void stop();
     void safeStop();
     void ensureThread();
-    MOCKABLE_VIRTUAL void clearQueue();
-    MOCKABLE_VIRTUAL bool areElementsReleased();
+    MOCKABLE_VIRTUAL void clearQueue(bool hostptrsOnly);
+    MOCKABLE_VIRTUAL bool areElementsReleased(bool hostptrsOnly);
     MOCKABLE_VIRTUAL bool shouldStop();
 
     static void *run(void *);
 
-    std::atomic<bool> doWorkInBackground;
-    std::atomic<int> elementsToRelease;
+    std::atomic<bool> doWorkInBackground = false;
+    std::atomic<int> elementsToRelease = 0;
+    std::atomic<int> hostptrsToRelease = 0;
     std::unique_ptr<Thread> worker;
     int32_t numClients = 0;
     IDList<DeferrableDeletion, true> queue;

shared/source/memory_manager/memory_manager.cpp

@@ -190,7 +190,7 @@ void *MemoryManager::allocateSystemMemory(size_t size, size_t alignment) {
 
 GraphicsAllocation *MemoryManager::allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) {
     if (deferredDeleter) {
-        deferredDeleter->drain(true);
+        deferredDeleter->drain(true, false);
     }
     GraphicsAllocation *graphicsAllocation = nullptr;
     auto osStorage = hostPtrManager->prepareOsStorageForAllocation(*this, allocationData.size, allocationData.hostPtr, allocationData.rootDeviceIndex);
@@ -307,7 +307,7 @@ void MemoryManager::checkGpuUsageAndDestroyGraphicsAllocations(GraphicsAllocatio
     if (gfxAllocation->isUsed()) {
         if (gfxAllocation->isUsedByManyOsContexts()) {
             multiContextResourceDestructor->deferDeletion(new DeferrableAllocationDeletion{*this, *gfxAllocation});
-            multiContextResourceDestructor->drain(false);
+            multiContextResourceDestructor->drain(false, false);
             return;
         }
         for (auto &engine : getRegisteredEngines(gfxAllocation->getRootDeviceIndex())) {
@@ -337,7 +337,7 @@ bool MemoryManager::isLimitedRange(uint32_t rootDeviceIndex) {
 
 void MemoryManager::waitForDeletions() {
     if (deferredDeleter) {
-        deferredDeleter->drain(false);
+        deferredDeleter->drain(false, false);
     }
     deferredDeleter.reset(nullptr);
 }
@@ -727,6 +727,12 @@ bool MemoryManager::mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) {
 }
 
 GraphicsAllocation *MemoryManager::allocateGraphicsMemory(const AllocationData &allocationData) {
+    if (allocationData.type == AllocationType::externalHostPtr &&
+        allocationData.hostPtr &&
+        this->getDeferredDeleter()) {
+        this->getDeferredDeleter()->drain(true, true);
+    }
+
     if (allocationData.type == AllocationType::image || allocationData.type == AllocationType::sharedResourceCopy) {
         UNRECOVERABLE_IF(allocationData.imgInfo == nullptr);
         return allocateGraphicsMemoryForImage(allocationData);

shared/source/os_interface/windows/deferrable_deletion_win.cpp

@@ -7,6 +7,7 @@
 
 #include "shared/source/os_interface/windows/deferrable_deletion_win.h"
 
+#include "shared/source/memory_manager/allocation_type.h"
 #include "shared/source/os_interface/windows/wddm/wddm.h"
 
 namespace NEO {
@@ -15,9 +16,10 @@ template <typename... Args>
 DeferrableDeletion *DeferrableDeletion::create(Args... args) {
     return new DeferrableDeletionImpl(std::forward<Args>(args)...);
 }
-template DeferrableDeletion *DeferrableDeletion::create(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
 
-DeferrableDeletionImpl::DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle)
+template DeferrableDeletion *DeferrableDeletion::create(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, AllocationType type);
+
+DeferrableDeletionImpl::DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, AllocationType type)
     : wddm(wddm), allocationCount(allocationCount), resourceHandle(resourceHandle) {
     if (handles) {
         this->handles = new D3DKMT_HANDLE[allocationCount];
@@ -25,12 +27,15 @@ DeferrableDeletionImpl::DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *
             this->handles[i] = handles[i];
         }
     }
+    this->externalHostptr = type == AllocationType::externalHostPtr;
 }
+
 bool DeferrableDeletionImpl::apply() {
     [[maybe_unused]] bool destroyStatus = wddm->destroyAllocations(handles, allocationCount, resourceHandle);
     DEBUG_BREAK_IF(!destroyStatus);
     return true;
 }
+
 DeferrableDeletionImpl::~DeferrableDeletionImpl() {
     if (handles) {
         delete[] handles;

shared/source/os_interface/windows/deferrable_deletion_win.h

@@ -15,10 +15,11 @@ namespace NEO {
 
 class OsContextWin;
 class Wddm;
+enum class AllocationType;
 
 class DeferrableDeletionImpl : public DeferrableDeletion {
   public:
-    DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
+    DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, AllocationType type);
     bool apply() override;
     ~DeferrableDeletionImpl() override;
 

shared/source/os_interface/windows/wddm_memory_manager.cpp

@@ -811,8 +811,8 @@ void WddmMemoryManager::handleFenceCompletion(GraphicsAllocation *allocation) {
 
 bool WddmMemoryManager::tryDeferDeletions(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, uint32_t rootDeviceIndex, AllocationType type) {
     bool status = true;
-    if (deferredDeleter && type != AllocationType::externalHostPtr) {
-        deferredDeleter->deferDeletion(DeferrableDeletion::create(&getWddm(rootDeviceIndex), handles, allocationCount, resourceHandle));
+    if (deferredDeleter) {
+        deferredDeleter->deferDeletion(DeferrableDeletion::create(&getWddm(rootDeviceIndex), handles, allocationCount, resourceHandle, type));
     } else {
         status = getWddm(rootDeviceIndex).destroyAllocations(handles, allocationCount, resourceHandle);
     }
@@ -1035,7 +1035,7 @@ bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocatio
     auto status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), minimumAddress, maximumAddress, addressToMap, allocation->getGpuAddressToModify(), allocation->getAllocationType());
 
     if (!status && deferredDeleter) {
-        deferredDeleter->drain(true);
+        deferredDeleter->drain(true, false);
         status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), minimumAddress, maximumAddress, addressToMap, allocation->getGpuAddressToModify(), allocation->getAllocationType());
     }
     if (!status) {
@@ -1081,7 +1081,7 @@ bool WddmMemoryManager::mapMultiHandleAllocationWithRetry(WddmAllocation *alloca
                                                 gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, gpuAddress, allocation->getAllocationType());
 
         if (!status && deferredDeleter) {
-            deferredDeleter->drain(true);
+            deferredDeleter->drain(true, false);
             status = wddm.mapGpuVirtualAddress(allocation->getGmm(currentHandle), allocation->getHandles()[currentHandle],
                                                gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, gpuAddress, allocation->getAllocationType());
         }
@@ -1104,7 +1104,7 @@ bool WddmMemoryManager::createGpuAllocationsWithRetry(WddmAllocation *allocation
         auto gmm = allocation->getGmm(handleId);
         auto status = getWddm(allocation->getRootDeviceIndex()).createAllocation(gmm->gmmResourceInfo->getSystemMemPointer(), gmm, allocation->getHandleToModify(handleId), allocation->getResourceHandleToModify(), allocation->getSharedHandleToModify());
         if (status == STATUS_GRAPHICS_NO_VIDEO_MEMORY && deferredDeleter) {
-            deferredDeleter->drain(true);
+            deferredDeleter->drain(true, false);
             status = getWddm(allocation->getRootDeviceIndex()).createAllocation(gmm->gmmResourceInfo->getSystemMemPointer(), gmm, allocation->getHandleToModify(handleId), allocation->getResourceHandleToModify(), allocation->getSharedHandleToModify());
         }
         if (status != STATUS_SUCCESS) {

shared/test/common/mocks/mock_deferred_deleter.cpp

@@ -34,19 +34,20 @@ void MockDeferredDeleter::removeClient() {
     --numClients;
 }
 
-void MockDeferredDeleter::drain(bool blocking) {
+void MockDeferredDeleter::drain(bool blocking, bool hostptrsOnly) {
     if (expectDrainCalled) {
         EXPECT_EQ(expectedDrainValue, blocking);
     }
-    DeferredDeleter::drain(blocking);
+    DeferredDeleter::drain(blocking, hostptrsOnly);
     drainCalled++;
 }
 
 void MockDeferredDeleter::drain() {
-    return drain(true);
+    return drain(true, false);
 }
 
-bool MockDeferredDeleter::areElementsReleased() {
+bool MockDeferredDeleter::areElementsReleased(bool hostptrsOnly) {
+    this->areElementsReleasedCalledForHostptrs = hostptrsOnly;
     areElementsReleasedCalled++;
     return areElementsReleasedCalled != 1;
 }
@@ -65,8 +66,8 @@ bool MockDeferredDeleter::shouldStop() {
     return shouldStopCalled > 1;
 }
 
-void MockDeferredDeleter::clearQueue() {
-    DeferredDeleter::clearQueue();
+void MockDeferredDeleter::clearQueue(bool hostptrsOnly) {
+    DeferredDeleter::clearQueue(hostptrsOnly);
     clearCalled++;
 }
 
@@ -100,7 +101,7 @@ void MockDeferredDeleter::setDoWorkInBackgroundValue(bool value) {
 }
 
 bool MockDeferredDeleter::baseAreElementsReleased() {
-    return DeferredDeleter::areElementsReleased();
+    return DeferredDeleter::areElementsReleased(false);
 }
 
 bool MockDeferredDeleter::baseShouldStop() {

shared/test/common/mocks/mock_deferred_deleter.h

@@ -22,9 +22,9 @@ class MockDeferredDeleter : public DeferredDeleter {
 
     void removeClient() override;
 
-    void drain(bool blocking) override;
+    void drain(bool blocking, bool hostptrsOnly) override;
 
-    bool areElementsReleased() override;
+    bool areElementsReleased(bool hostptrsOnly) override;
 
     bool shouldStop() override;
 
@@ -56,6 +56,8 @@ class MockDeferredDeleter : public DeferredDeleter {
 
     int areElementsReleasedCalled = 0;
 
+    bool areElementsReleasedCalledForHostptrs = false;
+
     std::atomic<int> shouldStopCalled;
 
     std::atomic<int> clearCalled;
@@ -73,6 +75,6 @@ class MockDeferredDeleter : public DeferredDeleter {
 
     bool expectDrainCalled = false;
 
-    void clearQueue() override;
+    void clearQueue(bool hostptrsOnly) override;
 };
 } // namespace NEO

shared/test/unit_test/memory_manager/deferred_deleter_tests.cpp

@@ -23,14 +23,23 @@ TEST(DeferredDeleter, WhenDeferredDeleterIsCreatedThenItIsNotAssignable) {
 
 TEST(DeferredDeleter, givenDeferredDeleterWhenBlockingDrainIsCalledThenArElementsReleasedIsCalled) {
     auto deleter = std::make_unique<MockDeferredDeleter>();
-    deleter->drain(true);
+    deleter->drain(true, false);
     EXPECT_NE(0, deleter->areElementsReleasedCalled);
+    EXPECT_FALSE(deleter->areElementsReleasedCalledForHostptrs);
+    EXPECT_EQ(1, deleter->drainCalled);
+}
+
+TEST(DeferredDeleter, givenDeferredDeleterWhenBlockingDrainOnlyForHostptrsIsCalledThenArElementsReleasedIsCalledWithHostptrsOnly) {
+    auto deleter = std::make_unique<MockDeferredDeleter>();
+    deleter->drain(true, true);
+    EXPECT_NE(0, deleter->areElementsReleasedCalled);
+    EXPECT_TRUE(deleter->areElementsReleasedCalledForHostptrs);
     EXPECT_EQ(1, deleter->drainCalled);
 }
 
 TEST(DeferredDeleter, givenDeferredDeleterWhenNonBlockingDrainIsCalledThenArElementsReleasedIsNotCalled) {
     auto deleter = std::make_unique<MockDeferredDeleter>();
-    deleter->drain(false);
+    deleter->drain(false, false);
     EXPECT_EQ(0, deleter->areElementsReleasedCalled);
     EXPECT_EQ(1, deleter->drainCalled);
 }

shared/test/unit_test/os_interface/windows/deferrable_deletion_win_tests.cpp

@@ -51,18 +51,21 @@ class DeferrableDeletionTest : public ::testing::Test {
 };
 
 TEST_F(DeferrableDeletionTest, givenDeferrableDeletionWhenIsCreatedThenObjectMembersAreSetProperly) {
-    MockDeferrableDeletion deletion(wddm.get(), &handle, allocationCount, resourceHandle);
+    MockDeferrableDeletion deletion(wddm.get(), &handle, allocationCount, resourceHandle, AllocationType::buffer);
     EXPECT_EQ(wddm.get(), deletion.wddm);
     EXPECT_NE(nullptr, deletion.handles);
     EXPECT_EQ(handle, *deletion.handles);
     EXPECT_NE(&handle, deletion.handles);
     EXPECT_EQ(allocationCount, deletion.allocationCount);
     EXPECT_EQ(resourceHandle, deletion.resourceHandle);
+    EXPECT_FALSE(deletion.isExternalHostptr());
+    MockDeferrableDeletion deletion2(wddm.get(), &handle, allocationCount, resourceHandle, AllocationType::externalHostPtr);
+    EXPECT_TRUE(deletion2.isExternalHostptr());
 }
 
 TEST_F(DeferrableDeletionTest, givenDeferrableDeletionWhenApplyIsCalledThenDeletionIsApplied) {
     wddm->callBaseDestroyAllocations = false;
-    std::unique_ptr<DeferrableDeletion> deletion(DeferrableDeletion::create((Wddm *)wddm.get(), &handle, allocationCount, resourceHandle));
+    std::unique_ptr<DeferrableDeletion> deletion(DeferrableDeletion::create((Wddm *)wddm.get(), &handle, allocationCount, resourceHandle, AllocationType::buffer));
     EXPECT_EQ(0u, wddm->destroyAllocationResult.called);
     deletion->apply();
     EXPECT_EQ(1u, wddm->destroyAllocationResult.called);

shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp

@@ -4032,13 +4032,6 @@ struct WddmMemoryManagerWithAsyncDeleterTest : public ::testing::Test {
     WddmMock *wddm;
 };
 
-TEST_F(WddmMemoryManagerWithAsyncDeleterTest, givenWddmWhenAsyncDeleterIsEnabledThenDoNotDeferExternalHostptrDeletions) {
-    EXPECT_EQ(0, deleter->deferDeletionCalled);
-    memoryManager->tryDeferDeletions(nullptr, 0, 0, 0, AllocationType::externalHostPtr);
-    EXPECT_EQ(0, deleter->deferDeletionCalled);
-    EXPECT_EQ(1u, wddm->destroyAllocationResult.called);
-}
-
 TEST_F(WddmMemoryManagerWithAsyncDeleterTest, givenWddmWhenAsyncDeleterIsEnabledThenCanDeferDeletions) {
     EXPECT_EQ(0, deleter->deferDeletionCalled);
     memoryManager->tryDeferDeletions(nullptr, 0, 0, 0, AllocationType::unknown);