fix: use condition variables instead of busy waits in worker threads

Resolves: NEO-16085, GSD-11678, HSD-14025819208

Signed-off-by: Igor Venevtsev <igor.venevtsev@intel.com>

shared/source/memory_manager/unified_memory_manager.cpp

@@ -67,7 +67,7 @@ bool SVMAllocsManager::SvmAllocationCache::insert(size_t size, void *ptr, SvmAll
         return false;
     }
 
-    std::lock_guard<std::mutex> lock(this->mtx);
+    std::unique_lock<std::mutex> lock(this->mtx);
     if (svmData->device ? svmData->device->shouldLimitAllocationsReuse() : memoryManager->shouldLimitAllocationsReuse()) {
         return false;
     }
@@ -99,8 +99,11 @@ bool SVMAllocsManager::SvmAllocationCache::insert(size_t size, void *ptr, SvmAll
         }
         svmData->isSavedForReuse = true;
         allocations.emplace(std::lower_bound(allocations.begin(), allocations.end(), size), size, ptr, svmData, waitForCompletion);
-        if (memoryManager->peekExecutionEnvironment().unifiedMemoryReuseCleaner) {
-            memoryManager->peekExecutionEnvironment().unifiedMemoryReuseCleaner->startThread();
+        empty = false;
+        if (auto usmReuseCleaner = this->memoryManager->peekExecutionEnvironment().unifiedMemoryReuseCleaner.get()) {
+            lock.unlock();
+            usmReuseCleaner->startThread();
+            usmReuseCleaner->notifySvmAllocationsCacheUpdate();
         }
     }
     if (enablePerformanceLogging) {
@@ -110,6 +113,7 @@ bool SVMAllocsManager::SvmAllocationCache::insert(size_t size, void *ptr, SvmAll
                            .operationType = CacheOperationType::insert,
                            .isSuccess = isSuccess});
     }
+
     return isSuccess;
 }
 
@@ -181,6 +185,7 @@ void *SVMAllocsManager::SvmAllocationCache::get(size_t size, const UnifiedMemory
                 svmAllocsManager->reinsertToAllocsForIndirectAccess(*allocationIter->svmData);
             }
             allocations.erase(allocationIter);
+            empty = allocations.empty();
             return allocationPtr;
         }
     }
@@ -215,6 +220,7 @@ void SVMAllocsManager::SvmAllocationCache::trim() {
         svmAllocsManager->freeSVMAllocImpl(cachedAllocationInfo.allocation, FreePolicyType::blocking, cachedAllocationInfo.svmData);
     }
     this->allocations.clear();
+    empty = true;
 }
 
 void SVMAllocsManager::SvmAllocationCache::cleanup() {
@@ -299,6 +305,7 @@ void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resol
     if (trimAll) {
         std::erase_if(allocations, SvmCacheAllocationInfo::isMarkedForDelete);
     }
+    empty = allocations.empty();
 }
 
 SvmAllocationData *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) {

shared/source/memory_manager/unified_memory_manager.h

@@ -221,6 +221,7 @@ class SVMAllocsManager {
         static bool allocUtilizationAllows(size_t requestedSize, size_t reuseCandidateSize);
         static bool alignmentAllows(void *ptr, size_t alignment);
         bool isInUse(SvmCacheAllocationInfo &cacheAllocInfo);
+        bool isEmpty() { return empty; };
         void *get(size_t size, const UnifiedMemoryProperties &unifiedMemoryProperties);
         void trim();
         void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool trimAll);
@@ -234,6 +235,7 @@ class SVMAllocsManager {
         MemoryManager *memoryManager = nullptr;
         bool enablePerformanceLogging = false;
         bool requireUpdatingAllocsForIndirectAccess = false;
+        std::atomic_bool empty = true;
     };
 
     enum class FreePolicyType : uint32_t {

shared/source/memory_manager/unified_memory_reuse_cleaner.cpp

@@ -23,8 +23,11 @@ UnifiedMemoryReuseCleaner::~UnifiedMemoryReuseCleaner() {
 }
 
 void UnifiedMemoryReuseCleaner::stopThread() {
-    keepCleaning.store(false);
-    runCleaning.store(false);
+    {
+        std::lock_guard<std::mutex> lock(condVarMutex);
+        keepCleaning.store(false);
+        condVar.notify_one();
+    }
     if (unifiedMemoryReuseCleanerThread) {
         unifiedMemoryReuseCleanerThread->join();
         unifiedMemoryReuseCleanerThread.reset();
@@ -33,26 +36,24 @@ void UnifiedMemoryReuseCleaner::stopThread() {
 
 void *UnifiedMemoryReuseCleaner::cleanUnifiedMemoryReuse(void *self) {
     auto cleaner = reinterpret_cast<UnifiedMemoryReuseCleaner *>(self);
-    while (!cleaner->runCleaning.load()) {
-        if (!cleaner->keepCleaning.load()) {
-            return nullptr;
-        }
-        NEO::sleep(sleepTime);
-    }
-
-    while (true) {
-        if (!cleaner->keepCleaning.load()) {
-            return nullptr;
-        }
-        NEO::sleep(sleepTime);
+    while (cleaner->keepCleaning.load()) {
+        std::unique_lock lock(cleaner->condVarMutex);
+        cleaner->wait(lock);
+        lock.unlock();
         cleaner->trimOldInCaches();
+        NEO::sleep(sleepTime);
     }
+    return nullptr;
+}
+
+void UnifiedMemoryReuseCleaner::notifySvmAllocationsCacheUpdate() {
+    std::lock_guard<std::mutex> lock(condVarMutex);
+    condVar.notify_one();
 }
 
 void UnifiedMemoryReuseCleaner::registerSvmAllocationCache(SvmAllocationCache *cache) {
     std::lock_guard<std::mutex> lockSvmAllocationCaches(this->svmAllocationCachesMutex);
     this->svmAllocationCaches.push_back(cache);
-    this->startCleaning();
 }
 
 void UnifiedMemoryReuseCleaner::unregisterSvmAllocationCache(SvmAllocationCache *cache) {
@@ -85,4 +86,4 @@ void UnifiedMemoryReuseCleaner::startThread() {
     });
 }
 
-} // namespace NEO
+} // namespace NEO

shared/source/memory_manager/unified_memory_reuse_cleaner.h

@@ -11,6 +11,7 @@
 #include "shared/source/memory_manager/unified_memory_manager.h"
 
 #include <chrono>
+#include <condition_variable>
 #include <memory>
 #include <mutex>
 #include <vector>
@@ -33,9 +34,17 @@ class UnifiedMemoryReuseCleaner : NEO::NonCopyableAndNonMovableClass {
 
     void registerSvmAllocationCache(SvmAllocationCache *cache);
     void unregisterSvmAllocationCache(SvmAllocationCache *cache);
+    bool waitPredicate() { return !keepCleaning || !isEmpty(); };
+    MOCKABLE_VIRTUAL void wait(std::unique_lock<std::mutex> &lock) {
+        condVar.wait(lock, [&]() { return waitPredicate(); });
+    }
+    MOCKABLE_VIRTUAL bool isEmpty() {
+        std::unique_lock<std::mutex> lock(svmAllocationCachesMutex);
+        return std::all_of(svmAllocationCaches.begin(), svmAllocationCaches.end(), [](const auto &it) { return it->isEmpty(); });
+    }
+    void notifySvmAllocationsCacheUpdate();
 
   protected:
-    void startCleaning() { runCleaning.store(true); };
     static void *cleanUnifiedMemoryReuse(void *self);
     MOCKABLE_VIRTUAL void trimOldInCaches();
     std::unique_ptr<Thread> unifiedMemoryReuseCleanerThread;
@@ -44,7 +53,8 @@ class UnifiedMemoryReuseCleaner : NEO::NonCopyableAndNonMovableClass {
     std::mutex svmAllocationCachesMutex;
     std::once_flag startThreadOnce;
 
-    std::atomic_bool runCleaning = false;
+    std::mutex condVarMutex;
+    std::condition_variable condVar;
     std::atomic_bool keepCleaning = true;
 
     bool trimAllAllocations = false;