[1/n] Unified Shared Memory

- Add Internal Allocation type to differentiate SVM allocs from UM allocs.
- Add API to make internal allocations resident.
- Add API to allocate UM.

Related-To: NEO-3148

Change-Id: I9787891c5a0ffccac45c43bc5fde4ea50f37d703
Signed-off-by: Mrozek, Michal <michal.mrozek@intel.com>

runtime/memory_manager/svm_memory_manager.cpp

@@ -68,6 +68,15 @@ SvmMapOperation *SVMAllocsManager::MapOperationsTracker::get(const void *regionP
     return &iter->second;
 }
 
+void SVMAllocsManager::makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver) {
+    std::unique_lock<SpinLock> lock(mtx);
+    for (auto &allocation : this->SVMAllocs.allocations) {
+        if (allocation.second.memoryType == InternalMemoryType::DEVICE_UNIFIED_MEMORY) {
+            commandStreamReceiver.makeResident(*allocation.second.gpuAllocation);
+        }
+    }
+}
+
 SVMAllocsManager::SVMAllocsManager(MemoryManager *memoryManager) : memoryManager(memoryManager) {
 }
 
@@ -83,6 +92,22 @@ void *SVMAllocsManager::createSVMAlloc(size_t size, const SvmAllocationPropertie
     }
 }
 
+void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties svmProperties) {
+    size_t alignedSize = alignUp<size_t>(size, MemoryConstants::pageSize64k);
+    AllocationProperties unifiedMemoryProperties{true, alignedSize, GraphicsAllocation::AllocationType::BUFFER};
+    GraphicsAllocation *unifiedMemoryAllocation = memoryManager->allocateGraphicsMemoryWithProperties(unifiedMemoryProperties);
+
+    SvmAllocationData allocData;
+    allocData.gpuAllocation = unifiedMemoryAllocation;
+    allocData.cpuAllocation = nullptr;
+    allocData.size = size;
+    allocData.memoryType = InternalMemoryType::DEVICE_UNIFIED_MEMORY;
+
+    std::unique_lock<SpinLock> lock(mtx);
+    this->SVMAllocs.insert(allocData);
+    return reinterpret_cast<void *>(unifiedMemoryAllocation->getGpuAddress());
+}
+
 SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) {
     std::unique_lock<SpinLock> lock(mtx);
     return SVMAllocs.get(ptr);

runtime/memory_manager/svm_memory_manager.h

@@ -18,10 +18,17 @@ class Device;
 class GraphicsAllocation;
 class MemoryManager;
 
+enum class InternalMemoryType : uint32_t {
+    SVM = 0,
+    DEVICE_UNIFIED_MEMORY,
+    NOT_SPECIFIED
+};
+
 struct SvmAllocationData {
     GraphicsAllocation *cpuAllocation = nullptr;
     GraphicsAllocation *gpuAllocation = nullptr;
     size_t size = 0;
+    InternalMemoryType memoryType = InternalMemoryType::SVM;
 };
 
 struct SvmMapOperation {
@@ -35,6 +42,8 @@ struct SvmMapOperation {
 class SVMAllocsManager {
   public:
     class MapBasedAllocationTracker {
+        friend class SVMAllocsManager;
+
       public:
         using SvmAllocationContainer = std::map<const void *, SvmAllocationData>;
         void insert(SvmAllocationData);
@@ -63,8 +72,13 @@ class SVMAllocsManager {
         bool readOnly = false;
     };
 
+    struct UnifiedMemoryProperties {
+        InternalMemoryType memoryType = InternalMemoryType::NOT_SPECIFIED;
+    };
+
     SVMAllocsManager(MemoryManager *memoryManager);
     void *createSVMAlloc(size_t size, const SvmAllocationProperties svmProperties);
+    void *createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties svmProperties);
     SvmAllocationData *getSVMAlloc(const void *ptr);
     void freeSVMAlloc(void *ptr);
     size_t getNumAllocs() const { return SVMAllocs.getNumAllocs(); }
@@ -72,10 +86,12 @@ class SVMAllocsManager {
     void insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap);
     void removeSvmMapOperation(const void *regionSvmPtr);
     SvmMapOperation *getSvmMapOperation(const void *regionPtr);
+    void makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver);
 
   protected:
     void *createZeroCopySvmAllocation(size_t size, const SvmAllocationProperties &svmProperties);
     void *createSvmAllocationWithDeviceStorage(size_t size, const SvmAllocationProperties &svmProperties);
+
     void freeZeroCopySvmAllocation(SvmAllocationData *svmData);
     void freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData);
 

unit_tests/command_queue/enqueue_svm_tests.cpp

@@ -19,6 +19,7 @@
 #include "unit_tests/fixtures/device_fixture.h"
 #include "unit_tests/helpers/debug_manager_state_restore.h"
 #include "unit_tests/helpers/hw_parse.h"
+#include "unit_tests/libult/ult_command_stream_receiver.h"
 #include "unit_tests/mocks/mock_command_queue.h"
 #include "unit_tests/mocks/mock_context.h"
 #include "unit_tests/mocks/mock_kernel.h"
@@ -1051,6 +1052,30 @@ struct FailCsr : public CommandStreamReceiverHw<GfxFamily> {
     }
 };
 
+HWTEST_F(EnqueueSvmTest, whenInternalAllocationsAreMadeResidentThenOnlyNonSvmAllocationsAreAdded) {
+    SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties;
+    unifiedMemoryProperties.memoryType = InternalMemoryType::DEVICE_UNIFIED_MEMORY;
+    auto allocationSize = 4096u;
+    auto svmManager = this->context->getSVMAllocsManager();
+    EXPECT_NE(0u, svmManager->getNumAllocs());
+    auto unifiedMemoryPtr = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
+    EXPECT_NE(nullptr, unifiedMemoryPtr);
+    EXPECT_EQ(2u, svmManager->getNumAllocs());
+
+    auto &commandStreamReceiver = pDevice->getUltCommandStreamReceiver<FamilyType>();
+
+    auto &residentAllocations = commandStreamReceiver.getResidencyAllocations();
+    EXPECT_EQ(0u, residentAllocations.size());
+
+    svmManager->makeInternalAllocationsResident(commandStreamReceiver);
+
+    //only unified memory allocation is made resident
+    EXPECT_EQ(1u, residentAllocations.size());
+    EXPECT_EQ(residentAllocations[0]->getGpuAddress(), castToUint64(unifiedMemoryPtr));
+
+    svmManager->freeSVMAlloc(unifiedMemoryPtr);
+}
+
 HWTEST_F(EnqueueSvmTest, GivenDstHostPtrWhenHostPtrAllocationCreationFailsThenReturnOutOfResource) {
     char dstHostPtr[260];
     void *pDstSVM = dstHostPtr;

unit_tests/memory_manager/svm_memory_manager_tests.cpp

@@ -5,6 +5,7 @@
  *
  */
 
+#include "runtime/command_stream/command_stream_receiver.h"
 #include "runtime/mem_obj/mem_obj_helper.h"
 #include "test.h"
 #include "unit_tests/mocks/mock_execution_environment.h"
@@ -141,6 +142,24 @@ TEST_F(SVMMemoryAllocatorTest, whenCoherentFlagIsPassedThenAllocationIsCoherent)
     svmManager->freeSVMAlloc(ptr);
 }
 
+TEST_F(SVMMemoryAllocatorTest, whenDeviceAllocationIsCreatedThenItIsStoredWithProperTypeInAllocationMap) {
+    SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties;
+    unifiedMemoryProperties.memoryType = InternalMemoryType::DEVICE_UNIFIED_MEMORY;
+    auto allocationSize = 4096u;
+    auto ptr = svmManager->createUnifiedMemoryAllocation(4096u, unifiedMemoryProperties);
+    EXPECT_NE(nullptr, ptr);
+    auto allocation = svmManager->getSVMAlloc(ptr);
+    EXPECT_EQ(nullptr, allocation->cpuAllocation);
+    EXPECT_NE(nullptr, allocation->gpuAllocation);
+    EXPECT_EQ(InternalMemoryType::DEVICE_UNIFIED_MEMORY, allocation->memoryType);
+    EXPECT_EQ(allocationSize, allocation->size);
+
+    EXPECT_EQ(alignUp(allocationSize, MemoryConstants::pageSize64k), allocation->gpuAllocation->getUnderlyingBufferSize());
+    EXPECT_EQ(GraphicsAllocation::AllocationType::BUFFER, allocation->gpuAllocation->getAllocationType());
+
+    svmManager->freeSVMAlloc(ptr);
+}
+
 TEST(SvmAllocationPropertiesTests, givenDifferentMemFlagsWhenGettingSvmAllocationPropertiesThenPropertiesAreCorrectlySet) {
     SVMAllocsManager::SvmAllocationProperties allocationProperties = MemObjHelper::getSvmAllocationProperties(0);
     EXPECT_FALSE(allocationProperties.coherent);