[2/n] Unified Shared Memory

- Add kernel flags specifying indirect usage of device allocations.
- make device usm allocation resident when kernel requires this

Related-To: NEO-3148

Change-Id: I689347a0ea9b0f84c83f7883ca2381be63c61af9
Signed-off-by: Mrozek, Michal <michal.mrozek@intel.com>

public/cl_ext_private.h

@@ -59,3 +59,12 @@ using cl_mem_flags_intel = cl_mem_flags;
 #define CL_MEM_FORCE_SHARED_PHYSICAL_MEMORY_INTEL (1 << 20)
 
 #define CL_MEM_ALLOCATION_HANDLE_INTEL 0x10050
+
+/******************************
+*        UNIFIED MEMORY       *
+*******************************/
+
+#define CL_KERNEL_EXEC_INFO_INDIRECT_HOST_ACCESS_INTEL 0x10000
+#define CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL 0x10001
+#define CL_KERNEL_EXEC_INFO_INDIRECT_SHARED_ACCESS_INTEL 0x10002
+#define CL_KERNEL_EXEC_INFO_USM_PTRS_INTEL 0x10003

runtime/kernel/kernel.cpp

@@ -34,6 +34,7 @@
 #include "runtime/mem_obj/pipe.h"
 #include "runtime/memory_manager/memory_manager.h"
 #include "runtime/memory_manager/surface.h"
+#include "runtime/memory_manager/svm_memory_manager.h"
 #include "runtime/os_interface/debug_settings_manager.h"
 #include "runtime/platform/platform.h"
 #include "runtime/program/kernel_info.h"
@@ -940,6 +941,12 @@ void Kernel::clearKernelExecInfo() {
     svmAllocationsRequireCacheFlush = false;
 }
 
+void Kernel::setUnifiedMemoryProperty(cl_kernel_exec_info infoType, bool infoValue) {
+    if (infoType == CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL) {
+        this->unifiedMemoryControls.indirectDeviceAllocationsAllowed = infoValue;
+    }
+}
+
 inline void Kernel::makeArgsResident(CommandStreamReceiver &commandStreamReceiver) {
 
     auto numArgs = kernelInfo.kernelArgInfo.size();
@@ -989,6 +996,10 @@ void Kernel::makeResident(CommandStreamReceiver &commandStreamReceiver) {
     }
 
     gtpinNotifyMakeResident(this, &commandStreamReceiver);
+
+    if (unifiedMemoryControls.indirectDeviceAllocationsAllowed) {
+        this->getContext().getSVMAllocsManager()->makeInternalAllocationsResident(commandStreamReceiver);
+    }
 }
 
 void Kernel::getResidency(std::vector<Surface *> &dst) {

runtime/kernel/kernel.h

@@ -61,6 +61,10 @@ class Kernel : public BaseObject<_cl_kernel> {
         bool isUncacheable = false;
     };
 
+    struct UnifiedMemoryControls {
+        bool indirectDeviceAllocationsAllowed = false;
+    };
+
     typedef int32_t (Kernel::*KernelArgHandler)(uint32_t argIndex,
                                                 size_t argSize,
                                                 const void *argVal);
@@ -388,6 +392,7 @@ class Kernel : public BaseObject<_cl_kernel> {
     void setAuxTranslationFlag(bool auxTranslationFlag) {
         this->auxTranslationKernel = auxTranslationFlag;
     }
+    void setUnifiedMemoryProperty(cl_kernel_exec_info infoType, bool infoValue);
 
   protected:
     struct ObjectCounts {
@@ -457,8 +462,8 @@ class Kernel : public BaseObject<_cl_kernel> {
                                                        uint64_t privateMemoryCurbeOffset, uint32_t privateMemoryPatchSize, uint64_t privateMemoryGpuAddress);
     };
 
-  protected:
-    void makeArgsResident(CommandStreamReceiver &commandStreamReceiver);
+    void
+    makeArgsResident(CommandStreamReceiver &commandStreamReceiver);
 
     void *patchBufferOffset(const KernelArgInfo &argInfo, void *svmPtr, GraphicsAllocation *svmAlloc);
 
@@ -515,5 +520,6 @@ class Kernel : public BaseObject<_cl_kernel> {
     bool specialPipelineSelectMode = false;
     bool svmAllocationsRequireCacheFlush = false;
     std::vector<GraphicsAllocation *> kernelArgRequiresCacheFlush;
+    UnifiedMemoryControls unifiedMemoryControls;
 };
 } // namespace NEO

runtime/memory_manager/svm_memory_manager.h

@@ -73,6 +73,8 @@ class SVMAllocsManager {
     };
 
     struct UnifiedMemoryProperties {
+        UnifiedMemoryProperties() = default;
+        UnifiedMemoryProperties(InternalMemoryType memoryType) : memoryType(memoryType){};
         InternalMemoryType memoryType = InternalMemoryType::NOT_SPECIFIED;
     };
 

unit_tests/kernel/kernel_tests.cpp

@@ -16,6 +16,7 @@
 #include "runtime/mem_obj/image.h"
 #include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/os_agnostic_memory_manager.h"
+#include "runtime/memory_manager/svm_memory_manager.h"
 #include "runtime/os_interface/debug_settings_manager.h"
 #include "runtime/os_interface/os_context.h"
 #include "test.h"
@@ -24,6 +25,7 @@
 #include "unit_tests/fixtures/memory_management_fixture.h"
 #include "unit_tests/helpers/debug_manager_state_restore.h"
 #include "unit_tests/helpers/gtest_helpers.h"
+#include "unit_tests/libult/ult_command_stream_receiver.h"
 #include "unit_tests/mocks/mock_command_queue.h"
 #include "unit_tests/mocks/mock_context.h"
 #include "unit_tests/mocks/mock_graphics_allocation.h"
@@ -1614,6 +1616,30 @@ HWTEST_F(KernelResidencyTest, givenKernelWhenMakeResidentIsCalledThenKernelIsaIs
     memoryManager->freeGraphicsMemory(pKernelInfo->kernelAllocation);
 }
 
+HWTEST_F(KernelResidencyTest, givenKernelWhenItUsesIndirectUnifiedMemoryDeviceAllocationThenTheyAreMadeResident) {
+    MockKernelWithInternals mockKernel(*this->pDevice);
+    auto &commandStreamReceiver = this->pDevice->getUltCommandStreamReceiver<FamilyType>();
+
+    auto svmAllocationsManager = mockKernel.mockContext->getSVMAllocsManager();
+    auto unifiedMemoryAllocation = svmAllocationsManager->createUnifiedMemoryAllocation(4096u, SVMAllocsManager::UnifiedMemoryProperties(InternalMemoryType::DEVICE_UNIFIED_MEMORY));
+
+    mockKernel.mockKernel->makeResident(this->pDevice->getCommandStreamReceiver());
+
+    EXPECT_EQ(0u, commandStreamReceiver.getResidencyAllocations().size());
+
+    mockKernel.mockKernel->setUnifiedMemoryProperty(CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL, true);
+
+    mockKernel.mockKernel->makeResident(this->pDevice->getCommandStreamReceiver());
+
+    EXPECT_EQ(1u, commandStreamReceiver.getResidencyAllocations().size());
+
+    EXPECT_EQ(commandStreamReceiver.getResidencyAllocations()[0]->getGpuAddress(), castToUint64(unifiedMemoryAllocation));
+
+    mockKernel.mockKernel->setUnifiedMemoryProperty(CL_KERNEL_EXEC_INFO_SVM_PTRS, true);
+
+    svmAllocationsManager->freeSVMAlloc(unifiedMemoryAllocation);
+}
+
 TEST(KernelImageDetectionTests, givenKernelWithImagesOnlyWhenItIsAskedIfItHasImagesOnlyThenTrueIsReturned) {
     auto pKernelInfo = std::make_unique<KernelInfo>();
     pKernelInfo->kernelArgInfo.resize(3);