diff --git a/public/cl_ext_private.h b/public/cl_ext_private.h index c5281891b5..e400677eb4 100644 --- a/public/cl_ext_private.h +++ b/public/cl_ext_private.h @@ -59,3 +59,12 @@ using cl_mem_flags_intel = cl_mem_flags; #define CL_MEM_FORCE_SHARED_PHYSICAL_MEMORY_INTEL (1 << 20) #define CL_MEM_ALLOCATION_HANDLE_INTEL 0x10050 + +/****************************** +* UNIFIED MEMORY * +*******************************/ + +#define CL_KERNEL_EXEC_INFO_INDIRECT_HOST_ACCESS_INTEL 0x10000 +#define CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL 0x10001 +#define CL_KERNEL_EXEC_INFO_INDIRECT_SHARED_ACCESS_INTEL 0x10002 +#define CL_KERNEL_EXEC_INFO_USM_PTRS_INTEL 0x10003 diff --git a/runtime/kernel/kernel.cpp b/runtime/kernel/kernel.cpp index 92fa31cb27..a1abfc103f 100644 --- a/runtime/kernel/kernel.cpp +++ b/runtime/kernel/kernel.cpp @@ -34,6 +34,7 @@ #include "runtime/mem_obj/pipe.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/memory_manager/surface.h" +#include "runtime/memory_manager/svm_memory_manager.h" #include "runtime/os_interface/debug_settings_manager.h" #include "runtime/platform/platform.h" #include "runtime/program/kernel_info.h" @@ -940,6 +941,12 @@ void Kernel::clearKernelExecInfo() { svmAllocationsRequireCacheFlush = false; } +void Kernel::setUnifiedMemoryProperty(cl_kernel_exec_info infoType, bool infoValue) { + if (infoType == CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL) { + this->unifiedMemoryControls.indirectDeviceAllocationsAllowed = infoValue; + } +} + inline void Kernel::makeArgsResident(CommandStreamReceiver &commandStreamReceiver) { auto numArgs = kernelInfo.kernelArgInfo.size(); @@ -989,6 +996,10 @@ void Kernel::makeResident(CommandStreamReceiver &commandStreamReceiver) { } gtpinNotifyMakeResident(this, &commandStreamReceiver); + + if (unifiedMemoryControls.indirectDeviceAllocationsAllowed) { + this->getContext().getSVMAllocsManager()->makeInternalAllocationsResident(commandStreamReceiver); + } } void Kernel::getResidency(std::vector &dst) { diff --git a/runtime/kernel/kernel.h b/runtime/kernel/kernel.h index 0b3a8968b3..0c30335570 100644 --- a/runtime/kernel/kernel.h +++ b/runtime/kernel/kernel.h @@ -61,6 +61,10 @@ class Kernel : public BaseObject<_cl_kernel> { bool isUncacheable = false; }; + struct UnifiedMemoryControls { + bool indirectDeviceAllocationsAllowed = false; + }; + typedef int32_t (Kernel::*KernelArgHandler)(uint32_t argIndex, size_t argSize, const void *argVal); @@ -388,6 +392,7 @@ class Kernel : public BaseObject<_cl_kernel> { void setAuxTranslationFlag(bool auxTranslationFlag) { this->auxTranslationKernel = auxTranslationFlag; } + void setUnifiedMemoryProperty(cl_kernel_exec_info infoType, bool infoValue); protected: struct ObjectCounts { @@ -457,8 +462,8 @@ class Kernel : public BaseObject<_cl_kernel> { uint64_t privateMemoryCurbeOffset, uint32_t privateMemoryPatchSize, uint64_t privateMemoryGpuAddress); }; - protected: - void makeArgsResident(CommandStreamReceiver &commandStreamReceiver); + void + makeArgsResident(CommandStreamReceiver &commandStreamReceiver); void *patchBufferOffset(const KernelArgInfo &argInfo, void *svmPtr, GraphicsAllocation *svmAlloc); @@ -515,5 +520,6 @@ class Kernel : public BaseObject<_cl_kernel> { bool specialPipelineSelectMode = false; bool svmAllocationsRequireCacheFlush = false; std::vector kernelArgRequiresCacheFlush; + UnifiedMemoryControls unifiedMemoryControls; }; } // namespace NEO diff --git a/runtime/memory_manager/svm_memory_manager.h b/runtime/memory_manager/svm_memory_manager.h index 44a24c638c..79413be4b5 100644 --- a/runtime/memory_manager/svm_memory_manager.h +++ b/runtime/memory_manager/svm_memory_manager.h @@ -73,6 +73,8 @@ class SVMAllocsManager { }; struct UnifiedMemoryProperties { + UnifiedMemoryProperties() = default; + UnifiedMemoryProperties(InternalMemoryType memoryType) : memoryType(memoryType){}; InternalMemoryType memoryType = InternalMemoryType::NOT_SPECIFIED; }; diff --git a/unit_tests/kernel/kernel_tests.cpp b/unit_tests/kernel/kernel_tests.cpp index 8d2da85f46..78d4d59a67 100644 --- a/unit_tests/kernel/kernel_tests.cpp +++ b/unit_tests/kernel/kernel_tests.cpp @@ -16,6 +16,7 @@ #include "runtime/mem_obj/image.h" #include "runtime/memory_manager/allocations_list.h" #include "runtime/memory_manager/os_agnostic_memory_manager.h" +#include "runtime/memory_manager/svm_memory_manager.h" #include "runtime/os_interface/debug_settings_manager.h" #include "runtime/os_interface/os_context.h" #include "test.h" @@ -24,6 +25,7 @@ #include "unit_tests/fixtures/memory_management_fixture.h" #include "unit_tests/helpers/debug_manager_state_restore.h" #include "unit_tests/helpers/gtest_helpers.h" +#include "unit_tests/libult/ult_command_stream_receiver.h" #include "unit_tests/mocks/mock_command_queue.h" #include "unit_tests/mocks/mock_context.h" #include "unit_tests/mocks/mock_graphics_allocation.h" @@ -1614,6 +1616,30 @@ HWTEST_F(KernelResidencyTest, givenKernelWhenMakeResidentIsCalledThenKernelIsaIs memoryManager->freeGraphicsMemory(pKernelInfo->kernelAllocation); } +HWTEST_F(KernelResidencyTest, givenKernelWhenItUsesIndirectUnifiedMemoryDeviceAllocationThenTheyAreMadeResident) { + MockKernelWithInternals mockKernel(*this->pDevice); + auto &commandStreamReceiver = this->pDevice->getUltCommandStreamReceiver(); + + auto svmAllocationsManager = mockKernel.mockContext->getSVMAllocsManager(); + auto unifiedMemoryAllocation = svmAllocationsManager->createUnifiedMemoryAllocation(4096u, SVMAllocsManager::UnifiedMemoryProperties(InternalMemoryType::DEVICE_UNIFIED_MEMORY)); + + mockKernel.mockKernel->makeResident(this->pDevice->getCommandStreamReceiver()); + + EXPECT_EQ(0u, commandStreamReceiver.getResidencyAllocations().size()); + + mockKernel.mockKernel->setUnifiedMemoryProperty(CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL, true); + + mockKernel.mockKernel->makeResident(this->pDevice->getCommandStreamReceiver()); + + EXPECT_EQ(1u, commandStreamReceiver.getResidencyAllocations().size()); + + EXPECT_EQ(commandStreamReceiver.getResidencyAllocations()[0]->getGpuAddress(), castToUint64(unifiedMemoryAllocation)); + + mockKernel.mockKernel->setUnifiedMemoryProperty(CL_KERNEL_EXEC_INFO_SVM_PTRS, true); + + svmAllocationsManager->freeSVMAlloc(unifiedMemoryAllocation); +} + TEST(KernelImageDetectionTests, givenKernelWithImagesOnlyWhenItIsAskedIfItHasImagesOnlyThenTrueIsReturned) { auto pKernelInfo = std::make_unique(); pKernelInfo->kernelArgInfo.resize(3);