Use imported host pointers in the driver when available

Related-To: NEO-5126

Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>

level_zero/core/source/cmdlist/cmdlist_hw.inl

@@ -1101,18 +1101,20 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemoryFill(void *ptr,
 
     NEO::SvmAllocationData *allocData = nullptr;
     bool dstAllocFound = device->getDriverHandle()->findAllocationDataForRange(ptr, size, &allocData);
-    if (dstAllocFound == false) {
+    if (dstAllocFound) {
-        return ZE_RESULT_ERROR_INVALID_ARGUMENT;
-    } else {
         if (allocData->memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY ||
             allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {
             hostPointerNeedsFlush = true;
         }
+    } else {
+        if (device->getDriverHandle()->getHostPointerBaseAddress(ptr, nullptr) != ZE_RESULT_SUCCESS) {
+            return ZE_RESULT_ERROR_INVALID_ARGUMENT;
+        } else {
+            hostPointerNeedsFlush = true;
+        }
     }
 
-    uintptr_t dstPtr = reinterpret_cast<uintptr_t>(ptr);
+    auto dstAllocation = this->getAlignedAllocation(this->device, ptr, size);
-
-    auto dstAllocation = this->getAlignedAllocation(this->device, reinterpret_cast<void *>(dstPtr), size);
 
     uintptr_t srcPtr = reinterpret_cast<uintptr_t>(const_cast<void *>(pattern));
     size_t srcOffset = 0;
@@ -1212,12 +1214,12 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendBlitFill(void *ptr,
         return ZE_RESULT_ERROR_INVALID_SIZE;
     } else {
         appendEventForProfiling(hEvent, true);
-        NEO::SvmAllocationData *allocData = nullptr;
+        NEO::GraphicsAllocation *gpuAllocation = device->getDriverHandle()->getDriverSystemMemoryAllocation(ptr,
-        bool dstAllocFound = device->getDriverHandle()->findAllocationDataForRange(ptr, size, &allocData);
+                                                                                                            size,
-        if (dstAllocFound == false) {
+                                                                                                            neoDevice->getRootDeviceIndex());
+        if (gpuAllocation == nullptr) {
             return ZE_RESULT_ERROR_INVALID_ARGUMENT;
         }
-        auto gpuAllocation = allocData->gpuAllocations.getGraphicsAllocation(device->getRootDeviceIndex());
         commandContainer.addToResidencyContainer(gpuAllocation);
         uint32_t patternToCommand[4] = {};
         memcpy_s(&patternToCommand, sizeof(patternToCommand), pattern, patternSize);
@@ -1286,25 +1288,30 @@ inline AlignedAllocationData CommandListCoreFamily<gfxCoreFamily>::getAlignedAll
                                                                                         uint64_t bufferSize) {
 
     NEO::SvmAllocationData *allocData = nullptr;
-    bool srcAllocFound = device->getDriverHandle()->findAllocationDataForRange(const_cast<void *>(buffer),
+    void *ptr = const_cast<void *>(buffer);
+    bool srcAllocFound = device->getDriverHandle()->findAllocationDataForRange(ptr,
                                                                                bufferSize, &allocData);
     NEO::GraphicsAllocation *alloc = nullptr;
 
-    uintptr_t sourcePtr = reinterpret_cast<uintptr_t>(const_cast<void *>(buffer));
+    uintptr_t sourcePtr = reinterpret_cast<uintptr_t>(ptr);
     size_t offset = 0;
     NEO::EncodeSurfaceState<GfxFamily>::getSshAlignedPointer(sourcePtr, offset);
     uintptr_t alignedPtr = 0u;
     bool hostPointerNeedsFlush = false;
 
     if (srcAllocFound == false) {
-        alloc = getHostPtrAlloc(buffer, bufferSize, &offset);
+        alloc = device->getDriverHandle()->findHostPointerAllocation(ptr, static_cast<size_t>(bufferSize), device->getRootDeviceIndex());
-
+        if (alloc != nullptr) {
+            offset += ptrDiff(buffer, alloc->getUnderlyingBuffer());
+        } else {
+            alloc = getHostPtrAlloc(buffer, bufferSize, &offset);
+        }
         alignedPtr = static_cast<uintptr_t>(alignDown(alloc->getGpuAddress(), NEO::EncodeSurfaceState<GfxFamily>::getSurfaceBaseAddressAlignment()));
+
         hostPointerNeedsFlush = true;
     } else {
         alloc = allocData->gpuAllocations.getGraphicsAllocation(device->getRootDeviceIndex());
-
+        alignedPtr = sourcePtr - offset;
-        alignedPtr = reinterpret_cast<uintptr_t>(buffer) - offset;
 
         if (allocData->memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY ||
             allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {

level_zero/core/source/context/context_imp.cpp

@@ -84,27 +84,28 @@ ze_result_t ContextImp::freeMem(const void *ptr) {
 }
 
 ze_result_t ContextImp::makeMemoryResident(ze_device_handle_t hDevice, void *ptr, size_t size) {
-    auto alloc = getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
+    Device *device = L0::Device::fromHandle(hDevice);
-    if (alloc == nullptr) {
+    NEO::Device *neoDevice = device->getNEODevice();
+    auto allocation = device->getDriverHandle()->getDriverSystemMemoryAllocation(ptr, size, neoDevice->getRootDeviceIndex());
+    if (allocation == nullptr) {
         return ZE_RESULT_ERROR_INVALID_ARGUMENT;
     }
 
-    NEO::Device *neoDevice = L0::Device::fromHandle(hDevice)->getNEODevice();
     NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get();
-    auto gpuAllocation = alloc->gpuAllocations.getGraphicsAllocation(neoDevice->getRootDeviceIndex());
+    auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&allocation, 1));
-    auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef<NEO::GraphicsAllocation *>(&gpuAllocation, 1));
     return changeMemoryOperationStatusToL0ResultType(success);
 }
 
 ze_result_t ContextImp::evictMemory(ze_device_handle_t hDevice, void *ptr, size_t size) {
-    auto alloc = getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
+    Device *device = L0::Device::fromHandle(hDevice);
-    if (alloc == nullptr) {
+    NEO::Device *neoDevice = device->getNEODevice();
+    auto allocation = device->getDriverHandle()->getDriverSystemMemoryAllocation(ptr, size, neoDevice->getRootDeviceIndex());
+    if (allocation == nullptr) {
         return ZE_RESULT_ERROR_INVALID_ARGUMENT;
     }
 
-    NEO::Device *neoDevice = L0::Device::fromHandle(hDevice)->getNEODevice();
     NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get();
-    auto success = memoryOperationsIface->evict(neoDevice, *alloc->gpuAllocations.getGraphicsAllocation(neoDevice->getRootDeviceIndex()));
+    auto success = memoryOperationsIface->evict(neoDevice, *allocation);
     return changeMemoryOperationStatusToL0ResultType(success);
 }
 

level_zero/core/source/kernel/kernel_imp.cpp

@@ -521,13 +521,14 @@ ze_result_t KernelImp::setArgBuffer(uint32_t argIndex, size_t argSize, const voi
     }
 
     auto requestedAddress = *reinterpret_cast<void *const *>(argVal);
-    auto svmAllocsManager = module->getDevice()->getDriverHandle()->getSvmAllocsManager();
+    uintptr_t gpuAddress = 0u;
-    if (nullptr == svmAllocsManager->getSVMAlloc(requestedAddress)) {
+    NEO::GraphicsAllocation *alloc = module->getDevice()->getDriverHandle()->getDriverSystemMemoryAllocation(requestedAddress,
+                                                                                                             1u,
+                                                                                                             module->getDevice()->getRootDeviceIndex());
+    if (nullptr == alloc) {
         return ZE_RESULT_ERROR_INVALID_ARGUMENT;
     }
-    uint32_t rootDeviceIndex = module->getDevice()->getRootDeviceIndex();
+    gpuAddress = static_cast<uintptr_t>(alloc->getGpuAddress());
-    NEO::GraphicsAllocation *alloc = svmAllocsManager->getSVMAlloc(requestedAddress)->gpuAllocations.getGraphicsAllocation(rootDeviceIndex);
-    auto gpuAddress = reinterpret_cast<uintptr_t>(requestedAddress);
     return setArgBufferWithAlloc(argIndex, gpuAddress, alloc);
 }
 

level_zero/core/test/unit_tests/fixtures/host_pointer_manager_fixture.h

@@ -8,10 +8,13 @@
 #pragma once
 
 #include "shared/source/helpers/constants.h"
+#include "shared/source/os_interface/device_factory.h"
 #include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
 #include "shared/test/unit_test/helpers/default_hw_info.h"
 #include "shared/test/unit_test/mocks/mock_device.h"
 
+#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h"
+
 #include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_driver_handle.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_host_pointer_manager.h"
@@ -25,6 +28,10 @@ struct HostPointerManagerFixure {
         neoDevice = NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(NEO::defaultHwInfo.get());
         auto mockBuiltIns = new MockBuiltins();
         neoDevice->executionEnvironment->rootDeviceEnvironments[0]->builtins.reset(mockBuiltIns);
+        neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface =
+            std::make_unique<NEO::MockMemoryOperationsHandlerTests>();
+        mockMemoryInterface = static_cast<NEO::MockMemoryOperationsHandlerTests *>(
+            neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface.get());
         devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
 
         DebugManager.flags.EnableHostPointerImport.set(1);
@@ -33,18 +40,33 @@ struct HostPointerManagerFixure {
         device = hostDriverHandle->devices[0];
         EXPECT_NE(nullptr, hostDriverHandle->hostPointerManager.get());
         openHostPointerManager = static_cast<L0::ult::HostPointerManager *>(hostDriverHandle->hostPointerManager.get());
+
         heapPointer = hostDriverHandle->getMemoryManager()->allocateSystemMemory(4 * MemoryConstants::pageSize, MemoryConstants::pageSize);
         ASSERT_NE(nullptr, heapPointer);
+
+        ze_context_desc_t desc;
+        ze_result_t ret = hostDriverHandle->createContext(&desc, &hContext);
+        EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+        context = L0::Context::fromHandle(hContext);
     }
 
     void TearDown() {
+        context->destroy();
+
         hostDriverHandle->getMemoryManager()->freeSystemMemory(heapPointer);
     }
-    L0::ult::HostPointerManager *openHostPointerManager = nullptr;
+    DebugManagerStateRestore debugRestore;
+
     std::unique_ptr<L0::ult::DriverHandle> hostDriverHandle;
-    void *heapPointer = nullptr;
+
+    L0::ult::HostPointerManager *openHostPointerManager = nullptr;
     NEO::MockDevice *neoDevice = nullptr;
     L0::Device *device = nullptr;
+    NEO::MockMemoryOperationsHandlerTests *mockMemoryInterface = nullptr;
+    ze_context_handle_t hContext;
+    L0::Context *context;
+
+    void *heapPointer = nullptr;
 };
 
 } // namespace ult

level_zero/core/test/unit_tests/fixtures/module_fixture.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "shared/source/helpers/file_io.h"
+#include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
 #include "shared/test/unit_test/helpers/test_files.h"
 
 #include "level_zero/core/source/module/module.h"
@@ -104,5 +105,22 @@ struct MultiDeviceModuleFixture : public MultiDeviceFixture {
     std::vector<std::unique_ptr<L0::Module>> modules;
 };
 
+struct ImportHostPointerModuleFixture : public ModuleFixture {
+    void SetUp() override {
+        DebugManager.flags.EnableHostPointerImport.set(1);
+        ModuleFixture::SetUp();
+
+        hostPointer = driverHandle->getMemoryManager()->allocateSystemMemory(MemoryConstants::pageSize, MemoryConstants::pageSize);
+    }
+
+    void TearDown() override {
+        driverHandle->getMemoryManager()->freeSystemMemory(hostPointer);
+        ModuleFixture::TearDown();
+    }
+
+    DebugManagerStateRestore debugRestore;
+    void *hostPointer = nullptr;
+};
+
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp

@@ -17,6 +17,7 @@
 #include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/image/image_hw.h"
 #include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
+#include "level_zero/core/test/unit_tests/fixtures/host_pointer_manager_fixture.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_cmdlist.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_event.h"
@@ -1102,5 +1103,41 @@ HWTEST_F(CommandListCreate, GivenCommandListWhenUnalignedPtrThenLeftMiddleAndRig
     itor = find<XY_COPY_BLT *>(++itor, cmdList.end());
     EXPECT_NE(cmdList.end(), itor);
 }
+
+using HostPointerManagerCommandListTest = Test<HostPointerManagerFixure>;
+HWTEST2_F(HostPointerManagerCommandListTest,
+          givenImportedHostPointerWhenAppendMemoryFillUsingHostPointerThenAppendFillUsingHostPointerAllocation,
+          Platforms) {
+    auto commandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
+    commandList->initialize(device, NEO::EngineGroupType::RenderCompute);
+
+    auto ret = hostDriverHandle->importExternalPointer(heapPointer, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+
+    int pattern = 1;
+    ret = commandList->appendMemoryFill(heapPointer, reinterpret_cast<void *>(&pattern), sizeof(pattern), 64u, nullptr);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+
+    ret = hostDriverHandle->releaseImportedPointer(heapPointer);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+}
+
+HWTEST2_F(HostPointerManagerCommandListTest,
+          givenImportedHostPointerAndCopyEngineWhenAppendMemoryFillUsingHostPointerThenAppendFillUsingHostPointerAllocation,
+          Platforms) {
+    auto commandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
+    commandList->initialize(device, NEO::EngineGroupType::Copy);
+
+    auto ret = hostDriverHandle->importExternalPointer(heapPointer, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+
+    int pattern = 1;
+    ret = commandList->appendMemoryFill(heapPointer, reinterpret_cast<void *>(&pattern), sizeof(pattern), 64u, nullptr);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+
+    ret = hostDriverHandle->releaseImportedPointer(heapPointer);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+}
+
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/sources/driver/host_pointer_manager_tests.cpp

@@ -5,8 +5,6 @@
  *
  */
 
-#include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
-
 #include "opencl/test/unit_test/mocks/mock_memory_manager.h"
 #include "test.h"
 
@@ -14,6 +12,7 @@
 #include "level_zero/core/test/unit_tests/fixtures/host_pointer_manager_fixture.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_host_pointer_manager.h"
 
+using ::testing::_;
 using ::testing::Return;
 
 namespace L0 {
@@ -313,5 +312,34 @@ TEST_F(HostPointerManagerTest, givenNoPointerRegisteredWhenAllocationCreationFai
     EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, result);
 }
 
+TEST_F(HostPointerManagerTest, givenHostAllocationImportedWhenMakingResidentAddressThenAllocationMadeResident) {
+    void *testPtr = heapPointer;
+
+    EXPECT_CALL(*mockMemoryInterface, makeResident(_, _))
+        .Times(1)
+        .WillRepeatedly(::testing::Return(NEO::MemoryOperationsStatus::SUCCESS));
+    EXPECT_CALL(*mockMemoryInterface, evict(_, _))
+        .Times(1)
+        .WillRepeatedly(::testing::Return(NEO::MemoryOperationsStatus::SUCCESS));
+
+    auto result = context->makeMemoryResident(device, testPtr, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, result);
+
+    result = hostDriverHandle->importExternalPointer(testPtr, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, result);
+
+    result = context->makeMemoryResident(device, testPtr, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, result);
+
+    result = context->evictMemory(device, testPtr, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, result);
+
+    result = hostDriverHandle->releaseImportedPointer(testPtr);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, result);
+
+    result = context->evictMemory(device, testPtr, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, result);
+}
+
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp

@@ -869,5 +869,19 @@ HWTEST2_F(SetKernelArg, givenImageAndBindfulKernelWhenSetArgImageThenCopySurface
     EXPECT_EQ(imageHW->passedSurfaceStateOffset, imageArg.bindful);
 }
 
+using ImportHostPointerSetKernelArg = Test<ImportHostPointerModuleFixture>;
+HWTEST_F(ImportHostPointerSetKernelArg, givenHostPointerImportedWhenSettingKernelArgThenUseHostPointerAllocation) {
+    createKernel();
+
+    auto ret = driverHandle->importExternalPointer(hostPointer, MemoryConstants::pageSize);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+
+    ret = kernel->setArgBuffer(0, sizeof(hostPointer), &hostPointer);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+
+    ret = driverHandle->releaseImportedPointer(hostPointer);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, ret);
+}
+
 } // namespace ult
 } // namespace L0