From 1ed273dab8dde1cd9d681f0bfb9a457bb639ed44 Mon Sep 17 00:00:00 2001 From: Raiyan Latif Date: Tue, 12 Jul 2022 16:43:14 +0000 Subject: [PATCH] Ensure shared allocations made resident across multiple devices Signed-off-by: Raiyan Latif --- level_zero/core/source/cmdlist/cmdlist_hw.inl | 9 ++++ level_zero/core/source/driver/driver_handle.h | 3 ++ .../core/source/driver/driver_handle_imp.cpp | 22 +++++++++ .../core/source/driver/driver_handle_imp.h | 4 ++ .../sources/context/test_context.cpp | 15 +++--- .../unit_tests/sources/memory/test_memory.cpp | 48 +++++++++++++++++++ 6 files changed, 95 insertions(+), 6 deletions(-) diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index 682520bf69..7d2bc13d16 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -1496,6 +1496,10 @@ ze_result_t CommandListCoreFamily::appendMemoryFill(void *ptr, NEO::SvmAllocationData *allocData = nullptr; bool dstAllocFound = device->getDriverHandle()->findAllocationDataForRange(ptr, size, &allocData); if (dstAllocFound) { + if (allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) { + DriverHandleImp *driverHandleImp = static_cast(device->getDriverHandle()); + driverHandleImp->makeMemoryResident(device->toHandle(), ptr, size); + } if (allocData->memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY || allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) { hostPointerNeedsFlush = true; @@ -1843,6 +1847,11 @@ inline AlignedAllocationData CommandListCoreFamily::getAlignedAll alignedPtr = sourcePtr; } + if (allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) { + DriverHandleImp *driverHandleImp = static_cast(device->getDriverHandle()); + driverHandleImp->makeMemoryResident(device->toHandle(), ptr, bufferSize); + } + if (allocData->memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY || allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) { hostPointerNeedsFlush = true; diff --git a/level_zero/core/source/driver/driver_handle.h b/level_zero/core/source/driver/driver_handle.h index e94b336e96..6db0815f2b 100644 --- a/level_zero/core/source/driver/driver_handle.h +++ b/level_zero/core/source/driver/driver_handle.h @@ -65,6 +65,9 @@ struct DriverHandle : _ze_driver_handle_t { size_t size, uint32_t rootDeviceIndex, uintptr_t *gpuAddress) = 0; + virtual ze_result_t makeMemoryResident(ze_device_handle_t hDevice, + void *ptr, + size_t size) = 0; static DriverHandle *fromHandle(ze_driver_handle_t handle) { return static_cast(handle); } inline ze_driver_handle_t toHandle() { return this; } diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp index 5bcf27a2ab..b725c61d95 100644 --- a/level_zero/core/source/driver/driver_handle_imp.cpp +++ b/level_zero/core/source/driver/driver_handle_imp.cpp @@ -578,6 +578,28 @@ NEO::GraphicsAllocation *DriverHandleImp::getPeerAllocation(Device *device, return alloc; } +ze_result_t DriverHandleImp::makeMemoryResident(ze_device_handle_t hDevice, void *ptr, size_t size) { + Device *device = L0::Device::fromHandle(hDevice); + NEO::Device *neoDevice = device->getNEODevice(); + auto allocation = device->getDriverHandle()->getDriverSystemMemoryAllocation( + ptr, + size, + neoDevice->getRootDeviceIndex(), + nullptr); + if (allocation == nullptr) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + + auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); + if (allocData && allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) { + DriverHandleImp *driverHandleImp = static_cast(device->getDriverHandle()); + std::lock_guard lock(driverHandleImp->sharedMakeResidentAllocationsLock); + driverHandleImp->sharedMakeResidentAllocations.insert({ptr, allocation}); + } + + return ZE_RESULT_SUCCESS; +} + void *DriverHandleImp::importNTHandle(ze_device_handle_t hDevice, void *handle) { auto neoDevice = Device::fromHandle(hDevice)->getNEODevice(); diff --git a/level_zero/core/source/driver/driver_handle_imp.h b/level_zero/core/source/driver/driver_handle_imp.h index 0a0df1ec49..ce349066df 100644 --- a/level_zero/core/source/driver/driver_handle_imp.h +++ b/level_zero/core/source/driver/driver_handle_imp.h @@ -83,6 +83,10 @@ struct DriverHandleImp : public DriverHandle { std::mutex sharedMakeResidentAllocationsLock; std::map sharedMakeResidentAllocations; + ze_result_t makeMemoryResident(ze_device_handle_t hDevice, + void *ptr, + size_t size) override; + std::vector devices; // Spec extensions const std::vector> extensionsSupported = { diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index 90d575627e..09e1f1cf3b 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -459,14 +459,14 @@ struct ContextMakeMemoryResidentAndMigrationTests : public ContextMakeMemoryResi struct MockResidentTestsPageFaultManager : public MockPageFaultManager { void moveAllocationToGpuDomain(void *ptr) override { moveAllocationToGpuDomainCalledTimes++; - migratedAddress = ptr; + migratedAddress.push_back(ptr); } void moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager) override { moveAllocationsWithinUMAllocsManagerToGpuDomainCalled++; } uint32_t moveAllocationToGpuDomainCalledTimes = 0; uint32_t moveAllocationsWithinUMAllocsManagerToGpuDomainCalled = 0; - void *migratedAddress = nullptr; + std::vector migratedAddress; }; void SetUp() override { @@ -544,7 +544,7 @@ HWTEST_F(ContextMakeMemoryResidentAndMigrationTests, EXPECT_EQ(ZE_RESULT_SUCCESS, res); EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 1u); - EXPECT_EQ(mockPageFaultManager->migratedAddress, ptr); + EXPECT_EQ(mockPageFaultManager->migratedAddress[0], ptr); mockMemoryInterface->evictResult = NEO::MemoryOperationsStatus::SUCCESS; res = context->evictMemory(device, ptr, size); @@ -642,7 +642,7 @@ HWTEST_F(ContextMakeMemoryResidentAndMigrationTests, EXPECT_EQ(ZE_RESULT_SUCCESS, res); EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 0u); - EXPECT_EQ(mockPageFaultManager->migratedAddress, nullptr); + EXPECT_EQ(mockPageFaultManager->migratedAddress.empty(), true); mockMemoryInterface->evictResult = NEO::MemoryOperationsStatus::SUCCESS; res = context->evictMemory(device, ptr, size); @@ -694,8 +694,11 @@ HWTEST_F(ContextMakeMemoryResidentAndMigrationTests, nullptr, 0, nullptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 1u); - EXPECT_EQ(mockPageFaultManager->migratedAddress, ptr); + EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 2u); + + if (mockPageFaultManager->migratedAddress[0] != ptr) { + EXPECT_EQ(mockPageFaultManager->migratedAddress[1], ptr); + } mockMemoryInterface->evictResult = NEO::MemoryOperationsStatus::SUCCESS; res = context->evictMemory(device, ptr, size); diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 4eb6c7670e..6e2598d926 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -3033,6 +3033,54 @@ TEST_F(MemoryTest, givenNoDeviceWhenAllocatingSharedMemoryThenDeviceInAllocation ASSERT_EQ(result, ZE_RESULT_SUCCESS); } +TEST_F(MemoryTest, givenCallToMakeMemoryResidentWithInvalidPointerThenInvalidArgumentIsReturned) { + void *ptr = nullptr; + ze_result_t res = driverHandle->makeMemoryResident(device, ptr, 1); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); +} + +TEST_F(MemoryTest, + givenCallToMakeMemoryResidentWithDeviceMemoryThenAllocationIsNotAddedToVectorOfResidentAllocations) { + const size_t size = 4096; + void *ptr = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t res = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, + 0, + &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + DriverHandleImp *driverHandleImp = static_cast(context->getDriverHandle()); + size_t previousSize = driverHandleImp->sharedMakeResidentAllocations.size(); + + res = driverHandle->makeMemoryResident(device, ptr, size); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + size_t currentSize = driverHandleImp->sharedMakeResidentAllocations.size(); + EXPECT_EQ(previousSize, currentSize); + + context->freeMem(ptr); +} + +TEST_F(MemoryTest, + givenCallToMakeMemoryResidentWithHeapPointerThenSuccessIsReturned) { + size_t size = 4 * MemoryConstants::pageSize; + void *ptr = driverHandle->getMemoryManager()->allocateSystemMemory(size, MemoryConstants::pageSize); + ASSERT_NE(nullptr, ptr); + + ze_result_t res = driverHandle->importExternalPointer(ptr, MemoryConstants::pageSize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = driverHandle->makeMemoryResident(device, ptr, MemoryConstants::pageSize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = driverHandle->releaseImportedPointer(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + driverHandle->getMemoryManager()->freeSystemMemory(ptr); +} + TEST_F(MemoryTest, givenCallToCheckMemoryAccessFromDeviceWithInvalidPointerThenInvalidArgumentIsReturned) { void *ptr = nullptr; ze_result_t res = driverHandle->checkMemoryAccessFromDevice(device, ptr);