From ebb1474210a67b18449d7646e8d63c43c6ffe517 Mon Sep 17 00:00:00 2001 From: Jaime Arteaga Date: Tue, 13 Apr 2021 18:26:50 +0000 Subject: [PATCH] Isolate shared allocations with respect to context Related-To: LOCI-1996 Signed-off-by: Jaime Arteaga --- .../core/source/context/context_imp.cpp | 63 +++- level_zero/core/source/context/context_imp.h | 2 +- level_zero/core/source/driver/driver_handle.h | 5 - .../core/source/driver/driver_handle_imp.cpp | 4 +- .../core/source/driver/driver_handle_imp.h | 7 - level_zero/core/source/event/event.cpp | 2 +- level_zero/core/source/memory/memory.cpp | 55 ---- .../sources/cmdlist/test_cmdlist_3.cpp | 2 +- .../test_cmdlist_append_launch_kernel.cpp | 2 +- .../sources/context/test_context.cpp | 23 ++ .../unit_tests/sources/event/test_event.cpp | 2 +- .../unit_tests/sources/memory/test_memory.cpp | 306 ++++++++++++++---- 12 files changed, 334 insertions(+), 139 deletions(-) diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index b214691def..839ba6ed63 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -57,7 +57,7 @@ ze_result_t ContextImp::allocHostMem(const ze_host_mem_alloc_desc_t *hostDesc, NEO::SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::HOST_UNIFIED_MEMORY, this->rootDeviceIndices, - this->subDeviceBitfields); + this->deviceBitfields); auto usmPtr = this->driverHandle->svmAllocsManager->createHostUnifiedMemoryAllocation(size, unifiedMemoryProperties); @@ -153,13 +153,60 @@ ze_result_t ContextImp::allocSharedMem(ze_device_handle_t hDevice, size_t size, size_t alignment, void **ptr) { - DEBUG_BREAK_IF(nullptr == this->driverHandle); - return this->driverHandle->allocSharedMem(hDevice, - deviceDesc, - hostDesc, - size, - alignment, - ptr); + bool relaxedSizeAllowed = false; + if (deviceDesc->pNext) { + const ze_base_desc_t *extendedDesc = reinterpret_cast(deviceDesc->pNext); + if (extendedDesc->stype == ZE_STRUCTURE_TYPE_RELAXED_ALLOCATION_LIMITS_EXP_DESC) { + const ze_relaxed_allocation_limits_exp_desc_t *relaxedLimitsDesc = + reinterpret_cast(extendedDesc); + if (!(relaxedLimitsDesc->flags & ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE)) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + relaxedSizeAllowed = true; + } + } + + if (relaxedSizeAllowed == false && + (size > this->devices.begin()->second->getNEODevice()->getHardwareCapabilities().maxMemAllocSize)) { + *ptr = nullptr; + return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; + } + + auto neoDevice = this->devices.begin()->second->getNEODevice(); + + auto deviceBitfields = this->deviceBitfields; + NEO::Device *unifiedMemoryPropertiesDevice = nullptr; + if (hDevice) { + if (isDeviceDefinedForThisContext(Device::fromHandle(hDevice)) == false) { + return ZE_RESULT_ERROR_DEVICE_LOST; + } + + neoDevice = Device::fromHandle(hDevice)->getNEODevice(); + auto rootDeviceIndex = neoDevice->getRootDeviceIndex(); + unifiedMemoryPropertiesDevice = neoDevice; + deviceBitfields[rootDeviceIndex] = neoDevice->getDeviceBitfield(); + } + + NEO::SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::SHARED_UNIFIED_MEMORY, + this->rootDeviceIndices, + deviceBitfields); + unifiedMemoryProperties.device = unifiedMemoryPropertiesDevice; + + if (deviceDesc->flags & ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED) { + unifiedMemoryProperties.allocationFlags.flags.locallyUncachedResource = 1; + } + + auto usmPtr = + this->driverHandle->svmAllocsManager->createSharedUnifiedMemoryAllocation(size, + unifiedMemoryProperties, + static_cast(neoDevice->getSpecializedDevice())); + + if (usmPtr == nullptr) { + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; + } + *ptr = usmPtr; + + return ZE_RESULT_SUCCESS; } ze_result_t ContextImp::freeMem(const void *ptr) { diff --git a/level_zero/core/source/context/context_imp.h b/level_zero/core/source/context/context_imp.h index 51a7124ed0..f77b24d3d2 100644 --- a/level_zero/core/source/context/context_imp.h +++ b/level_zero/core/source/context/context_imp.h @@ -116,7 +116,7 @@ struct ContextImp : Context { } std::set rootDeviceIndices = {}; - std::map subDeviceBitfields; + std::map deviceBitfields; bool isDeviceDefinedForThisContext(Device *inDevice); diff --git a/level_zero/core/source/driver/driver_handle.h b/level_zero/core/source/driver/driver_handle.h index 4a7dafee5a..8dd4f5035c 100644 --- a/level_zero/core/source/driver/driver_handle.h +++ b/level_zero/core/source/driver/driver_handle.h @@ -38,11 +38,6 @@ struct DriverHandle : _ze_driver_handle_t { ze_memory_allocation_properties_t *pMemAllocProperties, ze_device_handle_t *phDevice) = 0; - virtual ze_result_t allocSharedMem(ze_device_handle_t hDevice, const ze_device_mem_alloc_desc_t *deviceDesc, - const ze_host_mem_alloc_desc_t *hostDesc, - size_t size, - size_t alignment, - void **ptr) = 0; virtual ze_result_t freeMem(const void *ptr) = 0; virtual NEO::MemoryManager *getMemoryManager() = 0; virtual void setMemoryManager(NEO::MemoryManager *memoryManager) = 0; diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp index 279360a503..5282f96c3f 100644 --- a/level_zero/core/source/driver/driver_handle_imp.cpp +++ b/level_zero/core/source/driver/driver_handle_imp.cpp @@ -56,8 +56,8 @@ ze_result_t DriverHandleImp::createContext(const ze_context_desc_t *desc, for (auto devicePair : context->getDevices()) { auto neoDevice = devicePair.second->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), - neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), + neoDevice->getDeviceBitfield()}); } return ZE_RESULT_SUCCESS; diff --git a/level_zero/core/source/driver/driver_handle_imp.h b/level_zero/core/source/driver/driver_handle_imp.h index b1a6ee050e..57f469627e 100644 --- a/level_zero/core/source/driver/driver_handle_imp.h +++ b/level_zero/core/source/driver/driver_handle_imp.h @@ -35,13 +35,6 @@ struct DriverHandleImp : public DriverHandle { ze_memory_allocation_properties_t *pMemAllocProperties, ze_device_handle_t *phDevice) override; - ze_result_t allocSharedMem(ze_device_handle_t hDevice, - const ze_device_mem_alloc_desc_t *deviceDesc, - const ze_host_mem_alloc_desc_t *hostDesc, - size_t size, - size_t alignment, - void **ptr) override; - ze_result_t getMemAddressRange(const void *ptr, void **pBase, size_t *pSize) override; ze_result_t freeMem(const void *ptr) override; NEO::MemoryManager *getMemoryManager() override; diff --git a/level_zero/core/source/event/event.cpp b/level_zero/core/source/event/event.cpp index 5b0c494205..fccd1e5e37 100644 --- a/level_zero/core/source/event/event.cpp +++ b/level_zero/core/source/event/event.cpp @@ -89,7 +89,7 @@ ze_result_t EventPoolImp::initialize(DriverHandle *driver, Context *context, uin } uint32_t rootDeviceIndex = rootDeviceIndices.at(0); - auto &deviceBitfield = contextImp->subDeviceBitfields.at(rootDeviceIndex); + auto &deviceBitfield = contextImp->deviceBitfields.at(rootDeviceIndex); NEO::AllocationProperties unifiedMemoryProperties{rootDeviceIndex, true, diff --git a/level_zero/core/source/memory/memory.cpp b/level_zero/core/source/memory/memory.cpp index 7e3234cb1f..d3280cd889 100644 --- a/level_zero/core/source/memory/memory.cpp +++ b/level_zero/core/source/memory/memory.cpp @@ -115,61 +115,6 @@ ze_result_t DriverHandleImp::getMemAddressRange(const void *ptr, void **pBase, s return ZE_RESULT_ERROR_UNKNOWN; } -ze_result_t DriverHandleImp::allocSharedMem(ze_device_handle_t hDevice, const ze_device_mem_alloc_desc_t *deviceDesc, - const ze_host_mem_alloc_desc_t *hostDesc, - size_t size, - size_t alignment, - void **ptr) { - bool relaxedSizeAllowed = false; - if (deviceDesc->pNext) { - const ze_base_desc_t *extendedDesc = reinterpret_cast(deviceDesc->pNext); - if (extendedDesc->stype == ZE_STRUCTURE_TYPE_RELAXED_ALLOCATION_LIMITS_EXP_DESC) { - const ze_relaxed_allocation_limits_exp_desc_t *relaxedLimitsDesc = - reinterpret_cast(extendedDesc); - if (!(relaxedLimitsDesc->flags & ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE)) { - return ZE_RESULT_ERROR_INVALID_ARGUMENT; - } - relaxedSizeAllowed = true; - } - } - - if (relaxedSizeAllowed == false && - (size > this->devices[0]->getNEODevice()->getHardwareCapabilities().maxMemAllocSize)) { - *ptr = nullptr; - return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; - } - - auto neoDevice = this->devices[0]->getNEODevice(); - - auto deviceBitfields = this->deviceBitfields; - NEO::Device *unifiedMemoryPropertiesDevice = nullptr; - if (hDevice) { - neoDevice = Device::fromHandle(hDevice)->getNEODevice(); - auto rootDeviceIndex = neoDevice->getRootDeviceIndex(); - unifiedMemoryPropertiesDevice = neoDevice; - deviceBitfields[rootDeviceIndex] = neoDevice->getDeviceBitfield(); - } - - NEO::SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::SHARED_UNIFIED_MEMORY, this->rootDeviceIndices, deviceBitfields); - unifiedMemoryProperties.device = unifiedMemoryPropertiesDevice; - - if (deviceDesc->flags & ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED) { - unifiedMemoryProperties.allocationFlags.flags.locallyUncachedResource = 1; - } - - auto usmPtr = - svmAllocsManager->createSharedUnifiedMemoryAllocation(size, - unifiedMemoryProperties, - static_cast(neoDevice->getSpecializedDevice())); - - if (usmPtr == nullptr) { - return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; - } - *ptr = usmPtr; - - return ZE_RESULT_SUCCESS; -} - ze_result_t DriverHandleImp::freeMem(const void *ptr) { auto allocation = svmAllocsManager->getSVMAlloc(ptr); if (allocation == nullptr) { diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_3.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_3.cpp index b46a164ad2..0ad9a26059 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_3.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_3.cpp @@ -438,7 +438,7 @@ HWTEST2_F(CommandListCreate, givenCommandListWhenMemoryFillHavingSharedMemoryWit void *dst_buffer = nullptr; ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - result = device->getDriverHandle()->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, 16384u, 4096u, &dst_buffer); + result = context->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, 16384u, 4096u, &dst_buffer); ASSERT_EQ(ZE_RESULT_SUCCESS, result); ze_event_pool_desc_t eventPoolDesc = {}; diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel.cpp index e91b7eeb06..69258e0e89 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel.cpp @@ -578,7 +578,7 @@ HWTEST_F(CommandListDualStroage, givenIndirectDispatchWithSharedDualStorageMemor void *alloc = nullptr; ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - auto result = device->getDriverHandle()->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, 16384u, 4096u, &alloc); + auto result = context->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, 16384u, 4096u, &alloc); ASSERT_EQ(ZE_RESULT_SUCCESS, result); ze_group_count_t *pThreadGroupDimensions = static_cast(ptrOffset(alloc, sizeof(ze_group_count_t))); diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index 4db0f57045..9ccaff359b 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -113,6 +113,29 @@ TEST_F(MultiDeviceContextTests, EXPECT_EQ(ZE_RESULT_SUCCESS, res); } +TEST_F(MultiDeviceContextTests, + whenAllocatingSharedMemoryWithDeviceNotDefinedForContextThenDeviceLostIsReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc; + + ze_device_handle_t device = driverHandle->devices[1]->toHandle(); + + ze_result_t res = driverHandle->createContext(&desc, 1u, &device, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(Context::fromHandle(hContext)); + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_host_mem_alloc_desc_t hostDesc = {}; + size_t size = 4096; + void *ptr = nullptr; + res = contextImp->allocSharedMem(driverHandle->devices[0]->toHandle(), &deviceDesc, &hostDesc, size, 0u, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_DEVICE_LOST, res); + + res = L0::Context::fromHandle(hContext)->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + struct SVMAllocsManagerContextMock : public NEO::SVMAllocsManager { SVMAllocsManagerContextMock(MemoryManager *memoryManager) : NEO::SVMAllocsManager(memoryManager, false) {} void *createHostUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties &memoryProperties) override { diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index 42db3ae1d6..6ca27d3aa6 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -80,7 +80,7 @@ struct EventPoolFailTests : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 8a9f9a683b..7e55d73b6b 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -77,10 +77,10 @@ TEST_F(MemoryTest, whenAllocatingSharedMemoryWithUncachedFlagThenLocallyUncached ze_device_mem_alloc_desc_t deviceDesc = {}; deviceDesc.flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -124,7 +124,7 @@ struct OutOfMemoryTests : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -194,7 +194,7 @@ struct MemoryRelaxedSizeTests : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -227,7 +227,7 @@ TEST_F(MemoryRelaxedSizeTests, } TEST_F(MemoryRelaxedSizeTests, - givenCallToDeviceAllocWithLargerThanAllowdSizeAndWithoutRelaxedFlagThenAllocationIsNotMade) { + givenCallToDeviceAllocWithLargerThanAllowedSizeAndWithoutRelaxedFlagThenAllocationIsNotMade) { size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; size_t alignment = 1u; void *ptr = nullptr; @@ -241,7 +241,7 @@ TEST_F(MemoryRelaxedSizeTests, } TEST_F(MemoryRelaxedSizeTests, - givenCallToDeviceAllocWithLargerThanAllowdSizeAndRelaxedFlagThenAllocationIsMade) { + givenCallToDeviceAllocWithLargerThanAllowedSizeAndRelaxedFlagThenAllocationIsMade) { size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; size_t alignment = 1u; void *ptr = nullptr; @@ -263,7 +263,7 @@ TEST_F(MemoryRelaxedSizeTests, } TEST_F(MemoryRelaxedSizeTests, - givenCallToDeviceAllocWithLargerThanAllowdSizeAndRelaxedFlagWithIncorrectFlagThenAllocationIsNotMade) { + givenCallToDeviceAllocWithLargerThanAllowedSizeAndRelaxedFlagWithIncorrectFlagThenAllocationIsNotMade) { size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; size_t alignment = 1u; void *ptr = nullptr; @@ -281,6 +281,25 @@ TEST_F(MemoryRelaxedSizeTests, EXPECT_EQ(nullptr, ptr); } +TEST_F(MemoryRelaxedSizeTests, + givenCallToDeviceAllocWithLargerThanAllowedSizeAndRelaxedDescriptorWithWrongStypeThenUnsupportedSizeIsReturned) { + size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + deviceDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + ze_relaxed_allocation_limits_exp_desc_t relaxedSizeDesc = {}; + relaxedSizeDesc.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; + relaxedSizeDesc.flags = static_cast(ZE_BIT(1)); + deviceDesc.pNext = &relaxedSizeDesc; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, result); + EXPECT_EQ(nullptr, ptr); +} + TEST_F(MemoryRelaxedSizeTests, givenCallToSharedAllocWithAllowedSizeAndWithoutRelaxedFlagThenAllocationIsMade) { size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize - 1; @@ -289,10 +308,10 @@ TEST_F(MemoryRelaxedSizeTests, ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -308,16 +327,16 @@ TEST_F(MemoryRelaxedSizeTests, ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, result); EXPECT_EQ(nullptr, ptr); } TEST_F(MemoryRelaxedSizeTests, - givenCallToSharedAllocWithLargerThanAllowdSizeAndRelaxedFlagThenAllocationIsMade) { + givenCallToSharedAllocWithLargerThanAllowedSizeAndRelaxedFlagThenAllocationIsMade) { size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; size_t alignment = 1u; void *ptr = nullptr; @@ -329,10 +348,10 @@ TEST_F(MemoryRelaxedSizeTests, relaxedSizeDesc.flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE; deviceDesc.pNext = &relaxedSizeDesc; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -341,7 +360,7 @@ TEST_F(MemoryRelaxedSizeTests, } TEST_F(MemoryRelaxedSizeTests, - givenCallToSharedAllocWithLargerThanAllowdSizeAndRelaxedFlagWithIncorrectFlagThenAllocationIsNotMade) { + givenCallToSharedAllocWithLargerThanAllowedSizeAndRelaxedFlagWithIncorrectFlagThenAllocationIsNotMade) { size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; size_t alignment = 1u; void *ptr = nullptr; @@ -353,14 +372,35 @@ TEST_F(MemoryRelaxedSizeTests, relaxedSizeDesc.flags = static_cast(ZE_BIT(1)); deviceDesc.pNext = &relaxedSizeDesc; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, result); EXPECT_EQ(nullptr, ptr); } +TEST_F(MemoryRelaxedSizeTests, + givenCallToSharedAllocWithLargerThanAllowedSizeAndRelaxedDescriptorWithWrongStypeThenUnsupportedSizeIsReturned) { + size_t size = device->getNEODevice()->getHardwareCapabilities().maxMemAllocSize + 1; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + deviceDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + ze_relaxed_allocation_limits_exp_desc_t relaxedSizeDesc = {}; + relaxedSizeDesc.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; + relaxedSizeDesc.flags = static_cast(ZE_BIT(1)); + deviceDesc.pNext = &relaxedSizeDesc; + ze_host_mem_alloc_desc_t hostDesc = {}; + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, result); + EXPECT_EQ(nullptr, ptr); +} + struct DriverHandleGetFdMock : public L0::DriverHandleImp { void *importFdHandle(ze_device_handle_t hDevice, ze_ipc_memory_flags_t flags, uint64_t handle) override { if (mockFd == allocationMap.second) { @@ -426,7 +466,7 @@ struct MemoryExportImportTest : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -845,7 +885,7 @@ struct MemoryGetIpcHandleTest : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -967,7 +1007,7 @@ struct MemoryOpenIpcHandleTest : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -1033,7 +1073,7 @@ struct MemoryFailedOpenIpcHandleTest : public ::testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -1096,10 +1136,10 @@ TEST_F(MemoryTest, givenSharedPointerThenDriverGetAllocPropertiesReturnsDeviceHa ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -1162,10 +1202,10 @@ TEST_F(MemoryTest, givenSharedPointerAndDeviceHandleAsNullThenDriverReturnsSucce ASSERT_NE(nullptr, device->toHandle()); ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(nullptr, - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(nullptr, + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -1181,10 +1221,10 @@ TEST_F(MemoryTest, givenNoDeviceWhenAllocatingSharedMemoryThenDeviceInAllocation ASSERT_NE(nullptr, device->toHandle()); ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t result = driverHandle->allocSharedMem(nullptr, - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t result = context->allocSharedMem(nullptr, + &deviceDesc, + &hostDesc, + size, alignment, &ptr); auto alloc = driverHandle->svmAllocsManager->getSVMAlloc(ptr); EXPECT_EQ(alloc->device, nullptr); @@ -1227,10 +1267,10 @@ TEST_F(MemoryTest, givenCallToCheckMemoryAccessFromDeviceWithValidSharedAllocati ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - ze_result_t res = driverHandle->allocSharedMem(device->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + ze_result_t res = context->allocSharedMem(device->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); EXPECT_NE(nullptr, ptr); @@ -1284,7 +1324,7 @@ struct MemoryBitfieldTest : testing::Test { context->getDevices().insert(std::make_pair(device->toHandle(), device)); auto neoDevice = device->getNEODevice(); context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); - context->subDeviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); } void TearDown() override { @@ -1337,16 +1377,25 @@ TEST(MemoryBitfieldTests, givenDeviceWithValidBitfieldWhenAllocatingSharedMemory auto driverHandle = std::make_unique>(); driverHandle->initialize(std::move(devices)); + auto device = driverHandle->devices[0]; + std::unique_ptr context; + context = std::make_unique(driverHandle.get()); + EXPECT_NE(context, nullptr); + context->getDevices().insert(std::make_pair(device->toHandle(), device)); + auto neoDevice = device->getNEODevice(); + context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + memoryManager->recentlyPassedDeviceBitfield = {}; ASSERT_NE(nullptr, driverHandle->devices[1]->toHandle()); EXPECT_NE(neoDevice0->getDeviceBitfield(), neoDevice1->getDeviceBitfield()); EXPECT_NE(neoDevice0->getDeviceBitfield(), memoryManager->recentlyPassedDeviceBitfield); ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; - auto result = driverHandle->allocSharedMem(nullptr, - &deviceDesc, - &hostDesc, - size, alignment, &ptr); + auto result = context->allocSharedMem(nullptr, + &deviceDesc, + &hostDesc, + size, alignment, &ptr); EXPECT_EQ(neoDevice0->getDeviceBitfield(), memoryManager->recentlyPassedDeviceBitfield); EXPECT_EQ(ZE_RESULT_SUCCESS, result); @@ -1356,11 +1405,11 @@ TEST(MemoryBitfieldTests, givenDeviceWithValidBitfieldWhenAllocatingSharedMemory memoryManager->recentlyPassedDeviceBitfield = {}; EXPECT_NE(neoDevice1->getDeviceBitfield(), memoryManager->recentlyPassedDeviceBitfield); - result = driverHandle->allocSharedMem(driverHandle->devices[1]->toHandle(), - &deviceDesc, - &hostDesc, - size, alignment, &ptr); - EXPECT_EQ(neoDevice1->getDeviceBitfield(), memoryManager->recentlyPassedDeviceBitfield); + result = context->allocSharedMem(driverHandle->devices[0]->toHandle(), + &deviceDesc, + &hostDesc, + size, alignment, &ptr); + EXPECT_EQ(neoDevice0->getDeviceBitfield(), memoryManager->recentlyPassedDeviceBitfield); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -1655,5 +1704,148 @@ TEST_F(ImportFdUncachedTests, driverHandle->freeMem(ptr); } +struct SVMAllocsManagerSharedAllocFailMock : public NEO::SVMAllocsManager { + SVMAllocsManagerSharedAllocFailMock(MemoryManager *memoryManager) : NEO::SVMAllocsManager(memoryManager, false) {} + void *createSharedUnifiedMemoryAllocation(size_t size, + const UnifiedMemoryProperties &svmProperties, + void *cmdQ) override { + return nullptr; + } +}; + +struct SharedAllocFailTests : public ::testing::Test { + void SetUp() override { + NEO::MockCompilerEnableGuard mock(true); + neoDevice = + NEO::MockDevice::createWithNewExecutionEnvironment(NEO::defaultHwInfo.get()); + auto mockBuiltIns = new MockBuiltins(); + neoDevice->executionEnvironment->rootDeviceEnvironments[0]->builtins.reset(mockBuiltIns); + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + driverHandle = std::make_unique(); + driverHandle->initialize(std::move(devices)); + prevSvmAllocsManager = driverHandle->svmAllocsManager; + currSvmAllocsManager = new SVMAllocsManagerSharedAllocFailMock(driverHandle->memoryManager); + driverHandle->svmAllocsManager = currSvmAllocsManager; + device = driverHandle->devices[0]; + + context = std::make_unique(driverHandle.get()); + EXPECT_NE(context, nullptr); + context->getDevices().insert(std::make_pair(device->toHandle(), device)); + auto neoDevice = device->getNEODevice(); + context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + } + + void TearDown() override { + driverHandle->svmAllocsManager = prevSvmAllocsManager; + delete currSvmAllocsManager; + } + NEO::SVMAllocsManager *prevSvmAllocsManager; + NEO::SVMAllocsManager *currSvmAllocsManager; + std::unique_ptr driverHandle; + NEO::MockDevice *neoDevice = nullptr; + L0::Device *device = nullptr; + std::unique_ptr context; +}; + +TEST_F(SharedAllocFailTests, whenAllocatinSharedMemoryAndAllocationFailsThenOutOfDeviceMemoryIsReturned) { + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_host_mem_alloc_desc_t hostDesc = {}; + void *ptr = nullptr; + size_t size = 1024; + ze_result_t res = context->allocSharedMem(nullptr, &deviceDesc, &hostDesc, size, 0u, &ptr); + EXPECT_EQ(res, ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY); +} + +struct SVMAllocsManagerSharedAllocMultiDeviceMock : public NEO::SVMAllocsManager { + SVMAllocsManagerSharedAllocMultiDeviceMock(MemoryManager *memoryManager) : NEO::SVMAllocsManager(memoryManager, false) {} + void *createHostUnifiedMemoryAllocation(size_t size, + const UnifiedMemoryProperties &memoryProperties) override { + createHostUnifiedMemoryAllocationTimes++; + return alignedMalloc(4096u, 4096u); + } + + uint32_t createHostUnifiedMemoryAllocationTimes = 0; +}; + +struct DriverHandleMultiDeviceMock : public DriverHandleImp { + ze_result_t freeMem(const void *ptr) override { + SVMAllocsManagerSharedAllocMultiDeviceMock *currSvmAllocsManager = + static_cast(this->svmAllocsManager); + if (currSvmAllocsManager->createHostUnifiedMemoryAllocationTimes == 0) { + return DriverHandleImp::freeMem(ptr); + } + alignedFree(const_cast(ptr)); + return ZE_RESULT_SUCCESS; + } +}; + +struct SharedAllocMultiDeviceTests : public ::testing::Test { + void SetUp() override { + NEO::MockCompilerEnableGuard mock(true); + DebugManager.flags.CreateMultipleRootDevices.set(numRootDevices); + auto executionEnvironment = new NEO::ExecutionEnvironment; + auto devices = NEO::DeviceFactory::createDevices(*executionEnvironment); + driverHandle = std::make_unique(); + ze_result_t res = driverHandle->initialize(std::move(devices)); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + prevSvmAllocsManager = driverHandle->svmAllocsManager; + currSvmAllocsManager = new SVMAllocsManagerSharedAllocMultiDeviceMock(driverHandle->memoryManager); + driverHandle->svmAllocsManager = currSvmAllocsManager; + + context = std::make_unique(driverHandle.get()); + EXPECT_NE(context, nullptr); + + for (uint32_t i = 0; i < numRootDevices; i++) { + auto device = driverHandle->devices[i]; + context->getDevices().insert(std::make_pair(device->toHandle(), device)); + auto neoDevice = device->getNEODevice(); + context->rootDeviceIndices.insert(neoDevice->getRootDeviceIndex()); + context->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + } + } + + void TearDown() override { + driverHandle->svmAllocsManager = prevSvmAllocsManager; + delete currSvmAllocsManager; + } + + DebugManagerStateRestore restorer; + NEO::SVMAllocsManager *prevSvmAllocsManager; + SVMAllocsManagerSharedAllocMultiDeviceMock *currSvmAllocsManager; + std::unique_ptr driverHandle; + std::unique_ptr context; + const uint32_t numRootDevices = 4u; +}; + +TEST_F(SharedAllocMultiDeviceTests, whenAllocatinSharedMemoryWithNullDeviceInAMultiDeviceSystemThenHostAllocationIsCreated) { + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_host_mem_alloc_desc_t hostDesc = {}; + void *ptr = nullptr; + size_t size = 1024; + EXPECT_EQ(currSvmAllocsManager->createHostUnifiedMemoryAllocationTimes, 0u); + ze_result_t res = context->allocSharedMem(nullptr, &deviceDesc, &hostDesc, size, 0u, &ptr); + EXPECT_EQ(res, ZE_RESULT_SUCCESS); + EXPECT_EQ(currSvmAllocsManager->createHostUnifiedMemoryAllocationTimes, 1u); + + res = context->freeMem(ptr); + EXPECT_EQ(res, ZE_RESULT_SUCCESS); +} + +TEST_F(SharedAllocMultiDeviceTests, whenAllocatinSharedMemoryWithNonNullDeviceInAMultiDeviceSystemThenSharedAllocationIsCreated) { + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_host_mem_alloc_desc_t hostDesc = {}; + void *ptr = nullptr; + size_t size = 1024; + EXPECT_EQ(currSvmAllocsManager->createHostUnifiedMemoryAllocationTimes, 0u); + ze_result_t res = context->allocSharedMem(driverHandle->devices[0]->toHandle(), &deviceDesc, &hostDesc, size, 0u, &ptr); + EXPECT_EQ(res, ZE_RESULT_SUCCESS); + EXPECT_EQ(currSvmAllocsManager->createHostUnifiedMemoryAllocationTimes, 0u); + + res = context->freeMem(ptr); + EXPECT_EQ(res, ZE_RESULT_SUCCESS); +} + } // namespace ult } // namespace L0