diff --git a/level_zero/core/source/cmdlist/cmdlist.h b/level_zero/core/source/cmdlist/cmdlist.h index aec23da606..65ffa6c074 100644 --- a/level_zero/core/source/cmdlist/cmdlist.h +++ b/level_zero/core/source/cmdlist/cmdlist.h @@ -51,6 +51,15 @@ struct CmdListReturnPoint { NEO::GraphicsAllocation *currentCmdBuffer = nullptr; }; +struct MemAdviseOperation { + ze_device_handle_t hDevice; + const void *ptr; + const size_t size; + ze_memory_advice_t advice; +}; + +using AppendedMemAdviseOperations = std::vector; + struct CommandList : _ze_command_list_handle_t { static constexpr uint32_t defaultNumIddsPerBlock = 64u; static constexpr uint32_t commandListimmediateIddsPerBlock = 1u; @@ -114,6 +123,9 @@ struct CommandList : _ze_command_list_handle_t { uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, bool relaxedOrderingDispatch) = 0; virtual ze_result_t appendMemAdvise(ze_device_handle_t hDevice, const void *ptr, size_t size, ze_memory_advice_t advice) = 0; + virtual ze_result_t executeMemAdvise(ze_device_handle_t hDevice, + const void *ptr, size_t size, + ze_memory_advice_t advice) = 0; virtual ze_result_t appendMemoryCopy(void *dstptr, const void *srcptr, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) = 0; @@ -329,6 +341,10 @@ struct CommandList : _ze_command_list_handle_t { return prefetchContext; } + AppendedMemAdviseOperations &getMemAdviseOperations() { + return memAdviseOperations; + } + NEO::HeapAddressModel getCmdListHeapAddressModel() const { return this->cmdListHeapAddressModel; } @@ -437,6 +453,7 @@ struct CommandList : _ze_command_list_handle_t { CommandsToPatch commandsToPatch{}; UnifiedMemoryControls unifiedMemoryControls; NEO::PrefetchContext prefetchContext; + AppendedMemAdviseOperations memAdviseOperations; NEO::L1CachePolicy l1CachePolicyData{}; NEO::EncodeDummyBlitWaArgs dummyBlitWa{}; diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.h b/level_zero/core/source/cmdlist/cmdlist_hw.h index 69c89ea3b8..8149d74d5e 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.h +++ b/level_zero/core/source/cmdlist/cmdlist_hw.h @@ -135,6 +135,9 @@ struct CommandListCoreFamily : public CommandListImp { ze_result_t appendMemAdvise(ze_device_handle_t hDevice, const void *ptr, size_t size, ze_memory_advice_t advice) override; + ze_result_t executeMemAdvise(ze_device_handle_t hDevice, + const void *ptr, size_t size, + ze_memory_advice_t advice) override; ze_result_t appendMemoryCopy(void *dstptr, const void *srcptr, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) override; diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index cb6a18a646..f25bc31a28 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -1247,68 +1247,121 @@ template ze_result_t CommandListCoreFamily::appendMemAdvise(ze_device_handle_t hDevice, const void *ptr, size_t size, ze_memory_advice_t advice) { - NEO::MemAdviseFlags flags{}; - auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - if (allocData) { - DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(hDevice))); - - if (deviceImp->memAdviseSharedAllocations.find(allocData) != deviceImp->memAdviseSharedAllocations.end()) { - flags = deviceImp->memAdviseSharedAllocations[allocData]; - } - - switch (advice) { - case ZE_MEMORY_ADVICE_SET_READ_MOSTLY: - flags.readOnly = 1; - break; - case ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY: - flags.readOnly = 0; - break; - case ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION: - flags.devicePreferredLocation = 1; - break; - case ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION: - flags.devicePreferredLocation = 0; - break; - case ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION: - flags.systemPreferredLocation = 1; - break; - case ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION: - flags.systemPreferredLocation = 0; - break; - case ZE_MEMORY_ADVICE_BIAS_CACHED: - flags.cachedMemory = 1; - break; - case ZE_MEMORY_ADVICE_BIAS_UNCACHED: - flags.cachedMemory = 0; - break; - case ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY: - case ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY: - default: - break; - } - - auto memoryManager = device->getDriverHandle()->getMemoryManager(); - auto pageFaultManager = memoryManager->getPageFaultManager(); - if (pageFaultManager) { - /* If Read Only and Device Preferred Hints have been cleared, then cpu_migration of Shared memory can be re-enabled*/ - if (flags.cpuMigrationBlocked) { - if (flags.readOnly == 0 && flags.devicePreferredLocation == 0) { - pageFaultManager->protectCPUMemoryAccess(const_cast(ptr), size); - flags.cpuMigrationBlocked = 0; - } - } - /* Given MemAdvise hints, use different gpu Domain Handler for the Page Fault Handling */ - pageFaultManager->setGpuDomainHandler(L0::transferAndUnprotectMemoryWithHints); - } - - auto alloc = allocData->gpuAllocations.getGraphicsAllocation(deviceImp->getRootDeviceIndex()); - memoryManager->setMemAdvise(alloc, flags, deviceImp->getRootDeviceIndex()); - - deviceImp->memAdviseSharedAllocations[allocData] = flags; - return ZE_RESULT_SUCCESS; + if (ptr == nullptr) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; } - return ZE_RESULT_ERROR_INVALID_ARGUMENT; + + this->memAdviseOperations.push_back(MemAdviseOperation(hDevice, ptr, size, advice)); + + return ZE_RESULT_SUCCESS; +} + +template +ze_result_t CommandListCoreFamily::executeMemAdvise(ze_device_handle_t hDevice, + const void *ptr, size_t size, + ze_memory_advice_t advice) { + + auto driverHandle = device->getDriverHandle(); + auto allocData = driverHandle->getSvmAllocsManager()->getSVMAlloc(ptr); + + if (!allocData) { + if (device->getNEODevice()->areSharedSystemAllocationsAllowed()) { + NEO::MemAdvise memAdviseOp = NEO::MemAdvise::invalidAdvise; + + switch (advice) { + case ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION: + memAdviseOp = NEO::MemAdvise::setPreferredLocation; + break; + case ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION: + memAdviseOp = NEO::MemAdvise::clearPreferredLocation; + break; + case ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION: + memAdviseOp = NEO::MemAdvise::setSystemMemoryPreferredLocation; + break; + case ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION: + memAdviseOp = NEO::MemAdvise::clearSystemMemoryPreferredLocation; + break; + case ZE_MEMORY_ADVICE_SET_READ_MOSTLY: + case ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY: + case ZE_MEMORY_ADVICE_BIAS_CACHED: + case ZE_MEMORY_ADVICE_BIAS_UNCACHED: + case ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY: + case ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY: + default: + return ZE_RESULT_SUCCESS; + } + + DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(hDevice))); + auto memoryManager = device->getDriverHandle()->getMemoryManager(); + + memoryManager->setSharedSystemMemAdvise(ptr, size, memAdviseOp, deviceImp->getRootDeviceIndex()); + + return ZE_RESULT_SUCCESS; + } else { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + } + + NEO::MemAdviseFlags flags{}; + DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(hDevice))); + + if (deviceImp->memAdviseSharedAllocations.find(allocData) != deviceImp->memAdviseSharedAllocations.end()) { + flags = deviceImp->memAdviseSharedAllocations[allocData]; + } + + switch (advice) { + case ZE_MEMORY_ADVICE_SET_READ_MOSTLY: + flags.readOnly = 1; + break; + case ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY: + flags.readOnly = 0; + break; + case ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION: + flags.devicePreferredLocation = 1; + break; + case ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION: + flags.devicePreferredLocation = 0; + break; + case ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION: + flags.systemPreferredLocation = 1; + break; + case ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION: + flags.systemPreferredLocation = 0; + break; + case ZE_MEMORY_ADVICE_BIAS_CACHED: + flags.cachedMemory = 1; + break; + case ZE_MEMORY_ADVICE_BIAS_UNCACHED: + flags.cachedMemory = 0; + break; + case ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY: + case ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY: + default: + break; + } + + auto memoryManager = device->getDriverHandle()->getMemoryManager(); + auto pageFaultManager = memoryManager->getPageFaultManager(); + + if (pageFaultManager) { + /* If Read Only and Device Preferred Hints have been cleared, then cpu_migration of Shared memory can be re-enabled*/ + if (flags.cpuMigrationBlocked) { + if (flags.readOnly == 0 && flags.devicePreferredLocation == 0) { + pageFaultManager->protectCPUMemoryAccess(const_cast(ptr), size); + flags.cpuMigrationBlocked = 0; + } + } + /* Given MemAdvise hints, use different gpu Domain Handler for the Page Fault Handling */ + pageFaultManager->setGpuDomainHandler(L0::transferAndUnprotectMemoryWithHints); + } + + auto alloc = allocData->gpuAllocations.getGraphicsAllocation(deviceImp->getRootDeviceIndex()); + memoryManager->setMemAdvise(alloc, flags, deviceImp->getRootDeviceIndex()); + + deviceImp->memAdviseSharedAllocations[allocData] = flags; + + return ZE_RESULT_SUCCESS; } template diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl b/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl index 3f9894e963..308ad5b19b 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl @@ -495,6 +495,11 @@ inline ze_result_t CommandListCoreFamilyImmediate::executeCommand cmdQImp->makeResidentForResidencyContainer(this->commandContainer.getResidencyContainer()); } + for (auto &operation : this->memAdviseOperations) { + this->executeMemAdvise(operation.hDevice, operation.ptr, operation.size, operation.advice); + } + this->memAdviseOperations.clear(); + NEO::CompletionStamp completionStamp; if (cmdQ->peekIsCopyOnlyCommandQueue()) { completionStamp = flushBcsTask(*commandStream, commandStreamStart, hasStallingCmds, hasRelaxedOrderingDependencies, requireTaskCountUpdate, appendOperation, csr); diff --git a/level_zero/core/source/cmdqueue/cmdqueue_hw.h b/level_zero/core/source/cmdqueue/cmdqueue_hw.h index 5aa483304c..3fdaf5363d 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_hw.h +++ b/level_zero/core/source/cmdqueue/cmdqueue_hw.h @@ -114,6 +114,8 @@ struct CommandQueueHw : public CommandQueueImp { bool containsParentImmediateStream = false; }; + inline void processMemAdviseOperations(CommandList *commandList); + ze_result_t executeCommandListsRegularHeapless(CommandListExecutionContext &ctx, uint32_t numCommandLists, ze_command_list_handle_t *commandListHandles, diff --git a/level_zero/core/source/cmdqueue/cmdqueue_hw.inl b/level_zero/core/source/cmdqueue/cmdqueue_hw.inl index 8459f1d8d9..7960d1fccb 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_hw.inl +++ b/level_zero/core/source/cmdqueue/cmdqueue_hw.inl @@ -133,6 +133,15 @@ ze_result_t CommandQueueHw::executeCommandLists( return ret; } +template +void CommandQueueHw::processMemAdviseOperations(CommandList *commandList) { + auto &memAdviseOperations = commandList->getMemAdviseOperations(); + for (auto &operation : memAdviseOperations) { + commandList->executeMemAdvise(operation.hDevice, operation.ptr, operation.size, operation.advice); + } + memAdviseOperations.clear(); +} + template ze_result_t CommandQueueHw::executeCommandListsRegularHeapless( CommandListExecutionContext &ctx, @@ -208,6 +217,7 @@ ze_result_t CommandQueueHw::executeCommandListsRegularHeapless( cmdListWithAssertExecuted.exchange(true); } + this->processMemAdviseOperations(commandList); this->collectPrintfContentsFromCommandsList(commandList); } @@ -433,6 +443,7 @@ ze_result_t CommandQueueHw::executeCommandListsRegular( cmdListWithAssertExecuted.exchange(true); } + this->processMemAdviseOperations(commandList); this->collectPrintfContentsFromCommandsList(commandList); } @@ -525,6 +536,7 @@ ze_result_t CommandQueueHw::executeCommandListsCopyOnly( this->programOneCmdListBatchBufferStart(commandList, *streamForDispatch, ctx); this->prefetchMemoryToDeviceAssociatedWithCmdList(commandList); } + this->migrateSharedAllocationsIfRequested(ctx.isMigrationRequested, ctx.firstCommandList); this->assignCsrTaskCountToFenceIfAvailable(hFence); diff --git a/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h b/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h index 131bd419f9..4d0c5d522c 100644 --- a/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h +++ b/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h @@ -446,6 +446,12 @@ struct MockCommandList : public CommandList { size_t size, ze_memory_advice_t advice)); + ADDMETHOD_NOBASE(executeMemAdvise, ze_result_t, ZE_RESULT_SUCCESS, + (ze_device_handle_t hDevice, + const void *ptr, + size_t size, + ze_memory_advice_t advice)); + ADDMETHOD_NOBASE(appendMemoryCopy, ze_result_t, ZE_RESULT_SUCCESS, (void *dstptr, const void *srcptr, @@ -623,6 +629,13 @@ class MockCommandListCoreFamily : public CommandListCoreFamily { using BaseClass::ownedPrivateAllocations; using BaseClass::taskCountUpdateFenceRequired; + ze_result_t executeMemAdvise(ze_device_handle_t hDevice, + const void *ptr, size_t size, + ze_memory_advice_t advice) override { + executeMemAdviseCallCount++; + return ZE_RESULT_SUCCESS; + } + ADDMETHOD(appendMemoryCopyKernelWithGA, ze_result_t, false, ZE_RESULT_SUCCESS, (void *dstPtr, NEO::GraphicsAllocation *dstPtrAlloc, uint64_t dstOffset, void *srcPtr, @@ -699,6 +712,7 @@ class MockCommandListCoreFamily : public CommandListCoreFamily { uintptr_t dstAlignedPtr; size_t srcBlitCopyRegionOffset = 0; size_t dstBlitCopyRegionOffset = 0; + uint32_t executeMemAdviseCallCount = 0; }; template @@ -830,5 +844,25 @@ class MockCommandListForAppendLaunchKernel : public WhiteBox<::L0::CommandListCo } }; +template +class MockCommandListForExecuteMemAdvise : public WhiteBox<::L0::CommandListCoreFamilyImmediate> { + public: + using BaseClass = WhiteBox<::L0::CommandListCoreFamilyImmediate>; + + using BaseClass::cmdQImmediate; + using BaseClass::indirectAllocationsAllowed; + + using BaseClass::executeCommandListImmediateWithFlushTaskImpl; + + ze_result_t executeMemAdvise(ze_device_handle_t hDevice, + const void *ptr, size_t size, + ze_memory_advice_t advice) override { + executeMemAdviseCallCount++; + return ZE_RESULT_SUCCESS; + } + + uint32_t executeMemAdviseCallCount = 0; +}; + } // namespace ult } // namespace L0 diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp index 0206bf4892..7d232867df 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp @@ -11,6 +11,7 @@ #include "shared/source/helpers/compiler_product_helper.h" #include "shared/source/indirect_heap/indirect_heap.h" #include "shared/source/memory_manager/internal_allocation_storage.h" +#include "shared/source/unified_memory/usm_memory_support.h" #include "shared/test/common/cmd_parse/gen_cmd_parse.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/helpers/relaxed_ordering_commands_helper.h" @@ -184,7 +185,7 @@ TEST_F(CommandListCreateTests, givenNonExistingPtrThenAppendMemoryPrefetchReturn EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); } -TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseFailsThenReturnSuccess) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenExecuteMemAdviseFailsThenReturnSuccess) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -203,14 +204,183 @@ TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseFailsThenReturnSu auto memoryManager = static_cast(device->getDriverHandle()->getMemoryManager()); memoryManager->failSetMemAdvise = true; + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = context->freeMem(ptr); + ASSERT_EQ(res, ZE_RESULT_SUCCESS); +} + +TEST_F(CommandListCreateTests, givenValidSystemAlloctedPtrAndNotSharedSystemAllocationsAllowedWhenExecuteMemAdviseFailsThenReturnError) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableSharedSystemUsmSupport.set(1u); + debugManager.flags.EnableRecoverablePageFaults.set(1u); + + size_t size = 10; + void *ptr = nullptr; + + ptr = malloc(size); + EXPECT_NE(nullptr, ptr); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); + ASSERT_NE(nullptr, commandList); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + + sharedSystemMemCapabilities = 0; // enables return false for Device::areSharedSystemAllocationsAllowed() + + auto res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + free(ptr); +} + +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenAppendMemAdviseSuccedsThenMemAdviseOperationsGrows) { + + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + auto res = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_NE(nullptr, ptr); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); + ASSERT_NE(nullptr, commandList); + res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + EXPECT_EQ(1u, commandList->getMemAdviseOperations().size()); EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = context->freeMem(ptr); ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseSucceedsThenReturnSuccess) { +TEST_F(CommandListCreateTests, givenValidSystemAlloctedPtrAndSharedSystemAllocationsAllowedWhenAppendMemAdviseSuccedsThenMemAdviseOperationsGrows) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableSharedSystemUsmSupport.set(1u); + debugManager.flags.EnableRecoverablePageFaults.set(1u); + + size_t size = 10; + void *ptr = nullptr; + + ptr = malloc(size); + EXPECT_NE(nullptr, ptr); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); + ASSERT_NE(nullptr, commandList); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + + sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled; + + auto res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + EXPECT_EQ(1u, commandList->getMemAdviseOperations().size()); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + free(ptr); +} + +using ::testing::ValuesIn; + +class SupportedMemAdviceSystemAllocatorTests : public CommandListCreateTests, public ::testing::WithParamInterface {}; + +TEST_P(SupportedMemAdviceSystemAllocatorTests, givenValidSystemAlloctedPtrWhenExecuteMemAdviseWithSupportedAdviceFailsThenReturnSuccess) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableSharedSystemUsmSupport.set(1u); + debugManager.flags.EnableRecoverablePageFaults.set(1u); + + size_t size = 10; + void *ptr = nullptr; + + ptr = malloc(size); + EXPECT_NE(nullptr, ptr); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); + ASSERT_NE(nullptr, commandList); + + auto memoryManager = static_cast(device->getDriverHandle()->getMemoryManager()); + memoryManager->failSetSharedSystemMemAdvise = true; + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + + sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled; + + auto res = commandList->executeMemAdvise(device, ptr, size, GetParam()); + EXPECT_EQ(1u, memoryManager->setSharedSystemMemAdviseCalledCount); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + free(ptr); +} + +INSTANTIATE_TEST_SUITE_P( + SupportedMemAdviceTests, + SupportedMemAdviceSystemAllocatorTests, + ValuesIn(std::vector{ + ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION, + ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION, + ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION, + ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION})); + +class UnSupportedMemAdviceSystemAllocatorTests : public CommandListCreateTests, public ::testing::WithParamInterface {}; + +TEST_P(UnSupportedMemAdviceSystemAllocatorTests, givenValidSystemAlloctedPtrWhenExecuteMemAdviseWithUnSupportedAdviceFailsThenReturnSuccess) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableSharedSystemUsmSupport.set(1u); + debugManager.flags.EnableRecoverablePageFaults.set(1u); + + size_t size = 10; + void *ptr = nullptr; + + ptr = malloc(size); + EXPECT_NE(nullptr, ptr); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); + ASSERT_NE(nullptr, commandList); + + auto memoryManager = static_cast(device->getDriverHandle()->getMemoryManager()); + memoryManager->failSetSharedSystemMemAdvise = true; + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + + sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled; + + auto res = commandList->executeMemAdvise(device, ptr, size, GetParam()); + EXPECT_EQ(0u, memoryManager->setSharedSystemMemAdviseCalledCount); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + free(ptr); +} + +INSTANTIATE_TEST_SUITE_P( + UnSupportedMemAdviceTests, + UnSupportedMemAdviceSystemAllocatorTests, + ValuesIn(std::vector{ + ZE_MEMORY_ADVICE_SET_READ_MOSTLY, + ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY, + ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY, + ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY, + ZE_MEMORY_ADVICE_BIAS_CACHED, + ZE_MEMORY_ADVICE_BIAS_UNCACHED, + ZE_MEMORY_ADVICE_FORCE_UINT32})); + +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenExecuteMemAdviseSucceedsThenReturnSuccess) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -226,14 +396,14 @@ TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseSucceedsThenRetur std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = context->freeMem(ptr); ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetWithMaxHintThenSuccessReturned) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrThenExecuteMemAdviseSetWithMaxHintThenSuccessReturned) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -249,14 +419,14 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetWithMaxHintThe std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_FORCE_UINT32); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_FORCE_UINT32); EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = context->freeMem(ptr); ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearReadMostlyThenMemAdviseReadOnlySet) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrThenExecuteMemAdviseSetAndClearReadMostlyThenMemAdviseReadOnlySet) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -273,13 +443,13 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearReadMo std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); L0::DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(device))); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.readOnly); @@ -288,7 +458,7 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearReadMo ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearPreferredLocationThenMemAdvisePreferredDeviceSet) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrThenExecuteMemAdviseSetAndClearPreferredLocationThenMemAdvisePreferredDeviceSet) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -305,13 +475,13 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearPrefer std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); L0::DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(device))); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.devicePreferredLocation); @@ -320,7 +490,7 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearPrefer ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseIsCalledWithSetAndClearSystemMemoryPreferredLocationThenMemAdviseSetPreferredSystemMemoryFlagIsSetCorrectly) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenExecuteMemAdviseIsCalledWithSetAndClearSystemMemoryPreferredLocationThenMemAdviseSetPreferredSystemMemoryFlagIsSetCorrectly) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -339,13 +509,13 @@ TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseIsCalledWithSetAn std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); L0::DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(device))); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.systemPreferredLocation); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.systemPreferredLocation); @@ -354,7 +524,7 @@ TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseIsCalledWithSetAn ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseSetAndClearNonAtomicMostlyThenMemAdviseNonAtomicIgnored) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrWhenExecuteMemAdviseSetAndClearNonAtomicMostlyThenMemAdviseNonAtomicIgnored) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -371,13 +541,13 @@ TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseSetAndClearNonAto std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); L0::DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(device))); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.nonAtomic); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.nonAtomic); @@ -386,7 +556,7 @@ TEST_F(CommandListCreateTests, givenValidPtrWhenAppendMemAdviseSetAndClearNonAto ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearCachingThenMemAdviseCachingSet) { +TEST_F(CommandListCreateTests, givenValidDeviceMemPtrThenExecuteMemAdviseSetAndClearCachingThenMemAdviseCachingSet) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -403,7 +573,7 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearCachin std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)); ASSERT_NE(nullptr, commandList); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_CACHED); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_CACHED); EXPECT_EQ(ZE_RESULT_SUCCESS, res); auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); L0::DeviceImp *deviceImp = static_cast((L0::Device::fromHandle(device))); @@ -411,7 +581,7 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearCachin EXPECT_EQ(1, flags.cachedMemory); auto memoryManager = static_cast(device->getDriverHandle()->getMemoryManager()); EXPECT_EQ(1, memoryManager->memAdviseFlags.cachedMemory); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_UNCACHED); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_UNCACHED); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.cachedMemory); @@ -423,7 +593,7 @@ TEST_F(CommandListCreateTests, givenValidPtrThenAppendMemAdviseSetAndClearCachin using CommandListMemAdvisePageFault = Test; -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenAppendMemAdviseWithReadOnlyAndDevicePreferredClearsMigrationBlocked) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerThenExecuteMemAdviseWithReadOnlyAndDevicePreferredClearsMigrationBlocked) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -447,20 +617,20 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenAppend flags.cpuMigrationBlocked = 1; deviceImp->memAdviseSharedAllocations[allocData] = flags; - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.readOnly); @@ -471,7 +641,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenAppend ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenGpuDomainHanlderWithHintsIsSet) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerThenGpuDomainHanlderWithHintsIsSet) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -494,12 +664,12 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenGpuDom flags = deviceImp->memAdviseSharedAllocations[allocData]; deviceImp->memAdviseSharedAllocations[allocData] = flags; - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); @@ -512,7 +682,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenGpuDom ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetThenHandlerBlocksCpuMigration) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetThenHandlerBlocksCpuMigration) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -533,12 +703,12 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); @@ -558,7 +728,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndOnlyReadOnlyOrDevicePreferredHintThenHandlerAllowsCpuMigration) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndOnlyReadOnlyOrDevicePreferredHintThenHandlerAllowsCpuMigration) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -579,7 +749,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); @@ -599,12 +769,12 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.cpuMigrationBlocked); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); @@ -613,7 +783,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.cpuMigrationBlocked); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(0, flags.devicePreferredLocation); @@ -626,7 +796,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndWithPrintUsmSharedMigrationDebugKeyThenMessageIsPrinted) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndWithPrintUsmSharedMigrationDebugKeyThenMessageIsPrinted) { DebugManagerStateRestore restorer; debugManager.flags.PrintUmdSharedMigration.set(1); @@ -652,7 +822,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); @@ -687,7 +857,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndInvalidHintsThenHandlerAllowsCpuMigration) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndInvalidHintsThenHandlerAllowsCpuMigration) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -708,7 +878,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_CACHED); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_CACHED); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.cachedMemory); @@ -729,7 +899,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndCpuDomainThenHandlerAllowsCpuMigration) { +TEST_F(CommandListMemAdvisePageFault, givenValidDeviceMemPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetAndCpuDomainThenHandlerAllowsCpuMigration) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -750,12 +920,12 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); @@ -776,7 +946,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma ASSERT_EQ(res, ZE_RESULT_SUCCESS); } -TEST_F(CommandListMemAdvisePageFault, givenInvalidPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetThenHandlerAllowsCpuMigration) { +TEST_F(CommandListMemAdvisePageFault, givenInvalidDeviceMemPtrAndPageFaultHandlerAndGpuDomainHandlerWithHintsSetThenHandlerAllowsCpuMigration) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -797,12 +967,12 @@ TEST_F(CommandListMemAdvisePageFault, givenInvalidPtrAndPageFaultHandlerAndGpuDo auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.readOnly); - res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + res = commandList->executeMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); EXPECT_EQ(ZE_RESULT_SUCCESS, res); flags = deviceImp->memAdviseSharedAllocations[allocData]; EXPECT_EQ(1, flags.devicePreferredLocation); diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_6.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_6.cpp index 0b7935ec9d..82a223433d 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_6.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_6.cpp @@ -216,6 +216,31 @@ HWTEST2_F(CommandListExecuteImmediate, whenExecutingCommandListImmediateWithFlus EXPECT_EQ(ZE_RESULT_SUCCESS, commandListImmediate.executeCommandListImmediateWithFlushTask(false, false, false, NEO::AppendOperations::nonKernel, false, false, nullptr, nullptr)); } +HWTEST2_F(CommandListExecuteImmediate, whenExecutingCommandListImmediateWithFlushTaskWithMemAdvicesThenMemAdvicesAreDispatchedAndSuccessIsReturned, MatchAny) { + + MockCommandListForExecuteMemAdvise commandList; + commandList.initialize(device, NEO::EngineGroupType::compute, 0u); + + ze_command_queue_desc_t desc = {}; + desc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS; + MockCommandStreamReceiver mockCommandStreamReceiver(*neoDevice->executionEnvironment, neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()); + Mock mockCommandQueue(device, &mockCommandStreamReceiver, &desc); + auto oldCommandQueue = commandList.cmdQImmediate; + commandList.cmdQImmediate = &mockCommandQueue; + commandList.indirectAllocationsAllowed = false; + + commandList.getMemAdviseOperations().push_back(MemAdviseOperation(0, 0, 16, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION)); + EXPECT_EQ(1u, commandList.getMemAdviseOperations().size()); + commandList.getMemAdviseOperations().push_back(MemAdviseOperation(0, 0, 8, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION)); + EXPECT_EQ(2u, commandList.getMemAdviseOperations().size()); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList.executeCommandListImmediateWithFlushTask(false, false, false, NEO::AppendOperations::none, false, false, nullptr, nullptr)); + EXPECT_EQ(0u, commandList.getMemAdviseOperations().size()); + EXPECT_EQ(2u, commandList.executeMemAdviseCallCount); + + commandList.cmdQImmediate = oldCommandQueue; +} + HWTEST2_F(CommandListExecuteImmediate, givenOutOfHostMemoryErrorOnFlushWhenExecutingCommandListImmediateWithFlushTaskThenProperErrorIsReturned, MatchAny) { std::unique_ptr commandList; const ze_command_queue_desc_t desc = {}; diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_memory.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_memory.cpp index fcd615a5d2..608a3529f9 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_memory.cpp @@ -1169,6 +1169,47 @@ HWTEST2_F(AppendMemoryCopyFenceTest, givenDeviceToHostCopyWhenProgrammingThenAdd context->freeMem(deviceBuffer); } +HWTEST2_F(AppendMemoryCopyFenceTest, givenAppendMemAdviseWithRegularAndHeapLessCmdListThenMemAdvicesAreDispatched, IsAtLeastXeHpcCore) { + + ze_result_t result = ZE_RESULT_SUCCESS; + + ze_command_queue_desc_t desc = {}; + auto mockCmdQHw = makeZeUniquePtr>(device, device->getNEODevice()->getDefaultEngine().commandStreamReceiver, &desc); + mockCmdQHw->initialize(false, false, false); + + MockCommandListCoreFamily cmdList; + cmdList.initialize(device, NEO::EngineGroupType::compute, 0u); + cmdList.isFlushTaskSubmissionEnabled = true; + + constexpr size_t allocSize = 1; + void *deviceBuffer = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + result = context->allocDeviceMem(device->toHandle(), &deviceDesc, allocSize, allocSize, &deviceBuffer); + ASSERT_EQ(ZE_RESULT_SUCCESS, result); + + auto cmdListHandle = cmdList.toHandle(); + + { + cmdList.appendMemAdvise(device->toHandle(), deviceBuffer, allocSize, ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION); + cmdList.close(); + mockCmdQHw->executeCommandLists(1, &cmdListHandle, nullptr, false, nullptr, nullptr); + EXPECT_EQ(1u, cmdList.executeMemAdviseCallCount); + EXPECT_EQ(0u, cmdList.getMemAdviseOperations().size()); + } + + { + mockCmdQHw->heaplessStateInitEnabled = true; + cmdList.reset(); + cmdList.appendMemAdvise(device->toHandle(), deviceBuffer, allocSize, ZE_MEMORY_ADVICE_CLEAR_SYSTEM_MEMORY_PREFERRED_LOCATION); + cmdList.close(); + mockCmdQHw->executeCommandLists(1, &cmdListHandle, nullptr, false, nullptr, nullptr); + EXPECT_EQ(2u, cmdList.executeMemAdviseCallCount); + EXPECT_EQ(0u, cmdList.getMemAdviseOperations().size()); + } + + context->freeMem(deviceBuffer); +} + HWTEST2_F(AppendMemoryCopyFenceTest, givenRegularCmdListWhenDeviceToHostCopyProgrammedWithoutEventThenAddFenceDuringTagUpdate, IsAtLeastXeHpcCore) { using MI_FLUSH_DW = typename FamilyType::MI_FLUSH_DW; diff --git a/shared/source/memory_manager/memadvise_flags.h b/shared/source/memory_manager/memadvise_flags.h index 34e6ba4f73..ed475b45d7 100644 --- a/shared/source/memory_manager/memadvise_flags.h +++ b/shared/source/memory_manager/memadvise_flags.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Intel Corporation + * Copyright (C) 2021-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -32,4 +32,24 @@ struct MemAdviseFlags { } }; static_assert(sizeof(MemAdviseFlags) == sizeof(uint8_t), ""); + +enum class MemAdvise : uint8_t { + setReadMostly = 0, /* hint that memory will be read from frequently and written to rarely */ + clearReadMostly, /* removes the effect of SetReadMostly */ + setPreferredLocation, /* hint that the preferred memory location is the specified device */ + clearPreferredLocation, /* removes the effect of SetPreferredLocation */ + setNonAtomicMostly, /* hints that memory will mostly be accessed non-atomically */ + clearNonAtomicMostly, /* removes the effect of SetNonAtomicMostly */ + biasCached, /* hints that memory should be cached */ + biasUncached, /* hints that memory should not be cached */ + setSystemMemoryPreferredLocation, /* hint that the preferred memory location is host memory */ + clearSystemMemoryPreferredLocation, /* removes the effect of SetSystemMemoryPreferredLocation */ + setAtomicDevice, /* hint that memory will be accessed atomically by the specified device */ + clearAtomicDevice, /* removes the effect of SetAtomicDevice */ + setAtomicGlobal, /* hint that memory will be accessed atomically by device on system memory */ + clearAtomicGlobal, /* removes the effect of SetAtomicGlobal */ + setAtomicCpu, /* hint that memory will be accessed atomically by CPU */ + clearAtomicCpu, /* removes the effect of SetAtomicCpu */ + invalidAdvise /* invalid advise */ +}; } // namespace NEO diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 2d3d8548cb..84bb15c0ac 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -276,6 +276,7 @@ class MemoryManager { virtual AllocationStatus registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex); virtual bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) { return true; } + virtual bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, uint32_t rootDeviceIndex) { return true; } virtual bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } virtual bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) { return true; } virtual bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) { return true; } diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 87b6738a81..35ff273123 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -48,7 +48,6 @@ #include #include #include - namespace NEO { using AllocationStatus = MemoryManager::AllocationStatus; @@ -294,6 +293,70 @@ bool DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdvise return drmAllocation->setMemAdvise(&this->getDrm(rootDeviceIndex), flags); } +bool DrmMemoryManager::setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, uint32_t rootDeviceIndex) { + + auto &drm = this->getDrm(rootDeviceIndex); + auto ioctlHelper = drm.getIoctlHelper(); + + uint32_t attribute = 0; + uint64_t param = 0; + + switch (memAdviseOp) { + case MemAdvise::setPreferredLocation: + attribute = ioctlHelper->getPreferredLocationAdvise(); + param = (static_cast(-1) << 32) //-1 as currently not supported and ignored. This will be useful in multi device settings. + | static_cast(ioctlHelper->getDrmParamValue(DrmParam::memoryClassDevice)); + break; + case MemAdvise::clearPreferredLocation: + // Assumes that the default location is VRAM, i.e. 1 == DrmParam::memoryClassDevice + attribute = ioctlHelper->getPreferredLocationAdvise(); + param = (static_cast(-1) << 32) | static_cast(ioctlHelper->getDrmParamValue(DrmParam::memoryClassDevice)); + break; + case MemAdvise::setSystemMemoryPreferredLocation: + attribute = ioctlHelper->getPreferredLocationAdvise(); + param = (static_cast(-1) << 32) | static_cast(ioctlHelper->getDrmParamValue(DrmParam::memoryClassSystem)); + break; + case MemAdvise::clearSystemMemoryPreferredLocation: + attribute = ioctlHelper->getPreferredLocationAdvise(); + param = (static_cast(-1) << 32) | static_cast(ioctlHelper->getDrmParamValue(DrmParam::memoryClassDevice)); + break; + case MemAdvise::setAtomicDevice: + attribute = ioctlHelper->getAtomicAdvise(false); + param = (static_cast(ioctlHelper->getDrmParamValue(DrmParam::atomicClassDevice)) << 32); + break; + case MemAdvise::clearAtomicDevice: + attribute = ioctlHelper->getAtomicAdvise(false); + param = (static_cast(ioctlHelper->getDrmParamValue(DrmParam::atomicClassUndefined)) << 32); + break; + case MemAdvise::setAtomicGlobal: + attribute = ioctlHelper->getAtomicAdvise(false); + param = (static_cast(ioctlHelper->getDrmParamValue(DrmParam::atomicClassGlobal)) << 32); + break; + case MemAdvise::clearAtomicGlobal: + attribute = ioctlHelper->getAtomicAdvise(false); + param = (static_cast(ioctlHelper->getDrmParamValue(DrmParam::atomicClassUndefined)) << 32); + break; + case MemAdvise::setAtomicCpu: + attribute = ioctlHelper->getAtomicAdvise(false); + param = (static_cast(ioctlHelper->getDrmParamValue(DrmParam::atomicClassSystem)) << 32); + break; + case MemAdvise::clearAtomicCpu: + attribute = ioctlHelper->getAtomicAdvise(false); + param = (static_cast(ioctlHelper->getDrmParamValue(DrmParam::atomicClassUndefined)) << 32); + break; + default: + return false; + } + + // Single vm_id for shared system allocation + uint32_t vmHandleId = 0; + auto vmId = drm.getVirtualMemoryAddressSpace(vmHandleId); + + auto result = ioctlHelper->setVmSharedSystemMemAdvise(reinterpret_cast(ptr), size, attribute, param, vmId); + + return result; +} + bool DrmMemoryManager::setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) { auto drmAllocation = static_cast(gfxAllocation); diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index d096d3c366..462df6787b 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -86,6 +86,7 @@ class DrmMemoryManager : public MemoryManager { bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override; bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override; + bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, uint32_t rootDeviceIndex) override; bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; bool prefetchSharedSystemAlloc(const void *ptr, const size_t size, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override; bool setAtomicAccess(GraphicsAllocation *gfxAllocation, size_t size, AtomicAccessMode mode, uint32_t rootDeviceIndex) override; diff --git a/shared/source/os_interface/linux/drm_wrappers.h b/shared/source/os_interface/linux/drm_wrappers.h index ce5c7a0d14..3e3c5e7042 100644 --- a/shared/source/os_interface/linux/drm_wrappers.h +++ b/shared/source/os_interface/linux/drm_wrappers.h @@ -299,6 +299,10 @@ enum class DrmIoctl { }; enum class DrmParam { + atomicClassUndefined, + atomicClassDevice, + atomicClassGlobal, + atomicClassSystem, contextCreateExtSetparam, contextCreateFlagsUseExtensions, contextEnginesExtLoadBalance, diff --git a/shared/source/os_interface/linux/ioctl_helper.h b/shared/source/os_interface/linux/ioctl_helper.h index 308be92c87..8ea49971ca 100644 --- a/shared/source/os_interface/linux/ioctl_helper.h +++ b/shared/source/os_interface/linux/ioctl_helper.h @@ -130,6 +130,7 @@ class IoctlHelper { virtual uint32_t getPreferredLocationAdvise() = 0; virtual std::optional getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) = 0; virtual bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) = 0; + virtual bool setVmSharedSystemMemAdvise(uint64_t handle, const size_t size, const uint32_t attribute, const uint64_t param, const uint32_t vmId) { return true; } virtual bool setVmBoAdviseForChunking(int32_t handle, uint64_t start, uint64_t length, uint32_t attribute, void *region) = 0; virtual bool setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) = 0; virtual bool setGemTiling(void *setTiling) = 0; diff --git a/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp b/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp index 80b1c7f5e4..de67167e15 100644 --- a/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp +++ b/shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp @@ -46,7 +46,6 @@ #ifndef DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR #define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 2) #endif - namespace NEO { const char *IoctlHelperXe::xeGetClassName(int className) const { @@ -81,6 +80,10 @@ const char *IoctlHelperXe::xeGetBindOperationName(int bindOperation) { return "Unknown operation"; } +const char *IoctlHelperXe::xeGetAdviseOperationName(int adviseOperation) { + return "Unknown operation"; +} + std::string IoctlHelperXe::xeGetBindFlagNames(int bindFlags) { if (bindFlags == 0) { return ""; @@ -819,7 +822,7 @@ int IoctlHelperXe::waitUserFence(uint32_t ctxId, uint64_t address, return 0; } -uint32_t IoctlHelperXe::getAtomicAdvise(bool isNonAtomic) { +uint32_t IoctlHelperXe::getAtomicAdvise(bool /* isNonAtomic */) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); return 0; } @@ -844,6 +847,12 @@ bool IoctlHelperXe::setVmBoAdvise(int32_t handle, uint32_t attribute, void *regi return true; } +bool IoctlHelperXe::setVmSharedSystemMemAdvise(uint64_t handle, const size_t size, const uint32_t attribute, const uint64_t param, const uint32_t vmId) { + xeLog(" -> IoctlHelperXe::%s h=0x%llx s=0x%llx vmid=0x%x\n", __FUNCTION__, handle, size, vmId); + // There is no vmAdvise attribute in Xe, so return success + return true; +} + bool IoctlHelperXe::setVmBoAdviseForChunking(int32_t handle, uint64_t start, uint64_t length, uint32_t attribute, void *region) { xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__); // There is no vmAdvise attribute in Xe, so return success @@ -1100,8 +1109,15 @@ bool IoctlHelperXe::isDebugAttachAvailable() { int IoctlHelperXe::getDrmParamValue(DrmParam drmParam) const { xeLog(" -> IoctlHelperXe::%s 0x%x %s\n", __FUNCTION__, drmParam, getDrmParamString(drmParam).c_str()); - switch (drmParam) { + case DrmParam::atomicClassUndefined: + return -1; + case DrmParam::atomicClassDevice: + return -1; + case DrmParam::atomicClassGlobal: + return -1; + case DrmParam::atomicClassSystem: + return -1; case DrmParam::memoryClassDevice: return DRM_XE_MEM_REGION_CLASS_VRAM; case DrmParam::memoryClassSystem: @@ -1486,6 +1502,14 @@ int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool isBind) { std::string IoctlHelperXe::getDrmParamString(DrmParam drmParam) const { switch (drmParam) { + case DrmParam::atomicClassUndefined: + return "AtomicClassUndefined"; + case DrmParam::atomicClassDevice: + return "AtomicClassDevice"; + case DrmParam::atomicClassGlobal: + return "AtomicClassGlobal"; + case DrmParam::atomicClassSystem: + return "AtomicClassSystem"; case DrmParam::contextCreateExtSetparam: return "ContextCreateExtSetparam"; case DrmParam::contextCreateFlagsUseExtensions: diff --git a/shared/source/os_interface/linux/xe/ioctl_helper_xe.h b/shared/source/os_interface/linux/xe/ioctl_helper_xe.h index bb8d6e6d44..9830e3a864 100644 --- a/shared/source/os_interface/linux/xe/ioctl_helper_xe.h +++ b/shared/source/os_interface/linux/xe/ioctl_helper_xe.h @@ -60,6 +60,7 @@ class IoctlHelperXe : public IoctlHelper { uint32_t getPreferredLocationAdvise() override; std::optional getPreferredLocationRegion(PreferredLocation memoryLocation, uint32_t memoryInstance) override; bool setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) override; + bool setVmSharedSystemMemAdvise(uint64_t handle, const size_t size, const uint32_t attribute, const uint64_t param, const uint32_t vmId) override; bool setVmBoAdviseForChunking(int32_t handle, uint64_t start, uint64_t length, uint32_t attribute, void *region) override; bool setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) override; bool setGemTiling(void *setTiling) override; @@ -144,6 +145,7 @@ class IoctlHelperXe : public IoctlHelper { virtual const char *xeGetClassName(int className) const; const char *xeGetBindOperationName(int bindOperation); + const char *xeGetAdviseOperationName(int adviseOperation); const char *xeGetengineClassName(uint32_t engineClass); template diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index eb42d03b77..582332bed2 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -212,6 +212,14 @@ class MockMemoryManager : public MemoryManagerCreate { return MemoryManager::setMemAdvise(gfxAllocation, flags, rootDeviceIndex); } + bool setSharedSystemMemAdvise(const void *ptr, const size_t size, MemAdvise memAdviseOp, uint32_t rootDeviceIndex) override { + setSharedSystemMemAdviseCalledCount++; + if (failSetSharedSystemMemAdvise) { + return false; + } + return MemoryManager::setSharedSystemMemAdvise(ptr, size, memAdviseOp, rootDeviceIndex); + } + bool setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDeviceIdsVec &subDeviceIds, uint32_t rootDeviceIndex) override { memPrefetchSubDeviceIds = subDeviceIds; setMemPrefetchCalled = true; @@ -320,6 +328,7 @@ class MockMemoryManager : public MemoryManagerCreate { uint32_t waitForEnginesCompletionCalled = 0u; uint32_t allocateGraphicsMemoryWithPropertiesCount = 0; uint32_t setMemPrefetchCalledCount = 0; + uint32_t setSharedSystemMemAdviseCalledCount = 0; osHandle capturedSharedHandle = 0u; bool allocationCreated = false; bool allocation64kbPageCreated = false; @@ -337,6 +346,7 @@ class MockMemoryManager : public MemoryManagerCreate { bool failAllocate32Bit = false; bool failLockResource = false; bool failSetMemAdvise = false; + bool failSetSharedSystemMemAdvise = false; bool setMemPrefetchCalled = false; bool prefetchSharedSystemAllocCalled = false; bool cpuCopyRequired = false; diff --git a/shared/test/common/os_interface/linux/xe/mock_ioctl_helper_xe.h b/shared/test/common/os_interface/linux/xe/mock_ioctl_helper_xe.h index 7c065296bc..4c9797ffbb 100644 --- a/shared/test/common/os_interface/linux/xe/mock_ioctl_helper_xe.h +++ b/shared/test/common/os_interface/linux/xe/mock_ioctl_helper_xe.h @@ -27,6 +27,7 @@ struct MockIoctlHelperXe : IoctlHelperXe { using IoctlHelperXe::tileIdToGtId; using IoctlHelperXe::updateBindInfo; using IoctlHelperXe::UserFenceExtension; + using IoctlHelperXe::xeGetAdviseOperationName; using IoctlHelperXe::xeGetBindFlagNames; using IoctlHelperXe::xeGetBindOperationName; using IoctlHelperXe::xeGetClassName; diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index f659e417ab..c58c10fdf9 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -50,6 +50,12 @@ enum class AtomicAccessMode : uint32_t; using namespace NEO; +TEST(MemoryManagerTest, WhenCallingSetSharedSystemMemAdviseThenReturnTrue) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + OsAgnosticMemoryManager memoryManager(executionEnvironment); + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, MemAdvise::invalidAdvise, 0u)); +} + TEST(MemoryManagerTest, WhenCallingHasPageFaultsEnabledThenReturnFalse) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); OsAgnosticMemoryManager memoryManager(executionEnvironment); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index ed2328d6ee..f6f7c0cd8b 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -5808,6 +5808,70 @@ HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetMemAdviseIs } } +HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetSharedSystemMemAdviseIsCalledThenAdviceSentToIoctlHelper) { + TestedDrmMemoryManager memoryManager(false, false, false, *executionEnvironment); + + class MyMockIoctlHelper : public MockIoctlHelper { + using MockIoctlHelper::MockIoctlHelper; + + public: + bool setVmSharedSystemMemAdvise(uint64_t handle, const size_t size, const uint32_t attribute, const uint64_t param, const uint32_t vmId) override { + setVmSharedSystemMemAdviseCalled++; + return true; + } + u_int32_t setVmSharedSystemMemAdviseCalled = 0; + }; + + auto mockIoctlHelper = new MyMockIoctlHelper(*mock); + + auto &drm = static_cast(memoryManager.getDrm(mockRootDeviceIndex)); + drm.ioctlHelper.reset(mockIoctlHelper); + + MemAdvise memAdviseOp = MemAdvise::setPreferredLocation; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(1u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::clearPreferredLocation; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(2u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::setSystemMemoryPreferredLocation; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(3u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::clearSystemMemoryPreferredLocation; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(4u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::setAtomicDevice; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(5u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::clearAtomicDevice; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(6u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::setAtomicGlobal; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(7u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::clearAtomicGlobal; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(8u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::setAtomicCpu; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(9u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::clearAtomicCpu; + EXPECT_TRUE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(10u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); + + memAdviseOp = MemAdvise::invalidAdvise; + EXPECT_FALSE(memoryManager.setSharedSystemMemAdvise(nullptr, 0u, memAdviseOp, 0u)); + EXPECT_EQ(10u, mockIoctlHelper->setVmSharedSystemMemAdviseCalled); +} + HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetAtomicAccessIsCalledThenTrueReturned) { TestedDrmMemoryManager memoryManager(false, false, false, *executionEnvironment); BufferObject bo(rootDeviceIndex, mock, 3, 1, 1024, 0); diff --git a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp index da25fb03d2..f649a69d66 100644 --- a/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp +++ b/shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp @@ -297,6 +297,10 @@ TEST_F(IoctlPrelimHelperTests, givenIoctlHelperisVmBindPatIndexExtSupportedRetur ASSERT_EQ(true, ioctlHelper.isVmBindPatIndexExtSupported()); } +TEST_F(IoctlPrelimHelperTests, givenIoctlHelperSetVmSharedSystemMemAdviseReturnsTrue) { + ASSERT_EQ(true, ioctlHelper.setVmSharedSystemMemAdvise(0u, 0u, 0u, 0u, 0u)); +} + TEST_F(IoctlPrelimHelperTests, whenGettingVmBindExtFromHandlesThenProperStructsAreReturned) { StackVec bindExtHandles; bindExtHandles.push_back(1u); diff --git a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp index 365d3475d9..79e736cbf4 100644 --- a/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/xe/ioctl_helper_xe_tests.cpp @@ -392,8 +392,12 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingAnyMethodThenDummyValueIs EXPECT_EQ(std::nullopt, xeIoctlHelper->getPreferredLocationRegion(PreferredLocation::none, 0)); + EXPECT_EQ(0u, xeIoctlHelper->getPreferredLocationAdvise()); + EXPECT_TRUE(xeIoctlHelper->setVmBoAdvise(0, 0, nullptr)); + EXPECT_TRUE(xeIoctlHelper->setVmSharedSystemMemAdvise(0, 0, 0, 0, 0)); + EXPECT_TRUE(xeIoctlHelper->setVmBoAdviseForChunking(0, 0, 0, 0, nullptr)); EXPECT_FALSE(xeIoctlHelper->isChunkingAvailable()); @@ -447,6 +451,10 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingAnyMethodThenDummyValueIs EXPECT_EQ(0, xeIoctlHelper->setContextDebugFlag(0)); + verifyDrmGetParamValue(-1, DrmParam::atomicClassUndefined); + verifyDrmGetParamValue(-1, DrmParam::atomicClassDevice); + verifyDrmGetParamValue(-1, DrmParam::atomicClassGlobal); + verifyDrmGetParamValue(-1, DrmParam::atomicClassSystem); verifyDrmGetParamValue(DRM_XE_MEM_REGION_CLASS_VRAM, DrmParam::memoryClassDevice); verifyDrmGetParamValue(DRM_XE_MEM_REGION_CLASS_SYSMEM, DrmParam::memoryClassSystem); verifyDrmGetParamValue(DRM_XE_ENGINE_CLASS_RENDER, DrmParam::engineClassRender); @@ -460,6 +468,10 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingAnyMethodThenDummyValueIs verifyDrmGetParamValue(DRM_XE_ENGINE_CLASS_COMPUTE, DrmParam::execDefault); // Expect stringify + verifyDrmParamString("AtomicClassUndefined", DrmParam::atomicClassUndefined); + verifyDrmParamString("AtomicClassDevice", DrmParam::atomicClassDevice); + verifyDrmParamString("AtomicClassGlobal", DrmParam::atomicClassGlobal); + verifyDrmParamString("AtomicClassSystem", DrmParam::atomicClassSystem); verifyDrmParamString("ContextCreateExtSetparam", DrmParam::contextCreateExtSetparam); verifyDrmParamString("ContextCreateExtSetparam", DrmParam::contextCreateExtSetparam); verifyDrmParamString("ContextCreateFlagsUseExtensions", DrmParam::contextCreateFlagsUseExtensions); @@ -502,7 +514,6 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingAnyMethodThenDummyValueIs verifyDrmParamString("TilingY", DrmParam::tilingY); verifyIoctlString(DrmIoctl::gemClose, "DRM_IOCTL_GEM_CLOSE"); - verifyIoctlString(DrmIoctl::gemCreate, "DRM_IOCTL_XE_GEM_CREATE"); verifyIoctlString(DrmIoctl::gemVmCreate, "DRM_IOCTL_XE_VM_CREATE"); verifyIoctlString(DrmIoctl::gemVmDestroy, "DRM_IOCTL_XE_VM_DESTROY"); verifyIoctlString(DrmIoctl::gemMmapOffset, "DRM_IOCTL_XE_GEM_MMAP_OFFSET"); @@ -515,6 +526,7 @@ TEST_F(IoctlHelperXeTest, givenIoctlHelperXeWhenCallingAnyMethodThenDummyValueIs verifyIoctlString(DrmIoctl::gemWaitUserFence, "DRM_IOCTL_XE_WAIT_USER_FENCE"); verifyIoctlString(DrmIoctl::primeFdToHandle, "DRM_IOCTL_PRIME_FD_TO_HANDLE"); verifyIoctlString(DrmIoctl::primeHandleToFd, "DRM_IOCTL_PRIME_HANDLE_TO_FD"); + verifyIoctlString(DrmIoctl::getResetStats, "DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY"); EXPECT_TRUE(xeIoctlHelper->completionFenceExtensionSupported(true)); @@ -592,6 +604,10 @@ TEST_F(IoctlHelperXeTest, verifyPublicFunctions) { EXPECT_STREQ(name, mockXeIoctlHelper->xeGetBindOperationName(bind)); }; + auto verifyXeOperationAdviseName = [&mockXeIoctlHelper](const char *name, auto advise) { + EXPECT_STREQ(name, mockXeIoctlHelper->xeGetAdviseOperationName(advise)); + }; + auto verifyXeFlagsBindName = [&mockXeIoctlHelper](const char *name, auto flags) { EXPECT_STREQ(name, mockXeIoctlHelper->xeGetBindFlagNames(flags).c_str()); }; @@ -613,6 +629,8 @@ TEST_F(IoctlHelperXeTest, verifyPublicFunctions) { verifyXeOperationBindName("PREFETCH", DRM_XE_VM_BIND_OP_PREFETCH); verifyXeOperationBindName("Unknown operation", -1); + verifyXeOperationAdviseName("Unknown operation", -1); + verifyXeFlagsBindName("", 0); verifyXeFlagsBindName("NULL", DRM_XE_VM_BIND_FLAG_NULL); verifyXeFlagsBindName("READONLY", DRM_XE_VM_BIND_FLAG_READONLY);