diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index 20ba78ab1c..e6cbd84ffc 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -3678,7 +3678,7 @@ bool CommandListCoreFamily::handleCounterBasedEventOperations(Eve } if (isInOrderExecutionEnabled() && (this->cmdListType == typeImmediate)) { - signalEvent->enableCounterBasedMode(false); + signalEvent->enableCounterBasedMode(false, ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE); } else { signalEvent->disableImplicitCounterBasedMode(); } diff --git a/level_zero/core/source/event/event.cpp b/level_zero/core/source/event/event.cpp index 715a1f9ed0..5ba743ef57 100644 --- a/level_zero/core/source/event/event.cpp +++ b/level_zero/core/source/event/event.cpp @@ -46,6 +46,11 @@ ze_result_t EventPool::initialize(DriverHandle *driver, Context *context, uint32 return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; } + constexpr uint32_t supportedCounterBasedFlags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE | ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE; + if (counterBased && ((counterBasedFlags & supportedCounterBasedFlags) == 0)) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + RootDeviceIndicesContainer rootDeviceIndices; uint32_t maxRootDeviceIndex = 0u; uint32_t currentNumDevices = numDevices; @@ -210,6 +215,8 @@ void EventPool::setupDescriptorFlags(const ze_event_pool_desc_t *desc) { auto pNext = reinterpret_cast(desc->pNext); if (pNext && pNext->stype == ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC) { + auto counterBasedDesc = reinterpret_cast(pNext); + counterBasedFlags = counterBasedDesc->flags; counterBased = true; } } @@ -354,9 +361,10 @@ ze_result_t Event::destroy() { return ZE_RESULT_SUCCESS; } -void Event::enableCounterBasedMode(bool apiRequest) { +void Event::enableCounterBasedMode(bool apiRequest, uint32_t flags) { if (counterBasedMode == CounterBasedMode::initiallyDisabled) { counterBasedMode = apiRequest ? CounterBasedMode::explicitlyEnabled : CounterBasedMode::implicitlyEnabled; + counterBasedFlags = flags; } } @@ -367,6 +375,7 @@ void Event::disableImplicitCounterBasedMode() { if (counterBasedMode == CounterBasedMode::implicitlyEnabled || counterBasedMode == CounterBasedMode::initiallyDisabled) { counterBasedMode = CounterBasedMode::implicitlyDisabled; + counterBasedFlags = 0; unsetInOrderExecInfo(); } } diff --git a/level_zero/core/source/event/event.h b/level_zero/core/source/event/event.h index 86ab53ece1..45fab8dc64 100644 --- a/level_zero/core/source/event/event.h +++ b/level_zero/core/source/event/event.h @@ -241,7 +241,7 @@ struct Event : _ze_event_handle_t { void updateInOrderExecState(std::shared_ptr &newInOrderExecInfo, uint64_t signalValue, uint32_t allocationOffset); bool isCounterBased() const { return ((counterBasedMode == CounterBasedMode::explicitlyEnabled) || (counterBasedMode == CounterBasedMode::implicitlyEnabled)); } bool isCounterBasedExplicitlyEnabled() const { return (counterBasedMode == CounterBasedMode::explicitlyEnabled); } - void enableCounterBasedMode(bool apiRequest); + void enableCounterBasedMode(bool apiRequest, uint32_t flags); void disableImplicitCounterBasedMode(); NEO::GraphicsAllocation *getInOrderExecDataAllocation() const; uint64_t getInOrderExecSignalValueWithSubmissionCounter() const; @@ -305,6 +305,7 @@ struct Event : _ze_event_handle_t { uint32_t kernelCount = 1u; uint32_t maxPacketCount = 0; uint32_t totalEventSize = 0; + uint32_t counterBasedFlags = 0; CounterBasedMode counterBasedMode = CounterBasedMode::initiallyDisabled; ze_event_scope_flags_t signalScope = 0u; @@ -386,6 +387,7 @@ struct EventPool : _ze_event_pool_handle_t { } bool isCounterBased() const { return counterBased; } + uint32_t getCounterBasedFlags() const { return counterBasedFlags; } bool isIpcPoolFlagSet() const { return isIpcPoolFlag; } protected: @@ -407,6 +409,8 @@ struct EventPool : _ze_event_pool_handle_t { uint32_t eventPackets = 0; uint32_t maxKernelCount = 0; + uint32_t counterBasedFlags = 0; + ze_event_pool_flags_t eventPoolFlags{}; bool isDeviceEventPoolAllocation = false; diff --git a/level_zero/core/source/event/event_impl.inl b/level_zero/core/source/event/event_impl.inl index 406cb0a56d..9ca41f1c63 100644 --- a/level_zero/core/source/event/event_impl.inl +++ b/level_zero/core/source/event/event_impl.inl @@ -87,7 +87,7 @@ Event *Event::create(EventPool *eventPool, const ze_event_desc_t *desc, Device * } if (eventPool->isCounterBased() || NEO::debugManager.flags.ForceInOrderEvents.get() == 1) { - event->enableCounterBasedMode(true); + event->enableCounterBasedMode(true, eventPool->getCounterBasedFlags()); } auto extendedDesc = reinterpret_cast(desc->pNext); diff --git a/level_zero/core/test/unit_tests/fixtures/in_order_cmd_list_fixture.h b/level_zero/core/test/unit_tests/fixtures/in_order_cmd_list_fixture.h index 0ea7697e29..328641f11c 100644 --- a/level_zero/core/test/unit_tests/fixtures/in_order_cmd_list_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/in_order_cmd_list_fixture.h @@ -24,6 +24,7 @@ namespace ult { struct InOrderCmdListFixture : public ::Test { struct FixtureMockEvent : public EventImp { using EventImp::Event::counterBasedMode; + using EventImp::Event::counterBasedFlags; using EventImp::maxPacketCount; using EventImp::inOrderExecInfo; using EventImp::inOrderExecSignalValue; @@ -33,10 +34,12 @@ struct InOrderCmdListFixture : public ::Test { void makeCounterBasedInitiallyDisabled() { counterBasedMode = CounterBasedMode::initiallyDisabled; + counterBasedFlags = 0; } void makeCounterBasedImplicitlyDisabled() { counterBasedMode = CounterBasedMode::implicitlyDisabled; + counterBasedFlags = 0; } }; @@ -62,6 +65,7 @@ struct InOrderCmdListFixture : public ::Test { eventPoolDesc.count = numEvents; ze_event_pool_counter_based_exp_desc_t counterBasedExtension = {ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC}; + counterBasedExtension.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE | ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE; eventPoolDesc.pNext = &counterBasedExtension; if (timestampEvent) { @@ -78,6 +82,7 @@ struct InOrderCmdListFixture : public ::Test { events.emplace_back(DestroyableZeUniquePtr(static_cast(Event::create(eventPool.get(), &eventDesc, device)))); EXPECT_EQ(Event::CounterBasedMode::explicitlyEnabled, events.back()->counterBasedMode); EXPECT_TRUE(events.back()->isCounterBased()); + EXPECT_EQ(counterBasedExtension.flags, events.back()->counterBasedFlags); } return eventPool; diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp index 3734cc8fe5..5c96b49528 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp @@ -813,6 +813,7 @@ HWTEST2_F(InOrderCmdListTests, givenIpcAndCounterBasedEventPoolFlagsWhenCreating eventPoolDesc.count = 1; ze_event_pool_counter_based_exp_desc_t counterBasedExtension = {ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC}; + counterBasedExtension.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE | ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE; eventPoolDesc.pNext = &counterBasedExtension; auto eventPool = EventPool::create(driverHandle.get(), context, 0, nullptr, &eventPoolDesc, returnValue); @@ -821,6 +822,36 @@ HWTEST2_F(InOrderCmdListTests, givenIpcAndCounterBasedEventPoolFlagsWhenCreating EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, returnValue); } +HWTEST2_F(InOrderCmdListTests, givenIncorrectFlagsWhenCreatingCounterBasedEventsThenReturnError, IsAtLeastSkl) { + ze_event_pool_desc_t eventPoolDesc = {}; + eventPoolDesc.count = 1; + + ze_event_pool_counter_based_exp_desc_t counterBasedExtension = {ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC}; + eventPoolDesc.pNext = &counterBasedExtension; + + counterBasedExtension.flags = 0; + auto eventPool = EventPool::create(driverHandle.get(), context, 0, nullptr, &eventPoolDesc, returnValue); + EXPECT_EQ(nullptr, eventPool); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, returnValue); + + counterBasedExtension.flags = static_cast(ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE) << 1; + eventPool = EventPool::create(driverHandle.get(), context, 0, nullptr, &eventPoolDesc, returnValue); + EXPECT_EQ(nullptr, eventPool); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, returnValue); + + counterBasedExtension.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE; + eventPool = EventPool::create(driverHandle.get(), context, 0, nullptr, &eventPoolDesc, returnValue); + EXPECT_EQ(counterBasedExtension.flags, eventPool->getCounterBasedFlags()); + EXPECT_NE(nullptr, eventPool); + eventPool->destroy(); + + counterBasedExtension.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE; + eventPool = EventPool::create(driverHandle.get(), context, 0, nullptr, &eventPoolDesc, returnValue); + EXPECT_EQ(counterBasedExtension.flags, eventPool->getCounterBasedFlags()); + EXPECT_NE(nullptr, eventPool); + eventPool->destroy(); +} + HWTEST2_F(InOrderCmdListTests, givenIpcPoolEventWhenTryingToImplicitlyConverToCounterBasedEventThenDisallow, IsAtLeastSkl) { ze_event_pool_desc_t eventPoolDesc = {}; eventPoolDesc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE; @@ -844,7 +875,7 @@ HWTEST2_F(InOrderCmdListTests, givenIpcPoolEventWhenTryingToImplicitlyConverToCo HWTEST2_F(InOrderCmdListTests, givenNotSignaledInOrderWhenWhenCallingQueryStatusThenReturnNotReady, IsAtLeastSkl) { auto eventPool = createEvents(1, false); - events[0]->enableCounterBasedMode(true); + events[0]->enableCounterBasedMode(true, eventPool->getCounterBasedFlags()); EXPECT_EQ(ZE_RESULT_NOT_READY, events[0]->queryStatus()); } @@ -1410,40 +1441,48 @@ HWTEST2_F(InOrderCmdListTests, givenImplicitEventConvertionEnabledWhenUsingImmed immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[0]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::initiallyDisabled, events[0]->counterBasedMode); + EXPECT_EQ(0u, events[0]->counterBasedFlags); EXPECT_FALSE(events[0]->isCounterBased()); regularCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[1]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::initiallyDisabled, events[1]->counterBasedMode); + EXPECT_EQ(0u, events[1]->counterBasedFlags); EXPECT_FALSE(events[1]->isCounterBased()); debugManager.flags.EnableImplicitConvertionToCounterBasedEvents.set(-1); immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[0]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::implicitlyEnabled, events[0]->counterBasedMode); + EXPECT_EQ(static_cast(ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE), events[0]->counterBasedFlags); EXPECT_TRUE(events[0]->isCounterBased()); regularCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[1]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::implicitlyDisabled, events[1]->counterBasedMode); + EXPECT_EQ(0u, events[1]->counterBasedFlags); EXPECT_FALSE(events[1]->isCounterBased()); outOfOrderImmCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[2]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::implicitlyDisabled, events[2]->counterBasedMode); + EXPECT_EQ(0u, events[2]->counterBasedFlags); EXPECT_FALSE(events[2]->isCounterBased()); // Reuse on Regular = disable regularCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[0]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::implicitlyDisabled, events[0]->counterBasedMode); + EXPECT_EQ(0u, events[0]->counterBasedFlags); EXPECT_FALSE(events[0]->isCounterBased()); // Reuse on non-inOrder = disable events[0]->counterBasedMode = Event::CounterBasedMode::implicitlyEnabled; regularCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[0]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::implicitlyDisabled, events[0]->counterBasedMode); + EXPECT_EQ(0u, events[0]->counterBasedFlags); EXPECT_FALSE(events[0]->isCounterBased()); // Reuse on already disabled immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[0]->toHandle(), 0, nullptr, launchParams, false); EXPECT_EQ(Event::CounterBasedMode::implicitlyDisabled, events[0]->counterBasedMode); + EXPECT_EQ(0u, events[0]->counterBasedFlags); EXPECT_FALSE(events[0]->isCounterBased()); // On explicitly enabled @@ -1458,17 +1497,18 @@ HWTEST2_F(InOrderCmdListTests, givenImplicitEventConvertionEnabledWhenUsingAppen auto eventPool = createEvents(1, false); events[0]->makeCounterBasedInitiallyDisabled(); - events[0]->enableCounterBasedMode(false); + events[0]->enableCounterBasedMode(false, eventPool->getCounterBasedFlags()); immCmdList->appendEventReset(events[0]->toHandle()); EXPECT_EQ(Event::CounterBasedMode::implicitlyDisabled, events[0]->counterBasedMode); + EXPECT_EQ(0u, events[0]->counterBasedFlags); } HWTEST2_F(InOrderCmdListTests, givenImplicitEventConvertionEnabledWhenCallingAppendThenHandleInOrderExecInfo, IsAtLeastSkl) { auto immCmdList = createImmCmdList(); auto eventPool = createEvents(1, false); events[0]->makeCounterBasedInitiallyDisabled(); - events[0]->enableCounterBasedMode(false); + events[0]->enableCounterBasedMode(false, eventPool->getCounterBasedFlags()); immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, events[0]->toHandle(), 0, nullptr, launchParams, false); diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index ad04fe7ba8..6f51fab1dd 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -3235,7 +3235,7 @@ HWTEST_F(EventTests, givenInOrderEventWhenHostSynchronizeIsCalledThenAllocationI auto inOrderExecInfo = std::make_shared(*syncAllocation, nullptr, *neoDevice->getMemoryManager(), 1, false, false); *inOrderExecInfo->getBaseHostAddress() = 1; - event->enableCounterBasedMode(true); + event->enableCounterBasedMode(true, ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE); event->updateInOrderExecState(inOrderExecInfo, 1, 0); constexpr uint64_t timeout = std::numeric_limits::max(); @@ -3247,7 +3247,7 @@ HWTEST_F(EventTests, givenInOrderEventWhenHostSynchronizeIsCalledThenAllocationI auto event2 = zeUniquePtr(whiteboxCast(getHelper().createEvent(eventPool.get(), &eventDesc, device))); - event2->enableCounterBasedMode(true); + event2->enableCounterBasedMode(true, ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE); event2->updateInOrderExecState(inOrderExecInfo, 1, 0); syncAllocation->updateTaskCount(0u, ultCsr->getOsContext().getContextId()); ultCsr->downloadAllocationsCalledCount = 0; @@ -3291,7 +3291,7 @@ HWTEST_F(EventTests, givenInOrderEventWithHostAllocWhenHostSynchronizeIsCalledTh auto inOrderExecInfo = std::make_shared(*deviceSyncAllocation, hostSyncAllocation, *neoDevice->getMemoryManager(), 1, false, false); *inOrderExecInfo->getBaseHostAddress() = 1; - event->enableCounterBasedMode(true); + event->enableCounterBasedMode(true, ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE); event->updateInOrderExecState(inOrderExecInfo, 1, 0); constexpr uint64_t timeout = std::numeric_limits::max(); @@ -3304,7 +3304,7 @@ HWTEST_F(EventTests, givenInOrderEventWithHostAllocWhenHostSynchronizeIsCalledTh auto event2 = zeUniquePtr(whiteboxCast(getHelper().createEvent(eventPool.get(), &eventDesc, device))); - event2->enableCounterBasedMode(true); + event2->enableCounterBasedMode(true, ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE); event2->updateInOrderExecState(inOrderExecInfo, 1, 0); hostSyncAllocation->updateTaskCount(0u, ultCsr->getOsContext().getContextId()); ultCsr->downloadAllocationsCalledCount = 0;