diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_7.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_7.cpp index 45dbc8285a..c6bfd8e513 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_7.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_7.cpp @@ -2031,6 +2031,9 @@ TEST(CommandList, givenContextGroupEnabledWhenCreatingImmediateCommandListWithIn mockOsContexts.push_back(newOsContext); newOsContext->incRefInternal(); + newOsContext->setIsPrimaryEngine(engine.osContext->getIsPrimaryEngine()); + newOsContext->setContextGroup(engine.osContext->isPartOfContextGroup()); + engine.osContext = newOsContext; engine.commandStreamReceiver->setupContext(*newOsContext); } @@ -2059,9 +2062,7 @@ TEST(CommandList, givenContextGroupEnabledWhenCreatingImmediateCommandListWithIn auto commandList3 = static_cast(L0::CommandList::fromHandle(commandListHandle3)); EXPECT_TRUE(static_cast(commandList1->getCsr(false)->getOsContext()).allocateInterruptPassed); - EXPECT_FALSE(static_cast(commandList1->getCsr(false)->getOsContext()).isPartOfContextGroup()); EXPECT_TRUE(static_cast(commandList2->getCsr(false)->getOsContext()).allocateInterruptPassed); - EXPECT_FALSE(static_cast(commandList2->getCsr(false)->getOsContext()).isPartOfContextGroup()); EXPECT_FALSE(static_cast(commandList3->getCsr(false)->getOsContext()).allocateInterruptPassed); commandList1->destroy(); diff --git a/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_3.cpp b/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_3.cpp index efa18bf08f..4efd930ce5 100644 --- a/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_3.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_3.cpp @@ -1404,6 +1404,9 @@ TEST(CommandQueue, givenContextGroupEnabledWhenCreatingCommandQueuesWithInterrup mockOsContexts.push_back(newOsContext); newOsContext->incRefInternal(); + newOsContext->setIsPrimaryEngine(engine.osContext->getIsPrimaryEngine()); + newOsContext->setContextGroup(engine.osContext->isPartOfContextGroup()); + engine.osContext = newOsContext; engine.commandStreamReceiver->setupContext(*newOsContext); } @@ -1432,9 +1435,7 @@ TEST(CommandQueue, givenContextGroupEnabledWhenCreatingCommandQueuesWithInterrup auto commandQueue3 = static_cast(L0::CommandQueue::fromHandle(commandQueueHandle3)); EXPECT_TRUE(static_cast(commandQueue1->getCsr()->getOsContext()).allocateInterruptPassed); - EXPECT_FALSE(static_cast(commandQueue1->getCsr()->getOsContext()).isPartOfContextGroup()); EXPECT_TRUE(static_cast(commandQueue2->getCsr()->getOsContext()).allocateInterruptPassed); - EXPECT_FALSE(static_cast(commandQueue2->getCsr()->getOsContext()).isPartOfContextGroup()); EXPECT_FALSE(static_cast(commandQueue3->getCsr()->getOsContext()).allocateInterruptPassed); commandQueue1->destroy(); diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp index f8fa0c74a7..3ea4eccf6a 100644 --- a/shared/source/device/device.cpp +++ b/shared/source/device/device.cpp @@ -394,6 +394,7 @@ bool Device::createEngines() { secondaryEnginesForType.highPriorityEnginesTotal = highPriorityContextCount; secondaryEnginesForType.regularCounter = 0; secondaryEnginesForType.highPriorityCounter = 0; + secondaryEnginesForType.assignedContextsCounter = 1; NEO::EngineTypeUsage engineTypeUsage; engineTypeUsage.first = primaryEngine.getEngineType(); @@ -591,24 +592,20 @@ EngineControl *Device::getSecondaryEngineCsr(EngineTypeUsage engineTypeUsage, bo auto &secondaryEnginesForType = secondaryEngines[engineTypeUsage.first]; - auto secondaryEngineIndex = 0; - if (engineTypeUsage.second == EngineUsage::highPriority) { - secondaryEngineIndex = (secondaryEnginesForType.highPriorityCounter.fetch_add(1)) % (secondaryEnginesForType.highPriorityEnginesTotal); - secondaryEngineIndex += secondaryEnginesForType.regularEnginesTotal; - } else if (engineTypeUsage.second == EngineUsage::regular) { - secondaryEngineIndex = (secondaryEnginesForType.regularCounter.fetch_add(1)) % (secondaryEnginesForType.regularEnginesTotal); + auto engineControl = secondaryEnginesForType.getEngine(engineTypeUsage.second); - if (secondaryEngineIndex == 0 && allocateInterrupt) { - // Context 0 is already pre-initialized. We need non-initialized context, to pass context creation flag. - // If all contexts are already initialized, just take next available. Interrupt request is only a hint. - secondaryEngineIndex = (secondaryEnginesForType.regularCounter.fetch_add(1)) % (secondaryEnginesForType.regularEnginesTotal); - } - } else { - DEBUG_BREAK_IF(true); + bool isPrimaryContextInGroup = engineControl->osContext->getIsPrimaryEngine() && engineControl->osContext->isPartOfContextGroup(); + + if (isPrimaryContextInGroup && allocateInterrupt) { + // Context 0 is already pre-initialized. We need non-initialized context, to pass context creation flag. + // If all contexts are already initialized, just take next available. Interrupt request is only a hint. + engineControl = secondaryEnginesForType.getEngine(engineTypeUsage.second); } - if (secondaryEngineIndex > 0) { - auto commandStreamReceiver = secondaryEnginesForType.engines[secondaryEngineIndex].commandStreamReceiver; + isPrimaryContextInGroup = engineControl->osContext->getIsPrimaryEngine() && engineControl->osContext->isPartOfContextGroup(); + + if (!isPrimaryContextInGroup) { + auto commandStreamReceiver = engineControl->commandStreamReceiver; auto lock = commandStreamReceiver->obtainUniqueOwnership(); @@ -633,7 +630,7 @@ EngineControl *Device::getSecondaryEngineCsr(EngineTypeUsage engineTypeUsage, bo } } } - return &secondaryEnginesForType.engines[secondaryEngineIndex]; + return engineControl; } const HardwareInfo &Device::getHardwareInfo() const { return *getRootDeviceEnvironment().getHardwareInfo(); } @@ -1200,4 +1197,57 @@ const EngineGroupT *Device::tryGetRegularEngineGroup(EngineGroupType engineGroup } return nullptr; } + +EngineControl *SecondaryContexts::getEngine(EngineUsage usage) { + auto secondaryEngineIndex = 0; + + std::lock_guard guard(mutex); + + if (usage == EngineUsage::highPriority) { + + // Use index from reserved HP pool + if (hpIndices.size() < highPriorityEnginesTotal) { + secondaryEngineIndex = (highPriorityCounter.fetch_add(1)) % (highPriorityEnginesTotal); + secondaryEngineIndex += regularEnginesTotal; + hpIndices.push_back(secondaryEngineIndex); + } + // Check if there is free index + else if (assignedContextsCounter < regularEnginesTotal) { + secondaryEngineIndex = assignedContextsCounter.fetch_add(1); + highPriorityCounter.fetch_add(1); + hpIndices.push_back(secondaryEngineIndex); + } + // Assign from existing indices + else { + auto index = (highPriorityCounter.fetch_add(1)) % (hpIndices.size()); + secondaryEngineIndex = hpIndices[index]; + } + + if (engines[secondaryEngineIndex].osContext->getEngineUsage() != EngineUsage::highPriority) { + engines[secondaryEngineIndex].osContext->overrideEngineUsage(EngineUsage::highPriority); + } + + } else if (usage == EngineUsage::regular) { + if (npIndices.size() == 0) { + regularCounter.fetch_add(1); + npIndices.push_back(secondaryEngineIndex); + } + // Check if there is free index + else if (assignedContextsCounter < regularEnginesTotal) { + secondaryEngineIndex = assignedContextsCounter.fetch_add(1); + regularCounter.fetch_add(1); + npIndices.push_back(secondaryEngineIndex); + } + // Assign from existing indices + else { + auto index = (regularCounter.fetch_add(1)) % (npIndices.size()); + secondaryEngineIndex = npIndices[index]; + } + } else { + DEBUG_BREAK_IF(true); + } + + return &engines[secondaryEngineIndex]; +} + } // namespace NEO diff --git a/shared/source/device/device.h b/shared/source/device/device.h index 1260bc0a6f..a3686ff830 100644 --- a/shared/source/device/device.h +++ b/shared/source/device/device.h @@ -18,6 +18,7 @@ #include "shared/source/utilities/reference_tracked_object.h" #include +#include namespace NEO { class BindlessHeapsHelper; @@ -63,11 +64,18 @@ struct SecondaryContexts { SecondaryContexts(const SecondaryContexts &in) = delete; SecondaryContexts &operator=(const SecondaryContexts &) = delete; - EnginesT engines; // vector of secondary EngineControls - std::atomic regularCounter = 0; // Counter used to assign next regular EngineControl - std::atomic highPriorityCounter = 0; // Counter used to assign next highPriority EngineControl + EngineControl *getEngine(const EngineUsage usage); + + EnginesT engines; // vector of secondary EngineControls + std::atomic regularCounter = 0; // Counter used to assign next regular EngineControl + std::atomic highPriorityCounter = 0; // Counter used to assign next highPriority EngineControl + std::atomic assignedContextsCounter = 0; // Counter of assigned contexts in group uint32_t regularEnginesTotal; uint32_t highPriorityEnginesTotal; + + std::vector npIndices; + std::vector hpIndices; + std::mutex mutex; }; struct RTDispatchGlobalsInfo { diff --git a/shared/source/os_interface/os_context.h b/shared/source/os_interface/os_context.h index 00f6890f7c..7b00b6edfe 100644 --- a/shared/source/os_interface/os_context.h +++ b/shared/source/os_interface/os_context.h @@ -34,6 +34,8 @@ class OsContext : public ReferenceTrackedObject { PreemptionMode getPreemptionMode() const { return preemptionMode; } const aub_stream::EngineType &getEngineType() const { return engineType; } EngineUsage getEngineUsage() { return engineUsage; } + void overrideEngineUsage(EngineUsage usage) { engineUsage = usage; } + bool isRegular() const { return engineUsage == EngineUsage::regular; } bool isLowPriority() const { return engineUsage == EngineUsage::lowPriority; } bool isHighPriority() const { return engineUsage == EngineUsage::highPriority; } @@ -110,7 +112,7 @@ class OsContext : public ReferenceTrackedObject { const PreemptionMode preemptionMode; const uint32_t numSupportedDevices; aub_stream::EngineType engineType = aub_stream::ENGINE_RCS; - const EngineUsage engineUsage; + EngineUsage engineUsage; const bool rootDevice = false; bool defaultContext = false; bool directSubmissionActive = false; diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp index 4022b5e384..d66ff925dc 100644 --- a/shared/test/unit_test/device/neo_device_tests.cpp +++ b/shared/test/unit_test/device/neo_device_tests.cpp @@ -1262,6 +1262,107 @@ HWTEST_F(DeviceTests, givenCCSEnginesAndContextGroupSizeEnabledWhenDeviceIsCreat EXPECT_NE(internalEngine.commandStreamReceiver, device->getSecondaryEngineCsr({aub_stream::EngineType::ENGINE_CCS, EngineUsage::internal}, false)->commandStreamReceiver); } +HWTEST_F(DeviceTests, givenContextGroupSizeEnabledWhenMoreHpEnginesCreatedThenFreeEnginesAreAssignedUpToHalfOfContextGroup) { + DebugManagerStateRestore dbgRestorer; + const uint32_t contextGroupSize = 14; + debugManager.flags.ContextGroupSize.set(contextGroupSize); + + HardwareInfo hwInfo = *defaultHwInfo; + hwInfo.featureTable.flags.ftrCCSNode = true; + hwInfo.featureTable.ftrBcsInfo = 0; + hwInfo.capabilityTable.defaultEngineType = aub_stream::ENGINE_CCS; + hwInfo.gtSystemInfo.CCSInfo.NumberOfCCSEnabled = 1; + + auto device = std::unique_ptr(MockDevice::createWithNewExecutionEnvironment(&hwInfo)); + auto &engineGroups = device->getRegularEngineGroups(); + + auto engineGroupType = EngineGroupType::compute; + size_t computeEnginesCount = 0; + for (const auto &engine : engineGroups) { + if (engine.engineGroupType == engineGroupType) { + computeEnginesCount = engine.engines.size(); + } + } + + if (computeEnginesCount == 0) { + GTEST_SKIP(); + } + + ASSERT_EQ(computeEnginesCount, device->secondaryEngines.size()); + ASSERT_EQ(contextGroupSize, device->secondaryEngines[aub_stream::EngineType::ENGINE_CCS].engines.size()); + + auto defaultEngine = device->getDefaultEngine(); + EXPECT_EQ(defaultEngine.commandStreamReceiver, device->secondaryEngines[aub_stream::EngineType::ENGINE_CCS].engines[0].commandStreamReceiver); + + const uint32_t maxHpContextCount = contextGroupSize / 2; + + for (uint32_t ccsIndex = 0; ccsIndex < computeEnginesCount; ccsIndex++) { + auto &secondaryEngines = device->secondaryEngines[EngineHelpers::mapCcsIndexToEngineType(ccsIndex)]; + + EXPECT_TRUE(secondaryEngines.engines[0].osContext->isPartOfContextGroup()); + EXPECT_EQ(nullptr, secondaryEngines.engines[0].osContext->getPrimaryContext()); + + for (size_t i = 1; i < device->secondaryEngines[aub_stream::EngineType::ENGINE_CCS].engines.size(); i++) { + EXPECT_EQ(secondaryEngines.engines[0].osContext, secondaryEngines.engines[i].osContext->getPrimaryContext()); + EXPECT_TRUE(secondaryEngines.engines[i].osContext->isPartOfContextGroup()); + } + + EXPECT_EQ(0u, secondaryEngines.regularCounter.load()); + EXPECT_EQ(0u, secondaryEngines.highPriorityCounter.load()); + + auto regularContextCount = secondaryEngines.regularEnginesTotal; + EXPECT_EQ(contextGroupSize - regularContextCount, secondaryEngines.highPriorityEnginesTotal); + + uint32_t npCounter = 0; + uint32_t hpCounter = 0; + std::vector hpEngines; + + for (size_t contextId = 0; contextId < maxHpContextCount + 2; contextId++) { + + if (contextId == 2) { + auto engine = device->getSecondaryEngineCsr({EngineHelpers::mapCcsIndexToEngineType(ccsIndex), EngineUsage::regular}, false); + ASSERT_NE(nullptr, engine); + + EXPECT_EQ(1, secondaryEngines.regularCounter.load()); + EXPECT_EQ(&secondaryEngines.engines[npCounter], engine); + EXPECT_FALSE(secondaryEngines.engines[npCounter].osContext->isHighPriority()); + + npCounter++; + } + if (contextId == 6) { + auto engine = device->getSecondaryEngineCsr({EngineHelpers::mapCcsIndexToEngineType(ccsIndex), EngineUsage::regular}, false); + ASSERT_NE(nullptr, engine); + + EXPECT_EQ(2, secondaryEngines.regularCounter.load()); + EXPECT_EQ(&secondaryEngines.engines[npCounter], engine); + EXPECT_FALSE(secondaryEngines.engines[npCounter].osContext->isHighPriority()); + + npCounter++; + } + + auto engine = device->getSecondaryEngineCsr({EngineHelpers::mapCcsIndexToEngineType(ccsIndex), EngineUsage::highPriority}, false); + ASSERT_NE(nullptr, engine); + hpEngines.push_back(engine); + hpCounter++; + + if (contextId < secondaryEngines.highPriorityEnginesTotal) { + EXPECT_EQ(&secondaryEngines.engines[regularContextCount + hpCounter - 1], engine); + EXPECT_TRUE(secondaryEngines.engines[regularContextCount + hpCounter - 1].osContext->isHighPriority()); + } else if (contextId >= secondaryEngines.highPriorityEnginesTotal) { + + if (hpCounter <= maxHpContextCount) { + EXPECT_EQ(&secondaryEngines.engines[npCounter], engine); + EXPECT_TRUE(secondaryEngines.engines[npCounter].osContext->isHighPriority()); + npCounter++; + } else { + EXPECT_EQ(hpEngines[hpCounter - 1 % maxHpContextCount], engine); + EXPECT_TRUE(hpEngines[hpCounter - 1 % maxHpContextCount]->osContext->isHighPriority()); + } + } + } + } +} + HWTEST_F(DeviceTests, givenDebugFlagSetWhenCreatingSecondaryEnginesThenCreateCorrectNumberOfHighPriorityContexts) { DebugManagerStateRestore dbgRestorer; constexpr uint32_t contextGroupSize = 16;