From c2c622d695c84e76b7653f67628e3e67b9ff5bdb Mon Sep 17 00:00:00 2001 From: Igor Venevtsev Date: Tue, 27 Jun 2023 15:50:33 +0000 Subject: [PATCH] fix: stop direct submission on platform destruction Related-To: NEO-8072 Signed-off-by: Igor Venevtsev --- opencl/source/platform/platform.cpp | 1 + .../unit_test/platform/platform_tests.cpp | 22 ++++++++++ shared/source/device/device.cpp | 9 ++++ shared/source/device/device.h | 1 + .../libult/ult_command_stream_receiver.h | 6 +++ shared/test/common/mocks/mock_device.h | 6 +++ .../direct_submission_tests_1.cpp | 42 +++++++++++++++++++ 7 files changed, 87 insertions(+) diff --git a/opencl/source/platform/platform.cpp b/opencl/source/platform/platform.cpp index 093748830f..1891579028 100644 --- a/opencl/source/platform/platform.cpp +++ b/opencl/source/platform/platform.cpp @@ -44,6 +44,7 @@ Platform::~Platform() { for (auto clDevice : this->clDevices) { clDevice->getDevice().getRootDeviceEnvironmentRef().debugger.reset(nullptr); + clDevice->getDevice().stopDirectSubmission(); clDevice->decRefInternal(); } diff --git a/opencl/test/unit_test/platform/platform_tests.cpp b/opencl/test/unit_test/platform/platform_tests.cpp index 3f19c14981..8093c5180d 100644 --- a/opencl/test/unit_test/platform/platform_tests.cpp +++ b/opencl/test/unit_test/platform/platform_tests.cpp @@ -89,6 +89,28 @@ TEST_F(PlatformTest, WhenGetDeviceIsCalledThenExpectedValuesAreReturned) { EXPECT_EQ(nullptr, pPlatform->getClDevice(numDevices)); } +TEST_F(PlatformTest, WhenPlatformIsDestroyedThenDirectSubmissionIsTerminated) { + VariableBackup createFuncBackup{&DeviceFactory::createRootDeviceFunc}; + DeviceFactory::createRootDeviceFunc = [](ExecutionEnvironment &executionEnvironment, uint32_t rootDeviceIndex) -> std::unique_ptr { + return std::unique_ptr(MockDevice::create(&executionEnvironment, rootDeviceIndex)); + }; + + pPlatform->initializeWithNewDevices(); + + auto clDevice = pPlatform->getClDevice(0); + EXPECT_NE(nullptr, clDevice); + + auto mockDevice = static_cast(&clDevice->getDevice()); + EXPECT_FALSE(mockDevice->stopDirectSubmissionCalled); + + clDevice->incRefInternal(); + + pPlatform = nullptr; + EXPECT_TRUE(mockDevice->stopDirectSubmissionCalled); + + clDevice->decRefInternal(); +} + TEST_F(PlatformTest, WhenGetClDevicesIsCalledThenExpectedValuesAreReturned) { EXPECT_EQ(nullptr, pPlatform->getClDevices()); diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp index 0599033891..7054759d8d 100644 --- a/shared/source/device/device.cpp +++ b/shared/source/device/device.cpp @@ -865,6 +865,15 @@ const ReleaseHelper *Device::getReleaseHelper() const { return getRootDeviceEnvironment().getReleaseHelper(); } +void Device::stopDirectSubmission() { + for (auto &engine : allEngines) { + auto csr = engine.commandStreamReceiver; + if (csr->isAnyDirectSubmissionEnabled()) { + csr->stopDirectSubmission(); + } + } +} + void Device::allocateRTDispatchGlobals(uint32_t maxBvhLevels) { UNRECOVERABLE_IF(rtDispatchGlobalsInfos.size() < maxBvhLevels + 1); UNRECOVERABLE_IF(rtDispatchGlobalsInfos[maxBvhLevels] != nullptr); diff --git a/shared/source/device/device.h b/shared/source/device/device.h index eb0867c135..56d8576580 100644 --- a/shared/source/device/device.h +++ b/shared/source/device/device.h @@ -165,6 +165,7 @@ class Device : public ReferenceTrackedObject { uint32_t getNumberOfRegularContextsPerEngine() const { return numberOfRegularContextsPerEngine; } bool isMultiRegularContextSelectionAllowed(aub_stream::EngineType engineType, EngineUsage engineUsage) const; + MOCKABLE_VIRTUAL void stopDirectSubmission(); std::atomic debugExecutionCounter = 0; diff --git a/shared/test/common/libult/ult_command_stream_receiver.h b/shared/test/common/libult/ult_command_stream_receiver.h index ccda7fcbed..2a21cb6f82 100644 --- a/shared/test/common/libult/ult_command_stream_receiver.h +++ b/shared/test/common/libult/ult_command_stream_receiver.h @@ -404,6 +404,11 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw, publ return *flushReturnValue; } + void stopDirectSubmission() override { + stopDirectSubmissionCalled = true; + BaseClass::stopDirectSubmission(); + } + std::vector aubCommentMessages; BatchBuffer latestFlushedBatchBuffer = {}; @@ -458,6 +463,7 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw, publ bool forceReturnGpuHang = false; bool callBaseIsKmdWaitOnTaskCountAllowed = false; bool isKmdWaitOnTaskCountAllowedValue = false; + bool stopDirectSubmissionCalled = false; }; } // namespace NEO diff --git a/shared/test/common/mocks/mock_device.h b/shared/test/common/mocks/mock_device.h index c992ca05f1..1d0212d468 100644 --- a/shared/test/common/mocks/mock_device.h +++ b/shared/test/common/mocks/mock_device.h @@ -162,6 +162,11 @@ class MockDevice : public RootDevice { void setRTDispatchGlobalsForceAllocation() { rtDispatchGlobalsForceAllocation = true; } + + void stopDirectSubmission() override { + stopDirectSubmissionCalled = true; + Device::stopDirectSubmission(); + } static ExecutionEnvironment *prepareExecutionEnvironment(const HardwareInfo *pHwInfo); static decltype(&createCommandStream) createCommandStreamReceiverFunc; @@ -172,6 +177,7 @@ class MockDevice : public RootDevice { bool verifyAdapterLuidReturnValue = true; size_t maxParameterSizeFromIGC = 0u; bool rtDispatchGlobalsForceAllocation = true; + bool stopDirectSubmissionCalled = false; }; template <> diff --git a/shared/test/unit_test/direct_submission/direct_submission_tests_1.cpp b/shared/test/unit_test/direct_submission/direct_submission_tests_1.cpp index 0ddfb899bb..6a1c5b6bec 100644 --- a/shared/test/unit_test/direct_submission/direct_submission_tests_1.cpp +++ b/shared/test/unit_test/direct_submission/direct_submission_tests_1.cpp @@ -97,6 +97,48 @@ HWTEST_F(DirectSubmissionTest, givenBlitterDirectSubmissionWhenStopThenRingIsNot csr.blitterDirectSubmission.release(); } +HWTEST_F(DirectSubmissionTest, givenDeviceStopDirectSubmissionCalledThenCsrStopDirecttSubmissionCalled) { + VariableBackup backup(&ultHwConfig); + ultHwConfig.csrBaseCallDirectSubmissionAvailable = true; + ultHwConfig.csrBaseCallBlitterDirectSubmissionAvailable = true; + + MockDirectSubmissionHw> directSubmission(*pDevice->getDefaultEngine().commandStreamReceiver); + auto &csr = pDevice->getUltCommandStreamReceiver(); + csr.directSubmission.reset(&directSubmission); + + bool ret = directSubmission.initialize(true, false); + EXPECT_TRUE(ret); + + EXPECT_FALSE(csr.stopDirectSubmissionCalled); + pDevice->stopDirectSubmission(); + EXPECT_TRUE(csr.stopDirectSubmissionCalled); + + csr.directSubmission.release(); +} + +HWTEST_F(DirectSubmissionTest, givenDeviceStopDirectSubmissionCalledThenBcsStopDirecttSubmissionCalled) { + VariableBackup backup(&ultHwConfig); + ultHwConfig.csrBaseCallDirectSubmissionAvailable = true; + ultHwConfig.csrBaseCallBlitterDirectSubmissionAvailable = true; + + MockDirectSubmissionHw> directSubmission(*pDevice->getDefaultEngine().commandStreamReceiver); + auto &csr = pDevice->getUltCommandStreamReceiver(); + std::unique_ptr osContext(OsContext::create(pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->osInterface.get(), pDevice->getRootDeviceIndex(), 0, + EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_BCS, EngineUsage::Regular}, + PreemptionMode::ThreadGroup, pDevice->getDeviceBitfield()))); + csr.blitterDirectSubmission.reset(&directSubmission); + csr.setupContext(*osContext); + + bool ret = directSubmission.initialize(true, false); + EXPECT_TRUE(ret); + + EXPECT_FALSE(csr.stopDirectSubmissionCalled); + pDevice->stopDirectSubmission(); + EXPECT_TRUE(csr.stopDirectSubmissionCalled); + + csr.blitterDirectSubmission.release(); +} + HWTEST_F(DirectSubmissionTest, givenDirectSubmissionWhenMakingResourcesResidentThenCorrectContextIsUsed) { auto mockMemoryOperations = std::make_unique();