refactor: Add optional user fence during unbind

Add optional fence and wait operations after unbind operation.

Signed-off-by: Aravind Gopalakrishnan <aravind.gopalakrishnan@intel.com>

shared/source/debug_settings/debug_variables_base.inl

@@ -275,6 +275,8 @@ DECLARE_DEBUG_VARIABLE(int32_t, SetThreadPriority, -1, "-1: default, 0: Disabled
 DECLARE_DEBUG_VARIABLE(int32_t, OverrideCpuCaching, -1, "-1: default, 1: DRM_XE_GEM_CPU_CACHING_WB, 2: DRM_XE_GEM_CPU_CACHING_WC")
 DECLARE_DEBUG_VARIABLE(int32_t, FlushTlbBeforeCopy, -1, "-1: default, 0: Dont flush, 1: flush TLB as part of MI_FLUSH_DW/PIPE_CONTROL command before copy operation")
 DECLARE_DEBUG_VARIABLE(int32_t, PrintMmapAndMunMapCalls, -1, "-1: default, If set, print all system mmap and munmap calls")
+DECLARE_DEBUG_VARIABLE(int32_t, EnableUserFenceUponUnbind, -1, "-1: default, 0: Dont enable fence, 1: Enable user fence on Vm_Unbind call")
+DECLARE_DEBUG_VARIABLE(int32_t, EnableWaitOnUserFenceAfterBindAndUnbind, -1, "-1: default, 0: Dont wait on fence, 1: Wait on user fence after Vm_Unbind call to ensure fence completion")
 
 /*LOGGING FLAGS*/
 DECLARE_DEBUG_VARIABLE(int32_t, PrintDriverDiagnostics, -1, "prints driver diagnostics messages to standard output, value corresponds to hint level")

shared/source/os_interface/linux/drm_neo.cpp

@@ -1246,6 +1246,23 @@ uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion c
     return patIndex;
 }
 
+void programUserFence(Drm *drm, OsContext *osContext, BufferObject *bo, VmBindExtUserFenceT &vmBindExtUserFence, uint32_t vmHandleId, uint64_t nextExtension) {
+    auto ioctlHelper = drm->getIoctlHelper();
+    uint64_t address = 0;
+    uint64_t value = 0;
+
+    if (drm->isPerContextVMRequired()) {
+        auto osContextLinux = static_cast<OsContextLinux *>(osContext);
+        address = castToUint64(osContextLinux->getFenceAddr(vmHandleId));
+        value = osContextLinux->getNextFenceVal(vmHandleId);
+    } else {
+        address = castToUint64(drm->getFenceAddr(vmHandleId));
+        value = drm->getNextFenceVal(vmHandleId);
+    }
+
+    ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, address, value, nextExtension);
+}
+
 int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, bool bind) {
     auto vmId = drm->getVirtualMemoryAddressSpace(vmHandleId);
     auto ioctlHelper = drm->getIoctlHelper();
@@ -1322,21 +1339,8 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI
 
                 if (!drm->hasPageFaultSupport() || bo->isExplicitResidencyRequired()) {
                     auto nextExtension = vmBind.extensions;
-
-                    uint64_t address = 0;
-                    uint64_t value = 0;
-
-                    if (drm->isPerContextVMRequired()) {
-                        auto osContextLinux = static_cast<OsContextLinux *>(osContext);
-                        address = castToUint64(osContextLinux->getFenceAddr(vmHandleId));
-                        value = osContextLinux->getNextFenceVal(vmHandleId);
-                    } else {
-                        address = castToUint64(drm->getFenceAddr(vmHandleId));
-                        value = drm->getNextFenceVal(vmHandleId);
-                    }
-
                     incrementFenceValue = true;
-                    ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, address, value, nextExtension);
+                    programUserFence(drm, osContext, bo, vmBindExtUserFence, vmHandleId, nextExtension);
                     ioctlHelper->setVmBindUserFence(vmBind, vmBindExtUserFence);
                 }
             }
@@ -1355,6 +1359,14 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI
                 break;
             }
         }
+        bool waitOnUserFenceAfterBindAndUnbind = false;
+        if (debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get() != -1) {
+            waitOnUserFenceAfterBindAndUnbind = !!debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get();
+        }
+        if (ioctlHelper->isWaitBeforeBindRequired(bind) && waitOnUserFenceAfterBindAndUnbind && drm->useVMBindImmediate()) {
+            auto osContextLinux = static_cast<OsContextLinux *>(osContext);
+            osContextLinux->waitForPagingFence();
+        }
         if (incrementFenceValue) {
             if (drm->isPerContextVMRequired()) {
                 auto osContextLinux = static_cast<OsContextLinux *>(osContext);

shared/source/os_interface/linux/ioctl_helper_prelim.cpp

@@ -936,7 +936,12 @@ bool IoctlHelperPrelim20::getFabricLatency(uint32_t fabricId, uint32_t &latency,
 }
 
 bool IoctlHelperPrelim20::isWaitBeforeBindRequired(bool bind) const {
-    return bind;
+    bool userFenceOnUnbind = false;
+    if (debugManager.flags.EnableUserFenceUponUnbind.get() != -1) {
+        userFenceOnUnbind = !!debugManager.flags.EnableUserFenceUponUnbind.get();
+    }
+
+    return (bind || userFenceOnUnbind);
 }
 
 void *IoctlHelperPrelim20::pciBarrierMmap() {

shared/source/os_interface/linux/ioctl_helper_upstream.cpp

@@ -310,6 +310,10 @@ bool IoctlHelperUpstream::getFabricLatency(uint32_t fabricId, uint32_t &latency,
 }
 
 bool IoctlHelperUpstream::isWaitBeforeBindRequired(bool bind) const {
-    return false;
+    bool userFenceOnUnbind = false;
+    if (debugManager.flags.EnableUserFenceUponUnbind.get() != -1) {
+        userFenceOnUnbind = !!debugManager.flags.EnableUserFenceUponUnbind.get();
+    }
+    return userFenceOnUnbind;
 }
 } // namespace NEO

shared/source/os_interface/linux/os_context_linux.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2023 Intel Corporation
+ * Copyright (C) 2018-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -29,7 +29,7 @@ class OsContextLinux : public OsContext {
     const std::vector<uint32_t> &getDrmVmIds() const { return drmVmIds; }
     bool isDirectSubmissionSupported() const override;
     Drm &getDrm() const;
-    void waitForPagingFence();
+    virtual void waitForPagingFence();
     static OsContext *create(OSInterface *osInterface, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor);
     void reInitializeContext() override;
     void setHangDetected() {

shared/test/common/test_files/igdrcl.config

@@ -586,6 +586,8 @@ ExperimentalEnableHostAllocationCache = -1
 OverridePatIndexForUncachedTypes = -1
 OverridePatIndexForCachedTypes = -1
 FlushTlbBeforeCopy = -1
+EnableUserFenceUponUnbind = -1
+EnableWaitOnUserFenceAfterBindAndUnbind = -1
 UseGemCreateExtInAllocateMemoryByKMD = -1
 PrintMmapAndMunMapCalls = -1
 UseLocalPreferredForCacheableBuffers = -1

shared/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2023 Intel Corporation
+ * Copyright (C) 2018-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -701,6 +701,210 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsThenFenceValueGrow) {
     EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1);
 }
 
+TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsAndForceUserFenceUponUnbindAndDisableFenceWaitThenFenceValueGrow) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableUserFenceUponUnbind.set(1);
+    debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(0);
+
+    auto executionEnvironment = new ExecutionEnvironment;
+    executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online);
+    executionEnvironment->prepareRootDeviceEnvironments(1);
+    executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get());
+    executionEnvironment->rootDeviceEnvironments[0]->initGmm();
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique<OSInterface>();
+
+    auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]);
+    drm->requirePerContextVM = false;
+    drm->isVMBindImmediateSupported = true;
+    auto ioctlHelper = std::make_unique<MockIoctlHelper>(*drm);
+    ioctlHelper->failBind = false;
+    ioctlHelper->waitBeforeBindRequired = true;
+    drm->ioctlHelper.reset(ioctlHelper.release());
+
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr<DriverModel>(drm));
+    executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false);
+    uint64_t initFenceValue = 10u;
+    drm->fenceVal[0] = initFenceValue;
+    std::unique_ptr<Device> device(MockDevice::createWithExecutionEnvironment<MockDevice>(defaultHwInfo.get(), executionEnvironment, 0));
+
+    auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex());
+    auto osContextCount = engines.size();
+    auto contextId = osContextCount / 2;
+    auto osContext = engines[contextId].osContext;
+    MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount);
+    drm->unbindBufferObject(osContext, 0, &bo);
+
+    EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1);
+}
+
+TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsAndForceFenceWaitThenFenceValueGrow) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableUserFenceUponUnbind.set(1);
+    debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(1);
+
+    class MockOsContextLinuxUnbind : public OsContextLinux {
+      public:
+        using OsContextLinux::drmContextIds;
+        using OsContextLinux::fenceVal;
+        using OsContextLinux::pagingFence;
+
+        MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor)
+            : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {}
+
+        void waitForPagingFence() override {
+            waitForPagingFenceCalled = true;
+        }
+
+        bool waitForPagingFenceCalled = false;
+    };
+
+    auto executionEnvironment = new ExecutionEnvironment;
+    executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online);
+    executionEnvironment->prepareRootDeviceEnvironments(1);
+    executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get());
+    executionEnvironment->rootDeviceEnvironments[0]->initGmm();
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique<OSInterface>();
+
+    auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]);
+    drm->requirePerContextVM = false;
+    drm->isVMBindImmediateSupported = true;
+    auto ioctlHelper = std::make_unique<MockIoctlHelper>(*drm);
+    ioctlHelper->failBind = false;
+    ioctlHelper->waitBeforeBindRequired = true;
+    drm->ioctlHelper.reset(ioctlHelper.release());
+
+    auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor());
+    osContext->ensureContextInitialized();
+
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr<DriverModel>(drm));
+    executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false);
+    uint64_t initFenceValue = 10u;
+    drm->fenceVal[0] = initFenceValue;
+    std::unique_ptr<Device> device(MockDevice::createWithExecutionEnvironment<MockDevice>(defaultHwInfo.get(), executionEnvironment, 0));
+
+    auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex());
+    auto osContextCount = engines.size();
+
+    MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount);
+    drm->unbindBufferObject(static_cast<OsContext *>(osContext), 0, &bo);
+
+    EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1);
+    EXPECT_TRUE(osContext->waitForPagingFenceCalled);
+    delete osContext;
+}
+
+TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsWaitBeforeBindFalseAndForceFenceWaitButWaitNotTrueThenFenceDoesNotGrow) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableUserFenceUponUnbind.set(0);
+    debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(1);
+
+    class MockOsContextLinuxUnbind : public OsContextLinux {
+      public:
+        using OsContextLinux::drmContextIds;
+        using OsContextLinux::fenceVal;
+        using OsContextLinux::pagingFence;
+
+        MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor)
+            : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {}
+
+        void waitForPagingFence() override {
+            waitForPagingFenceCalled = true;
+        }
+
+        bool waitForPagingFenceCalled = false;
+    };
+
+    auto executionEnvironment = new ExecutionEnvironment;
+    executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online);
+    executionEnvironment->prepareRootDeviceEnvironments(1);
+    executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get());
+    executionEnvironment->rootDeviceEnvironments[0]->initGmm();
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique<OSInterface>();
+
+    auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]);
+    drm->requirePerContextVM = false;
+    drm->isVMBindImmediateSupported = true;
+    auto ioctlHelper = std::make_unique<MockIoctlHelper>(*drm);
+    ioctlHelper->failBind = false;
+    ioctlHelper->waitBeforeBindRequired = false;
+    drm->ioctlHelper.reset(ioctlHelper.release());
+
+    auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor());
+    osContext->ensureContextInitialized();
+
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr<DriverModel>(drm));
+    executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false);
+    uint64_t initFenceValue = 10u;
+    drm->fenceVal[0] = initFenceValue;
+    std::unique_ptr<Device> device(MockDevice::createWithExecutionEnvironment<MockDevice>(defaultHwInfo.get(), executionEnvironment, 0));
+
+    auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex());
+    auto osContextCount = engines.size();
+
+    MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount);
+    drm->unbindBufferObject(static_cast<OsContext *>(osContext), 0, &bo);
+
+    EXPECT_EQ(drm->fenceVal[0], initFenceValue);
+    EXPECT_FALSE(osContext->waitForPagingFenceCalled);
+    delete osContext;
+}
+
+TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsWaitBeforeBindTrueAndForceFenceWaitButNotVmBindImmediateThenWaitPagingFenceNotCalled) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableUserFenceUponUnbind.set(1);
+    debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(1);
+
+    class MockOsContextLinuxUnbind : public OsContextLinux {
+      public:
+        using OsContextLinux::drmContextIds;
+        using OsContextLinux::fenceVal;
+        using OsContextLinux::pagingFence;
+
+        MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor)
+            : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {}
+
+        void waitForPagingFence() override {
+            waitForPagingFenceCalled = true;
+        }
+
+        bool waitForPagingFenceCalled = false;
+    };
+
+    auto executionEnvironment = new ExecutionEnvironment;
+    executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online);
+    executionEnvironment->prepareRootDeviceEnvironments(1);
+    executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get());
+    executionEnvironment->rootDeviceEnvironments[0]->initGmm();
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique<OSInterface>();
+
+    auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]);
+    drm->requirePerContextVM = false;
+    drm->isVMBindImmediateSupported = false;
+    auto ioctlHelper = std::make_unique<MockIoctlHelper>(*drm);
+    ioctlHelper->failBind = false;
+    ioctlHelper->waitBeforeBindRequired = true;
+    drm->ioctlHelper.reset(ioctlHelper.release());
+
+    auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor());
+    osContext->ensureContextInitialized();
+
+    executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr<DriverModel>(drm));
+    executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false);
+    uint64_t initFenceValue = 10u;
+    drm->fenceVal[0] = initFenceValue;
+    std::unique_ptr<Device> device(MockDevice::createWithExecutionEnvironment<MockDevice>(defaultHwInfo.get(), executionEnvironment, 0));
+
+    auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex());
+    auto osContextCount = engines.size();
+
+    MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount);
+    drm->unbindBufferObject(static_cast<OsContext *>(osContext), 0, &bo);
+
+    EXPECT_EQ(drm->fenceVal[0], initFenceValue);
+    EXPECT_FALSE(osContext->waitForPagingFenceCalled);
+    delete osContext;
+}
+
 TEST(DrmBufferObject, whenBindExtHandleAddedThenItIsStored) {
     auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
     DrmMockResources drm(*executionEnvironment->rootDeviceEnvironments[0]);

shared/test/unit_test/os_interface/linux/ioctl_helper_tests_prelim.cpp

@@ -680,6 +680,30 @@ TEST_F(IoctlPrelimHelperTests, whenChangingBufferBindingThenWaitIsNeededOnlyBefo
     EXPECT_FALSE(ioctlHelper.isWaitBeforeBindRequired(false));
 }
 
+TEST_F(IoctlPrelimHelperTests, whenChangingBufferBindingAndForcingFenceWaitThenCallReturnsTrueForBindAndUnbind) {
+    DebugManagerStateRestore restorer;
+    MockExecutionEnvironment executionEnvironment{};
+    std::unique_ptr<Drm> drm{Drm::create(std::make_unique<HwDeviceIdDrm>(0, ""), *executionEnvironment.rootDeviceEnvironments[0])};
+
+    IoctlHelperPrelim20 ioctlHelper{*drm};
+
+    debugManager.flags.EnableUserFenceUponUnbind.set(1);
+    EXPECT_TRUE(ioctlHelper.isWaitBeforeBindRequired(true));
+    EXPECT_TRUE(ioctlHelper.isWaitBeforeBindRequired(false));
+}
+
+TEST_F(IoctlPrelimHelperTests, whenChangingBufferBindingAndNotForcingFenceWaitThenCallReturnsTrueForBindOnly) {
+    DebugManagerStateRestore restorer;
+    MockExecutionEnvironment executionEnvironment{};
+    std::unique_ptr<Drm> drm{Drm::create(std::make_unique<HwDeviceIdDrm>(0, ""), *executionEnvironment.rootDeviceEnvironments[0])};
+
+    IoctlHelperPrelim20 ioctlHelper{*drm};
+
+    debugManager.flags.EnableUserFenceUponUnbind.set(0);
+    EXPECT_TRUE(ioctlHelper.isWaitBeforeBindRequired(true));
+    EXPECT_FALSE(ioctlHelper.isWaitBeforeBindRequired(false));
+}
+
 TEST_F(IoctlPrelimHelperTests, whenGettingPreferredLocationRegionThenReturnCorrectMemoryClassAndInstance) {
     DebugManagerStateRestore restorer;
 

shared/test/unit_test/os_interface/linux/ioctl_helper_tests_upstream.cpp

@@ -143,6 +143,19 @@ TEST(IoctlHelperUpstreamTest, whenChangingBufferBindingThenWaitIsNeverNeeded) {
     EXPECT_FALSE(ioctlHelper.isWaitBeforeBindRequired(true));
     EXPECT_FALSE(ioctlHelper.isWaitBeforeBindRequired(false));
 }
+
+TEST(IoctlHelperUpstreamTest, whenChangingBufferBindingThenWaitIsAddedWhenForced) {
+    DebugManagerStateRestore restorer;
+    MockExecutionEnvironment executionEnvironment{};
+    std::unique_ptr<Drm> drm{Drm::create(std::make_unique<HwDeviceIdDrm>(0, ""), *executionEnvironment.rootDeviceEnvironments[0])};
+
+    IoctlHelperUpstream ioctlHelper{*drm};
+
+    debugManager.flags.EnableUserFenceUponUnbind.set(1);
+    EXPECT_TRUE(ioctlHelper.isWaitBeforeBindRequired(true));
+    EXPECT_TRUE(ioctlHelper.isWaitBeforeBindRequired(false));
+}
+
 TEST(IoctlHelperUpstreamTest, whenGettingIoctlRequestStringThenProperStringIsReturned) {
     auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
     auto drm = std::make_unique<DrmTipMock>(*executionEnvironment->rootDeviceEnvironments[0]);