Capability to insert WA MMIO for BCS dispatch

Signed-off-by: Bartosz Dunajski <bartosz.dunajski@intel.com>

shared/source/command_stream/command_stream_receiver_hw.h

@@ -189,6 +189,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     std::unique_ptr<DirectSubmissionHw<GfxFamily, BlitterDispatcher<GfxFamily>>> blitterDirectSubmission;
 
     size_t cmdStreamStart = 0;
+    uint32_t latestSentBcsWaValue = std::numeric_limits<uint32_t>::max();
 };
 
 } // namespace NEO

shared/source/command_stream/command_stream_receiver_hw_base.inl

@@ -1075,6 +1075,8 @@ std::optional<uint32_t> CommandStreamReceiverHw<GfxFamily>::flushBcsTask(const B
         TimestampPacketHelper::programCsrDependenciesForTimestampPacketContainer<GfxFamily>(commandStream, blitProperties.csrDependencies);
         TimestampPacketHelper::programCsrDependenciesForForTaskCountContainer<GfxFamily>(commandStream, blitProperties.csrDependencies);
 
+        BlitCommandsHelper<GfxFamily>::encodeWa(commandStream, blitProperties, latestSentBcsWaValue);
+
         if (blitProperties.outputTimestampPacket && profilingEnabled) {
             BlitCommandsHelper<GfxFamily>::encodeProfilingStartMmios(commandStream, *blitProperties.outputTimestampPacket);
         }

shared/source/debug_settings/debug_variables_base.inl

@@ -398,6 +398,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, EnableDrmCompletionFence, -1, "Enables DRM compl
 DECLARE_DEBUG_VARIABLE(int32_t, UseDrmCompletionFenceForAllAllocations, -1, "Uses DRM completion fence for all allocations, -1:default (disabled), 0:disable, 1:enable")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableChipsetUniqueUUID, -1, "Enables retrieving chipset unique UUID using telemetry, -1:default (disabled), 0:disable, 1:enable")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableFlushTaskSubmission, -1, "Driver uses csr flushTask for immediate commandlist submissions, -1:default (enabled), 0:disabled, 1:enabled")
+DECLARE_DEBUG_VARIABLE(int32_t, EnableBcsSwControlWa, -1, "Enable BCS WA via BCSSWCONTROL MMIO. -1: default, 0: disabled, 1: if src in system mem, 2: if dst in system mem, 3: if src and dst in system mem, 4: always")
 
 /*EXPERIMENTAL TOGGLES*/
 DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalSetWalkerPartitionCount, 0, "Experimental implementation: Set number of COMPUTE_WALKERs for a given Partition Type, 0 - do not set the feature.")

shared/source/helpers/blit_commands_helper.h

@@ -178,5 +178,8 @@ struct BlitCommandsHelper {
     static void encodeProfilingStartMmios(LinearStream &cmdStream, const TagNodeBase &timestampPacketNode);
     static void encodeProfilingEndMmios(LinearStream &cmdStream, const TagNodeBase &timestampPacketNode);
     static size_t getProfilingMmioCmdsSize();
+
+    static void encodeWa(LinearStream &cmdStream, const BlitProperties &blitProperties, uint32_t &latestSentBcsWaValue);
+    static size_t getWaCmdsSize(const BlitPropertiesContainer &blitPropertiesContainer);
 };
 } // namespace NEO

shared/source/helpers/blit_commands_helper_base.inl

@@ -142,6 +142,7 @@ size_t BlitCommandsHelper<GfxFamily>::estimateBlitCommandsSize(const BlitPropert
         size += BlitCommandsHelper<GfxFamily>::estimateBlitCommandSize(blitProperties.copySize, blitProperties.csrDependencies, updateTimestampPacket,
                                                                        profilingEnabled, isImage, rootDeviceEnvironment);
     }
+    size += BlitCommandsHelper<GfxFamily>::getWaCmdsSize(blitPropertiesContainer);
     size += 2 * MemorySynchronizationCommands<GfxFamily>::getSizeForAdditonalSynchronization(*rootDeviceEnvironment.getHardwareInfo());
     size += EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite();
     size += blitterDirectSubmission ? sizeof(typename GfxFamily::MI_BATCH_BUFFER_START) : sizeof(typename GfxFamily::MI_BATCH_BUFFER_END);
@@ -509,4 +510,13 @@ size_t BlitCommandsHelper<GfxFamily>::getProfilingMmioCmdsSize() {
 
 template <typename GfxFamily>
 void BlitCommandsHelper<GfxFamily>::appendBaseAddressOffset(const BlitProperties &blitProperties, typename GfxFamily::XY_BLOCK_COPY_BLT &blitCmd, const uint32_t originalSliceIndex, const bool isSource) {}
+
+template <typename GfxFamily>
+void BlitCommandsHelper<GfxFamily>::encodeWa(LinearStream &cmdStream, const BlitProperties &blitProperties, uint32_t &latestSentBcsWaValue) {
+}
+
+template <typename GfxFamily>
+size_t BlitCommandsHelper<GfxFamily>::getWaCmdsSize(const BlitPropertiesContainer &blitPropertiesContainer) {
+    return 0;
+}
 } // namespace NEO

shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp

@@ -205,6 +205,56 @@ template <>
 template <>
 void BlitCommandsHelper<Family>::appendColorDepth(const BlitProperties &blitProperites, typename Family::XY_COPY_BLT &blitCmd) {}
 
+template <>
+void BlitCommandsHelper<Family>::encodeWa(LinearStream &cmdStream, const BlitProperties &blitProperties, uint32_t &latestSentBcsWaValue) {
+    using MI_LOAD_REGISTER_IMM = typename Family::MI_LOAD_REGISTER_IMM;
+
+    if (DebugManager.flags.EnableBcsSwControlWa.get() <= 0) {
+        return;
+    }
+
+    constexpr int32_t srcInSystemMemOnly = 1;
+    constexpr int32_t dstInSystemMemOnly = 2;
+    constexpr int32_t enableAlways = 4;
+    constexpr uint32_t waEnabledMMioValue = 0x40004;
+    constexpr uint32_t waDisabledMMioValue = 0x40000;
+
+    const bool applyForSrc = (DebugManager.flags.EnableBcsSwControlWa.get() & srcInSystemMemOnly);
+    const bool applyForDst = (DebugManager.flags.EnableBcsSwControlWa.get() & dstInSystemMemOnly);
+    const bool applyAlways = (DebugManager.flags.EnableBcsSwControlWa.get() == enableAlways);
+
+    const bool enableWa = (!blitProperties.srcAllocation->isAllocatedInLocalMemoryPool() && applyForSrc) ||
+                          (!blitProperties.dstAllocation->isAllocatedInLocalMemoryPool() && applyForDst) ||
+                          applyAlways;
+
+    uint32_t newValue = enableWa ? waEnabledMMioValue : waDisabledMMioValue;
+
+    if (newValue == latestSentBcsWaValue) {
+        return;
+    }
+
+    latestSentBcsWaValue = newValue;
+
+    MI_LOAD_REGISTER_IMM cmd = Family::cmdInitLoadRegisterImm;
+    cmd.setRegisterOffset(0x22200);
+    cmd.setDataDword(newValue);
+    cmd.setMmioRemapEnable(true);
+
+    auto lri = cmdStream.getSpaceForCmd<MI_LOAD_REGISTER_IMM>();
+    *lri = cmd;
+}
+
+template <>
+size_t BlitCommandsHelper<Family>::getWaCmdsSize(const BlitPropertiesContainer &blitPropertiesContainer) {
+    using MI_LOAD_REGISTER_IMM = typename Family::MI_LOAD_REGISTER_IMM;
+
+    if (DebugManager.flags.EnableBcsSwControlWa.get() <= 0) {
+        return 0;
+    }
+
+    return (blitPropertiesContainer.size() * sizeof(MI_LOAD_REGISTER_IMM));
+}
+
 template class CommandStreamReceiverHw<Family>;
 template struct BlitCommandsHelper<Family>;
 template void BlitCommandsHelper<Family>::appendBlitCommandsForBuffer<typename Family::XY_COPY_BLT>(const BlitProperties &blitProperties, typename Family::XY_COPY_BLT &blitCmd, const RootDeviceEnvironment &rootDeviceEnvironment);

shared/test/common/test_files/igdrcl.config

@@ -431,4 +431,5 @@ ForceUncachedGmmUsageType = 0
 OverrideDeviceName = unk
 EnablePrivateBO = 0
 ExperimentalEnableDeviceAllocationCache = 0 
-OverrideL1CachePolicyInSurfaceStateAndStateless = -1
+OverrideL1CachePolicyInSurfaceStateAndStateless = -1
+EnableBcsSwControlWa = -1