refactor: pass InOrderExecInfo to encode method

Related-To: NEO-7966

Signed-off-by: Dunajski, Bartosz <bartosz.dunajski@intel.com>

shared/source/command_container/command_encoder.h

@@ -26,6 +26,7 @@ class BindlessHeapsHelper;
 class Gmm;
 class GmmHelper;
 class IndirectHeap;
+class InOrderExecInfo;
 class ProductHelper;
 
 struct DeviceInfo;
@@ -43,9 +44,11 @@ struct StateBaseAddressProperties;
 struct StateComputeModeProperties;
 
 struct EncodeDispatchKernelArgs {
-    uint64_t eventAddress = 0ull;
+    uint64_t eventAddress = 0;
     uint64_t postSyncImmValue = 0;
+    uint64_t inOrderCounterValue = 0;
     Device *device = nullptr;
+    NEO::InOrderExecInfo *inOrderExecInfo = nullptr;
     DispatchKernelEncoderI *dispatchInterface = nullptr;
     IndirectHeap *surfaceStateHeap = nullptr;
     IndirectHeap *dynamicStateHeap = nullptr;

shared/source/command_container/command_encoder_xehp_and_later.inl

@@ -20,6 +20,7 @@
 #include "shared/source/helpers/constants.h"
 #include "shared/source/helpers/gfx_core_helper.h"
 #include "shared/source/helpers/hw_walk_order.h"
+#include "shared/source/helpers/in_order_cmd_helpers.h"
 #include "shared/source/helpers/pause_on_gpu_properties.h"
 #include "shared/source/helpers/pipe_control_args.h"
 #include "shared/source/helpers/ray_tracing_helper.h"
@@ -345,21 +346,31 @@ void EncodeDispatchKernel<Family>::encode(CommandContainer &container, EncodeDis
                                                    rootDeviceEnvironment);
 
     auto &postSync = walkerCmd.getPostSync();
-    if (args.eventAddress != 0) {
+    if (args.eventAddress != 0 || args.inOrderExecInfo) {
         postSync.setDataportPipelineFlush(true);
         postSync.setDataportSubsliceCacheFlush(true);
 
-        if (args.isTimestampEvent) {
-            UNRECOVERABLE_IF(!(isAligned<timestampDestinationAddressAlignment>(args.eventAddress)));
+        auto operationType = POSTSYNC_DATA::OPERATION_WRITE_IMMEDIATE_DATA;
+        uint64_t gpuVa = args.eventAddress;
+        uint64_t immData = args.postSyncImmValue;
 
-            postSync.setOperation(POSTSYNC_DATA::OPERATION_WRITE_TIMESTAMP);
-        } else {
-            UNRECOVERABLE_IF(!(isAligned<immWriteDestinationAddressAlignment>(args.eventAddress)));
+        if (args.inOrderExecInfo) {
+            gpuVa = args.inOrderExecInfo->getDeviceCounterAllocation().getGpuAddress() + args.inOrderExecInfo->getAllocationOffset();
+            immData = args.inOrderCounterValue;
+        } else if (args.isTimestampEvent) {
+            operationType = POSTSYNC_DATA::OPERATION_WRITE_TIMESTAMP;
+            immData = 0;
 
-            postSync.setOperation(POSTSYNC_DATA::OPERATION_WRITE_IMMEDIATE_DATA);
-            postSync.setImmediateData(args.postSyncImmValue);
+            UNRECOVERABLE_IF(!(isAligned<timestampDestinationAddressAlignment>(gpuVa)));
         }
-        postSync.setDestinationAddress(args.eventAddress);
+
+        if (operationType == POSTSYNC_DATA::OPERATION_WRITE_IMMEDIATE_DATA) {
+            UNRECOVERABLE_IF(!(isAligned<immWriteDestinationAddressAlignment>(gpuVa)));
+        }
+
+        postSync.setOperation(operationType);
+        postSync.setImmediateData(immData);
+        postSync.setDestinationAddress(gpuVa);
 
         EncodeDispatchKernel<Family>::setupPostSyncMocs(walkerCmd, rootDeviceEnvironment, args.dcFlushEnable);
         EncodeDispatchKernel<Family>::adjustTimestampPacket(walkerCmd, args);

shared/source/helpers/CMakeLists.txt

@@ -99,6 +99,8 @@ set(NEO_CORE_HELPERS
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_ip_version.h
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_mapper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/hw_walk_order.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/in_order_cmd_helpers.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/in_order_cmd_helpers.h
     ${CMAKE_CURRENT_SOURCE_DIR}/kernel_helpers.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/kernel_helpers.h
     ${CMAKE_CURRENT_SOURCE_DIR}/kmd_notify_properties.cpp

shared/source/helpers/in_order_cmd_helpers.cpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/helpers/in_order_cmd_helpers.h"
+
+#include "shared/source/memory_manager/memory_manager.h"
+
+#include <cstdint>
+#include <string.h>
+#include <vector>
+
+namespace NEO {
+
+InOrderExecInfo::~InOrderExecInfo() {
+    memoryManager.freeGraphicsMemory(&deviceCounterAllocation);
+    memoryManager.freeGraphicsMemory(hostCounterAllocation);
+}
+
+InOrderExecInfo::InOrderExecInfo(NEO::GraphicsAllocation &deviceCounterAllocation, NEO::GraphicsAllocation *hostCounterAllocation, NEO::MemoryManager &memoryManager, uint32_t partitionCount, bool regularCmdList, bool atomicDeviceSignalling)
+    : deviceCounterAllocation(deviceCounterAllocation), memoryManager(memoryManager), hostCounterAllocation(hostCounterAllocation), regularCmdList(regularCmdList) {
+
+    numDevicePartitionsToWait = atomicDeviceSignalling ? 1 : partitionCount;
+    numHostPartitionsToWait = partitionCount;
+
+    if (hostCounterAllocation) {
+        hostAddress = reinterpret_cast<uint64_t *>(hostCounterAllocation->getUnderlyingBuffer());
+        duplicatedHostStorage = true;
+    } else {
+        hostAddress = reinterpret_cast<uint64_t *>(deviceCounterAllocation.getUnderlyingBuffer());
+    }
+
+    reset();
+}
+
+void InOrderExecInfo::reset() {
+    resetCounterValue();
+    regularCmdListSubmissionCounter = 0;
+    allocationOffset = 0;
+
+    memset(deviceCounterAllocation.getUnderlyingBuffer(), 0, deviceCounterAllocation.getUnderlyingBufferSize());
+
+    if (hostCounterAllocation) {
+        memset(hostCounterAllocation->getUnderlyingBuffer(), 0, hostCounterAllocation->getUnderlyingBufferSize());
+    }
+}
+
+} // namespace NEO

shared/source/helpers/in_order_cmd_helpers.h

@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "shared/source/helpers/non_copyable_or_moveable.h"
+#include "shared/source/helpers/ptr_math.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+namespace NEO {
+class GraphicsAllocation;
+class MemoryManager;
+
+class InOrderExecInfo : public NEO::NonCopyableClass {
+  public:
+    ~InOrderExecInfo();
+
+    InOrderExecInfo() = delete;
+
+    InOrderExecInfo(NEO::GraphicsAllocation &deviceCounterAllocation, NEO::GraphicsAllocation *hostCounterAllocation, NEO::MemoryManager &memoryManager, uint32_t partitionCount, bool regularCmdList, bool atomicDeviceSignalling);
+
+    NEO::GraphicsAllocation &getDeviceCounterAllocation() const { return deviceCounterAllocation; }
+    NEO::GraphicsAllocation *getHostCounterAllocation() const { return hostCounterAllocation; }
+    uint64_t *getBaseHostAddress() const { return hostAddress; }
+
+    uint64_t getCounterValue() const { return counterValue; }
+    void addCounterValue(uint64_t addValue) { counterValue += addValue; }
+    void resetCounterValue() { counterValue = 0; }
+
+    uint64_t getRegularCmdListSubmissionCounter() const { return regularCmdListSubmissionCounter; }
+    void addRegularCmdListSubmissionCounter(uint64_t addValue) { regularCmdListSubmissionCounter += addValue; }
+
+    bool isRegularCmdList() const { return regularCmdList; }
+    bool isHostStorageDuplicated() const { return duplicatedHostStorage; }
+
+    uint32_t getNumDevicePartitionsToWait() const { return numDevicePartitionsToWait; }
+    uint32_t getNumHostPartitionsToWait() const { return numHostPartitionsToWait; }
+
+    void addAllocationOffset(uint32_t addValue) { allocationOffset += addValue; }
+    uint32_t getAllocationOffset() const { return allocationOffset; }
+
+    void reset();
+
+  protected:
+    NEO::GraphicsAllocation &deviceCounterAllocation;
+    NEO::MemoryManager &memoryManager;
+    NEO::GraphicsAllocation *hostCounterAllocation = nullptr;
+    uint64_t counterValue = 0;
+    uint64_t regularCmdListSubmissionCounter = 0;
+    uint64_t *hostAddress = nullptr;
+    uint32_t numDevicePartitionsToWait = 0;
+    uint32_t numHostPartitionsToWait = 0;
+    uint32_t allocationOffset = 0;
+    bool regularCmdList = false;
+    bool duplicatedHostStorage = false;
+};
+
+namespace InOrderPatchCommandHelpers {
+inline uint64_t getAppendCounterValue(const InOrderExecInfo &inOrderExecInfo) {
+    if (inOrderExecInfo.isRegularCmdList() && inOrderExecInfo.getRegularCmdListSubmissionCounter() > 1) {
+        return inOrderExecInfo.getCounterValue() * (inOrderExecInfo.getRegularCmdListSubmissionCounter() - 1);
+    }
+
+    return 0;
+}
+
+enum class PatchCmdType {
+    None,
+    Lri64b,
+    Sdi,
+    Semaphore,
+    Walker
+};
+
+template <typename GfxFamily>
+struct PatchCmd {
+    PatchCmd(std::shared_ptr<InOrderExecInfo> *inOrderExecInfo, void *cmd1, void *cmd2, uint64_t baseCounterValue, PatchCmdType patchCmdType)
+        : cmd1(cmd1), cmd2(cmd2), baseCounterValue(baseCounterValue), patchCmdType(patchCmdType) {
+        if (inOrderExecInfo) {
+            this->inOrderExecInfo = *inOrderExecInfo;
+        }
+    }
+
+    void patch(uint64_t appendCounterValue) {
+        switch (patchCmdType) {
+        case PatchCmdType::Sdi:
+            patchSdi(appendCounterValue);
+            break;
+        case PatchCmdType::Semaphore:
+            patchSemaphore(appendCounterValue);
+            break;
+        case PatchCmdType::Walker:
+            patchComputeWalker(appendCounterValue);
+            break;
+        case PatchCmdType::Lri64b:
+            patchLri64b(appendCounterValue);
+            break;
+        default:
+            UNRECOVERABLE_IF(true);
+            break;
+        }
+    }
+
+    bool isExternalDependency() const { return inOrderExecInfo.get(); }
+
+    std::shared_ptr<InOrderExecInfo> inOrderExecInfo;
+    void *cmd1 = nullptr;
+    void *cmd2 = nullptr;
+    const uint64_t baseCounterValue = 0;
+    const PatchCmdType patchCmdType = PatchCmdType::None;
+
+  protected:
+    void patchSdi(uint64_t appendCounterValue) {
+        auto sdiCmd = reinterpret_cast<typename GfxFamily::MI_STORE_DATA_IMM *>(cmd1);
+        sdiCmd->setDataDword0(getLowPart(baseCounterValue + appendCounterValue));
+        sdiCmd->setDataDword1(getHighPart(baseCounterValue + appendCounterValue));
+    }
+
+    void patchSemaphore(uint64_t appendCounterValue) {
+        if (isExternalDependency()) {
+            appendCounterValue = InOrderPatchCommandHelpers::getAppendCounterValue(*inOrderExecInfo);
+            if (appendCounterValue == 0) {
+                return;
+            }
+        }
+
+        auto semaphoreCmd = reinterpret_cast<typename GfxFamily::MI_SEMAPHORE_WAIT *>(cmd1);
+        semaphoreCmd->setSemaphoreDataDword(static_cast<uint32_t>(baseCounterValue + appendCounterValue));
+    }
+
+    void patchComputeWalker(uint64_t appendCounterValue) {
+        if constexpr (GfxFamily::walkerPostSyncSupport) {
+            auto walkerCmd = reinterpret_cast<typename GfxFamily::COMPUTE_WALKER *>(cmd1);
+            auto &postSync = walkerCmd->getPostSync();
+            postSync.setImmediateData(baseCounterValue + appendCounterValue);
+        } else {
+            UNRECOVERABLE_IF(true);
+        }
+    }
+
+    void patchLri64b(uint64_t appendCounterValue) {
+        if (isExternalDependency()) {
+            appendCounterValue = InOrderPatchCommandHelpers::getAppendCounterValue(*inOrderExecInfo);
+            if (appendCounterValue == 0) {
+                return;
+            }
+        }
+
+        const uint64_t counterValue = baseCounterValue + appendCounterValue;
+
+        auto lri1 = reinterpret_cast<typename GfxFamily::MI_LOAD_REGISTER_IMM *>(cmd1);
+        lri1->setDataDword(getLowPart(counterValue));
+
+        auto lri2 = reinterpret_cast<typename GfxFamily::MI_LOAD_REGISTER_IMM *>(cmd2);
+        lri2->setDataDword(getHighPart(counterValue));
+    }
+
+    PatchCmd() = delete;
+};
+
+} // namespace InOrderPatchCommandHelpers
+
+template <typename GfxFamily>
+using InOrderPatchCommandsContainer = std::vector<NEO::InOrderPatchCommandHelpers::PatchCmd<GfxFamily>>;
+
+} // namespace NEO