From d20c9264e5a5a7a778f4ed9d8adaade3c23f729f Mon Sep 17 00:00:00 2001 From: Kamil Kopryk Date: Wed, 10 Sep 2025 05:32:19 +0000 Subject: [PATCH] feature: add hostFunctions data layout Related-To: NEO-14577 Signed-off-by: Kamil Kopryk --- shared/source/command_stream/CMakeLists.txt | 3 + shared/source/command_stream/host_function.h | 47 ++++++++++++ .../source/command_stream/host_function.inl | 74 ++++++++++++++++++ .../command_stream/host_function_enablers.inl | 15 ++++ shared/source/enable_cores.cmake | 1 + .../source/gen12lp/host_function_gen12lp.cpp | 14 ++++ .../host_function_xe2_hpg_core.cpp | 14 ++++ .../xe3_core/host_function_xe3_core.cpp | 14 ++++ .../xe_hpc_core/host_function_xe_hpc_core.cpp | 14 ++++ .../xe_hpg_core/host_function_xe_hpg_core.cpp | 14 ++++ .../unit_test/command_stream/CMakeLists.txt | 1 + .../command_stream/host_function_tests.cpp | 75 +++++++++++++++++++ 12 files changed, 286 insertions(+) create mode 100644 shared/source/command_stream/host_function.h create mode 100644 shared/source/command_stream/host_function.inl create mode 100644 shared/source/command_stream/host_function_enablers.inl create mode 100644 shared/source/gen12lp/host_function_gen12lp.cpp create mode 100644 shared/source/xe2_hpg_core/host_function_xe2_hpg_core.cpp create mode 100644 shared/source/xe3_core/host_function_xe3_core.cpp create mode 100644 shared/source/xe_hpc_core/host_function_xe_hpc_core.cpp create mode 100644 shared/source/xe_hpg_core/host_function_xe_hpg_core.cpp create mode 100644 shared/test/unit_test/command_stream/host_function_tests.cpp diff --git a/shared/source/command_stream/CMakeLists.txt b/shared/source/command_stream/CMakeLists.txt index 60792dccfb..ab55819178 100644 --- a/shared/source/command_stream/CMakeLists.txt +++ b/shared/source/command_stream/CMakeLists.txt @@ -29,6 +29,9 @@ set(NEO_CORE_COMMAND_STREAM ${CMAKE_CURRENT_SOURCE_DIR}/definitions${BRANCH_DIR_SUFFIX}command_stream_receiver_hw_ext.inl ${CMAKE_CURRENT_SOURCE_DIR}/definitions${BRANCH_DIR_SUFFIX}stream_properties.inl ${CMAKE_CURRENT_SOURCE_DIR}/device_command_stream.h + ${CMAKE_CURRENT_SOURCE_DIR}/host_function.h + ${CMAKE_CURRENT_SOURCE_DIR}/host_function.inl + ${CMAKE_CURRENT_SOURCE_DIR}/host_function_enablers.inl ${CMAKE_CURRENT_SOURCE_DIR}/linear_stream.cpp ${CMAKE_CURRENT_SOURCE_DIR}/linear_stream.h ${CMAKE_CURRENT_SOURCE_DIR}/preemption.cpp diff --git a/shared/source/command_stream/host_function.h b/shared/source/command_stream/host_function.h new file mode 100644 index 0000000000..8ce3d2818f --- /dev/null +++ b/shared/source/command_stream/host_function.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include +#include + +namespace NEO { + +class LinearStream; + +struct HostFunctionData { + volatile uint64_t *entry = nullptr; + volatile uint64_t *userData = nullptr; + volatile uint32_t *internalTag = nullptr; +}; + +enum class HostFunctionTagStatus : uint32_t { + completed = 0, + pending = 1 +}; + +struct HostFunctionHelper { + + constexpr static size_t entryOffset = offsetof(HostFunctionData, entry); + constexpr static size_t userDataOffset = offsetof(HostFunctionData, userData); + constexpr static size_t internalTagOffset = offsetof(HostFunctionData, internalTag); + + template + static void programHostFunction(LinearStream &commandStream, const HostFunctionData &hostFunctionData, uint64_t userHostFunctionAddress, uint64_t userDataAddress); + + template + static void programHostFunctionUserData(LinearStream &commandStream, const HostFunctionData &hostFunctionData, uint64_t userHostFunctionAddress, uint64_t userDataAddress); + + template + static void programSignalHostFunctionStart(LinearStream &commandStream, const HostFunctionData &hostFunctionData); + + template + static void programWaitForHostFunctionCompletion(LinearStream &commandStream, const HostFunctionData &hostFunctionData); +}; + +} // namespace NEO diff --git a/shared/source/command_stream/host_function.inl b/shared/source/command_stream/host_function.inl new file mode 100644 index 0000000000..add5b04f15 --- /dev/null +++ b/shared/source/command_stream/host_function.inl @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_container/command_encoder.h" +#include "shared/source/command_stream/host_function.h" +#include "shared/source/memory_manager/multi_graphics_allocation.h" + +namespace NEO { + +template +void HostFunctionHelper::programHostFunction(LinearStream &commandStream, const HostFunctionData &hostFunctionData, uint64_t userHostFunctionAddress, uint64_t userDataAddress) { + + HostFunctionHelper::programHostFunctionUserData(commandStream, hostFunctionData, userHostFunctionAddress, userDataAddress); + HostFunctionHelper::programSignalHostFunctionStart(commandStream, hostFunctionData); + HostFunctionHelper::programWaitForHostFunctionCompletion(commandStream, hostFunctionData); +} + +template +void HostFunctionHelper::programHostFunctionUserData(LinearStream &commandStream, const HostFunctionData &hostFunctionData, uint64_t userHostFunctionAddress, uint64_t userDataAddress) { + + auto hostFunctionAddressDst = reinterpret_cast(hostFunctionData.entry); + auto userDataAddressDst = reinterpret_cast(hostFunctionData.userData); + + EncodeStoreMemory::programStoreDataImm(commandStream, + hostFunctionAddressDst, + getLowPart(userHostFunctionAddress), + getHighPart(userHostFunctionAddress), + true, + false, + nullptr); + + EncodeStoreMemory::programStoreDataImm(commandStream, + userDataAddressDst, + getLowPart(userDataAddress), + getHighPart(userDataAddress), + true, + false, + nullptr); +} + +template +void HostFunctionHelper::programSignalHostFunctionStart(LinearStream &commandStream, const HostFunctionData &hostFunctionData) { + + auto internalTagAddress = reinterpret_cast(hostFunctionData.internalTag); + EncodeStoreMemory::programStoreDataImm(commandStream, + internalTagAddress, + static_cast(HostFunctionTagStatus::pending), + 0u, + false, + false, + nullptr); +} + +template +void HostFunctionHelper::programWaitForHostFunctionCompletion(LinearStream &commandStream, const HostFunctionData &hostFunctionData) { + + auto internalTagAddress = reinterpret_cast(hostFunctionData.internalTag); + EncodeSemaphore::addMiSemaphoreWaitCommand( + commandStream, + internalTagAddress, + static_cast(HostFunctionTagStatus::completed), + GfxFamily::MI_SEMAPHORE_WAIT::COMPARE_OPERATION::COMPARE_OPERATION_SAD_EQUAL_SDD, + false, + false, + false, + false, + nullptr); +} + +} // namespace NEO diff --git a/shared/source/command_stream/host_function_enablers.inl b/shared/source/command_stream/host_function_enablers.inl new file mode 100644 index 0000000000..3aaf3a1a74 --- /dev/null +++ b/shared/source/command_stream/host_function_enablers.inl @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" + +namespace NEO { +template void HostFunctionHelper::programHostFunction(LinearStream &commandStream, const HostFunctionData &hostFunctionData, uint64_t userHostFunctionAddress, uint64_t userDataAddress); +template void HostFunctionHelper::programHostFunctionUserData(LinearStream &commandStream, const HostFunctionData &hostFunctionData, uint64_t userHostFunctionAddress, uint64_t userDataAddress); +template void HostFunctionHelper::programSignalHostFunctionStart(LinearStream &commandStream, const HostFunctionData &hostFunctionData); +template void HostFunctionHelper::programWaitForHostFunctionCompletion(LinearStream &commandStream, const HostFunctionData &hostFunctionData); +} // namespace NEO diff --git a/shared/source/enable_cores.cmake b/shared/source/enable_cores.cmake index f46624f95d..bdd6ccede8 100644 --- a/shared/source/enable_cores.cmake +++ b/shared/source/enable_cores.cmake @@ -21,6 +21,7 @@ set(CORE_RUNTIME_SRCS_COREX_CPP_BASE debugger direct_submission experimental_command_buffer + host_function implicit_scaling gfx_core_helper gmm_callbacks diff --git a/shared/source/gen12lp/host_function_gen12lp.cpp b/shared/source/gen12lp/host_function_gen12lp.cpp new file mode 100644 index 0000000000..06d61dbab7 --- /dev/null +++ b/shared/source/gen12lp/host_function_gen12lp.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" +#include "shared/source/command_stream/host_function.inl" +#include "shared/source/gen12lp/hw_cmds_base.h" + +using Family = NEO::Gen12LpFamily; + +#include "shared/source/command_stream/host_function_enablers.inl" diff --git a/shared/source/xe2_hpg_core/host_function_xe2_hpg_core.cpp b/shared/source/xe2_hpg_core/host_function_xe2_hpg_core.cpp new file mode 100644 index 0000000000..a23808e80f --- /dev/null +++ b/shared/source/xe2_hpg_core/host_function_xe2_hpg_core.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" +#include "shared/source/command_stream/host_function.inl" +#include "shared/source/xe2_hpg_core/hw_cmds_base.h" + +using Family = NEO::Xe2HpgCoreFamily; + +#include "shared/source/command_stream/host_function_enablers.inl" diff --git a/shared/source/xe3_core/host_function_xe3_core.cpp b/shared/source/xe3_core/host_function_xe3_core.cpp new file mode 100644 index 0000000000..1287a271f6 --- /dev/null +++ b/shared/source/xe3_core/host_function_xe3_core.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" +#include "shared/source/command_stream/host_function.inl" +#include "shared/source/xe3_core/hw_cmds_base.h" + +using Family = NEO::Xe3CoreFamily; + +#include "shared/source/command_stream/host_function_enablers.inl" diff --git a/shared/source/xe_hpc_core/host_function_xe_hpc_core.cpp b/shared/source/xe_hpc_core/host_function_xe_hpc_core.cpp new file mode 100644 index 0000000000..c3118787b7 --- /dev/null +++ b/shared/source/xe_hpc_core/host_function_xe_hpc_core.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" +#include "shared/source/command_stream/host_function.inl" +#include "shared/source/xe_hpc_core/hw_cmds_xe_hpc_core_base.h" + +using Family = NEO::XeHpcCoreFamily; + +#include "shared/source/command_stream/host_function_enablers.inl" diff --git a/shared/source/xe_hpg_core/host_function_xe_hpg_core.cpp b/shared/source/xe_hpg_core/host_function_xe_hpg_core.cpp new file mode 100644 index 0000000000..a590a59f53 --- /dev/null +++ b/shared/source/xe_hpg_core/host_function_xe_hpg_core.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" +#include "shared/source/command_stream/host_function.inl" +#include "shared/source/xe_hpg_core/hw_cmds_xe_hpg_core_base.h" + +using Family = NEO::XeHpgCoreFamily; + +#include "shared/source/command_stream/host_function_enablers.inl" diff --git a/shared/test/unit_test/command_stream/CMakeLists.txt b/shared/test/unit_test/command_stream/CMakeLists.txt index 7af090176c..5036be6306 100644 --- a/shared/test/unit_test/command_stream/CMakeLists.txt +++ b/shared/test/unit_test/command_stream/CMakeLists.txt @@ -18,6 +18,7 @@ target_sources(neo_shared_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/compute_mode_tests.h ${CMAKE_CURRENT_SOURCE_DIR}/csr_deps_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/get_devices_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/host_function_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/linear_stream_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/stream_properties_tests_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/stream_properties_tests_common.h diff --git a/shared/test/unit_test/command_stream/host_function_tests.cpp b/shared/test/unit_test/command_stream/host_function_tests.cpp new file mode 100644 index 0000000000..1f530582bd --- /dev/null +++ b/shared/test/unit_test/command_stream/host_function_tests.cpp @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/command_stream/host_function.h" +#include "shared/test/common/cmd_parse/hw_parse.h" +#include "shared/test/common/fixtures/device_fixture.h" +#include "shared/test/common/test_macros/hw_test.h" + +#include + +using namespace NEO; + +using HostFunctionTests = Test; + +HWTEST_F(HostFunctionTests, givenHostFunctionDataStoredWhenProgramHostFunctionIsCalledThenMiStoresAndSemaphoreWaitAreProgrammedCorrectlyInCorrectOrder) { + using MI_STORE_DATA_IMM = typename FamilyType::MI_STORE_DATA_IMM; + using MI_SEMAPHORE_WAIT = typename FamilyType::MI_SEMAPHORE_WAIT; + constexpr auto size = 1024u; + std::byte buff[size] = {}; + LinearStream stream(buff, size); + + uint64_t userHostFunctionStored = 10u; + uint64_t userDataStored = 20u; + uint32_t tagStored = 0; + + HostFunctionData hostFunctionData{ + .entry = &userHostFunctionStored, + .userData = &userDataStored, + .internalTag = &tagStored}; + + uint64_t userCallback = 0xAAAA'0000ull; + uint64_t userCallbackData = 0xBBBB'000ull; + + HostFunctionHelper::programHostFunction(stream, hostFunctionData, userCallback, userCallbackData); + + HardwareParse hwParser; + hwParser.parseCommands(stream, 0); + + auto miStores = findAll(hwParser.cmdList.begin(), hwParser.cmdList.end()); + EXPECT_EQ(3u, miStores.size()); + + auto miWait = findAll(hwParser.cmdList.begin(), hwParser.cmdList.end()); + EXPECT_EQ(1u, miWait.size()); + + // program callback address + auto miStoreUserHostFunction = genCmdCast(*miStores[0]); + EXPECT_EQ(reinterpret_cast(&userHostFunctionStored), miStoreUserHostFunction->getAddress()); + EXPECT_EQ(getLowPart(userCallback), miStoreUserHostFunction->getDataDword0()); + EXPECT_EQ(getHighPart(userCallback), miStoreUserHostFunction->getDataDword1()); + EXPECT_TRUE(miStoreUserHostFunction->getStoreQword()); + + // program callback data + auto miStoreUserData = genCmdCast(*miStores[1]); + EXPECT_EQ(reinterpret_cast(&userDataStored), miStoreUserData->getAddress()); + EXPECT_EQ(getLowPart(userCallbackData), miStoreUserData->getDataDword0()); + EXPECT_EQ(getHighPart(userCallbackData), miStoreUserData->getDataDword1()); + EXPECT_TRUE(miStoreUserData->getStoreQword()); + + // signal pending job + auto miStoreSignalTag = genCmdCast(*miStores[2]); + EXPECT_EQ(reinterpret_cast(&tagStored), miStoreSignalTag->getAddress()); + EXPECT_EQ(static_cast(HostFunctionTagStatus::pending), miStoreSignalTag->getDataDword0()); + EXPECT_FALSE(miStoreSignalTag->getStoreQword()); + + // wait for completion + auto miWaitTag = genCmdCast(*miWait[0]); + EXPECT_EQ(reinterpret_cast(&tagStored), miWaitTag->getSemaphoreGraphicsAddress()); + EXPECT_EQ(static_cast(HostFunctionTagStatus::completed), miWaitTag->getSemaphoreDataDword()); + EXPECT_EQ(MI_SEMAPHORE_WAIT::COMPARE_OPERATION_SAD_EQUAL_SDD, miWaitTag->getCompareOperation()); + EXPECT_EQ(MI_SEMAPHORE_WAIT::WAIT_MODE_POLLING_MODE, miWaitTag->getWaitMode()); +}