refactor: move gmm callbacks definition to common place

- merge all the callbacks to single template class
- store callback functions in global arrays
- partially reuse callbacks on Linux

Related-To: NEO-11080
Signed-off-by: Mateusz Jablonski <mateusz.jablonski@intel.com>
This commit is contained in:
Mateusz Jablonski
2025-08-14 10:43:51 +00:00
committed by Compute-Runtime-Automation
parent a7e40d1543
commit ec36dd6275
38 changed files with 286 additions and 243 deletions

View File

@@ -926,6 +926,7 @@ include_directories(${NEO_SHARED_DIRECTORY}/debug_settings/definitions${BRANCH_D
include_directories(${NEO_SHARED_DIRECTORY}/dll/devices${BRANCH_DIR_SUFFIX})
include_directories(${NEO_SHARED_DIRECTORY}/kernel/definitions${BRANCH_DIR_SUFFIX})
include_directories(${NEO_SHARED_DIRECTORY}/gen_common${BRANCH_DIR_SUFFIX})
include_directories(${NEO_SHARED_DIRECTORY}/gmm_helper/${DRIVER_MODEL})
if(WIN32)
include_directories(${NEO_SHARED_DIRECTORY}/gmm_helper/windows/gmm_memory${BRANCH_DIR_SUFFIX})
include_directories(${NEO_SHARED_DIRECTORY}/os_interface/windows/wddm/definitions)

View File

@@ -1,5 +1,5 @@
#
# Copyright (C) 2020-2023 Intel Corporation
# Copyright (C) 2020-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@@ -23,6 +23,7 @@ set(CORE_RUNTIME_SRCS_COREX_CPP_BASE
experimental_command_buffer
implicit_scaling
gfx_core_helper
gmm_callbacks
hw_info
preamble
preemption
@@ -33,7 +34,6 @@ set(CORE_RUNTIME_SRCS_COREX_CPP_BASE
set(CORE_RUNTIME_SRCS_COREX_CPP_WDDM
windows/command_stream_receiver
windows/direct_submission
windows/gmm_callbacks
)
set(CORE_RUNTIME_SRCS_COREX_CPP_LINUX

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2023 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -20,6 +20,9 @@ extern GfxCoreHelperCreateFunctionType gfxCoreHelperFactory[IGFX_MAX_CORE];
using Family = Gen12LpFamily;
static auto gfxFamily = IGFX_GEN12LP_CORE;
template <typename GfxFamily>
struct GmmCallbacks;
struct EnableCoreGen12LP {
EnableCoreGen12LP() {
gfxCoreHelperFactory[gfxFamily] = GfxCoreHelperHw<Family>::create;
@@ -27,6 +30,7 @@ struct EnableCoreGen12LP {
populateFactoryTable<CommandStreamReceiverHw<Family>>();
populateFactoryTable<TbxCommandStreamReceiverHw<Family>>();
populateFactoryTable<DebuggerL0Hw<Family>>();
populateFactoryTable<GmmCallbacks<Family>>();
}
};

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/gen12lp/hw_cmds_base.h"
using Family = NEO::Gen12LpFamily;
static auto gfxCore = IGFX_GEN12LP_CORE;
#include "gmm_callbacks.inl"

View File

@@ -1,14 +0,0 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/gen12lp/hw_cmds_base.h"
#include "shared/source/helpers/windows/gmm_callbacks.inl"
namespace NEO {
template struct DeviceCallbacks<Gen12LpFamily>;
template struct TTCallbacks<Gen12LpFamily>;
} // namespace NEO

View File

@@ -1,5 +1,5 @@
#
# Copyright (C) 2019-2023 Intel Corporation
# Copyright (C) 2019-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@@ -15,6 +15,10 @@ set(NEO_CORE_GMM_HELPER
${CMAKE_CURRENT_SOURCE_DIR}/cache_settings_helper.h
${CMAKE_CURRENT_SOURCE_DIR}/gmm.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm.h
${CMAKE_CURRENT_SOURCE_DIR}/gmm_callbacks.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm_callbacks.h
${CMAKE_CURRENT_SOURCE_DIR}/gmm_callbacks_base.inl
${CMAKE_CURRENT_SOURCE_DIR}/${DRIVER_MODEL}/gmm_callbacks.inl
${CMAKE_CURRENT_SOURCE_DIR}/gmm_helper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm_helper.h
${CMAKE_CURRENT_SOURCE_DIR}/gmm_interface.h

View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/gmm_helper/gmm_callbacks_base.inl"
#include "shared/source/helpers/array_count.h"
#include "shared/source/helpers/populate_factory.h"
namespace NEO {
template <typename GfxFamily>
long __stdcall GmmCallbacks<GfxFamily>::notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate) {
return 1;
}
template <>
void populateFactoryTable<GmmCallbacks<Family>>() {
UNRECOVERABLE_IF(!isInRange(gfxCore, writeL3AddressFuncFactory));
writeL3AddressFuncFactory[gfxCore] = GmmCallbacks<Family>::writeL3Address;
}
template struct GmmCallbacks<Family>;
} // namespace NEO

View File

@@ -0,0 +1,8 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/gmm_helper/wddm/gmm_callbacks.inl"

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/gmm_helper/gmm_callbacks.h"
namespace NEO {
NotifyAubCaptureFunc notifyAubCaptureFuncFactory[IGFX_MAX_CORE]{};
WriteL3AddressFunc writeL3AddressFuncFactory[IGFX_MAX_CORE]{};
} // namespace NEO

View File

@@ -0,0 +1,26 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/gmm_helper/gmm_lib.h"
#include <cstdint>
namespace NEO {
using NotifyAubCaptureFunc = long(__stdcall *)(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate);
using WriteL3AddressFunc = int(__stdcall *)(void *queueHandle, uint64_t l3GfxAddress, uint64_t regOffset);
extern NotifyAubCaptureFunc notifyAubCaptureFuncFactory[IGFX_MAX_CORE];
extern WriteL3AddressFunc writeL3AddressFuncFactory[IGFX_MAX_CORE];
template <typename GfxFamily>
struct GmmCallbacks {
static long __stdcall notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate);
static int __stdcall writeL3Address(void *queueHandle, uint64_t l3GfxAddress, uint64_t regOffset);
};
} // namespace NEO

View File

@@ -0,0 +1,33 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/helpers/gfx_core_helper.h"
namespace NEO {
template <typename GfxFamily>
int __stdcall GmmCallbacks<GfxFamily>::writeL3Address(void *queueHandle, uint64_t l3GfxAddress, uint64_t regOffset) {
auto csr = reinterpret_cast<CommandStreamReceiver *>(queueHandle);
LriHelper<GfxFamily>::program(&csr->getCS(0),
static_cast<uint32_t>(regOffset & 0xFFFFFFFF),
static_cast<uint32_t>(l3GfxAddress & 0xFFFFFFFF),
true,
false);
LriHelper<GfxFamily>::program(&csr->getCS(0),
static_cast<uint32_t>(regOffset >> 32),
static_cast<uint32_t>(l3GfxAddress >> 32),
true,
false);
return 1;
}
} // namespace NEO

View File

@@ -0,0 +1,46 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/command_stream/aub_command_stream_receiver_hw.h"
#include "shared/source/command_stream/command_stream_receiver_with_aub_dump.h"
#include "shared/source/gmm_helper/gmm_callbacks_base.inl"
#include "shared/source/helpers/array_count.h"
#include "shared/source/helpers/populate_factory.h"
#include "shared/source/os_interface/windows/wddm_device_command_stream.h"
namespace NEO {
template <typename GfxFamily>
long __stdcall GmmCallbacks<GfxFamily>::notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate) {
auto csr = reinterpret_cast<CommandStreamReceiverHw<GfxFamily> *>(csrHandle);
if (obtainCsrTypeFromIntegerValue(debugManager.flags.SetCommandStreamReceiver.get(), CommandStreamReceiverType::hardware) == CommandStreamReceiverType::hardwareWithAub) {
auto csrWithAub = static_cast<CommandStreamReceiverWithAUBDump<WddmCommandStreamReceiver<GfxFamily>> *>(csr);
auto aubCsr = static_cast<AUBCommandStreamReceiverHw<GfxFamily> *>(csrWithAub->aubCSR.get());
if (allocate) {
AllocationView externalAllocation(gfxAddress, gfxSize);
aubCsr->makeResidentExternal(externalAllocation);
} else {
aubCsr->makeNonResidentExternal(gfxAddress);
}
}
return 1;
}
template <>
void populateFactoryTable<GmmCallbacks<Family>>() {
UNRECOVERABLE_IF(!isInRange(gfxCore, writeL3AddressFuncFactory));
writeL3AddressFuncFactory[gfxCore] = GmmCallbacks<Family>::writeL3Address;
UNRECOVERABLE_IF(!isInRange(gfxCore, notifyAubCaptureFuncFactory));
notifyAubCaptureFuncFactory[gfxCore] = GmmCallbacks<Family>::notifyAubCapture;
}
template struct GmmCallbacks<Family>;
} // namespace NEO

View File

@@ -244,13 +244,6 @@ if(SUPPORT_XE3_AND_BEFORE)
)
endif()
set(NEO_CORE_HELPERS_GMM_CALLBACKS_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/windows/gmm_callbacks.cpp
${CMAKE_CURRENT_SOURCE_DIR}/windows/gmm_callbacks.h
${CMAKE_CURRENT_SOURCE_DIR}/windows/gmm_callbacks.inl
)
set_property(GLOBAL PROPERTY NEO_CORE_HELPERS_GMM_CALLBACKS_WINDOWS ${NEO_CORE_HELPERS_GMM_CALLBACKS_WINDOWS})
set_property(GLOBAL PROPERTY NEO_CORE_HELPERS ${NEO_CORE_HELPERS})
add_subdirectories()

View File

@@ -1,14 +0,0 @@
/*
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/windows/windows_wrapper.h"
#include <cstdint>
namespace NEO {
long(__stdcall *notifyAubCaptureImpl)(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate) = nullptr;
} // namespace NEO

View File

@@ -1,28 +0,0 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/os_interface/windows/windows_wrapper.h"
#include <cstdint>
namespace NEO {
extern long(__stdcall *notifyAubCaptureImpl)(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate);
template <typename GfxFamily>
struct DeviceCallbacks {
static long __stdcall notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate);
};
template <typename GfxFamily>
struct TTCallbacks {
using MI_LOAD_REGISTER_IMM = typename GfxFamily::MI_LOAD_REGISTER_IMM;
static int __stdcall writeL3Address(void *queueHandle, uint64_t l3GfxAddress, uint64_t regOffset);
};
} // namespace NEO

View File

@@ -1,54 +0,0 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/command_stream/aub_command_stream_receiver_hw.h"
#include "shared/source/command_stream/command_stream_receiver_hw.h"
#include "shared/source/command_stream/command_stream_receiver_with_aub_dump.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/os_interface/windows/wddm_device_command_stream.h"
namespace NEO {
template <typename GfxFamily>
long __stdcall DeviceCallbacks<GfxFamily>::notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate) {
auto csr = reinterpret_cast<CommandStreamReceiverHw<GfxFamily> *>(csrHandle);
if (obtainCsrTypeFromIntegerValue(debugManager.flags.SetCommandStreamReceiver.get(), CommandStreamReceiverType::hardware) == CommandStreamReceiverType::hardwareWithAub) {
auto csrWithAub = static_cast<CommandStreamReceiverWithAUBDump<WddmCommandStreamReceiver<GfxFamily>> *>(csr);
auto aubCsr = static_cast<AUBCommandStreamReceiverHw<GfxFamily> *>(csrWithAub->aubCSR.get());
if (allocate) {
AllocationView externalAllocation(gfxAddress, gfxSize);
aubCsr->makeResidentExternal(externalAllocation);
} else {
aubCsr->makeNonResidentExternal(gfxAddress);
}
}
return 1;
}
template <typename GfxFamily>
int __stdcall TTCallbacks<GfxFamily>::writeL3Address(void *queueHandle, uint64_t l3GfxAddress, uint64_t regOffset) {
auto csr = reinterpret_cast<CommandStreamReceiverHw<GfxFamily> *>(queueHandle);
LriHelper<GfxFamily>::program(&csr->getCS(0),
static_cast<uint32_t>(regOffset & 0xFFFFFFFF),
static_cast<uint32_t>(l3GfxAddress & 0xFFFFFFFF),
true,
false);
LriHelper<GfxFamily>::program(&csr->getCS(0),
static_cast<uint32_t>(regOffset >> 32),
static_cast<uint32_t>(l3GfxAddress >> 32),
true,
false);
return 1;
}
} // namespace NEO

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2023 Intel Corporation
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -9,12 +9,12 @@
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
#include "shared/source/gmm_helper/gmm.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/gmm_helper/gmm_helper.h"
#include "shared/source/gmm_helper/page_table_mngr.h"
#include "shared/source/gmm_helper/resource_info.h"
#include "shared/source/helpers/aligned_memory.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/memory_manager/gfx_partition.h"
#include "shared/source/os_interface/linux/allocator_helper.h"
#include "shared/source/os_interface/product_helper.h"

View File

@@ -6,9 +6,9 @@
*/
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/os_interface/windows/gdi_interface.h"
#include "shared/source/os_interface/windows/wddm/wddm.h"
@@ -16,10 +16,6 @@
namespace NEO {
long __stdcall notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate) {
return notifyAubCaptureImpl(csrHandle, gfxAddress, gfxSize, allocate);
}
bool Wddm::configureDeviceAddressSpace() {
GMM_DEVICE_CALLBACKS_INT deviceCallbacks{};
deviceCallbacks.Adapter.KmtHandle = getAdapter();
@@ -41,7 +37,10 @@ bool Wddm::configureDeviceAddressSpace() {
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnUnLock = getGdi()->unlock2;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnEscape = getGdi()->escape;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnFreeGPUVA = getGdi()->freeGpuVirtualAddress;
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture = notifyAubCapture;
auto hwInfo = rootDeviceEnvironment.getHardwareInfo();
deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture = notifyAubCaptureFuncFactory[hwInfo->platform.eRenderCoreFamily];
GMM_DEVICE_INFO deviceInfo{};
deviceInfo.pGfxPartition = &gfxPartition;
@@ -63,7 +62,7 @@ bool Wddm::configureDeviceAddressSpace() {
? maximumApplicationAddress + 1u
: 0u;
bool obtainMinAddress = rootDeviceEnvironment.getHardwareInfo()->platform.eRenderCoreFamily == IGFX_GEN12LP_CORE;
bool obtainMinAddress = hwInfo->platform.eRenderCoreFamily == IGFX_GEN12LP_CORE;
return gmmMemory->configureDevice(getAdapter(), device, getGdi()->escape, svmSize, featureTable->flags.ftrL3IACoherency, minAddress, obtainMinAddress);
}

View File

@@ -13,6 +13,7 @@
#include "shared/source/gmm_helper/client_context/gmm_handle_allocator.h"
#include "shared/source/gmm_helper/client_context/map_gpu_va_gmm.h"
#include "shared/source/gmm_helper/gmm.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/gmm_helper/gmm_helper.h"
#include "shared/source/gmm_helper/page_table_mngr.h"
#include "shared/source/gmm_helper/resource_info.h"
@@ -23,7 +24,6 @@
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/mt_helpers.h"
#include "shared/source/helpers/string.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/memory_manager/gfx_partition.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/os_interface/sys_calls_common.h"

View File

@@ -14,10 +14,11 @@
#include "shared/source/direct_submission/dispatchers/blitter_dispatcher.h"
#include "shared/source/direct_submission/dispatchers/render_dispatcher.h"
#include "shared/source/direct_submission/windows/wddm_direct_submission.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/gmm_helper/page_table_mngr.h"
#include "shared/source/helpers/flush_stamp.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/os_interface/windows/wddm/wddm.h"
#include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h"
#include "shared/source/os_interface/windows/wddm_device_command_stream.h"
@@ -40,7 +41,6 @@ WddmCommandStreamReceiver<GfxFamily>::WddmCommandStreamReceiver(ExecutionEnviron
const DeviceBitfield deviceBitfield)
: BaseClass(executionEnvironment, rootDeviceIndex, deviceBitfield) {
notifyAubCaptureImpl = DeviceCallbacks<GfxFamily>::notifyAubCapture;
this->wddm = executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface->getDriverModel()->as<Wddm>();
PreemptionMode preemptionMode = PreemptionHelper::getDefaultPreemptionMode(this->peekHwInfo());
@@ -175,9 +175,10 @@ bool WddmCommandStreamReceiver<GfxFamily>::isTlbFlushRequiredForStateCacheFlush(
template <typename GfxFamily>
GmmPageTableMngr *WddmCommandStreamReceiver<GfxFamily>::createPageTableManager() {
GMM_TRANSLATIONTABLE_CALLBACKS ttCallbacks = {};
ttCallbacks.pfWriteL3Adr = TTCallbacks<GfxFamily>::writeL3Address;
auto rootDeviceEnvironment = this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex].get();
auto hwInfo = rootDeviceEnvironment->getHardwareInfo();
ttCallbacks.pfWriteL3Adr = writeL3AddressFuncFactory[hwInfo->platform.eRenderCoreFamily];
GmmPageTableMngr *gmmPageTableMngr = GmmPageTableMngr::create(rootDeviceEnvironment->getGmmClientContext(), TT_TYPE::AUXTT, &ttCallbacks);
gmmPageTableMngr->setCsrHandle(this);

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2024 Intel Corporation
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -20,6 +20,9 @@ extern GfxCoreHelperCreateFunctionType gfxCoreHelperFactory[IGFX_MAX_CORE];
using Family = Xe2HpgCoreFamily;
static auto gfxFamily = IGFX_XE2_HPG_CORE;
template <typename GfxFamily>
struct GmmCallbacks;
struct EnableCoreXe2HpgCore {
EnableCoreXe2HpgCore() {
gfxCoreHelperFactory[gfxFamily] = GfxCoreHelperHw<Family>::create;
@@ -27,6 +30,7 @@ struct EnableCoreXe2HpgCore {
populateFactoryTable<CommandStreamReceiverHw<Family>>();
populateFactoryTable<TbxCommandStreamReceiverHw<Family>>();
populateFactoryTable<DebuggerL0Hw<Family>>();
populateFactoryTable<GmmCallbacks<Family>>();
}
};

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/xe2_hpg_core/hw_cmds.h"
using Family = NEO::Xe2HpgCoreFamily;
static auto gfxCore = IGFX_XE2_HPG_CORE;
#include "gmm_callbacks.inl"

View File

@@ -1,16 +0,0 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/windows/gmm_callbacks.inl"
#include "shared/source/xe2_hpg_core/hw_cmds.h"
namespace NEO {
template struct DeviceCallbacks<Xe2HpgCoreFamily>;
template struct TTCallbacks<Xe2HpgCoreFamily>;
} // namespace NEO

View File

@@ -20,6 +20,9 @@ extern GfxCoreHelperCreateFunctionType gfxCoreHelperFactory[IGFX_MAX_CORE];
using Family = Xe3CoreFamily;
static auto gfxFamily = IGFX_XE3_CORE;
template <typename GfxFamily>
struct GmmCallbacks;
struct EnableCoreXe3Core {
EnableCoreXe3Core() {
gfxCoreHelperFactory[gfxFamily] = GfxCoreHelperHw<Family>::create;
@@ -27,6 +30,7 @@ struct EnableCoreXe3Core {
populateFactoryTable<CommandStreamReceiverHw<Family>>();
populateFactoryTable<TbxCommandStreamReceiverHw<Family>>();
populateFactoryTable<DebuggerL0Hw<Family>>();
populateFactoryTable<GmmCallbacks<Family>>();
}
};

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/xe3_core/hw_cmds_base.h"
static auto gfxCore = IGFX_XE3_CORE;
using Family = NEO::Xe3CoreFamily;
#include "gmm_callbacks.inl"

View File

@@ -1,16 +0,0 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/windows/gmm_callbacks.inl"
#include "shared/source/xe3_core/hw_cmds_base.h"
namespace NEO {
template struct DeviceCallbacks<Xe3CoreFamily>;
template struct TTCallbacks<Xe3CoreFamily>;
} // namespace NEO

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2023 Intel Corporation
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -20,6 +20,9 @@ extern GfxCoreHelperCreateFunctionType gfxCoreHelperFactory[IGFX_MAX_CORE];
using Family = XeHpcCoreFamily;
static auto gfxFamily = IGFX_XE_HPC_CORE;
template <typename GfxFamily>
struct GmmCallbacks;
struct EnableCoreXeHpcCore {
EnableCoreXeHpcCore() {
gfxCoreHelperFactory[gfxFamily] = GfxCoreHelperHw<Family>::create;
@@ -27,6 +30,7 @@ struct EnableCoreXeHpcCore {
populateFactoryTable<CommandStreamReceiverHw<Family>>();
populateFactoryTable<TbxCommandStreamReceiverHw<Family>>();
populateFactoryTable<DebuggerL0Hw<Family>>();
populateFactoryTable<GmmCallbacks<Family>>();
}
};

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/xe_hpc_core/hw_cmds_xe_hpc_core_base.h"
static auto gfxCore = IGFX_XE_HPC_CORE;
using Family = NEO::XeHpcCoreFamily;
#include "gmm_callbacks.inl"

View File

@@ -1,16 +0,0 @@
/*
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/windows/gmm_callbacks.inl"
#include "shared/source/xe_hpc_core/hw_cmds_xe_hpc_core_base.h"
namespace NEO {
template struct DeviceCallbacks<XeHpcCoreFamily>;
template struct TTCallbacks<XeHpcCoreFamily>;
} // namespace NEO

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2023 Intel Corporation
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -20,6 +20,9 @@ extern GfxCoreHelperCreateFunctionType gfxCoreHelperFactory[IGFX_MAX_CORE];
using Family = XeHpgCoreFamily;
static auto gfxFamily = IGFX_XE_HPG_CORE;
template <typename GfxFamily>
struct GmmCallbacks;
struct EnableCoreXeHpgCore {
EnableCoreXeHpgCore() {
gfxCoreHelperFactory[gfxFamily] = GfxCoreHelperHw<Family>::create;
@@ -27,6 +30,7 @@ struct EnableCoreXeHpgCore {
populateFactoryTable<CommandStreamReceiverHw<Family>>();
populateFactoryTable<TbxCommandStreamReceiverHw<Family>>();
populateFactoryTable<DebuggerL0Hw<Family>>();
populateFactoryTable<GmmCallbacks<Family>>();
}
};

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/xe_hpg_core/hw_cmds_xe_hpg_core_base.h"
using Family = NEO::XeHpgCoreFamily;
static auto gfxCore = IGFX_XE_HPG_CORE;
#include "gmm_callbacks.inl"

View File

@@ -1,16 +0,0 @@
/*
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/windows/gmm_callbacks.inl"
#include "shared/source/xe_hpg_core/hw_cmds_xe_hpg_core_base.h"
namespace NEO {
template struct DeviceCallbacks<XeHpgCoreFamily>;
template struct TTCallbacks<XeHpgCoreFamily>;
} // namespace NEO

View File

@@ -1,5 +1,5 @@
#
# Copyright (C) 2022-2023 Intel Corporation
# Copyright (C) 2022-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@@ -7,7 +7,6 @@
if(WIN32)
target_sources(neo_shared_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/gmm_callbacks_tests_gen12lp.cpp
${CMAKE_CURRENT_SOURCE_DIR}/product_helper_tests_gen12lp.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_tests_gen12lp.cpp
)

View File

@@ -7,6 +7,7 @@
target_sources(neo_shared_tests PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/compression_tests_xe2_and_later.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm_callbacks_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm_helper_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm_resource_info_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gmm_tests.cpp

View File

@@ -1,46 +1,30 @@
/*
* Copyright (C) 2019-2023 Intel Corporation
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/command_stream/aub_command_stream_receiver_hw.h"
#include "shared/source/command_stream/command_stream_receiver_with_aub_dump.h"
#include "shared/source/command_stream/linear_stream.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/os_interface/windows/wddm_device_command_stream.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/test/common/cmd_parse/hw_parse.h"
#include "shared/test/common/fixtures/gmm_callbacks_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/helpers/default_hw_info.h"
#include "shared/test/common/libult/ult_command_stream_receiver.h"
#include "shared/test/common/test_macros/hw_test.h"
using namespace NEO;
using GmmCallbacksTests = ::Test<GmmCallbacksFixture>;
using Gen12LpGmmCallbacksTests = ::Test<GmmCallbacksFixture>;
template <typename GfxFamily>
struct MockAubCsrToTestNotifyAubCapture : public AUBCommandStreamReceiverHw<GfxFamily> {
using AUBCommandStreamReceiverHw<GfxFamily>::AUBCommandStreamReceiverHw;
using AUBCommandStreamReceiverHw<GfxFamily>::externalAllocations;
};
GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCsrWithoutAubDumpWhenNotifyAubCaptureCallbackIsCalledThenDoNothing) {
auto csr = std::make_unique<WddmCommandStreamReceiver<FamilyType>>(*executionEnvironment, 0, 1);
HWTEST_F(GmmCallbacksTests, givenCsrWithoutAubDumpWhenNotifyAubCaptureCallbackIsCalledThenDoNothing) {
UltCommandStreamReceiver<FamilyType> csr(*executionEnvironment, 0, 1);
uint64_t address = 0xFEDCBA9876543210;
size_t size = 1024;
auto res = DeviceCallbacks<FamilyType>::notifyAubCapture(csr.get(), address, size, true);
auto res = GmmCallbacks<FamilyType>::notifyAubCapture(&csr, address, size, true);
EXPECT_EQ(1, res);
}
GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenWddmCsrWhenWriteL3CalledThenWriteTwoMmio) {
HWTEST_F(GmmCallbacksTests, whenWriteL3CalledThenWriteTwoMmio) {
using MI_LOAD_REGISTER_IMM = typename FamilyType::MI_LOAD_REGISTER_IMM;
UltCommandStreamReceiver<FamilyType> csr(*executionEnvironment, 0, 1);
@@ -50,7 +34,7 @@ GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenWddmCsrWhenWriteL3CalledThenWriteTw
uint64_t address = 0x00234564002BCDEC;
uint64_t value = 0xFEDCBA987654321C;
auto res = TTCallbacks<FamilyType>::writeL3Address(&csr, value, address);
auto res = GmmCallbacks<FamilyType>::writeL3Address(&csr, value, address);
EXPECT_EQ(1, res);
EXPECT_EQ(2 * sizeof(MI_LOAD_REGISTER_IMM), csr.commandStream.getUsed());
@@ -69,8 +53,8 @@ GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenWddmCsrWhenWriteL3CalledThenWriteTw
EXPECT_EQ(value >> 32, cmd->getDataDword());
}
GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCcsEnabledhenWriteL3CalledThenSetRemapBit) {
typedef typename FamilyType::MI_LOAD_REGISTER_IMM MI_LOAD_REGISTER_IMM;
HWTEST_F(GmmCallbacksTests, givenCcsEnabledhenWriteL3CalledThenSetRemapBit) {
using MI_LOAD_REGISTER_IMM = typename FamilyType::MI_LOAD_REGISTER_IMM;
HardwareInfo localHwInfo = *defaultHwInfo;
localHwInfo.featureTable.flags.ftrCCSNode = true;
ExecutionEnvironment executionEnvironment;
@@ -81,7 +65,7 @@ GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCcsEnabledhenWriteL3CalledThenSetRe
uint8_t buffer[128] = {};
csr.commandStream.replaceBuffer(buffer, 128);
auto res = TTCallbacks<FamilyType>::writeL3Address(&csr, 1, 1);
auto res = GmmCallbacks<FamilyType>::writeL3Address(&csr, 1, 1);
EXPECT_EQ(1, res);
HardwareParse hwParse;
@@ -97,8 +81,8 @@ GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCcsEnabledhenWriteL3CalledThenSetRe
EXPECT_TRUE(cmd->getMmioRemapEnable());
}
GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCcsDisabledhenWriteL3CalledThenSetRemapBitToTrue) {
typedef typename FamilyType::MI_LOAD_REGISTER_IMM MI_LOAD_REGISTER_IMM;
HWTEST_F(GmmCallbacksTests, givenCcsDisabledhenWriteL3CalledThenSetRemapBitToTrue) {
using MI_LOAD_REGISTER_IMM = typename FamilyType::MI_LOAD_REGISTER_IMM;
HardwareInfo localHwInfo = *defaultHwInfo;
localHwInfo.featureTable.flags.ftrCCSNode = false;
ExecutionEnvironment executionEnvironment;
@@ -109,7 +93,7 @@ GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCcsDisabledhenWriteL3CalledThenSetR
uint8_t buffer[128] = {};
csr.commandStream.replaceBuffer(buffer, 128);
auto res = TTCallbacks<FamilyType>::writeL3Address(&csr, 1, 1);
auto res = GmmCallbacks<FamilyType>::writeL3Address(&csr, 1, 1);
EXPECT_EQ(1, res);
HardwareParse hwParse;
@@ -123,4 +107,4 @@ GEN12LPTEST_F(Gen12LpGmmCallbacksTests, givenCcsDisabledhenWriteL3CalledThenSetR
cmd = genCmdCast<MI_LOAD_REGISTER_IMM *>(*(++hwParse.cmdList.begin()));
ASSERT_NE(nullptr, cmd);
EXPECT_TRUE(cmd->getMmioRemapEnable());
}
}

View File

@@ -9,9 +9,9 @@
#include "shared/source/command_stream/preemption.h"
#include "shared/source/direct_submission/direct_submission_controller.h"
#include "shared/source/direct_submission/relaxed_ordering_helper.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/helpers/compiler_product_helper.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/indirect_heap/indirect_heap.h"
#include "shared/source/memory_manager/internal_allocation_storage.h"
#include "shared/source/os_interface/sys_calls_common.h"
@@ -1046,7 +1046,7 @@ HWTEST_P(WddmCsrCompressionParameterizedTest, givenEnabledCompressionWhenInitial
GMM_TRANSLATIONTABLE_CALLBACKS expectedTTCallbacks = {};
unsigned int expectedFlags = TT_TYPE::AUXTT;
expectedTTCallbacks.pfWriteL3Adr = TTCallbacks<FamilyType>::writeL3Address;
expectedTTCallbacks.pfWriteL3Adr = GmmCallbacks<FamilyType>::writeL3Address;
EXPECT_TRUE(memcmp(&expectedTTCallbacks, &mockMngr->translationTableCb, sizeof(GMM_TRANSLATIONTABLE_CALLBACKS)) == 0);
EXPECT_TRUE(memcmp(&expectedFlags, &mockMngr->translationTableFlags, sizeof(unsigned int)) == 0);

View File

@@ -14,7 +14,6 @@
#include "shared/source/direct_submission/windows/wddm_direct_submission.h"
#include "shared/source/helpers/api_specific_config.h"
#include "shared/source/helpers/flush_stamp.h"
#include "shared/source/helpers/windows/gmm_callbacks.h"
#include "shared/source/memory_manager/internal_allocation_storage.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/os_interface.h"

View File

@@ -6,6 +6,7 @@
*/
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/gmm_helper/gmm_callbacks.h"
#include "shared/source/gmm_helper/gmm_helper.h"
#include "shared/source/os_interface/windows/driver_info_windows.h"
#include "shared/source/os_interface/windows/dxgi_wrapper.h"
@@ -184,10 +185,6 @@ TEST(WddmGfxPartitionTests, WhenInitializingGfxPartitionThen64KBHeapsAreUsed) {
EXPECT_EQ(wddm->gfxPartition.Standard64KB.Base + rootDeviceIndex * heapStandard64KBSize, gfxPartition.getHeapBase(HeapIndex::heapStandard64KB));
}
namespace NEO {
long __stdcall notifyAubCapture(void *csrHandle, uint64_t gfxAddress, size_t gfxSize, bool allocate);
}
TEST_F(Wddm20WithMockGdiDllTests, whenSetDeviceInfoSucceedsThenDeviceCallbacksArePassedToGmmMemory) {
GMM_DEVICE_CALLBACKS_INT expectedDeviceCb{};
wddm->init();
@@ -213,7 +210,7 @@ TEST_F(Wddm20WithMockGdiDllTests, whenSetDeviceInfoSucceedsThenDeviceCallbacksAr
expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnUnLock = gdi->unlock2;
expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnEscape = gdi->escape;
expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnFreeGPUVA = gdi->freeGpuVirtualAddress;
expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture = notifyAubCapture;
expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture = notifyAubCaptureFuncFactory[defaultHwInfo->platform.eRenderCoreFamily];
EXPECT_EQ(expectedDeviceCb.Adapter.KmtHandle, gmmMemory->deviceCallbacks.Adapter.KmtHandle);
EXPECT_EQ(expectedDeviceCb.hDevice.KmtHandle, gmmMemory->deviceCallbacks.hDevice.KmtHandle);
@@ -234,6 +231,7 @@ TEST_F(Wddm20WithMockGdiDllTests, whenSetDeviceInfoSucceedsThenDeviceCallbacksAr
EXPECT_EQ(expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnEscape, gmmMemory->deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnEscape);
EXPECT_EQ(expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnFreeGPUVA, gmmMemory->deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnFreeGPUVA);
EXPECT_EQ(expectedDeviceCb.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture, gmmMemory->deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture);
EXPECT_NE(nullptr, gmmMemory->deviceCallbacks.DevCbPtrs.KmtCbPtrs.pfnNotifyAubCapture);
}
class MockGmmMemoryWindows : public MockGmmMemoryBase {