intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-19 16:24:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Pass hwInfo to appendMiFlushDw

Browse Source 
Signed-off-by: Filip Hazubski <filip.hazubski@intel.com>
This commit is contained in:
Filip Hazubski
2021-12-21 18:13:53 +00:00
committed by Compute-Runtime-Automation
parent 15f102a7cb
commit 9a450d1b74
24 changed files with 103 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
level_zero/core/source/cmdlist/cmdlist_hw.inl
View File

@@ -318,14 +318,14 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendEventReset(ze_event_hand
}
event->resetPackets();
commandContainer.addToResidencyContainer(&event->getAllocation(this->device));
const auto &hwInfo = this->device->getHwInfo();
if (isCopyOnly()) {
NEO::MiFlushArgs args;
args.commandWithPostSync = true;
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(),
event->getGpuAddress(this->device),
Event::STATE_CLEARED, args);
Event::STATE_CLEARED, args, hwInfo);
} else {
const auto &hwInfo = this->device->getHwInfo();
NEO::PipeControlArgs args;
args.dcFlushEnable = NEO::MemorySynchronizationCommands<GfxFamily>::isDcFlushAllowed(event->signalScope, hwInfo);
size_t estimateSize = NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(hwInfo) * packetsToReset;
@@ -1695,13 +1695,14 @@ void CommandListCoreFamily<gfxCoreFamily>::appendSignalEventPostWalker(ze_event_
commandContainer.addToResidencyContainer(&event->getAllocation(this->device));
uint64_t baseAddr = event->getGpuAddress(this->device);
const auto &hwInfo = this->device->getHwInfo();
if (isCopyOnly()) {
NEO::MiFlushArgs args;
args.commandWithPostSync = true;
increaseCommandStreamSpace(NEO::EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), baseAddr, Event::STATE_SIGNALED, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), baseAddr, Event::STATE_SIGNALED,
args, hwInfo);
} else {
const auto &hwInfo = this->device->getHwInfo();
increaseCommandStreamSpace(NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(hwInfo));
NEO::PipeControlArgs args;
args.dcFlushEnable = NEO::MemorySynchronizationCommands<GfxFamily>::isDcFlushAllowed(event->signalScope, hwInfo);
@@ -1730,9 +1731,10 @@ void CommandListCoreFamily<gfxCoreFamily>::appendEventForProfilingCopyCommand(ze
}
commandContainer.addToResidencyContainer(&event->getAllocation(this->device));
const auto &hwInfo = this->device->getHwInfo();
if (!beforeWalker) {
NEO::MiFlushArgs args;
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), 0, 0, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), 0, 0, args, hwInfo);
}
appendWriteKernelTimestamp(hEvent, beforeWalker, false);
}
@@ -1848,13 +1850,14 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendSignalEvent(ze_event_han
eventSignalOffset = event->getContextEndOffset();
}
const auto &hwInfo = this->device->getHwInfo();
if (isCopyOnly()) {
NEO::MiFlushArgs args;
args.commandWithPostSync = true;
increaseCommandStreamSpace(NEO::EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), ptrOffset(baseAddr, eventSignalOffset), Event::STATE_SIGNALED, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), ptrOffset(baseAddr, eventSignalOffset),
Event::STATE_SIGNALED, args, hwInfo);
} else {
const auto &hwInfo = this->device->getHwInfo();
NEO::PipeControlArgs args;
bool applyScope = event->signalScope;
args.dcFlushEnable = NEO::MemorySynchronizationCommands<GfxFamily>::isDcFlushAllowed(applyScope, hwInfo);
@@ -1945,7 +1948,7 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendWaitOnEvents(uint32_t nu
if (dcFlushRequired) {
if (isCopyOnly()) {
NEO::MiFlushArgs args;
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), 0, 0, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), 0, 0, args, hwInfo);
} else {
NEO::PipeControlArgs args;
args.dcFlushEnable = true;
@@ -2077,6 +2080,7 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendWriteGlobalTimestamp(
}
}
const auto &hwInfo = this->device->getHwInfo();
if (isCopyOnly()) {
NEO::MiFlushArgs args;
args.timeStampOperation = true;
@@ -2084,7 +2088,8 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendWriteGlobalTimestamp(
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(),
reinterpret_cast<uint64_t>(dstptr),
0,
args);
args,
hwInfo);
} else {
NEO::PipeControlArgs args;
@@ -2360,13 +2365,14 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendBarrier(ze_event_handle_
}
appendEventForProfiling(hSignalEvent, true);
const auto &hwInfo = this->device->getHwInfo();
if (!hSignalEvent) {
if (isCopyOnly()) {
size_t estimatedSizeRequired = NEO::EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite();
increaseCommandStreamSpace(estimatedSizeRequired);
NEO::MiFlushArgs args;
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), 0, 0, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(*commandContainer.getCommandStream(), 0, 0, args, hwInfo);
} else {
appendComputeBarrierCommand();
}

8
level_zero/core/source/cmdqueue/cmdqueue_hw.inl
View File

@@ -221,7 +221,7 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
linearStreamSizeEstimate += csrHw->getCmdSizeForActivePartitionConfig();
}
auto &hwInfo = device->getHwInfo();
const auto &hwInfo = this->device->getHwInfo();
if (hFence) {
fence = Fence::fromHandle(hFence);
spaceForResidency += residencyContainerSpaceForFence;
@@ -432,7 +432,7 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
if (isCopyOnlyCommandQueue) {
NEO::MiFlushArgs args;
args.commandWithPostSync = true;
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(child, fence->getGpuAddress(), Fence::STATE_SIGNALED, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(child, fence->getGpuAddress(), Fence::STATE_SIGNALED, args, hwInfo);
} else {
NEO::PipeControlArgs args;
args.dcFlushEnable = NEO::MemorySynchronizationCommands<GfxFamily>::isDcFlushAllowed(true, hwInfo);
@@ -571,13 +571,13 @@ void CommandQueueHw<gfxCoreFamily>::dispatchTaskCountWrite(NEO::LinearStream &co
auto taskCountToWrite = csr->peekTaskCount() + 1;
auto gpuAddress = static_cast<uint64_t>(csr->getTagAllocation()->getGpuAddress());
const auto &hwInfo = this->device->getHwInfo();
if (isCopyOnlyCommandQueue) {
NEO::MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = csr->isUsedNotifyEnableForPostSync();
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, gpuAddress, taskCountToWrite, args);
NEO::EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, gpuAddress, taskCountToWrite, args, hwInfo);
} else {
const auto &hwInfo = this->device->getHwInfo();
NEO::PipeControlArgs args;
args.dcFlushEnable = NEO::MemorySynchronizationCommands<GfxFamily>::isDcFlushAllowed(true, hwInfo);
if (partitionCount > 1) {

3
opencl/source/command_queue/hardware_interface.h
View File

@@ -75,7 +75,8 @@ class HardwareInterface {
LinearStream *commandStream,
CommandQueue &commandQueue,
DebugPauseState confirmationTrigger,
DebugPauseState waitCondition);
DebugPauseState waitCondition,
const HardwareInfo &hwInfo);
static void programWalker(
LinearStream &commandStream,

10
opencl/source/command_queue/hardware_interface_base.inl
View File

@@ -139,8 +139,10 @@ void HardwareInterface<GfxFamily>::dispatchWalker(
dispatchProfilingPerfStartCommands(hwTimeStamps, hwPerfCounter, commandStream, commandQueue);
const auto &hwInfo = commandQueue.getDevice().getHardwareInfo();
if (PauseOnGpuProperties::pauseModeAllowed(DebugManager.flags.PauseOnEnqueue.get(), commandQueue.getGpgpuCommandStreamReceiver().peekTaskCount(), PauseOnGpuProperties::PauseMode::BeforeWorkload)) {
dispatchDebugPauseCommands(commandStream, commandQueue, DebugPauseState::waitingForUserStartConfirmation, DebugPauseState::hasUserStartConfirmation);
dispatchDebugPauseCommands(commandStream, commandQueue, DebugPauseState::waitingForUserStartConfirmation,
DebugPauseState::hasUserStartConfirmation, hwInfo);
}
mainKernel->performKernelTuning(commandQueue.getGpgpuCommandStreamReceiver(),
@@ -179,7 +181,8 @@ void HardwareInterface<GfxFamily>::dispatchWalker(
}
if (PauseOnGpuProperties::pauseModeAllowed(DebugManager.flags.PauseOnEnqueue.get(), commandQueue.getGpgpuCommandStreamReceiver().peekTaskCount(), PauseOnGpuProperties::PauseMode::AfterWorkload)) {
dispatchDebugPauseCommands(commandStream, commandQueue, DebugPauseState::waitingForUserEndConfirmation, DebugPauseState::hasUserEndConfirmation);
dispatchDebugPauseCommands(commandStream, commandQueue, DebugPauseState::waitingForUserEndConfirmation,
DebugPauseState::hasUserEndConfirmation, hwInfo);
}
dispatchProfilingPerfEndCommands(hwTimeStamps, hwPerfCounter, commandStream, commandQueue);
@@ -299,7 +302,8 @@ inline void HardwareInterface<GfxFamily>::dispatchDebugPauseCommands(
LinearStream *commandStream,
CommandQueue &commandQueue,
DebugPauseState confirmationTrigger,
DebugPauseState waitCondition) {
DebugPauseState waitCondition,
const HardwareInfo &hwInfo) {
if (!commandQueue.isSpecial()) {
auto address = commandQueue.getGpgpuCommandStreamReceiver().getDebugPauseStateGPUAddress();

5
shared/source/command_container/command_encoder.h
View File

@@ -359,9 +359,10 @@ struct EncodeBatchBufferStartOrEnd {
template <typename GfxFamily>
struct EncodeMiFlushDW {
using MI_FLUSH_DW = typename GfxFamily::MI_FLUSH_DW;
static void programMiFlushDw(LinearStream &commandStream, uint64_t immediateDataGpuAddress, uint64_t immediateData, MiFlushArgs &args);
static void programMiFlushDw(LinearStream &commandStream, uint64_t immediateDataGpuAddress, uint64_t immediateData,
MiFlushArgs &args, const HardwareInfo &hwInfo);
static void programMiFlushDwWA(LinearStream &commandStream);
static void appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd);
static void appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd, const HardwareInfo &hwInfo);
static size_t getMiFlushDwCmdSizeForDataWrite();
static size_t getMiFlushDwWaSize();
};

5
shared/source/command_container/command_encoder.inl
View File

@@ -784,7 +784,8 @@ void EncodeBatchBufferStartOrEnd<Family>::programBatchBufferEnd(CommandContainer
}
template <typename Family>
void EncodeMiFlushDW<Family>::programMiFlushDw(LinearStream &commandStream, uint64_t immediateDataGpuAddress, uint64_t immediateData, MiFlushArgs &args) {
void EncodeMiFlushDW<Family>::programMiFlushDw(LinearStream &commandStream, uint64_t immediateDataGpuAddress, uint64_t immediateData,
MiFlushArgs &args, const HardwareInfo &hwInfo) {
programMiFlushDwWA(commandStream);
auto miFlushDwCmd = commandStream.getSpaceForCmd<MI_FLUSH_DW>();
@@ -797,7 +798,7 @@ void EncodeMiFlushDW<Family>::programMiFlushDw(LinearStream &commandStream, uint
}
miFlush.setNotifyEnable(args.notifyEnable);
miFlush.setTlbInvalidate(args.tlbFlush);
appendMiFlushDw(&miFlush);
appendMiFlushDw(&miFlush, hwInfo);
*miFlushDwCmd = miFlush;
}

2
shared/source/command_container/command_encoder_bdw_and_later.inl
View File

@@ -432,7 +432,7 @@ void EncodeL3State<Family>::encode(CommandContainer &container, bool enableSLM)
}
template <typename GfxFamily>
void EncodeMiFlushDW<GfxFamily>::appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd) {}
void EncodeMiFlushDW<GfxFamily>::appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd, const HardwareInfo &hwInfo) {}
template <typename GfxFamily>
void EncodeMiFlushDW<GfxFamily>::programMiFlushDwWA(LinearStream &commandStream) {}

2
shared/source/command_container/command_encoder_xehp_and_later.inl
View File

@@ -599,7 +599,7 @@ inline void EncodeMediaInterfaceDescriptorLoad<Family>::encode(CommandContainer
}
template <typename Family>
void EncodeMiFlushDW<Family>::appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd) {
void EncodeMiFlushDW<Family>::appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd, const HardwareInfo &hwInfo) {
miFlushDwCmd->setFlushCcs(1);
miFlushDwCmd->setFlushLlc(1);
}

26
shared/source/command_stream/command_stream_receiver_hw_base.inl
View File

@@ -1011,8 +1011,11 @@ uint32_t CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitPropertiesCont
getOsContext().ensureContextInitialized();
this->initDirectSubmission(device, getOsContext());
const auto &hwInfo = this->peekHwInfo();
if (PauseOnGpuProperties::pauseModeAllowed(DebugManager.flags.PauseOnBlitCopy.get(), taskCount, PauseOnGpuProperties::PauseMode::BeforeWorkload)) {
BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(commandStream, getDebugPauseStateGPUAddress(), DebugPauseState::waitingForUserStartConfirmation, DebugPauseState::hasUserStartConfirmation);
BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(commandStream, getDebugPauseStateGPUAddress(),
DebugPauseState::waitingForUserStartConfirmation,
DebugPauseState::hasUserStartConfirmation, hwInfo);
}
programEnginePrologue(commandStream);
@@ -1034,14 +1037,14 @@ uint32_t CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitPropertiesCont
if (blitProperties.outputTimestampPacket) {
if (profilingEnabled) {
MiFlushArgs args;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0llu, newTaskCount, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0llu, newTaskCount, args, hwInfo);
BlitCommandsHelper<GfxFamily>::encodeProfilingEndMmios(commandStream, *blitProperties.outputTimestampPacket);
} else {
auto timestampPacketGpuAddress = TimestampPacketHelper::getContextEndGpuAddress(*blitProperties.outputTimestampPacket);
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, timestampPacketGpuAddress, 0, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, timestampPacketGpuAddress, 0, args, hwInfo);
}
makeResident(*blitProperties.outputTimestampPacket->getBaseGraphicsAllocation());
}
@@ -1065,13 +1068,15 @@ uint32_t CommandStreamReceiverHw<GfxFamily>::blitBuffer(const BlitPropertiesCont
MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = isUsedNotifyEnableForPostSync();
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), newTaskCount, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), newTaskCount, args, hwInfo);
MemorySynchronizationCommands<GfxFamily>::addAdditionalSynchronization(commandStream, tagAllocation->getGpuAddress(), peekHwInfo());
}
if (PauseOnGpuProperties::pauseModeAllowed(DebugManager.flags.PauseOnBlitCopy.get(), taskCount, PauseOnGpuProperties::PauseMode::AfterWorkload)) {
BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(commandStream, getDebugPauseStateGPUAddress(), DebugPauseState::waitingForUserEndConfirmation, DebugPauseState::hasUserEndConfirmation);
BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(commandStream, getDebugPauseStateGPUAddress(),
DebugPauseState::waitingForUserEndConfirmation,
DebugPauseState::hasUserEndConfirmation, hwInfo);
}
void *endingCmdPtr = nullptr;
@@ -1149,10 +1154,11 @@ inline void CommandStreamReceiverHw<GfxFamily>::flushMiFlushDW() {
auto &commandStream = getCS(EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
auto commandStreamStart = commandStream.getUsed();
const auto &hwInfo = this->peekHwInfo();
MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = isUsedNotifyEnableForPostSync();
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), taskCount + 1, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), taskCount + 1, args, hwInfo);
makeResident(*tagAllocation);
@@ -1169,13 +1175,14 @@ void CommandStreamReceiverHw<GfxFamily>::flushMiFlushDW(GraphicsAllocation *even
programHardwareContext(commandStream);
const auto &hwInfo = this->peekHwInfo();
MiFlushArgs args;
if (eventAlloc) {
args.commandWithPostSync = true;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, immediateGpuAddress, immediateData, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, immediateGpuAddress, immediateData, args, hwInfo);
makeResident(*eventAlloc);
} else {
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0, 0, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0, 0, args, hwInfo);
}
this->flushSmallTask(commandStream, commandStreamStart);
@@ -1238,12 +1245,13 @@ void CommandStreamReceiverHw<GfxFamily>::flushPipeControl(GraphicsAllocation *ev
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushSemaphoreWait(GraphicsAllocation *eventAlloc, uint64_t immediateGpuAddress, uint64_t immediateData, PipeControlArgs &args, bool isStartOfDispatch, bool isEndOfDispatch) {
auto lock = obtainUniqueOwnership();
const auto &hwInfo = this->peekHwInfo();
if (isStartOfDispatch && args.dcFlushEnable) {
if (this->osContext->getEngineType() == aub_stream::ENGINE_BCS) {
LinearStream &commandStream = getCS(EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
cmdStreamStart = commandStream.getUsed();
MiFlushArgs args;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0, 0, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0, 0, args, hwInfo);
} else {
LinearStream &commandStream = getCS(MemorySynchronizationCommands<GfxFamily>::getSizeForSinglePipeControl());
cmdStreamStart = commandStream.getUsed();

2
shared/source/direct_submission/dispatchers/blitter_dispatcher.h
View File

@@ -25,7 +25,7 @@ class BlitterDispatcher : public Dispatcher<GfxFamily> {
static size_t getSizeMonitorFence(const HardwareInfo &hwInfo);
static void dispatchCacheFlush(LinearStream &cmdBuffer, const HardwareInfo &hwInfo, uint64_t address);
static void dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address);
static void dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address, const HardwareInfo &hwInfo);
static size_t getSizeCacheFlush(const HardwareInfo &hwInfo);
static size_t getSizeTlbFlush();
static bool isMultiTileSynchronizationSupported() {

8
shared/source/direct_submission/dispatchers/blitter_dispatcher.inl
View File

@@ -32,7 +32,7 @@ inline void BlitterDispatcher<GfxFamily>::dispatchMonitorFence(LinearStream &cmd
MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = useNotifyEnable;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(cmdBuffer, gpuAddress, immediateData, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(cmdBuffer, gpuAddress, immediateData, args, hwInfo);
}
template <typename GfxFamily>
@@ -43,15 +43,15 @@ inline size_t BlitterDispatcher<GfxFamily>::getSizeMonitorFence(const HardwareIn
template <typename GfxFamily>
inline void BlitterDispatcher<GfxFamily>::dispatchCacheFlush(LinearStream &cmdBuffer, const HardwareInfo &hwInfo, uint64_t address) {
dispatchTlbFlush(cmdBuffer, address);
dispatchTlbFlush(cmdBuffer, address, hwInfo);
}
template <typename GfxFamily>
inline void BlitterDispatcher<GfxFamily>::dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address) {
inline void BlitterDispatcher<GfxFamily>::dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address, const HardwareInfo &hwInfo) {
MiFlushArgs args;
args.tlbFlush = true;
args.commandWithPostSync = true;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(cmdBuffer, address, 0ull, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(cmdBuffer, address, 0ull, args, hwInfo);
}
template <typename GfxFamily>

2
shared/source/direct_submission/dispatchers/render_dispatcher.h
View File

@@ -25,7 +25,7 @@ class RenderDispatcher : public Dispatcher<GfxFamily> {
static size_t getSizeMonitorFence(const HardwareInfo &hwInfo);
static void dispatchCacheFlush(LinearStream &cmdBuffer, const HardwareInfo &hwInfo, uint64_t address);
static void dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address);
static void dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address, const HardwareInfo &hwInfo);
static size_t getSizeCacheFlush(const HardwareInfo &hwInfo);
static size_t getSizeTlbFlush();
static bool isMultiTileSynchronizationSupported() {

2
shared/source/direct_submission/dispatchers/render_dispatcher.inl
View File

@@ -57,7 +57,7 @@ inline void RenderDispatcher<GfxFamily>::dispatchCacheFlush(LinearStream &cmdBuf
}
template <typename GfxFamily>
inline void RenderDispatcher<GfxFamily>::dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address) {
inline void RenderDispatcher<GfxFamily>::dispatchTlbFlush(LinearStream &cmdBuffer, uint64_t address, const HardwareInfo &hwInfo) {
PipeControlArgs args;
args.tlbInvalidation = true;
args.pipeControlFlushEnable = true;

2
shared/source/direct_submission/linux/drm_direct_submission.inl
View File

@@ -123,7 +123,7 @@ bool DrmDirectSubmission<GfxFamily, Dispatcher>::isNewResourceHandleNeeded() {
template <typename GfxFamily, typename Dispatcher>
void DrmDirectSubmission<GfxFamily, Dispatcher>::handleNewResourcesSubmission() {
if (isNewResourceHandleNeeded()) {
Dispatcher::dispatchTlbFlush(this->ringCommandStream, this->gpuVaForMiFlush);
Dispatcher::dispatchTlbFlush(this->ringCommandStream, this->gpuVaForMiFlush, *this->hwInfo);
}
auto osContextLinux = static_cast<OsContextLinux *>(&this->osContext);

7
shared/source/helpers/blit_commands_helper.h
View File

@@ -114,9 +114,9 @@ struct BlitCommandsHelper {
static uint64_t getMaxBlitWidthOverride(const RootDeviceEnvironment &rootDeviceEnvironment);
static uint64_t getMaxBlitHeight(const RootDeviceEnvironment &rootDeviceEnvironment);
static uint64_t getMaxBlitHeightOverride(const RootDeviceEnvironment &rootDeviceEnvironment);
static void dispatchPreBlitCommand(LinearStream &linearStream);
static void dispatchPreBlitCommand(LinearStream &linearStream, const HardwareInfo &hwInfo);
static size_t estimatePreBlitCommandSize();
static void dispatchPostBlitCommand(LinearStream &linearStream);
static void dispatchPostBlitCommand(LinearStream &linearStream, const HardwareInfo &hwInfo);
static size_t estimatePostBlitCommandSize();
static size_t estimateBlitCommandsSize(const Vec3<size_t> &copySize, const CsrDependencies &csrDependencies, bool updateTimestampPacket,
bool profilingEnabled, const RootDeviceEnvironment &rootDeviceEnvironment);
@@ -146,7 +146,8 @@ struct BlitCommandsHelper {
static void appendTilingType(const GMM_TILE_TYPE srcTilingType, const GMM_TILE_TYPE dstTilingType, typename GfxFamily::XY_COPY_BLT &blitCmd);
static void appendSliceOffsets(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd, uint32_t sliceIndex, const RootDeviceEnvironment &rootDeviceEnvironment, uint32_t srcSlicePitch, uint32_t dstSlicePitch);
static void getBlitAllocationProperties(const GraphicsAllocation &allocation, uint32_t &pitch, uint32_t &qPitch, GMM_TILE_TYPE &tileType, uint32_t &mipTailLod, uint32_t &compressionDetails, const RootDeviceEnvironment &rootDeviceEnvironment);
static void dispatchDebugPauseCommands(LinearStream &commandStream, uint64_t debugPauseStateGPUAddress, DebugPauseState confirmationTrigger, DebugPauseState waitCondition);
static void dispatchDebugPauseCommands(LinearStream &commandStream, uint64_t debugPauseStateGPUAddress, DebugPauseState confirmationTrigger,
DebugPauseState waitCondition, const HardwareInfo &hwInfo);
static size_t getSizeForDebugPauseCommands();
static bool useOneBlitCopyCommand(const Vec3<size_t> &copySize, uint32_t bytesPerPixel);
static uint32_t getAvailableBytesPerPixel(size_t copySize, uint32_t srcOrigin, uint32_t dstOrigin, size_t srcSize, size_t dstSize);

32
shared/source/helpers/blit_commands_helper_base.inl
View File

@@ -41,10 +41,10 @@ uint64_t BlitCommandsHelper<GfxFamily>::getMaxBlitHeight(const RootDeviceEnviron
}
template <typename GfxFamily>
void BlitCommandsHelper<GfxFamily>::dispatchPreBlitCommand(LinearStream &linearStream) {
void BlitCommandsHelper<GfxFamily>::dispatchPreBlitCommand(LinearStream &linearStream, const HardwareInfo &hwInfo) {
if (BlitCommandsHelper<GfxFamily>::preBlitCommandWARequired()) {
MiFlushArgs args;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(linearStream, 0, 0, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(linearStream, 0, 0, args, hwInfo);
}
}
@@ -58,7 +58,7 @@ size_t BlitCommandsHelper<GfxFamily>::estimatePreBlitCommandSize() {
}
template <typename GfxFamily>
void BlitCommandsHelper<GfxFamily>::dispatchPostBlitCommand(LinearStream &linearStream) {
void BlitCommandsHelper<GfxFamily>::dispatchPostBlitCommand(LinearStream &linearStream, const HardwareInfo &hwInfo) {
MiFlushArgs args;
if (DebugManager.flags.PostBlitCommand.get() != BlitterConstants::PostBlitMode::Default) {
switch (DebugManager.flags.PostBlitCommand.get()) {
@@ -66,7 +66,7 @@ void BlitCommandsHelper<GfxFamily>::dispatchPostBlitCommand(LinearStream &linear
EncodeMiArbCheck<GfxFamily>::program(linearStream);
return;
case BlitterConstants::PostBlitMode::MiFlush:
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(linearStream, 0, 0, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(linearStream, 0, 0, args, hwInfo);
return;
default:
return;
@@ -74,7 +74,7 @@ void BlitCommandsHelper<GfxFamily>::dispatchPostBlitCommand(LinearStream &linear
}
if (BlitCommandsHelper<GfxFamily>::miArbCheckWaRequired()) {
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(linearStream, 0, 0, args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(linearStream, 0, 0, args, hwInfo);
}
EncodeMiArbCheck<GfxFamily>::program(linearStream);
@@ -178,7 +178,8 @@ void BlitCommandsHelper<GfxFamily>::dispatchBlitCommandsForBufferPerRow(const Bl
PRINT_DEBUG_STRING(DebugManager.flags.PrintBlitDispatchDetails.get(), stdout,
"\nBlit dispatch with AuxTranslationDirection %u ", static_cast<uint32_t>(blitProperties.auxTranslationDirection));
dispatchPreBlitCommand(linearStream);
const auto &hwInfo = *rootDeviceEnvironment.getHardwareInfo();
dispatchPreBlitCommand(linearStream, hwInfo);
for (uint64_t slice = 0; slice < blitProperties.copySize.z; slice++) {
for (uint64_t row = 0; row < blitProperties.copySize.y; row++) {
@@ -218,7 +219,7 @@ void BlitCommandsHelper<GfxFamily>::dispatchBlitCommandsForBufferPerRow(const Bl
*bltStream = bltCmd;
}
dispatchPostBlitCommand(linearStream);
dispatchPostBlitCommand(linearStream, hwInfo);
auto blitSize = width * height;
sizeToBlit -= blitSize;
@@ -286,26 +287,30 @@ void BlitCommandsHelper<GfxFamily>::dispatchBlitCommandsRegion(const BlitPropert
bltCmd.setSourceX1CoordinateLeft(static_cast<uint32_t>(blitProperties.srcOffset.x));
bltCmd.setSourceY1CoordinateTop(static_cast<uint32_t>(blitProperties.srcOffset.y));
const auto &hwInfo = *rootDeviceEnvironment.getHardwareInfo();
appendBlitCommandsForBuffer(blitProperties, bltCmd, rootDeviceEnvironment);
appendBlitCommandsForImages(blitProperties, bltCmd, rootDeviceEnvironment, srcSlicePitch, dstSlicePitch);
appendColorDepth(blitProperties, bltCmd);
appendSurfaceType(blitProperties, bltCmd);
dispatchPreBlitCommand(linearStream);
dispatchPreBlitCommand(linearStream, hwInfo);
for (uint32_t i = 0; i < blitProperties.copySize.z; i++) {
appendSliceOffsets(blitProperties, bltCmd, i, rootDeviceEnvironment, srcSlicePitch, dstSlicePitch);
auto cmd = linearStream.getSpaceForCmd<typename GfxFamily::XY_COPY_BLT>();
*cmd = bltCmd;
dispatchPostBlitCommand(linearStream);
dispatchPostBlitCommand(linearStream, hwInfo);
}
}
template <typename GfxFamily>
void BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(LinearStream &commandStream, uint64_t debugPauseStateGPUAddress, DebugPauseState confirmationTrigger, DebugPauseState waitCondition) {
void BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(LinearStream &commandStream, uint64_t debugPauseStateGPUAddress,
DebugPauseState confirmationTrigger, DebugPauseState waitCondition,
const HardwareInfo &hwInfo) {
using MI_SEMAPHORE_WAIT = typename GfxFamily::MI_SEMAPHORE_WAIT;
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, debugPauseStateGPUAddress, static_cast<uint32_t>(confirmationTrigger), args);
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, debugPauseStateGPUAddress, static_cast<uint32_t>(confirmationTrigger),
args, hwInfo);
EncodeSempahore<GfxFamily>::addMiSemaphoreWaitCommand(commandStream, debugPauseStateGPUAddress, static_cast<uint32_t>(waitCondition), MI_SEMAPHORE_WAIT::COMPARE_OPERATION::COMPARE_OPERATION_SAD_EQUAL_SDD);
}
@@ -367,7 +372,8 @@ void BlitCommandsHelper<GfxFamily>::dispatchBlitCommandsForBufferRegion(const Bl
const auto maxWidthToCopy = getMaxBlitWidth(rootDeviceEnvironment);
const auto maxHeightToCopy = getMaxBlitHeight(rootDeviceEnvironment);
dispatchPreBlitCommand(linearStream);
const auto &hwInfo = *rootDeviceEnvironment.getHardwareInfo();
dispatchPreBlitCommand(linearStream, hwInfo);
for (size_t slice = 0u; slice < blitProperties.copySize.z; ++slice) {
auto srcAddress = calculateBlitCommandSourceBaseAddressCopyRegion(blitProperties, slice);
@@ -393,7 +399,7 @@ void BlitCommandsHelper<GfxFamily>::dispatchBlitCommandsForBufferRegion(const Bl
auto cmd = linearStream.getSpaceForCmd<typename GfxFamily::XY_COPY_BLT>();
*cmd = bltCmd;
dispatchPostBlitCommand(linearStream);
dispatchPostBlitCommand(linearStream, hwInfo);
srcAddress += width;
dstAddress += width;

2
shared/source/xe_hpc_core/command_encoder_xe_hpc_core.cpp
View File

@@ -152,7 +152,7 @@ size_t EncodeMemoryPrefetch<Family>::getSizeForMemoryPrefetch(size_t size) {
}
template <>
inline void EncodeMiFlushDW<Family>::appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd) {
inline void EncodeMiFlushDW<Family>::appendMiFlushDw(MI_FLUSH_DW *miFlushDwCmd, const HardwareInfo &hwInfo) {
miFlushDwCmd->setFlushLlc(1);
}

11
shared/test/common/helpers/blit_commands_helper_tests.cpp
View File

@@ -13,6 +13,7 @@
#include "shared/test/common/cmd_parse/hw_parse.h"
#include "shared/test/common/fixtures/device_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/helpers/default_hw_info.h"
#include "shared/test/common/mocks/mock_graphics_allocation.h"
#include "shared/test/common/mocks/ult_device_factory.h"
#include "shared/test/common/test_macros/test_checks_shared.h"
@@ -183,7 +184,7 @@ HWTEST_F(BlitTests, givenDebugVariableWhenDispatchingPostBlitsCommandThenUseCorr
}
// -1: default
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream, *defaultHwInfo);
EXPECT_EQ(expectedDefaultSize, linearStream.getUsed());
CmdParse<FamilyType>::parseCommandBuffer(commands, linearStream.getCpuBase(), linearStream.getUsed());
@@ -205,7 +206,7 @@ HWTEST_F(BlitTests, givenDebugVariableWhenDispatchingPostBlitsCommandThenUseCorr
linearStream.replaceBuffer(streamBuffer, sizeof(streamBuffer));
commands.clear();
DebugManager.flags.PostBlitCommand.set(BlitterConstants::PostBlitMode::MiArbCheck);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream, *defaultHwInfo);
CmdParse<FamilyType>::parseCommandBuffer(commands, linearStream.getCpuBase(), linearStream.getUsed());
arbCheck = find<MI_ARB_CHECK *>(commands.begin(), commands.end());
EXPECT_NE(commands.end(), arbCheck);
@@ -215,7 +216,7 @@ HWTEST_F(BlitTests, givenDebugVariableWhenDispatchingPostBlitsCommandThenUseCorr
linearStream.replaceBuffer(streamBuffer, sizeof(streamBuffer));
commands.clear();
DebugManager.flags.PostBlitCommand.set(BlitterConstants::PostBlitMode::MiFlush);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream, *defaultHwInfo);
CmdParse<FamilyType>::parseCommandBuffer(commands, linearStream.getCpuBase(), linearStream.getUsed());
auto miFlush = find<MI_FLUSH_DW *>(commands.begin(), commands.end());
EXPECT_NE(commands.end(), miFlush);
@@ -225,7 +226,7 @@ HWTEST_F(BlitTests, givenDebugVariableWhenDispatchingPostBlitsCommandThenUseCorr
linearStream.replaceBuffer(streamBuffer, sizeof(streamBuffer));
commands.clear();
DebugManager.flags.PostBlitCommand.set(BlitterConstants::PostBlitMode::None);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream);
BlitCommandsHelper<FamilyType>::dispatchPostBlitCommand(linearStream, *defaultHwInfo);
EXPECT_EQ(0u, linearStream.getUsed());
}
@@ -610,7 +611,7 @@ HWTEST2_F(BlitTests, givenPlatformWhenCallingDispatchPreBlitCommandThenNoneMiFlu
auto miFlushBuffer = std::make_unique<MI_FLUSH_DW>();
LinearStream linearStream(miFlushBuffer.get(), EncodeMiFlushDW<FamilyType>::getMiFlushDwCmdSizeForDataWrite());
BlitCommandsHelper<FamilyType>::dispatchPreBlitCommand(linearStream);
BlitCommandsHelper<FamilyType>::dispatchPreBlitCommand(linearStream, *defaultHwInfo);
HardwareParse hwParser;
hwParser.parseCommands<FamilyType>(linearStream);

2
shared/test/common/helpers/blit_commands_helper_tests_gen12lp.cpp
View File

@@ -255,7 +255,7 @@ HWTEST2_F(BlitTests, givenGen12LpPlatformWhenDispatchPreBlitCommandThenMiFlushDw
auto miFlushBuffer = std::make_unique<MI_FLUSH_DW>();
LinearStream linearStream(miFlushBuffer.get(), EncodeMiFlushDW<FamilyType>::getMiFlushDwCmdSizeForDataWrite());
BlitCommandsHelper<FamilyType>::dispatchPreBlitCommand(linearStream);
BlitCommandsHelper<FamilyType>::dispatchPreBlitCommand(linearStream, *defaultHwInfo);
HardwareParse hwParser;
hwParser.parseCommands<FamilyType>(linearStream);

2
shared/test/common/xe_hpc_core/test_encode_xe_hpc_core.cpp
View File

@@ -34,7 +34,7 @@ XE_HPC_CORETEST_F(CommandEncodeXeHpcCoreTest, whenMiFlushDwIsProgrammedThenSetAn
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, 0x1230000, 456, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, 0x1230000, 456, args, *defaultHwInfo);
auto miFlushDwCmd = reinterpret_cast<MI_FLUSH_DW *>(buffer);
EXPECT_EQ(0u, miFlushDwCmd->getFlushCcs());
EXPECT_EQ(1u, miFlushDwCmd->getFlushLlc());

4
shared/test/unit_test/command_container/command_encoder_tests.cpp
View File

@@ -23,7 +23,7 @@ HWTEST_F(CommandEncoderTests, givenImmDataWriteWhenProgrammingMiFlushDwThenSetAl
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, gpuAddress, immData, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, gpuAddress, immData, args, *defaultHwInfo);
auto miFlushDwCmd = reinterpret_cast<MI_FLUSH_DW *>(buffer);
unsigned int sizeMultiplier = 1;
@@ -79,7 +79,7 @@ HWTEST_F(CommandEncoderTests, givenNotify) {
MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, gpuAddress, immData, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, gpuAddress, immData, args, *defaultHwInfo);
auto miFlushDwCmd = reinterpret_cast<MI_FLUSH_DW *>(buffer);
unsigned int sizeMultiplier = 1;

2
shared/test/unit_test/direct_submission/dispatchers/blitter_dispatcher_tests.cpp
View File

@@ -77,7 +77,7 @@ HWTEST_F(BlitterDispatcheTest, givenBlitterWhenDispatchingTlbFlushThenDispatchMi
using MI_FLUSH_DW = typename FamilyType::MI_FLUSH_DW;
size_t expectedSize = EncodeMiFlushDW<FamilyType>::getMiFlushDwCmdSizeForDataWrite();
BlitterDispatcher<FamilyType>::dispatchTlbFlush(cmdBuffer, 0ull);
BlitterDispatcher<FamilyType>::dispatchTlbFlush(cmdBuffer, 0ull, *defaultHwInfo);
EXPECT_EQ(expectedSize, cmdBuffer.getUsed());

9
shared/test/unit_test/encoders/test_encode_mi_flush_dw.cpp
View File

@@ -9,6 +9,7 @@
#include "shared/source/command_stream/linear_stream.h"
#include "shared/source/helpers/blit_commands_helper.h"
#include "shared/test/common/cmd_parse/gen_cmd_parse.h"
#include "shared/test/common/helpers/default_hw_info.h"
#include "shared/test/common/mocks/mock_graphics_allocation.h"
#include "shared/test/common/test_macros/test.h"
@@ -23,7 +24,7 @@ HWTEST_F(EncodeMiFlushDWTest, GivenLinearStreamWhenCllaedEncodeWithNoPostSyncThe
LinearStream stream(&gfxAllocation);
MiFlushArgs args;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, 0, 0, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, 0, 0, args, *defaultHwInfo);
GenCmdList commands;
CmdParse<FamilyType>::parseCommandBuffer(commands, stream.getCpuBase(), stream.getUsed());
@@ -46,7 +47,7 @@ HWTEST_F(EncodeMiFlushDWTest, GivenLinearStreamWhenCllaedEncodeWithPostSyncDataT
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, address, data, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, address, data, args, *defaultHwInfo);
GenCmdList commands;
CmdParse<FamilyType>::parseCommandBuffer(commands, stream.getCpuBase(), stream.getUsed());
@@ -78,7 +79,7 @@ HWTEST_F(EncodeMiFlushDWTest, GivenLinearStreamWhenCllaedEncodeWithTimestampFasl
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, address, data, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, address, data, args, *defaultHwInfo);
GenCmdList commands;
CmdParse<FamilyType>::parseCommandBuffer(commands, stream.getCpuBase(), stream.getUsed());
@@ -110,7 +111,7 @@ HWTEST_F(EncodeMiFlushDWTest, GivenLinearStreamWhenCllaedEncodeWithTimestampTrue
MiFlushArgs args;
args.timeStampOperation = true;
args.commandWithPostSync = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, address, data, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(stream, address, data, args, *defaultHwInfo);
GenCmdList commands;
CmdParse<FamilyType>::parseCommandBuffer(commands, stream.getCpuBase(), stream.getUsed());

2
shared/test/unit_test/encoders/test_encode_mi_flush_dw_xehp_and_later.cpp
View File

@@ -19,7 +19,7 @@ HWCMDTEST_F(IGFX_XE_HP_CORE, XeHPAndLaterEncodeMiFlushDWTest, whenMiFlushDwIsPro
MiFlushArgs args;
args.commandWithPostSync = true;
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, 0x1230000, 456, args);
EncodeMiFlushDW<FamilyType>::programMiFlushDw(linearStream, 0x1230000, 456, args, *defaultHwInfo);
auto miFlushDwCmd = reinterpret_cast<MI_FLUSH_DW *>(buffer);
EXPECT_EQ(0u, miFlushDwCmd->getFlushCcs());
EXPECT_EQ(0u, miFlushDwCmd->getFlushLlc());