2017-12-21 07:45:38 +08:00
/*
2022-01-07 22:53:31 +08:00
* Copyright (C) 2019-2022 Intel Corporation
2017-12-21 07:45:38 +08:00
*
2018-09-19 03:29:07 +08:00
* SPDX-License-Identifier: MIT
2017-12-21 07:45:38 +08:00
*
*/
2021-11-18 03:51:43 +08:00
#include "shared/source/command_container/command_encoder.h"
2020-02-24 05:44:01 +08:00
#include "shared/source/command_stream/command_stream_receiver_hw.h"
#include "shared/source/command_stream/experimental_command_buffer.h"
#include "shared/source/command_stream/linear_stream.h"
#include "shared/source/command_stream/preemption.h"
#include "shared/source/command_stream/scratch_space_controller_base.h"
2021-05-18 10:46:21 +08:00
#include "shared/source/command_stream/stream_properties.h"
2022-02-22 20:51:29 +08:00
#include "shared/source/command_stream/wait_status.h"
2020-02-24 05:44:01 +08:00
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
2021-07-30 17:56:58 +08:00
#include "shared/source/direct_submission/direct_submission_controller.h"
2020-02-24 05:44:01 +08:00
#include "shared/source/direct_submission/direct_submission_hw.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/page_table_mngr.h"
#include "shared/source/helpers/blit_commands_helper.h"
#include "shared/source/helpers/cache_policy.h"
2021-07-22 16:56:08 +08:00
#include "shared/source/helpers/engine_node_helper.h"
2020-02-24 05:44:01 +08:00
#include "shared/source/helpers/flat_batch_buffer_helper_hw.h"
#include "shared/source/helpers/flush_stamp.h"
#include "shared/source/helpers/hw_helper.h"
2020-09-24 16:52:53 +08:00
#include "shared/source/helpers/pause_on_gpu_properties.h"
2020-02-24 05:44:01 +08:00
#include "shared/source/helpers/preamble.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/helpers/state_base_address.h"
#include "shared/source/helpers/timestamp_packet.h"
#include "shared/source/indirect_heap/indirect_heap.h"
#include "shared/source/memory_manager/internal_allocation_storage.h"
#include "shared/source/memory_manager/memory_manager.h"
2021-09-06 18:34:34 +08:00
#include "shared/source/os_interface/hw_info_config.h"
2020-02-24 05:44:01 +08:00
#include "shared/source/os_interface/os_context.h"
#include "shared/source/utilities/tag_allocator.h"
2017-12-21 07:45:38 +08:00
2019-10-11 12:54:10 +08:00
#include "command_stream_receiver_hw_ext.inl"
2019-03-26 18:59:46 +08:00
namespace NEO {
2017-12-21 07:45:38 +08:00
2020-01-16 00:02:47 +08:00
template <typename GfxFamily>
2021-07-30 17:56:58 +08:00
CommandStreamReceiverHw<GfxFamily>::~CommandStreamReceiverHw() {
2022-03-25 01:11:47 +08:00
auto directSubmissionController = executionEnvironment.directSubmissionController.get();
if (directSubmissionController) {
directSubmissionController->unregisterDirectSubmission(this);
}
2021-07-30 17:56:58 +08:00
}
2020-01-16 00:02:47 +08:00
2018-02-20 15:11:24 +08:00
template <typename GfxFamily>
2020-10-28 23:08:37 +08:00
CommandStreamReceiverHw<GfxFamily>::CommandStreamReceiverHw(ExecutionEnvironment &executionEnvironment,
uint32_t rootDeviceIndex,
2020-10-29 22:33:35 +08:00
const DeviceBitfield deviceBitfield)
2020-10-28 23:08:37 +08:00
: CommandStreamReceiver(executionEnvironment, rootDeviceIndex, deviceBitfield) {
2018-10-18 19:40:53 +08:00
2021-12-20 22:37:33 +08:00
const auto &hwInfo = peekHwInfo();
auto &hwHelper = HwHelper::get(hwInfo.platform.eRenderCoreFamily);
localMemoryEnabled = hwHelper.getEnableLocalMemory(hwInfo);
2018-10-18 19:40:53 +08:00
2021-12-20 22:37:33 +08:00
resetKmdNotifyHelper(new KmdNotifyHelper(&hwInfo.capabilityTable.kmdNotifyProperties));
2021-10-21 20:32:45 +08:00
if (DebugManager.flags.FlattenBatchBufferForAUBDump.get() || DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
flatBatchBufferHelper.reset(new FlatBatchBufferHelperHw<GfxFamily>(executionEnvironment));
}
2018-09-03 22:44:42 +08:00
defaultSshSize = getSshHeapSize();
2021-04-01 04:21:59 +08:00
canUse4GbHeaps = are4GbHeapsAvailable();
2018-10-18 19:40:53 +08:00
timestampPacketWriteEnabled = hwHelper.timestampPacketWriteSupported();
if (DebugManager.flags.EnableTimestampPacket.get() != -1) {
timestampPacketWriteEnabled = !!DebugManager.flags.EnableTimestampPacket.get();
}
2019-03-29 07:49:23 +08:00
createScratchSpaceController();
2021-11-30 22:41:26 +08:00
configurePostSyncWriteOffset();
2018-02-20 15:11:24 +08:00
}
2018-02-02 17:33:31 +08:00
template <typename GfxFamily>
2022-01-07 22:53:31 +08:00
SubmissionStatus CommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
return SubmissionStatus::SUCCESS;
2018-02-02 17:33:31 +08:00
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::addBatchBufferEnd(LinearStream &commandStream, void **patchLocation) {
2020-04-27 03:48:59 +08:00
using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END;
2017-12-21 07:45:38 +08:00
2020-04-27 03:48:59 +08:00
auto pCmd = commandStream.getSpaceForCmd<MI_BATCH_BUFFER_END>();
2017-12-21 07:45:38 +08:00
*pCmd = GfxFamily::cmdInitBatchBufferEnd;
if (patchLocation) {
*patchLocation = pCmd;
}
}
2020-01-16 00:02:47 +08:00
template <typename GfxFamily>
2021-01-29 01:30:56 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programEndingCmd(LinearStream &commandStream, Device &device, void **patchLocation, bool directSubmissionEnabled) {
2020-01-16 00:02:47 +08:00
if (directSubmissionEnabled) {
2022-03-11 02:44:28 +08:00
uint64_t startAddress = commandStream.getGraphicsAllocation()->getGpuAddress() + commandStream.getUsed();
if (DebugManager.flags.BatchBufferStartPrepatchingWaEnabled.get() == 0) {
startAddress = 0;
2022-03-10 21:34:16 +08:00
}
2020-01-16 00:02:47 +08:00
*patchLocation = commandStream.getSpace(sizeof(MI_BATCH_BUFFER_START));
auto bbStart = reinterpret_cast<MI_BATCH_BUFFER_START *>(*patchLocation);
2020-04-28 00:55:26 +08:00
MI_BATCH_BUFFER_START cmd = {};
2022-03-10 21:34:16 +08:00
addBatchBufferStart(&cmd, startAddress, false);
2020-04-28 00:55:26 +08:00
*bbStart = cmd;
2020-01-16 00:02:47 +08:00
} else {
2022-03-10 21:34:16 +08:00
if (!EngineHelpers::isBcs(osContext->getEngineType())) {
PreemptionHelper::programStateSipEndWa<GfxFamily>(commandStream, device);
}
2020-01-16 00:02:47 +08:00
this->addBatchBufferEnd(commandStream, patchLocation);
}
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
2018-07-05 17:23:28 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress, bool secondary) {
2020-04-28 00:55:26 +08:00
MI_BATCH_BUFFER_START cmd = GfxFamily::cmdInitBatchBufferStart;
2021-12-17 02:02:45 +08:00
cmd.setBatchBufferStartAddress(startAddress);
2020-04-28 00:55:26 +08:00
cmd.setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT);
2018-07-05 17:23:28 +08:00
if (secondary) {
2020-04-28 00:55:26 +08:00
cmd.setSecondLevelBatchBuffer(MI_BATCH_BUFFER_START::SECOND_LEVEL_BATCH_BUFFER_SECOND_LEVEL_BATCH);
2018-07-05 17:23:28 +08:00
}
2018-04-04 17:34:46 +08:00
if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
flatBatchBufferHelper->registerBatchBufferStartAddress(reinterpret_cast<uint64_t>(commandBufferMemory), startAddress);
}
2020-04-28 00:55:26 +08:00
*commandBufferMemory = cmd;
2017-12-21 07:45:38 +08:00
}
2019-05-16 23:17:53 +08:00
template <typename GfxFamily>
inline size_t CommandStreamReceiverHw<GfxFamily>::getRequiredCmdSizeForPreamble(Device &device) const {
size_t size = 0;
if (mediaVfeStateDirty) {
size += PreambleHelper<GfxFamily>::getVFECommandsSize();
}
if (!this->isPreambleSent) {
size += PreambleHelper<GfxFamily>::getAdditionalCommandsSize(device);
}
2020-06-24 20:53:51 +08:00
if (!this->isPreambleSent) {
if (DebugManager.flags.ForceSemaphoreDelayBetweenWaits.get() > -1) {
size += PreambleHelper<GfxFamily>::getSemaphoreDelayCommandSize();
}
}
2019-05-16 23:17:53 +08:00
return size;
}
2020-05-27 21:30:31 +08:00
template <typename GfxFamily>
2020-05-23 00:11:28 +08:00
void CommandStreamReceiverHw<GfxFamily>::programHardwareContext(LinearStream &cmdStream) {
programEnginePrologue(cmdStream);
2020-05-27 21:30:31 +08:00
}
template <typename GfxFamily>
size_t CommandStreamReceiverHw<GfxFamily>::getCmdsSizeForHardwareContext() const {
return getCmdSizeForPrologue();
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
LinearStream &commandStreamTask,
size_t commandStreamStartTask,
2018-04-17 19:50:50 +08:00
const IndirectHeap &dsh,
const IndirectHeap &ioh,
const IndirectHeap &ssh,
2017-12-21 07:45:38 +08:00
uint32_t taskLevel,
2018-08-01 16:01:41 +08:00
DispatchFlags &dispatchFlags,
Device &device) {
2017-12-21 07:45:38 +08:00
typedef typename GfxFamily::MI_BATCH_BUFFER_START MI_BATCH_BUFFER_START;
typedef typename GfxFamily::MI_BATCH_BUFFER_END MI_BATCH_BUFFER_END;
typedef typename GfxFamily::PIPE_CONTROL PIPE_CONTROL;
typedef typename GfxFamily::STATE_BASE_ADDRESS STATE_BASE_ADDRESS;
DEBUG_BREAK_IF(&commandStreamTask == &commandStream);
2018-08-01 16:01:41 +08:00
DEBUG_BREAK_IF(!(dispatchFlags.preemptionMode == PreemptionMode::Disabled ? device.getPreemptionMode() == PreemptionMode::Disabled : true));
2020-06-16 19:19:11 +08:00
DEBUG_BREAK_IF(taskLevel >= CompletionStamp::notReady);
2017-12-21 07:45:38 +08:00
DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "taskLevel", taskLevel);
auto levelClosed = false;
2020-09-17 14:58:06 +08:00
bool implicitFlush = dispatchFlags.implicitFlush || dispatchFlags.blocking || DebugManager.flags.ForceImplicitFlush.get();
2017-12-21 07:45:38 +08:00
void *currentPipeControlForNooping = nullptr;
2018-02-15 15:29:57 +08:00
void *epiloguePipeControlLocation = nullptr;
2017-12-21 07:45:38 +08:00
2020-11-18 21:56:18 +08:00
bool csrFlush = this->wasSubmittedToSingleSubdevice != dispatchFlags.useSingleSubdevice;
csrFlush |= DebugManager.flags.ForceCsrFlushing.get();
if (csrFlush) {
2018-06-13 02:33:03 +08:00
flushBatchedSubmissions();
}
2020-06-19 15:50:04 +08:00
2019-01-28 20:44:59 +08:00
if (detectInitProgrammingFlagsRequired(dispatchFlags)) {
2018-06-13 02:33:03 +08:00
initProgrammingFlags();
}
2021-12-20 22:37:33 +08:00
const auto &hwInfo = peekHwInfo();
2022-03-02 20:43:02 +08:00
auto &hwHelper = HwHelper::get(hwInfo.platform.eRenderCoreFamily);
2022-03-25 06:31:03 +08:00
bool updateTag = false;
2017-12-21 07:45:38 +08:00
if (dispatchFlags.blocking || dispatchFlags.dcFlush || dispatchFlags.guardCommandBufferWithPipeControl) {
2018-04-04 17:34:46 +08:00
if (this->dispatchMode == DispatchMode::ImmediateDispatch) {
2017-12-21 07:45:38 +08:00
//for ImmediateDispatch we will send this right away, therefore this pipe control will close the level
//for BatchedSubmissions it will be nooped and only last ppc in batch will be emitted.
levelClosed = true;
2018-02-13 17:01:20 +08:00
//if we guard with ppc, flush dc as well to speed up completion latency
2021-12-20 22:34:39 +08:00
if (dispatchFlags.guardCommandBufferWithPipeControl) {
2022-03-02 20:43:02 +08:00
const auto &hwInfoConfig = *NEO::HwInfoConfig::get(hwInfo.platform.eProductFamily);
if (hwInfoConfig.isDcFlushAllowed()) {
dispatchFlags.dcFlush = true;
}
2018-02-13 17:01:20 +08:00
}
2017-12-21 07:45:38 +08:00
}
2018-03-05 18:03:38 +08:00
epiloguePipeControlLocation = ptrOffset(commandStreamTask.getCpuBase(), commandStreamTask.getUsed());
2018-02-15 15:29:57 +08:00
2018-09-06 15:03:07 +08:00
if ((dispatchFlags.outOfOrderExecutionAllowed || timestampPacketWriteEnabled) &&
2018-08-30 17:05:18 +08:00
!dispatchFlags.dcFlush) {
2018-02-15 15:29:57 +08:00
currentPipeControlForNooping = epiloguePipeControlLocation;
2017-12-21 07:45:38 +08:00
}
2018-11-23 17:32:15 +08:00
auto address = getTagAllocation()->getGpuAddress();
2020-04-27 03:48:59 +08:00
2022-03-25 06:31:03 +08:00
updateTag = !isUpdateTagFromWaitEnabled();
updateTag |= dispatchFlags.blocking;
updateTag |= dispatchFlags.dcFlush;
if (updateTag) {
PipeControlArgs args;
args.dcFlushEnable = MemorySynchronizationCommands<GfxFamily>::getDcFlushEnable(dispatchFlags.dcFlush, hwInfo);
args.notifyEnable = isUsedNotifyEnableForPostSync();
args.tlbInvalidation |= dispatchFlags.memoryMigrationRequired;
args.textureCacheInvalidationEnable |= dispatchFlags.textureCacheFlush;
args.workloadPartitionOffset = isMultiTileOperationEnabled();
MemorySynchronizationCommands<GfxFamily>::addPipeControlAndProgramPostSyncOperation(
commandStreamTask,
PIPE_CONTROL::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA,
address,
taskCount + 1,
hwInfo,
args);
} else {
currentPipeControlForNooping = nullptr;
}
2017-12-21 07:45:38 +08:00
2022-03-25 06:31:03 +08:00
this->latestSentTaskCount = taskCount + 1;
2020-07-18 00:33:16 +08:00
DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "taskCount", peekTaskCount());
2018-04-04 17:34:46 +08:00
if (DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
flatBatchBufferHelper->setPatchInfoData(PatchInfoData(address, 0u,
PatchInfoAllocationType::TagAddress,
commandStreamTask.getGraphicsAllocation()->getGpuAddress(),
commandStreamTask.getUsed() - 2 * sizeof(uint64_t),
PatchInfoAllocationType::Default));
flatBatchBufferHelper->setPatchInfoData(PatchInfoData(address, 0u,
PatchInfoAllocationType::TagValue,
commandStreamTask.getGraphicsAllocation()->getGpuAddress(),
commandStreamTask.getUsed() - sizeof(uint64_t),
PatchInfoAllocationType::Default));
}
2017-12-21 07:45:38 +08:00
}
if (DebugManager.flags.ForceSLML3Config.get()) {
dispatchFlags.useSLM = true;
}
2021-12-20 22:37:33 +08:00
auto newL3Config = PreambleHelper<GfxFamily>::getL3Config(hwInfo, dispatchFlags.useSLM);
2021-09-29 18:28:33 +08:00
auto isSpecialPipelineSelectModeChanged = PreambleHelper<GfxFamily>::isSpecialPipelineSelectModeChanged(lastSpecialPipelineSelectMode,
dispatchFlags.pipelineSelectArgs.specialPipelineSelectMode,
2021-12-20 22:37:33 +08:00
hwInfo);
2017-12-21 07:45:38 +08:00
2021-12-21 00:09:41 +08:00
auto requiresCoherency = hwHelper.forceNonGpuCoherencyWA(dispatchFlags.requiresCoherency);
this->streamProperties.stateComputeMode.setProperties(requiresCoherency, dispatchFlags.numGrfRequired,
2022-03-08 04:29:02 +08:00
dispatchFlags.threadArbitrationPolicy, hwInfo);
2021-12-09 00:22:18 +08:00
2017-12-21 07:45:38 +08:00
csrSizeRequestFlags.l3ConfigChanged = this->lastSentL3Config != newL3Config;
csrSizeRequestFlags.preemptionRequestChanged = this->lastPreemptionMode != dispatchFlags.preemptionMode;
2019-09-10 22:13:11 +08:00
csrSizeRequestFlags.mediaSamplerConfigChanged = this->lastMediaSamplerConfig != static_cast<int8_t>(dispatchFlags.pipelineSelectArgs.mediaSamplerRequired);
2021-09-29 18:28:33 +08:00
csrSizeRequestFlags.specialPipelineSelectModeChanged = isSpecialPipelineSelectModeChanged;
2020-08-21 19:05:01 +08:00
2021-11-18 03:51:43 +08:00
csrSizeRequestFlags.activePartitionsChanged = isProgramActivePartitionConfigRequired();
2021-09-24 02:13:37 +08:00
2017-12-21 07:45:38 +08:00
auto force32BitAllocations = getMemoryManager()->peekForce32BitAllocations();
bool stateBaseAddressDirty = false;
2018-11-22 22:16:20 +08:00
bool checkVfeStateDirty = false;
2019-07-15 15:12:53 +08:00
if (requiredScratchSize || requiredPrivateScratchSize) {
2018-11-22 22:16:20 +08:00
scratchSpaceController->setRequiredScratchSpace(ssh.getCpuBase(),
2020-10-05 17:39:15 +08:00
0u,
2018-11-22 22:16:20 +08:00
requiredScratchSize,
2019-06-28 15:37:04 +08:00
requiredPrivateScratchSize,
2018-11-22 22:16:20 +08:00
this->taskCount,
2019-09-16 20:59:54 +08:00
*this->osContext,
2018-11-22 22:16:20 +08:00
stateBaseAddressDirty,
checkVfeStateDirty);
if (checkVfeStateDirty) {
2018-12-13 18:06:28 +08:00
setMediaVFEStateDirty(true);
2017-12-21 07:45:38 +08:00
}
2019-07-15 15:12:53 +08:00
if (scratchSpaceController->getScratchSpaceAllocation()) {
makeResident(*scratchSpaceController->getScratchSpaceAllocation());
}
if (scratchSpaceController->getPrivateScratchSpaceAllocation()) {
makeResident(*scratchSpaceController->getPrivateScratchSpaceAllocation());
}
2017-12-21 07:45:38 +08:00
}
2020-01-29 21:15:10 +08:00
if (dispatchFlags.usePerDssBackedBuffer) {
2019-09-04 22:44:27 +08:00
if (!perDssBackedBuffer) {
createPerDssBackedBuffer(device);
}
makeResident(*perDssBackedBuffer);
}
2020-09-02 17:38:54 +08:00
if (dispatchFlags.additionalKernelExecInfo != AdditionalKernelExecInfo::NotApplicable && lastAdditionalKernelExecInfo != dispatchFlags.additionalKernelExecInfo) {
setMediaVFEStateDirty(true);
}
2020-11-17 18:42:29 +08:00
if (dispatchFlags.kernelExecutionType != KernelExecutionType::NotApplicable && lastKernelExecutionType != dispatchFlags.kernelExecutionType) {
setMediaVFEStateDirty(true);
}
2018-08-07 15:07:50 +08:00
auto &commandStreamCSR = this->getCS(getRequiredCmdStreamSizeAligned(dispatchFlags, device));
2018-05-15 15:46:22 +08:00
auto commandStreamStartCSR = commandStreamCSR.getUsed();
2021-06-23 18:34:31 +08:00
TimestampPacketHelper::programCsrDependenciesForTimestampPacketContainer<GfxFamily>(commandStreamCSR, dispatchFlags.csrDependencies);
2021-06-14 23:33:53 +08:00
TimestampPacketHelper::programCsrDependenciesForForTaskCountContainer<GfxFamily>(commandStreamCSR, dispatchFlags.csrDependencies);
2019-01-25 17:20:32 +08:00
2021-11-18 03:51:43 +08:00
programActivePartitionConfigFlushTask(commandStreamCSR);
2019-10-11 12:54:10 +08:00
programEngineModeCommands(commandStreamCSR, dispatchFlags);
2022-02-03 00:30:03 +08:00
2021-09-29 23:59:41 +08:00
if (pageTableManager.get() && !pageTableManagerInitialized) {
pageTableManagerInitialized = pageTableManager->initPageTableManagerRegisters(this);
2019-11-07 01:14:30 +08:00
}
2020-05-27 21:30:31 +08:00
2020-05-23 00:11:28 +08:00
programHardwareContext(commandStreamCSR);
2021-12-20 22:37:33 +08:00
programComputeMode(commandStreamCSR, dispatchFlags, hwInfo);
2019-09-10 22:13:11 +08:00
programPipelineSelect(commandStreamCSR, dispatchFlags.pipelineSelectArgs);
2021-11-04 20:54:18 +08:00
programL3(commandStreamCSR, newL3Config);
programPreamble(commandStreamCSR, device, newL3Config);
2018-05-15 15:46:22 +08:00
programMediaSampler(commandStreamCSR, dispatchFlags);
2021-09-30 19:10:58 +08:00
addPipeControlBefore3dState(commandStreamCSR, dispatchFlags);
2020-11-27 03:02:18 +08:00
programPerDssBackedBuffer(commandStreamCSR, device, dispatchFlags);
2018-05-15 15:46:22 +08:00
2019-09-13 18:00:30 +08:00
stateBaseAddressDirty |= ((GSBAFor32BitProgrammed ^ dispatchFlags.gsba32BitRequired) && force32BitAllocations);
2017-12-21 07:45:38 +08:00
2019-05-29 21:37:54 +08:00
programVFEState(commandStreamCSR, dispatchFlags, device.getDeviceInfo().maxFrontEndThreads);
2017-12-21 07:45:38 +08:00
2019-10-08 01:54:39 +08:00
programPreemption(commandStreamCSR, dispatchFlags);
2022-02-03 00:30:03 +08:00
if (stallingCommandsOnNextFlushRequired) {
programStallingCommandsForBarrier(commandStreamCSR, dispatchFlags);
}
2017-12-21 07:45:38 +08:00
bool dshDirty = dshState.updateAndCheck(&dsh);
bool iohDirty = iohState.updateAndCheck(&ioh);
bool sshDirty = sshState.updateAndCheck(&ssh);
2018-03-27 18:55:20 +08:00
auto isStateBaseAddressDirty = dshDirty || iohDirty || sshDirty || stateBaseAddressDirty;
2017-12-21 07:45:38 +08:00
2020-08-20 17:48:10 +08:00
auto mocsIndex = latestSentStatelessMocsConfig;
if (dispatchFlags.l3CacheSettings != L3CachingSettings::NotApplicable) {
auto l3On = dispatchFlags.l3CacheSettings != L3CachingSettings::l3CacheOff;
auto l1On = dispatchFlags.l3CacheSettings == L3CachingSettings::l3AndL1On;
mocsIndex = hwHelper.getMocsIndex(*device.getGmmHelper(), l3On, l1On);
}
2017-12-21 07:45:38 +08:00
2019-08-26 21:15:22 +08:00
if (mocsIndex != latestSentStatelessMocsConfig) {
2017-12-21 07:45:38 +08:00
isStateBaseAddressDirty = true;
2019-08-27 20:18:06 +08:00
latestSentStatelessMocsConfig = mocsIndex;
2017-12-21 07:45:38 +08:00
}
2021-03-30 01:06:29 +08:00
if ((isMultiOsContextCapable() || dispatchFlags.areMultipleSubDevicesInContext) && (dispatchFlags.useGlobalAtomics != lastSentUseGlobalAtomics)) {
2021-03-03 20:25:26 +08:00
isStateBaseAddressDirty = true;
lastSentUseGlobalAtomics = dispatchFlags.useGlobalAtomics;
}
2020-06-22 23:13:30 +08:00
bool sourceLevelDebuggerActive = device.getSourceLevelDebugger() != nullptr ? true : false;
2021-01-19 21:19:33 +08:00
auto memoryCompressionState = lastMemoryCompressionState;
2020-12-17 08:36:45 +08:00
if (dispatchFlags.memoryCompressionState != MemoryCompressionState::NotApplicable) {
2021-01-19 21:19:33 +08:00
memoryCompressionState = dispatchFlags.memoryCompressionState;
}
if (memoryCompressionState != lastMemoryCompressionState) {
isStateBaseAddressDirty = true;
lastMemoryCompressionState = memoryCompressionState;
2020-12-17 08:36:45 +08:00
}
2017-12-21 07:45:38 +08:00
//Reprogram state base address if required
2020-06-22 23:13:30 +08:00
if (isStateBaseAddressDirty || sourceLevelDebuggerActive) {
2022-03-18 19:06:51 +08:00
EncodeWA<GfxFamily>::addPipeControlBeforeStateBaseAddress(commandStreamCSR, hwInfo, isRcs());
2022-03-10 01:15:48 +08:00
EncodeWA<GfxFamily>::encodeAdditionalPipelineSelect(commandStreamCSR, dispatchFlags.pipelineSelectArgs, true, hwInfo, isRcs());
2017-12-21 07:45:38 +08:00
uint64_t newGSHbase = 0;
GSBAFor32BitProgrammed = false;
2018-11-22 22:16:20 +08:00
if (is64bit && scratchSpaceController->getScratchSpaceAllocation() && !force32BitAllocations) {
newGSHbase = scratchSpaceController->calculateNewGSH();
2019-09-13 18:00:30 +08:00
} else if (is64bit && force32BitAllocations && dispatchFlags.gsba32BitRequired) {
2020-07-01 20:03:46 +08:00
bool useLocalMemory = scratchSpaceController->getScratchSpaceAllocation() ? scratchSpaceController->getScratchSpaceAllocation()->isAllocatedInLocalMemoryPool() : false;
newGSHbase = getMemoryManager()->getExternalHeapBaseAddress(rootDeviceIndex, useLocalMemory);
2017-12-21 07:45:38 +08:00
GSBAFor32BitProgrammed = true;
}
2018-03-14 18:07:51 +08:00
auto stateBaseAddressCmdOffset = commandStreamCSR.getUsed();
2020-08-03 19:13:58 +08:00
auto pCmd = static_cast<STATE_BASE_ADDRESS *>(commandStreamCSR.getSpace(sizeof(STATE_BASE_ADDRESS)));
STATE_BASE_ADDRESS cmd;
2021-05-25 02:57:32 +08:00
auto instructionHeapBaseAddress = getMemoryManager()->getInternalHeapBaseAddress(rootDeviceIndex, getMemoryManager()->isLocalMemoryUsedForIsa(rootDeviceIndex));
2017-12-21 07:45:38 +08:00
StateBaseAddressHelper<GfxFamily>::programStateBaseAddress(
2020-08-03 19:13:58 +08:00
&cmd,
2019-12-06 16:50:33 +08:00
&dsh,
&ioh,
&ssh,
2017-12-21 07:45:38 +08:00
newGSHbase,
2020-01-22 23:39:40 +08:00
true,
2019-08-26 21:15:22 +08:00
mocsIndex,
2020-07-01 20:03:46 +08:00
getMemoryManager()->getInternalHeapBaseAddress(rootDeviceIndex, ioh.getGraphicsAllocation()->isAllocatedInLocalMemoryPool()),
2020-09-22 21:48:24 +08:00
instructionHeapBaseAddress,
2020-12-04 19:28:18 +08:00
0,
2020-01-22 23:39:40 +08:00
true,
2020-12-04 19:28:18 +08:00
false,
2019-02-15 00:12:15 +08:00
device.getGmmHelper(),
2020-12-17 08:36:45 +08:00
isMultiOsContextCapable(),
2021-01-26 22:05:22 +08:00
memoryCompressionState,
dispatchFlags.useGlobalAtomics,
2021-03-30 01:06:29 +08:00
dispatchFlags.areMultipleSubDevicesInContext);
2020-08-03 19:13:58 +08:00
*pCmd = cmd;
2018-03-14 18:07:51 +08:00
2021-06-23 21:34:56 +08:00
programAdditionalStateBaseAddress(commandStreamCSR, cmd, device);
2018-08-29 19:39:27 +08:00
if (sshDirty) {
2019-04-26 23:04:03 +08:00
bindingTableBaseAddressRequired = true;
}
if (bindingTableBaseAddressRequired) {
2019-12-09 23:29:59 +08:00
StateBaseAddressHelper<GfxFamily>::programBindingTableBaseAddress(commandStreamCSR, ssh, device.getGmmHelper());
2019-04-26 23:04:03 +08:00
bindingTableBaseAddressRequired = false;
2018-08-29 19:39:27 +08:00
}
2022-03-10 01:15:48 +08:00
EncodeWA<GfxFamily>::encodeAdditionalPipelineSelect(commandStreamCSR, dispatchFlags.pipelineSelectArgs, false, hwInfo, isRcs());
2021-10-01 19:47:49 +08:00
addPipeControlBeforeStateSip(commandStreamCSR, device);
2018-11-05 18:52:19 +08:00
programStateSip(commandStreamCSR, device);
2018-03-14 18:07:51 +08:00
if (DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
2018-04-04 17:34:46 +08:00
collectStateBaseAddresPatchInfo(commandStream.getGraphicsAllocation()->getGpuAddress(), stateBaseAddressCmdOffset, dsh, ioh, ssh, newGSHbase);
2018-03-14 18:07:51 +08:00
}
2017-12-21 07:45:38 +08:00
}
DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "this->taskLevel", (uint32_t)this->taskLevel);
2021-11-25 17:31:14 +08:00
if (executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->getHardwareInfo()->workaroundTable.flags.waSamplerCacheFlushBetweenRedescribedSurfaceReads) {
2018-01-10 21:05:34 +08:00
if (this->samplerCacheFlushRequired != SamplerCacheFlushState::samplerCacheFlushNotRequired) {
2020-04-27 03:48:59 +08:00
PipeControlArgs args;
args.textureCacheInvalidationEnable = true;
2021-11-04 21:05:13 +08:00
MemorySynchronizationCommands<GfxFamily>::addPipeControl(commandStreamCSR, args);
2018-01-10 21:05:34 +08:00
if (this->samplerCacheFlushRequired == SamplerCacheFlushState::samplerCacheFlushBefore) {
this->samplerCacheFlushRequired = SamplerCacheFlushState::samplerCacheFlushAfter;
} else {
this->samplerCacheFlushRequired = SamplerCacheFlushState::samplerCacheFlushNotRequired;
}
}
}
2018-07-05 17:23:28 +08:00
if (experimentalCmdBuffer.get() != nullptr) {
size_t startingOffset = experimentalCmdBuffer->programExperimentalCommandBuffer<GfxFamily>();
experimentalCmdBuffer->injectBufferStart<GfxFamily>(commandStreamCSR, startingOffset);
}
2019-08-28 01:14:24 +08:00
if (requiresInstructionCacheFlush) {
2020-04-27 03:48:59 +08:00
PipeControlArgs args;
args.instructionCacheInvalidateEnable = true;
MemorySynchronizationCommands<GfxFamily>::addPipeControl(commandStreamCSR, args);
2019-08-28 01:14:24 +08:00
requiresInstructionCacheFlush = false;
}
2020-11-04 21:02:31 +08:00
// Add a Pipe Control if we have a dependency on a previous walker to avoid concurrency issues.
2017-12-21 07:45:38 +08:00
if (taskLevel > this->taskLevel) {
2021-12-10 00:18:18 +08:00
auto programPipeControl = !timestampPacketWriteEnabled;
if (DebugManager.flags.ResolveDependenciesViaPipeControls.get() == 1) {
programPipeControl = true;
}
if (programPipeControl) {
2020-04-27 03:48:59 +08:00
PipeControlArgs args;
MemorySynchronizationCommands<GfxFamily>::addPipeControl(commandStreamCSR, args);
2018-08-30 17:05:18 +08:00
}
2017-12-21 07:45:38 +08:00
this->taskLevel = taskLevel;
2020-07-18 00:33:16 +08:00
DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "this->taskCount", peekTaskCount());
2017-12-21 07:45:38 +08:00
}
2020-07-06 15:34:15 +08:00
if (DebugManager.flags.ForcePipeControlPriorToWalker.get()) {
forcePipeControl(commandStreamCSR);
}
2017-12-21 07:45:38 +08:00
auto dshAllocation = dsh.getGraphicsAllocation();
auto iohAllocation = ioh.getGraphicsAllocation();
auto sshAllocation = ssh.getGraphicsAllocation();
this->makeResident(*dshAllocation);
2018-04-12 16:05:35 +08:00
dshAllocation->setEvictable(false);
2017-12-21 07:45:38 +08:00
this->makeResident(*iohAllocation);
this->makeResident(*sshAllocation);
2018-04-12 16:05:35 +08:00
iohAllocation->setEvictable(false);
2017-12-21 07:45:38 +08:00
this->makeResident(*tagAllocation);
2020-02-06 03:00:08 +08:00
if (globalFenceAllocation) {
makeResident(*globalFenceAllocation);
}
if (preemptionAllocation) {
2019-06-28 03:33:05 +08:00
makeResident(*preemptionAllocation);
2020-02-06 03:00:08 +08:00
}
2017-12-21 07:45:38 +08:00
2020-06-22 23:13:30 +08:00
if (dispatchFlags.preemptionMode == PreemptionMode::MidThread || sourceLevelDebuggerActive) {
2021-04-16 20:52:30 +08:00
makeResident(*SipKernel::getSipKernel(device).getSipAllocation());
2018-11-15 23:08:22 +08:00
if (debugSurface) {
makeResident(*debugSurface);
}
2018-03-13 00:18:00 +08:00
}
2018-07-05 17:23:28 +08:00
if (experimentalCmdBuffer.get() != nullptr) {
experimentalCmdBuffer->makeResidentAllocations();
}
2021-03-04 20:37:56 +08:00
if (workPartitionAllocation) {
makeResident(*workPartitionAllocation);
}
2017-12-21 07:45:38 +08:00
// If the CSR has work in its CS, flush it before the task
bool submitTask = commandStreamStartTask != commandStreamTask.getUsed();
2019-04-01 23:26:32 +08:00
bool submitCSR = (commandStreamStartCSR != commandStreamCSR.getUsed()) || this->isMultiOsContextCapable();
2017-12-21 07:45:38 +08:00
bool submitCommandStreamFromCsr = false;
void *bbEndLocation = nullptr;
auto bbEndPaddingSize = this->dispatchMode == DispatchMode::ImmediateDispatch ? 0 : sizeof(MI_BATCH_BUFFER_START) - sizeof(MI_BATCH_BUFFER_END);
2018-03-01 05:50:41 +08:00
size_t chainedBatchBufferStartOffset = 0;
GraphicsAllocation *chainedBatchBuffer = nullptr;
2020-01-16 00:02:47 +08:00
bool directSubmissionEnabled = isDirectSubmissionEnabled();
2017-12-21 07:45:38 +08:00
if (submitTask) {
2021-01-29 01:30:56 +08:00
programEndingCmd(commandStreamTask, device, &bbEndLocation, directSubmissionEnabled);
2021-11-18 06:36:00 +08:00
EncodeNoop<GfxFamily>::emitNoop(commandStreamTask, bbEndPaddingSize);
EncodeNoop<GfxFamily>::alignToCacheLine(commandStreamTask);
2017-12-21 07:45:38 +08:00
if (submitCSR) {
2018-03-01 05:50:41 +08:00
chainedBatchBufferStartOffset = commandStreamCSR.getUsed();
chainedBatchBuffer = commandStreamTask.getGraphicsAllocation();
2017-12-21 07:45:38 +08:00
// Add MI_BATCH_BUFFER_START to chain from CSR -> Task
auto pBBS = reinterpret_cast<MI_BATCH_BUFFER_START *>(commandStreamCSR.getSpace(sizeof(MI_BATCH_BUFFER_START)));
2018-07-05 17:23:28 +08:00
addBatchBufferStart(pBBS, ptrOffset(commandStreamTask.getGraphicsAllocation()->getGpuAddress(), commandStreamStartTask), false);
2018-04-04 17:34:46 +08:00
if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
flatBatchBufferHelper->registerCommandChunk(commandStreamTask.getGraphicsAllocation()->getGpuAddress(),
reinterpret_cast<uint64_t>(commandStreamTask.getCpuBase()),
commandStreamStartTask,
static_cast<uint64_t>(ptrDiff(bbEndLocation,
commandStreamTask.getGraphicsAllocation()->getGpuAddress())) +
sizeof(MI_BATCH_BUFFER_START));
}
2017-12-21 07:45:38 +08:00
auto commandStreamAllocation = commandStreamTask.getGraphicsAllocation();
DEBUG_BREAK_IF(commandStreamAllocation == nullptr);
this->makeResident(*commandStreamAllocation);
2021-11-18 06:36:00 +08:00
EncodeNoop<GfxFamily>::alignToCacheLine(commandStreamCSR);
2017-12-21 07:45:38 +08:00
submitCommandStreamFromCsr = true;
2019-08-08 01:33:40 +08:00
} else if (dispatchFlags.epilogueRequired) {
this->makeResident(*commandStreamCSR.getGraphicsAllocation());
2017-12-21 07:45:38 +08:00
}
2021-01-29 01:30:56 +08:00
this->programEpilogue(commandStreamCSR, device, &bbEndLocation, dispatchFlags);
2019-08-08 01:33:40 +08:00
2017-12-21 07:45:38 +08:00
} else if (submitCSR) {
2021-01-29 01:30:56 +08:00
programEndingCmd(commandStreamCSR, device, &bbEndLocation, directSubmissionEnabled);
2021-11-18 06:36:00 +08:00
EncodeNoop<GfxFamily>::emitNoop(commandStreamCSR, bbEndPaddingSize);
EncodeNoop<GfxFamily>::alignToCacheLine(commandStreamCSR);
2017-12-21 07:45:38 +08:00
DEBUG_BREAK_IF(commandStreamCSR.getUsed() > commandStreamCSR.getMaxAvailableSpace());
submitCommandStreamFromCsr = true;
}
size_t startOffset = submitCommandStreamFromCsr ? commandStreamStartCSR : commandStreamStartTask;
auto &streamToSubmit = submitCommandStreamFromCsr ? commandStreamCSR : commandStreamTask;
2020-01-16 00:02:47 +08:00
BatchBuffer batchBuffer{streamToSubmit.getGraphicsAllocation(), startOffset, chainedBatchBufferStartOffset, chainedBatchBuffer,
dispatchFlags.requiresCoherency, dispatchFlags.lowPriority, dispatchFlags.throttle, dispatchFlags.sliceCount,
2020-11-18 21:56:18 +08:00
streamToSubmit.getUsed(), &streamToSubmit, bbEndLocation, dispatchFlags.useSingleSubdevice};
2021-06-22 18:09:40 +08:00
streamToSubmit.getGraphicsAllocation()->updateTaskCount(this->taskCount + 1, this->osContext->getContextId());
streamToSubmit.getGraphicsAllocation()->updateResidencyTaskCount(this->taskCount + 1, this->osContext->getContextId());
2017-12-21 07:45:38 +08:00
if (submitCSR | submitTask) {
if (this->dispatchMode == DispatchMode::ImmediateDispatch) {
2021-07-28 20:35:06 +08:00
flushHandler(batchBuffer, this->getResidencyAllocations());
2022-03-25 06:31:03 +08:00
if (updateTag) {
2021-02-23 16:48:08 +08:00
this->latestFlushedTaskCount = this->taskCount + 1;
}
2017-12-21 07:45:38 +08:00
} else {
2018-08-16 17:18:05 +08:00
auto commandBuffer = new CommandBuffer(device);
2017-12-21 07:45:38 +08:00
commandBuffer->batchBuffer = batchBuffer;
2018-09-12 15:47:01 +08:00
commandBuffer->surfaces.swap(this->getResidencyAllocations());
2017-12-21 07:45:38 +08:00
commandBuffer->batchBufferEndLocation = bbEndLocation;
commandBuffer->taskCount = this->taskCount + 1;
commandBuffer->flushStamp->replaceStampObject(dispatchFlags.flushStampReference);
2018-02-15 15:29:57 +08:00
commandBuffer->pipeControlThatMayBeErasedLocation = currentPipeControlForNooping;
commandBuffer->epiloguePipeControlLocation = epiloguePipeControlLocation;
2017-12-21 07:45:38 +08:00
this->submissionAggregator->recordCommandBuffer(commandBuffer);
}
} else {
2018-11-26 21:04:52 +08:00
this->makeSurfacePackNonResident(this->getResidencyAllocations());
2017-12-21 07:45:38 +08:00
}
2020-11-18 21:56:18 +08:00
this->wasSubmittedToSingleSubdevice = dispatchFlags.useSingleSubdevice;
2017-12-21 07:45:38 +08:00
//check if we are not over the budget, if we are do implicit flush
if (getMemoryManager()->isMemoryBudgetExhausted()) {
2018-08-01 16:01:41 +08:00
if (this->totalMemoryUsed >= device.getDeviceInfo().globalMemSize / 4) {
2020-09-17 14:58:06 +08:00
implicitFlush = true;
}
}
if (DebugManager.flags.PerformImplicitFlushEveryEnqueueCount.get() != -1) {
if ((taskCount + 1) % DebugManager.flags.PerformImplicitFlushEveryEnqueueCount.get() == 0) {
implicitFlush = true;
2017-12-21 07:45:38 +08:00
}
}
2020-09-22 22:29:34 +08:00
if (this->newResources) {
implicitFlush = true;
this->newResources = false;
}
implicitFlush |= checkImplicitFlushForGpuIdle();
2020-09-17 14:58:06 +08:00
if (this->dispatchMode == DispatchMode::BatchedDispatch && implicitFlush) {
2017-12-21 07:45:38 +08:00
this->flushBatchedSubmissions();
}
++taskCount;
2020-07-18 00:33:16 +08:00
DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "taskCount", peekTaskCount());
2017-12-21 07:45:38 +08:00
DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "Current taskCount:", tagAddress ? *tagAddress : 0);
CompletionStamp completionStamp = {
taskCount,
this->taskLevel,
2018-11-26 21:04:52 +08:00
flushStamp->peekStamp()};
2017-12-21 07:45:38 +08:00
this->taskLevel += levelClosed ? 1 : 0;
2018-02-08 23:00:20 +08:00
2017-12-21 07:45:38 +08:00
return completionStamp;
}
2020-07-06 15:34:15 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::forcePipeControl(NEO::LinearStream &commandStreamCSR) {
PipeControlArgs args;
2021-04-01 00:41:29 +08:00
MemorySynchronizationCommands<GfxFamily>::addPipeControlWithCSStallOnly(commandStreamCSR);
2020-07-06 15:34:15 +08:00
MemorySynchronizationCommands<GfxFamily>::addPipeControl(commandStreamCSR, args);
}
2022-03-08 22:18:31 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::programComputeMode(LinearStream &stream, DispatchFlags &dispatchFlags, const HardwareInfo &hwInfo) {
if (this->streamProperties.stateComputeMode.isDirty()) {
EncodeComputeMode<GfxFamily>::programComputeModeCommandWithSynchronization(
stream, this->streamProperties.stateComputeMode, dispatchFlags.pipelineSelectArgs,
hasSharedHandles(), hwInfo, isRcs());
}
}
2019-10-03 20:38:49 +08:00
template <typename GfxFamily>
2021-11-06 09:42:54 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programStallingCommandsForBarrier(LinearStream &cmdStream, DispatchFlags &dispatchFlags) {
stallingCommandsOnNextFlushRequired = false;
2019-10-03 20:38:49 +08:00
auto barrierTimestampPacketNodes = dispatchFlags.barrierTimestampPacketNodes;
if (barrierTimestampPacketNodes && barrierTimestampPacketNodes->peekNodes().size() != 0) {
2021-12-10 03:31:27 +08:00
programStallingPostSyncCommandsForBarrier(cmdStream, *barrierTimestampPacketNodes->peekNodes()[0]);
barrierTimestampPacketNodes->makeResident(*this);
2019-10-03 20:38:49 +08:00
} else {
2021-11-06 09:42:54 +08:00
programStallingNoPostSyncCommandsForBarrier(cmdStream);
2019-10-03 20:38:49 +08:00
}
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
2019-11-24 21:50:41 +08:00
inline bool CommandStreamReceiverHw<GfxFamily>::flushBatchedSubmissions() {
2018-04-04 17:34:46 +08:00
if (this->dispatchMode == DispatchMode::ImmediateDispatch) {
2019-11-24 21:50:41 +08:00
return true;
2017-12-21 07:45:38 +08:00
}
typedef typename GfxFamily::MI_BATCH_BUFFER_START MI_BATCH_BUFFER_START;
2018-02-13 17:01:20 +08:00
typedef typename GfxFamily::PIPE_CONTROL PIPE_CONTROL;
2018-08-06 20:55:04 +08:00
std::unique_lock<MutexType> lockGuard(ownershipMutex);
2019-11-24 21:50:41 +08:00
bool submitResult = true;
2017-12-21 07:45:38 +08:00
auto &commandBufferList = this->submissionAggregator->peekCmdBufferList();
if (!commandBufferList.peekIsEmpty()) {
2018-12-04 22:11:29 +08:00
const auto totalMemoryBudget = static_cast<size_t>(commandBufferList.peekHead()->device.getDeviceInfo().globalMemSize / 2);
2018-08-16 17:18:05 +08:00
2017-12-21 07:45:38 +08:00
ResidencyContainer surfacesForSubmit;
ResourcePackage resourcePackage;
2021-12-20 22:37:33 +08:00
const auto &hwInfo = peekHwInfo();
auto pipeControlLocationSize = MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(hwInfo);
2018-02-15 15:29:57 +08:00
void *currentPipeControlForNooping = nullptr;
void *epiloguePipeControlLocation = nullptr;
2017-12-21 07:45:38 +08:00
while (!commandBufferList.peekIsEmpty()) {
size_t totalUsedSize = 0u;
2018-12-04 22:11:29 +08:00
this->submissionAggregator->aggregateCommandBuffers(resourcePackage, totalUsedSize, totalMemoryBudget, osContext->getContextId());
2017-12-21 07:45:38 +08:00
auto primaryCmdBuffer = commandBufferList.removeFrontOne();
auto nextCommandBuffer = commandBufferList.peekHead();
auto currentBBendLocation = primaryCmdBuffer->batchBufferEndLocation;
auto lastTaskCount = primaryCmdBuffer->taskCount;
FlushStampUpdateHelper flushStampUpdateHelper;
flushStampUpdateHelper.insert(primaryCmdBuffer->flushStamp->getStampReference());
2018-02-15 15:29:57 +08:00
currentPipeControlForNooping = primaryCmdBuffer->pipeControlThatMayBeErasedLocation;
epiloguePipeControlLocation = primaryCmdBuffer->epiloguePipeControlLocation;
2017-12-21 07:45:38 +08:00
2018-04-04 17:34:46 +08:00
if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
flatBatchBufferHelper->registerCommandChunk(primaryCmdBuffer.get()->batchBuffer, sizeof(MI_BATCH_BUFFER_START));
}
2021-07-13 19:56:41 +08:00
2017-12-21 07:45:38 +08:00
while (nextCommandBuffer && nextCommandBuffer->inspectionId == primaryCmdBuffer->inspectionId) {
2021-07-13 19:56:41 +08:00
2017-12-21 07:45:38 +08:00
//noop pipe control
2018-02-15 15:29:57 +08:00
if (currentPipeControlForNooping) {
2018-04-04 17:34:46 +08:00
if (DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
2021-12-20 22:37:33 +08:00
flatBatchBufferHelper->removePipeControlData(pipeControlLocationSize, currentPipeControlForNooping, hwInfo);
2018-04-04 17:34:46 +08:00
}
2018-02-15 15:29:57 +08:00
memset(currentPipeControlForNooping, 0, pipeControlLocationSize);
2017-12-21 07:45:38 +08:00
}
//obtain next candidate for nooping
2018-02-15 15:29:57 +08:00
currentPipeControlForNooping = nextCommandBuffer->pipeControlThatMayBeErasedLocation;
//track epilogue pipe control
epiloguePipeControlLocation = nextCommandBuffer->epiloguePipeControlLocation;
2017-12-21 07:45:38 +08:00
flushStampUpdateHelper.insert(nextCommandBuffer->flushStamp->getStampReference());
2018-08-24 21:23:45 +08:00
auto nextCommandBufferAddress = nextCommandBuffer->batchBuffer.commandBufferAllocation->getGpuAddress();
2017-12-21 07:45:38 +08:00
auto offsetedCommandBuffer = (uint64_t)ptrOffset(nextCommandBufferAddress, nextCommandBuffer->batchBuffer.startOffset);
2021-07-13 19:56:41 +08:00
auto cpuAddressForCommandBufferDestination = ptrOffset(nextCommandBuffer->batchBuffer.commandBufferAllocation->getUnderlyingBuffer(), nextCommandBuffer->batchBuffer.startOffset);
auto cpuAddressForCurrentCommandBufferEndingSection = alignUp(ptrOffset(currentBBendLocation, sizeof(MI_BATCH_BUFFER_START)), MemoryConstants::cacheLineSize);
//if we point to exact same command buffer, then batch buffer start is not needed at all
if (cpuAddressForCurrentCommandBufferEndingSection == cpuAddressForCommandBufferDestination) {
memset(currentBBendLocation, 0u, ptrDiff(cpuAddressForCurrentCommandBufferEndingSection, currentBBendLocation));
} else {
addBatchBufferStart((MI_BATCH_BUFFER_START *)currentBBendLocation, offsetedCommandBuffer, false);
}
2018-04-04 17:34:46 +08:00
if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
flatBatchBufferHelper->registerCommandChunk(nextCommandBuffer->batchBuffer, sizeof(MI_BATCH_BUFFER_START));
}
2017-12-21 07:45:38 +08:00
currentBBendLocation = nextCommandBuffer->batchBufferEndLocation;
lastTaskCount = nextCommandBuffer->taskCount;
nextCommandBuffer = nextCommandBuffer->next;
2021-07-13 19:56:41 +08:00
2017-12-21 07:45:38 +08:00
commandBufferList.removeFrontOne();
}
surfacesForSubmit.reserve(resourcePackage.size() + 1);
for (auto &surface : resourcePackage) {
surfacesForSubmit.push_back(surface);
}
2019-02-05 22:11:38 +08:00
//make sure we flush DC if needed
2022-03-04 22:07:37 +08:00
if (epiloguePipeControlLocation && MemorySynchronizationCommands<GfxFamily>::getDcFlushEnable(true, hwInfo)) {
auto emitDcFlush = true;
if (DebugManager.flags.DisableDcFlushInEpilogue.get()) {
emitDcFlush = false;
}
((PIPE_CONTROL *)epiloguePipeControlLocation)->setDcFlushEnable(emitDcFlush);
2018-03-09 21:48:42 +08:00
}
2019-11-24 21:50:41 +08:00
2020-09-22 20:19:07 +08:00
primaryCmdBuffer->batchBuffer.endCmdPtr = currentBBendLocation;
2022-01-07 22:53:31 +08:00
if (this->flush(primaryCmdBuffer->batchBuffer, surfacesForSubmit) != SubmissionStatus::SUCCESS) {
2019-11-24 21:50:41 +08:00
submitResult = false;
break;
}
2017-12-21 07:45:38 +08:00
//after flush task level is closed
this->taskLevel++;
2019-11-24 21:50:41 +08:00
flushStampUpdateHelper.updateAll(flushStamp->peekStamp());
2017-12-21 07:45:38 +08:00
2021-10-28 17:21:44 +08:00
if (!isUpdateTagFromWaitEnabled()) {
this->latestFlushedTaskCount = lastTaskCount;
}
2018-11-26 21:04:52 +08:00
this->makeSurfacePackNonResident(surfacesForSubmit);
2017-12-21 07:45:38 +08:00
resourcePackage.clear();
}
this->totalMemoryUsed = 0;
}
2019-11-24 21:50:41 +08:00
return submitResult;
2017-12-21 07:45:38 +08:00
}
2018-01-24 17:11:37 +08:00
template <typename GfxFamily>
2018-08-07 15:07:50 +08:00
size_t CommandStreamReceiverHw<GfxFamily>::getRequiredCmdStreamSizeAligned(const DispatchFlags &dispatchFlags, Device &device) {
size_t size = getRequiredCmdStreamSize(dispatchFlags, device);
2018-01-24 17:11:37 +08:00
return alignUp(size, MemoryConstants::cacheLineSize);
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
2018-08-07 15:07:50 +08:00
size_t CommandStreamReceiverHw<GfxFamily>::getRequiredCmdStreamSize(const DispatchFlags &dispatchFlags, Device &device) {
2018-08-16 17:18:05 +08:00
size_t size = getRequiredCmdSizeForPreamble(device);
2021-07-08 23:08:37 +08:00
size += getRequiredStateBaseAddressSize(device);
2020-02-10 22:57:49 +08:00
if (!this->isStateSipSent || device.isDebuggerActive()) {
2021-10-17 20:21:29 +08:00
size += PreemptionHelper::getRequiredStateSipCmdSize<GfxFamily>(device, isRcs());
2018-11-05 18:52:19 +08:00
}
2020-02-17 19:45:24 +08:00
size += MemorySynchronizationCommands<GfxFamily>::getSizeForSinglePipeControl();
2018-04-20 19:55:54 +08:00
size += sizeof(typename GfxFamily::MI_BATCH_BUFFER_START);
2018-04-13 17:05:09 +08:00
size += getCmdSizeForL3Config();
2022-03-15 00:40:32 +08:00
if (this->streamProperties.stateComputeMode.isDirty()) {
2022-03-10 01:15:48 +08:00
size += getCmdSizeForComputeMode();
}
2019-09-10 22:13:11 +08:00
size += getCmdSizeForMediaSampler(dispatchFlags.pipelineSelectArgs.mediaSamplerRequired);
2018-04-13 17:05:09 +08:00
size += getCmdSizeForPipelineSelect();
size += getCmdSizeForPreemption(dispatchFlags);
2021-09-23 05:39:42 +08:00
if (dispatchFlags.usePerDssBackedBuffer && !isPerDssBackedBufferSent) {
size += getCmdSizeForPerDssBackedBuffer(device.getHardwareInfo());
}
2019-08-08 01:33:40 +08:00
size += getCmdSizeForEpilogue(dispatchFlags);
2020-05-27 21:30:31 +08:00
size += getCmdsSizeForHardwareContext();
2021-11-18 03:51:43 +08:00
if (csrSizeRequestFlags.activePartitionsChanged) {
size += getCmdSizeForActivePartitionConfig();
}
2017-12-21 07:45:38 +08:00
2021-11-25 17:31:14 +08:00
if (executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->getHardwareInfo()->workaroundTable.flags.waSamplerCacheFlushBetweenRedescribedSurfaceReads) {
2018-01-10 21:05:34 +08:00
if (this->samplerCacheFlushRequired != SamplerCacheFlushState::samplerCacheFlushNotRequired) {
size += sizeof(typename GfxFamily::PIPE_CONTROL);
}
}
2018-07-05 17:23:28 +08:00
if (experimentalCmdBuffer.get() != nullptr) {
size += experimentalCmdBuffer->getRequiredInjectionSize<GfxFamily>();
}
2019-01-25 17:20:32 +08:00
size += TimestampPacketHelper::getRequiredCmdStreamSize<GfxFamily>(dispatchFlags.csrDependencies);
2021-06-14 23:33:53 +08:00
size += TimestampPacketHelper::getRequiredCmdStreamSizeForTaskCountContainer<GfxFamily>(dispatchFlags.csrDependencies);
2019-01-25 17:20:32 +08:00
2021-11-06 09:42:54 +08:00
if (stallingCommandsOnNextFlushRequired) {
size += getCmdSizeForStallingCommands(dispatchFlags);
2018-10-15 16:35:45 +08:00
}
2019-10-03 20:38:49 +08:00
2019-08-28 01:14:24 +08:00
if (requiresInstructionCacheFlush) {
size += sizeof(typename GfxFamily::PIPE_CONTROL);
}
2020-07-06 15:34:15 +08:00
if (DebugManager.flags.ForcePipeControlPriorToWalker.get()) {
size += 2 * sizeof(PIPE_CONTROL);
}
2018-01-24 17:11:37 +08:00
return size;
2017-12-21 07:45:38 +08:00
}
2019-08-21 20:08:51 +08:00
template <typename GfxFamily>
inline size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForPipelineSelect() const {
2019-09-10 22:13:11 +08:00
size_t size = 0;
2020-03-19 22:15:51 +08:00
if ((csrSizeRequestFlags.mediaSamplerConfigChanged ||
csrSizeRequestFlags.specialPipelineSelectModeChanged ||
!isPreambleSent) &&
!isPipelineSelectAlreadyProgrammed()) {
2019-09-10 22:13:11 +08:00
size += PreambleHelper<GfxFamily>::getCmdSizeForPipelineSelect(peekHwInfo());
2019-08-21 20:08:51 +08:00
}
return size;
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
2022-03-23 22:36:07 +08:00
inline WaitStatus CommandStreamReceiverHw<GfxFamily>::waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep, QueueThrottle throttle) {
const auto params = kmdNotifyHelper->obtainTimeoutParams(useQuickKmdSleep, *getTagAddress(), taskCountToWait, flushStampToWait, throttle, this->isKmdWaitModeActive(),
this->isAnyDirectSubmissionEnabled());
2018-03-21 17:00:49 +08:00
2020-09-25 17:24:15 +08:00
PRINT_DEBUG_STRING(DebugManager.flags.LogWaitingForCompletion.get(), stdout,
"\nWaiting for task count %u at location %p. Current value: %u\n",
taskCountToWait, getTagAddress(), *getTagAddress());
2020-08-17 17:38:52 +08:00
2022-03-23 22:36:07 +08:00
auto status = waitForCompletionWithTimeout(params, taskCountToWait);
2022-01-21 00:56:19 +08:00
if (status == WaitStatus::NotReady) {
2021-09-17 21:05:26 +08:00
waitForFlushStamp(flushStampToWait);
2017-12-21 07:45:38 +08:00
//now call blocking wait, this is to ensure that task count is reached
2022-03-23 22:36:07 +08:00
status = waitForCompletionWithTimeout(WaitParams{false, false, 0}, taskCountToWait);
2017-12-21 07:45:38 +08:00
}
2022-01-21 00:56:19 +08:00
// If GPU hang occured, then propagate it to the caller.
if (status == WaitStatus::GpuHang) {
return status;
}
2017-12-21 07:45:38 +08:00
UNRECOVERABLE_IF(*getTagAddress() < taskCountToWait);
2018-03-22 16:41:17 +08:00
2018-04-10 16:26:59 +08:00
if (kmdNotifyHelper->quickKmdSleepForSporadicWaitsEnabled()) {
kmdNotifyHelper->updateLastWaitForCompletionTimestamp();
2018-03-22 16:41:17 +08:00
}
2020-08-17 17:38:52 +08:00
2020-09-25 17:24:15 +08:00
PRINT_DEBUG_STRING(DebugManager.flags.LogWaitingForCompletion.get(), stdout,
"\nWaiting completed. Current value: %u\n", *getTagAddress());
2022-01-21 00:56:19 +08:00
return WaitStatus::Ready;
2017-12-21 07:45:38 +08:00
}
template <typename GfxFamily>
2019-07-31 14:57:00 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programPreemption(LinearStream &csr, DispatchFlags &dispatchFlags) {
PreemptionHelper::programCmdStream<GfxFamily>(csr, dispatchFlags.preemptionMode, this->lastPreemptionMode, preemptionAllocation);
2018-01-08 22:58:02 +08:00
this->lastPreemptionMode = dispatchFlags.preemptionMode;
2017-12-21 07:45:38 +08:00
}
2018-04-13 17:05:09 +08:00
template <typename GfxFamily>
inline size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForPreemption(const DispatchFlags &dispatchFlags) const {
return PreemptionHelper::getRequiredCmdStreamSize<GfxFamily>(dispatchFlags.preemptionMode, this->lastPreemptionMode);
}
2018-11-05 18:52:19 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::programStateSip(LinearStream &cmdStream, Device &device) {
2020-02-10 22:57:49 +08:00
if (!this->isStateSipSent || device.isDebuggerActive()) {
2018-11-05 18:52:19 +08:00
PreemptionHelper::programStateSip<GfxFamily>(cmdStream, device);
this->isStateSipSent = true;
}
}
2017-12-21 07:45:38 +08:00
template <typename GfxFamily>
2021-11-04 20:54:18 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programPreamble(LinearStream &csr, Device &device, uint32_t &newL3Config) {
2017-12-21 07:45:38 +08:00
if (!this->isPreambleSent) {
2022-03-16 20:16:30 +08:00
PreambleHelper<GfxFamily>::programPreamble(&csr, device, newL3Config, this->preemptionAllocation);
2017-12-21 07:45:38 +08:00
this->isPreambleSent = true;
this->lastSentL3Config = newL3Config;
}
}
template <typename GfxFamily>
2019-05-29 21:37:54 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programVFEState(LinearStream &csr, DispatchFlags &dispatchFlags, uint32_t maxFrontEndThreads) {
2017-12-21 07:45:38 +08:00
if (mediaVfeStateDirty) {
2020-09-02 17:38:54 +08:00
if (dispatchFlags.additionalKernelExecInfo != AdditionalKernelExecInfo::NotApplicable) {
lastAdditionalKernelExecInfo = dispatchFlags.additionalKernelExecInfo;
}
2020-11-17 18:42:29 +08:00
if (dispatchFlags.kernelExecutionType != KernelExecutionType::NotApplicable) {
lastKernelExecutionType = dispatchFlags.kernelExecutionType;
}
2021-03-31 22:11:31 +08:00
auto &hwInfo = peekHwInfo();
auto &hwHelper = NEO::HwHelper::get(hwInfo.platform.eRenderCoreFamily);
2021-09-06 18:34:34 +08:00
const auto &hwInfoConfig = *NEO::HwInfoConfig::get(hwInfo.platform.eProductFamily);
2021-08-17 02:24:13 +08:00
auto engineGroupType = hwHelper.getEngineGroupType(getOsContext().getEngineType(), getOsContext().getEngineUsage(), hwInfo);
2021-04-02 02:26:29 +08:00
auto pVfeState = PreambleHelper<GfxFamily>::getSpaceForVfeState(&csr, hwInfo, engineGroupType);
2021-09-06 18:34:34 +08:00
auto disableOverdispatch = hwInfoConfig.isDisableOverdispatchAvailable(hwInfo) &&
2021-07-30 02:21:14 +08:00
(dispatchFlags.additionalKernelExecInfo != AdditionalKernelExecInfo::NotSet);
2021-05-31 20:04:57 +08:00
streamProperties.frontEndState.setProperties(lastKernelExecutionType == KernelExecutionType::Concurrent,
2022-02-11 07:33:40 +08:00
dispatchFlags.disableEUFusion, disableOverdispatch, osContext->isEngineInstanced(), hwInfo);
2021-04-02 02:26:29 +08:00
PreambleHelper<GfxFamily>::programVfeState(
pVfeState, hwInfo, requiredScratchSize, getScratchPatchAddress(),
2021-08-13 22:48:13 +08:00
maxFrontEndThreads, streamProperties);
2021-03-31 22:11:31 +08:00
auto commandOffset = PreambleHelper<GfxFamily>::getScratchSpaceAddressOffsetForVfeState(&csr, pVfeState);
2019-08-13 17:34:56 +08:00
if (DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
flatBatchBufferHelper->collectScratchSpacePatchInfo(getScratchPatchAddress(), commandOffset, csr);
}
2018-12-13 18:06:28 +08:00
setMediaVFEStateDirty(false);
2017-12-21 07:45:38 +08:00
}
}
2018-01-19 20:29:25 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::programMediaSampler(LinearStream &commandStream, DispatchFlags &dispatchFlags) {
}
template <typename GfxFamily>
size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForMediaSampler(bool mediaSamplerRequired) const {
return 0;
}
2018-03-23 02:02:58 +08:00
2018-03-14 18:07:51 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::collectStateBaseAddresPatchInfo(
uint64_t baseAddress,
uint64_t commandOffset,
const LinearStream &dsh,
const LinearStream &ioh,
const LinearStream &ssh,
2018-04-04 17:34:46 +08:00
uint64_t generalStateBase) {
2018-03-14 18:07:51 +08:00
typedef typename GfxFamily::STATE_BASE_ADDRESS STATE_BASE_ADDRESS;
2018-04-04 17:34:46 +08:00
PatchInfoData dynamicStatePatchInfo = {dsh.getGraphicsAllocation()->getGpuAddress(), 0u, PatchInfoAllocationType::DynamicStateHeap, baseAddress, commandOffset + STATE_BASE_ADDRESS::PATCH_CONSTANTS::DYNAMICSTATEBASEADDRESS_BYTEOFFSET, PatchInfoAllocationType::Default};
2018-03-14 18:07:51 +08:00
PatchInfoData generalStatePatchInfo = {generalStateBase, 0u, PatchInfoAllocationType::GeneralStateHeap, baseAddress, commandOffset + STATE_BASE_ADDRESS::PATCH_CONSTANTS::GENERALSTATEBASEADDRESS_BYTEOFFSET, PatchInfoAllocationType::Default};
2018-04-04 17:34:46 +08:00
PatchInfoData surfaceStatePatchInfo = {ssh.getGraphicsAllocation()->getGpuAddress(), 0u, PatchInfoAllocationType::SurfaceStateHeap, baseAddress, commandOffset + STATE_BASE_ADDRESS::PATCH_CONSTANTS::SURFACESTATEBASEADDRESS_BYTEOFFSET, PatchInfoAllocationType::Default};
flatBatchBufferHelper->setPatchInfoData(dynamicStatePatchInfo);
flatBatchBufferHelper->setPatchInfoData(generalStatePatchInfo);
flatBatchBufferHelper->setPatchInfoData(surfaceStatePatchInfo);
2021-09-28 18:56:22 +08:00
collectStateBaseAddresIohPatchInfo(baseAddress, commandOffset, ioh);
2018-03-14 18:07:51 +08:00
}
2018-04-10 16:26:59 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::resetKmdNotifyHelper(KmdNotifyHelper *newHelper) {
kmdNotifyHelper.reset(newHelper);
kmdNotifyHelper->updateAcLineStatus();
if (kmdNotifyHelper->quickKmdSleepForSporadicWaitsEnabled()) {
kmdNotifyHelper->updateLastWaitForCompletionTimestamp();
}
}
2018-05-21 16:57:28 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::addClearSLMWorkAround(typename GfxFamily::PIPE_CONTROL *pCmd) {
}
2018-09-07 20:31:37 +08:00
2018-11-22 22:16:20 +08:00
template <typename GfxFamily>
uint64_t CommandStreamReceiverHw<GfxFamily>::getScratchPatchAddress() {
return scratchSpaceController->getScratchPatchAddress();
2018-09-12 22:32:42 +08:00
}
2019-01-28 20:44:59 +08:00
template <typename GfxFamily>
bool CommandStreamReceiverHw<GfxFamily>::detectInitProgrammingFlagsRequired(const DispatchFlags &dispatchFlags) const {
return DebugManager.flags.ForceCsrReprogramming.get();
}
2019-03-21 00:08:05 +08:00
2019-04-03 21:59:31 +08:00
template <typename GfxFamily>
2022-02-03 21:29:48 +08:00
uint32_t CommandStreamReceiverHw<GfxFamily>::flushBcsTask(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking, bool profilingEnabled, Device &device) {
2019-04-03 21:59:31 +08:00
using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END;
using MI_FLUSH_DW = typename GfxFamily::MI_FLUSH_DW;
2019-04-15 14:54:38 +08:00
auto lock = obtainUniqueOwnership();
2020-10-19 21:36:57 +08:00
bool blitterDirectSubmission = this->isBlitterDirectSubmissionEnabled();
2022-02-08 00:52:08 +08:00
auto debugPauseEnabled = PauseOnGpuProperties::featureEnabled(DebugManager.flags.PauseOnBlitCopy.get());
auto &commandStream = getCS(BlitCommandsHelper<GfxFamily>::estimateBlitCommandsSize(blitPropertiesContainer, profilingEnabled, debugPauseEnabled, blitterDirectSubmission,
*this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]));
2019-04-15 14:54:38 +08:00
auto commandStreamStart = commandStream.getUsed();
auto newTaskCount = taskCount + 1;
2019-04-16 19:02:34 +08:00
latestSentTaskCount = newTaskCount;
2019-04-03 21:59:31 +08:00
2021-04-16 00:14:04 +08:00
getOsContext().ensureContextInitialized();
2021-07-05 19:39:42 +08:00
this->initDirectSubmission(device, getOsContext());
2021-04-16 00:14:04 +08:00
2021-12-22 02:13:53 +08:00
const auto &hwInfo = this->peekHwInfo();
2020-09-24 16:52:53 +08:00
if (PauseOnGpuProperties::pauseModeAllowed(DebugManager.flags.PauseOnBlitCopy.get(), taskCount, PauseOnGpuProperties::PauseMode::BeforeWorkload)) {
2021-12-22 02:13:53 +08:00
BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(commandStream, getDebugPauseStateGPUAddress(),
DebugPauseState::waitingForUserStartConfirmation,
DebugPauseState::hasUserStartConfirmation, hwInfo);
2020-06-17 20:58:28 +08:00
}
2020-02-07 22:36:15 +08:00
programEnginePrologue(commandStream);
2021-09-29 23:59:41 +08:00
if (pageTableManager.get() && !pageTableManagerInitialized) {
pageTableManagerInitialized = pageTableManager->initPageTableManagerRegisters(this);
}
2019-11-07 16:15:53 +08:00
for (auto &blitProperties : blitPropertiesContainer) {
2021-06-23 18:34:31 +08:00
TimestampPacketHelper::programCsrDependenciesForTimestampPacketContainer<GfxFamily>(commandStream, blitProperties.csrDependencies);
2021-06-14 23:33:53 +08:00
TimestampPacketHelper::programCsrDependenciesForForTaskCountContainer<GfxFamily>(commandStream, blitProperties.csrDependencies);
2019-05-20 18:00:02 +08:00
2020-04-29 20:06:01 +08:00
if (blitProperties.outputTimestampPacket && profilingEnabled) {
2021-04-01 19:54:16 +08:00
BlitCommandsHelper<GfxFamily>::encodeProfilingStartMmios(commandStream, *blitProperties.outputTimestampPacket);
2020-04-29 20:06:01 +08:00
}
2020-08-25 21:24:09 +08:00
BlitCommandsHelper<GfxFamily>::dispatchBlitCommands(blitProperties, commandStream, *this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]);
2019-04-03 21:59:31 +08:00
2019-11-07 16:15:53 +08:00
if (blitProperties.outputTimestampPacket) {
2020-04-29 20:06:01 +08:00
if (profilingEnabled) {
2021-06-17 19:55:28 +08:00
MiFlushArgs args;
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0llu, newTaskCount, args, hwInfo);
2020-10-20 00:10:53 +08:00
2021-04-01 19:54:16 +08:00
BlitCommandsHelper<GfxFamily>::encodeProfilingEndMmios(commandStream, *blitProperties.outputTimestampPacket);
2020-04-29 20:06:01 +08:00
} else {
auto timestampPacketGpuAddress = TimestampPacketHelper::getContextEndGpuAddress(*blitProperties.outputTimestampPacket);
2021-06-17 19:55:28 +08:00
MiFlushArgs args;
args.commandWithPostSync = true;
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, timestampPacketGpuAddress, 0, args, hwInfo);
2020-04-29 20:06:01 +08:00
}
2019-11-13 00:56:10 +08:00
makeResident(*blitProperties.outputTimestampPacket->getBaseGraphicsAllocation());
2019-11-07 16:15:53 +08:00
}
2019-04-03 21:59:31 +08:00
2019-11-07 16:15:53 +08:00
blitProperties.csrDependencies.makeResident(*this);
makeResident(*blitProperties.srcAllocation);
makeResident(*blitProperties.dstAllocation);
2020-11-19 07:58:42 +08:00
if (blitProperties.clearColorAllocation) {
makeResident(*blitProperties.clearColorAllocation);
}
2019-06-13 17:45:27 +08:00
}
2020-10-16 21:58:47 +08:00
BlitCommandsHelper<GfxFamily>::programGlobalSequencerFlush(commandStream);
2021-10-28 17:21:44 +08:00
auto updateTag = !isUpdateTagFromWaitEnabled();
updateTag |= blocking;
if (updateTag) {
MemorySynchronizationCommands<GfxFamily>::addAdditionalSynchronization(commandStream, tagAllocation->getGpuAddress(), peekHwInfo());
2020-02-07 22:36:15 +08:00
2021-10-28 17:21:44 +08:00
MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = isUsedNotifyEnableForPostSync();
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), newTaskCount, args, hwInfo);
2019-11-07 16:15:53 +08:00
2021-10-28 17:21:44 +08:00
MemorySynchronizationCommands<GfxFamily>::addAdditionalSynchronization(commandStream, tagAllocation->getGpuAddress(), peekHwInfo());
}
2020-02-12 01:25:21 +08:00
2020-09-24 16:52:53 +08:00
if (PauseOnGpuProperties::pauseModeAllowed(DebugManager.flags.PauseOnBlitCopy.get(), taskCount, PauseOnGpuProperties::PauseMode::AfterWorkload)) {
2021-12-22 02:13:53 +08:00
BlitCommandsHelper<GfxFamily>::dispatchDebugPauseCommands(commandStream, getDebugPauseStateGPUAddress(),
DebugPauseState::waitingForUserEndConfirmation,
DebugPauseState::hasUserEndConfirmation, hwInfo);
2020-06-17 20:58:28 +08:00
}
2020-10-19 21:36:57 +08:00
void *endingCmdPtr = nullptr;
2022-03-10 21:34:16 +08:00
programEndingCmd(commandStream, device, &endingCmdPtr, blitterDirectSubmission);
2019-04-03 21:59:31 +08:00
2021-11-18 06:36:00 +08:00
EncodeNoop<GfxFamily>::alignToCacheLine(commandStream);
2019-04-03 21:59:31 +08:00
2019-04-11 17:34:35 +08:00
makeResident(*tagAllocation);
2020-02-07 22:36:15 +08:00
if (globalFenceAllocation) {
makeResident(*globalFenceAllocation);
}
2019-04-11 17:34:35 +08:00
2019-08-21 18:50:47 +08:00
BatchBuffer batchBuffer{commandStream.getGraphicsAllocation(), commandStreamStart, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount,
2020-11-18 21:56:18 +08:00
commandStream.getUsed(), &commandStream, endingCmdPtr, false};
2019-04-15 14:54:38 +08:00
2021-07-01 00:06:28 +08:00
commandStream.getGraphicsAllocation()->updateTaskCount(newTaskCount, this->osContext->getContextId());
commandStream.getGraphicsAllocation()->updateResidencyTaskCount(newTaskCount, this->osContext->getContextId());
2019-11-24 21:50:41 +08:00
flush(batchBuffer, getResidencyAllocations());
2019-04-15 14:54:38 +08:00
makeSurfacePackNonResident(getResidencyAllocations());
2021-11-03 20:05:33 +08:00
if (updateTag) {
2021-10-28 17:21:44 +08:00
latestFlushedTaskCount = newTaskCount;
}
2019-04-15 14:54:38 +08:00
taskCount = newTaskCount;
2019-04-25 15:59:08 +08:00
auto flushStampToWait = flushStamp->peekStamp();
2019-04-15 14:54:38 +08:00
lock.unlock();
2019-11-07 16:15:53 +08:00
if (blocking) {
2022-03-23 22:36:07 +08:00
waitForTaskCountWithKmdNotifyFallback(newTaskCount, flushStampToWait, false, QueueThrottle::MEDIUM);
2019-06-04 19:37:22 +08:00
internalAllocationStorage->cleanAllocationList(newTaskCount, TEMPORARY_ALLOCATION);
}
2019-10-22 17:25:14 +08:00
return newTaskCount;
2019-04-03 21:59:31 +08:00
}
2021-02-23 16:48:08 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::flushTagUpdate() {
2021-10-28 10:11:32 +08:00
if (this->osContext != nullptr) {
2021-10-28 17:21:44 +08:00
if (EngineHelpers::isBcs(this->osContext->getEngineType())) {
2021-10-28 10:11:32 +08:00
this->flushMiFlushDW();
} else {
this->flushPipeControl();
}
2021-02-23 16:48:08 +08:00
}
}
2021-04-13 04:19:19 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushNonKernelTask(GraphicsAllocation *eventAlloc, uint64_t immediateGpuAddress, uint64_t immediateData, PipeControlArgs &args, bool isWaitOnEvent, bool isStartOfDispatch, bool isEndOfDispatch) {
if (isWaitOnEvent) {
this->flushSemaphoreWait(eventAlloc, immediateGpuAddress, immediateData, args, isStartOfDispatch, isEndOfDispatch);
} else {
2021-12-27 19:54:52 +08:00
if (EngineHelpers::isBcs(this->osContext->getEngineType())) {
2021-04-13 04:19:19 +08:00
this->flushMiFlushDW(eventAlloc, immediateGpuAddress, immediateData);
} else {
this->flushPipeControl(eventAlloc, immediateGpuAddress, immediateData, args);
}
}
}
2021-02-23 16:48:08 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::flushMiFlushDW() {
auto lock = obtainUniqueOwnership();
auto &commandStream = getCS(EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
auto commandStreamStart = commandStream.getUsed();
2021-12-22 02:13:53 +08:00
const auto &hwInfo = this->peekHwInfo();
2021-06-17 19:55:28 +08:00
MiFlushArgs args;
args.commandWithPostSync = true;
args.notifyEnable = isUsedNotifyEnableForPostSync();
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), taskCount + 1, args, hwInfo);
2021-02-23 16:48:08 +08:00
makeResident(*tagAllocation);
2021-04-13 04:19:19 +08:00
this->flushSmallTask(commandStream, commandStreamStart);
2021-10-28 17:21:44 +08:00
this->latestFlushedTaskCount = taskCount.load();
2021-04-13 04:19:19 +08:00
}
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushMiFlushDW(GraphicsAllocation *eventAlloc, uint64_t immediateGpuAddress, uint64_t immediateData) {
auto lock = obtainUniqueOwnership();
auto &commandStream = getCS(EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
auto commandStreamStart = commandStream.getUsed();
2021-05-06 17:56:07 +08:00
programHardwareContext(commandStream);
2021-12-22 02:13:53 +08:00
const auto &hwInfo = this->peekHwInfo();
2021-06-17 19:55:28 +08:00
MiFlushArgs args;
2021-04-13 04:19:19 +08:00
if (eventAlloc) {
2021-06-17 19:55:28 +08:00
args.commandWithPostSync = true;
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, immediateGpuAddress, immediateData, args, hwInfo);
2021-04-13 04:19:19 +08:00
makeResident(*eventAlloc);
} else {
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0, 0, args, hwInfo);
2021-04-13 04:19:19 +08:00
}
2021-02-23 16:48:08 +08:00
this->flushSmallTask(commandStream, commandStreamStart);
}
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushPipeControl() {
using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
auto lock = obtainUniqueOwnership();
2021-12-20 22:37:33 +08:00
const auto &hwInfo = peekHwInfo();
auto &commandStream = getCS(MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(hwInfo));
2021-02-23 16:48:08 +08:00
auto commandStreamStart = commandStream.getUsed();
2021-12-21 05:37:45 +08:00
PipeControlArgs args;
2021-12-22 22:25:58 +08:00
args.dcFlushEnable = MemorySynchronizationCommands<GfxFamily>::getDcFlushEnable(true, hwInfo);
2021-06-17 19:55:28 +08:00
args.notifyEnable = isUsedNotifyEnableForPostSync();
2021-12-11 05:31:34 +08:00
args.workloadPartitionOffset = isMultiTileOperationEnabled();
2021-02-23 16:48:08 +08:00
MemorySynchronizationCommands<GfxFamily>::addPipeControlAndProgramPostSyncOperation(commandStream,
2021-10-28 17:21:44 +08:00
PIPE_CONTROL::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA,
2021-02-23 16:48:08 +08:00
getTagAllocation()->getGpuAddress(),
2021-05-20 19:31:24 +08:00
taskCount + 1,
2021-12-20 22:37:33 +08:00
hwInfo,
2021-02-23 16:48:08 +08:00
args);
makeResident(*tagAllocation);
this->flushSmallTask(commandStream, commandStreamStart);
2021-10-28 17:21:44 +08:00
this->latestFlushedTaskCount = taskCount.load();
2021-02-23 16:48:08 +08:00
}
2021-04-13 04:19:19 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushPipeControl(GraphicsAllocation *eventAlloc, uint64_t immediateGpuAddress, uint64_t immediateData, PipeControlArgs &args) {
using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
auto lock = obtainUniqueOwnership();
auto &commandStream = getCS(MemorySynchronizationCommands<GfxFamily>::getSizeForSinglePipeControl());
auto commandStreamStart = commandStream.getUsed();
2021-05-06 17:56:07 +08:00
programHardwareContext(commandStream);
2021-12-20 22:37:33 +08:00
const auto &hwInfo = peekHwInfo();
2021-04-13 04:19:19 +08:00
if (eventAlloc) {
MemorySynchronizationCommands<GfxFamily>::addPipeControlAndProgramPostSyncOperation(commandStream,
PIPE_CONTROL::POST_SYNC_OPERATION::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA,
immediateGpuAddress,
immediateData,
2021-12-20 22:37:33 +08:00
hwInfo,
2021-04-13 04:19:19 +08:00
args);
makeResident(*eventAlloc);
} else {
NEO::PipeControlArgs args;
NEO::MemorySynchronizationCommands<GfxFamily>::addPipeControl(commandStream, args);
}
this->flushSmallTask(commandStream, commandStreamStart);
}
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushSemaphoreWait(GraphicsAllocation *eventAlloc, uint64_t immediateGpuAddress, uint64_t immediateData, PipeControlArgs &args, bool isStartOfDispatch, bool isEndOfDispatch) {
auto lock = obtainUniqueOwnership();
2021-12-22 02:13:53 +08:00
const auto &hwInfo = this->peekHwInfo();
2021-04-13 04:19:19 +08:00
if (isStartOfDispatch && args.dcFlushEnable) {
if (this->osContext->getEngineType() == aub_stream::ENGINE_BCS) {
LinearStream &commandStream = getCS(EncodeMiFlushDW<GfxFamily>::getMiFlushDwCmdSizeForDataWrite());
cmdStreamStart = commandStream.getUsed();
2021-06-17 19:55:28 +08:00
MiFlushArgs args;
2021-12-22 02:13:53 +08:00
EncodeMiFlushDW<GfxFamily>::programMiFlushDw(commandStream, 0, 0, args, hwInfo);
2021-04-13 04:19:19 +08:00
} else {
LinearStream &commandStream = getCS(MemorySynchronizationCommands<GfxFamily>::getSizeForSinglePipeControl());
cmdStreamStart = commandStream.getUsed();
NEO::MemorySynchronizationCommands<GfxFamily>::addPipeControl(commandStream, args);
}
}
using MI_SEMAPHORE_WAIT = typename GfxFamily::MI_SEMAPHORE_WAIT;
using COMPARE_OPERATION = typename GfxFamily::MI_SEMAPHORE_WAIT::COMPARE_OPERATION;
LinearStream &commandStream = getCS(NEO::EncodeSempahore<GfxFamily>::getSizeMiSemaphoreWait());
if (isStartOfDispatch && !args.dcFlushEnable) {
cmdStreamStart = commandStream.getUsed();
}
2021-05-06 17:56:07 +08:00
programHardwareContext(commandStream);
2021-04-13 04:19:19 +08:00
NEO::EncodeSempahore<GfxFamily>::addMiSemaphoreWaitCommand(commandStream,
immediateGpuAddress,
static_cast<uint32_t>(immediateData),
MI_SEMAPHORE_WAIT::COMPARE_OPERATION::COMPARE_OPERATION_SAD_NOT_EQUAL_SDD);
makeResident(*eventAlloc);
if (isEndOfDispatch) {
this->flushSmallTask(commandStream, cmdStreamStart);
cmdStreamStart = 0;
}
}
2021-02-23 16:48:08 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::flushSmallTask(LinearStream &commandStreamTask, size_t commandStreamStartTask) {
2021-12-27 19:41:16 +08:00
using MI_BATCH_BUFFER_START = typename GfxFamily::MI_BATCH_BUFFER_START;
2021-02-23 16:48:08 +08:00
using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END;
void *endingCmdPtr = nullptr;
2021-11-16 21:26:36 +08:00
if (isAnyDirectSubmissionEnabled()) {
2021-02-23 16:48:08 +08:00
endingCmdPtr = commandStreamTask.getSpace(0);
EncodeBatchBufferStartOrEnd<GfxFamily>::programBatchBufferStart(&commandStreamTask,
0ull,
false);
} else {
auto batchBufferEnd = reinterpret_cast<MI_BATCH_BUFFER_END *>(commandStreamTask.getSpace(sizeof(MI_BATCH_BUFFER_END)));
*batchBufferEnd = GfxFamily::cmdInitBatchBufferEnd;
}
2021-12-27 19:41:16 +08:00
auto bytesToPad = sizeof(MI_BATCH_BUFFER_START) - sizeof(MI_BATCH_BUFFER_END);
EncodeNoop<GfxFamily>::emitNoop(commandStreamTask, bytesToPad);
2021-11-18 06:36:00 +08:00
EncodeNoop<GfxFamily>::alignToCacheLine(commandStreamTask);
2021-02-23 16:48:08 +08:00
2021-05-13 04:20:22 +08:00
if (globalFenceAllocation) {
makeResident(*globalFenceAllocation);
}
2021-02-23 16:48:08 +08:00
BatchBuffer batchBuffer{commandStreamTask.getGraphicsAllocation(), commandStreamStartTask, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount,
commandStreamTask.getUsed(), &commandStreamTask, endingCmdPtr, false};
2021-10-28 17:21:44 +08:00
this->latestSentTaskCount = taskCount + 1;
2021-02-23 16:48:08 +08:00
flushHandler(batchBuffer, getResidencyAllocations());
2021-10-28 17:21:44 +08:00
taskCount++;
2021-02-23 16:48:08 +08:00
}
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::flushHandler(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
flush(batchBuffer, allocationsForResidency);
makeSurfacePackNonResident(allocationsForResidency);
}
template <typename GfxFamily>
inline bool CommandStreamReceiverHw<GfxFamily>::isUpdateTagFromWaitEnabled() {
2022-03-25 06:31:03 +08:00
bool enabled = false;
2021-02-23 16:48:08 +08:00
2021-11-17 23:05:48 +08:00
switch (DebugManager.flags.UpdateTaskCountFromWait.get()) {
case 0:
enabled = false;
break;
case 1:
enabled = this->isDirectSubmissionEnabled();
break;
case 2:
enabled = this->isAnyDirectSubmissionEnabled();
break;
case 3:
enabled = true;
break;
2021-02-23 16:48:08 +08:00
}
return enabled;
}
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::updateTagFromWait() {
if (isUpdateTagFromWaitEnabled()) {
flushTagUpdate();
}
}
2021-06-23 21:34:56 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::programAdditionalStateBaseAddress(LinearStream &csr, typename GfxFamily::STATE_BASE_ADDRESS &cmd, Device &device) {}
2020-12-17 08:36:45 +08:00
template <typename GfxFamily>
2021-08-02 18:44:48 +08:00
inline MemoryCompressionState CommandStreamReceiverHw<GfxFamily>::getMemoryCompressionState(bool auxTranslationRequired, const HardwareInfo &hwInfo) const {
2020-12-17 08:36:45 +08:00
return MemoryCompressionState::NotApplicable;
}
2020-03-19 22:15:51 +08:00
template <typename GfxFamily>
inline bool CommandStreamReceiverHw<GfxFamily>::isPipelineSelectAlreadyProgrammed() const {
2021-09-24 22:29:49 +08:00
const auto &hwInfoConfig = *HwInfoConfig::get(peekHwInfo().platform.eProductFamily);
2022-03-15 00:40:32 +08:00
return this->streamProperties.stateComputeMode.isDirty() && hwInfoConfig.is3DPipelineSelectWARequired() && isRcs();
2020-03-19 22:15:51 +08:00
}
2019-08-08 01:33:40 +08:00
template <typename GfxFamily>
2021-01-29 01:30:56 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programEpilogue(LinearStream &csr, Device &device, void **batchBufferEndLocation, DispatchFlags &dispatchFlags) {
2019-08-08 01:33:40 +08:00
if (dispatchFlags.epilogueRequired) {
auto currentOffset = ptrDiff(csr.getSpace(0u), csr.getCpuBase());
auto gpuAddress = ptrOffset(csr.getGraphicsAllocation()->getGpuAddress(), currentOffset);
addBatchBufferStart(reinterpret_cast<typename GfxFamily::MI_BATCH_BUFFER_START *>(*batchBufferEndLocation), gpuAddress, false);
2019-08-08 17:44:23 +08:00
this->programEpliogueCommands(csr, dispatchFlags);
2021-01-29 01:30:56 +08:00
programEndingCmd(csr, device, batchBufferEndLocation, isDirectSubmissionEnabled());
2021-11-18 06:36:00 +08:00
EncodeNoop<GfxFamily>::alignToCacheLine(csr);
2019-08-08 01:33:40 +08:00
}
}
template <typename GfxFamily>
inline size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForEpilogue(const DispatchFlags &dispatchFlags) const {
if (dispatchFlags.epilogueRequired) {
2020-01-16 00:02:47 +08:00
size_t terminateCmd = sizeof(typename GfxFamily::MI_BATCH_BUFFER_END);
if (isDirectSubmissionEnabled()) {
terminateCmd = sizeof(typename GfxFamily::MI_BATCH_BUFFER_START);
}
auto size = getCmdSizeForEpilogueCommands(dispatchFlags) + terminateCmd;
2019-08-08 17:44:23 +08:00
return alignUp(size, MemoryConstants::cacheLineSize);
2019-08-08 01:33:40 +08:00
}
return 0u;
}
2020-01-31 15:50:12 +08:00
template <typename GfxFamily>
2020-02-07 22:36:15 +08:00
inline void CommandStreamReceiverHw<GfxFamily>::programEnginePrologue(LinearStream &csr) {
2020-01-31 15:50:12 +08:00
}
template <typename GfxFamily>
2020-05-27 21:30:31 +08:00
inline size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForPrologue() const {
2020-01-31 15:50:12 +08:00
return 0u;
}
2021-07-30 17:56:58 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::stopDirectSubmission() {
if (EngineHelpers::isBcs(this->osContext->getEngineType())) {
this->blitterDirectSubmission->stopRingBuffer();
} else {
this->directSubmission->stopRingBuffer();
}
}
2020-07-17 17:28:59 +08:00
template <typename GfxFamily>
inline bool CommandStreamReceiverHw<GfxFamily>::initDirectSubmission(Device &device, OsContext &osContext) {
bool ret = true;
2021-02-10 23:13:50 +08:00
bool submitOnInit = false;
auto startDirect = osContext.isDirectSubmissionAvailable(device.getHardwareInfo(), submitOnInit);
2020-07-17 17:28:59 +08:00
2021-02-10 23:13:50 +08:00
if (startDirect) {
2021-08-24 17:47:25 +08:00
auto lock = this->obtainUniqueOwnership();
2021-11-16 21:26:36 +08:00
if (!this->isAnyDirectSubmissionEnabled()) {
2021-07-30 17:56:58 +08:00
if (EngineHelpers::isBcs(osContext.getEngineType())) {
2021-07-05 19:39:42 +08:00
blitterDirectSubmission = DirectSubmissionHw<GfxFamily, BlitterDispatcher<GfxFamily>>::create(device, osContext);
2021-12-23 03:28:02 +08:00
ret = blitterDirectSubmission->initialize(submitOnInit, this->isUsedNotifyEnableForPostSync());
2021-07-30 17:56:58 +08:00
} else {
2021-07-05 19:39:42 +08:00
directSubmission = DirectSubmissionHw<GfxFamily, RenderDispatcher<GfxFamily>>::create(device, osContext);
2021-12-23 03:28:02 +08:00
ret = directSubmission->initialize(submitOnInit, this->isUsedNotifyEnableForPostSync());
2021-07-05 19:39:42 +08:00
}
2021-10-25 23:33:38 +08:00
auto directSubmissionController = executionEnvironment.initializeDirectSubmissionController();
2021-07-30 17:56:58 +08:00
if (directSubmissionController) {
directSubmissionController->registerDirectSubmission(this);
}
2020-07-17 17:28:59 +08:00
}
2021-02-10 23:13:50 +08:00
osContext.setDirectSubmissionActive();
2020-07-17 17:28:59 +08:00
}
return ret;
}
2021-04-28 18:49:38 +08:00
template <typename GfxFamily>
TagAllocatorBase *CommandStreamReceiverHw<GfxFamily>::getTimestampPacketAllocator() {
if (timestampPacketAllocator.get() == nullptr) {
2021-05-14 18:20:32 +08:00
auto &hwHelper = HwHelper::get(peekHwInfo().platform.eRenderCoreFamily);
const std::vector<uint32_t> rootDeviceIndices = {rootDeviceIndex};
timestampPacketAllocator = hwHelper.createTimestampPacketAllocator(rootDeviceIndices, getMemoryManager(), getPreferredTagPoolSize(), getType(), osContext->getDeviceBitfield());
2021-03-29 22:46:41 +08:00
}
return timestampPacketAllocator.get();
}
2021-05-20 05:41:59 +08:00
template <typename GfxFamily>
void CommandStreamReceiverHw<GfxFamily>::postInitFlagsSetup() {
useNewResourceImplicitFlush = checkPlatformSupportsNewResourceImplicitFlush();
int32_t overrideNewResourceImplicitFlush = DebugManager.flags.PerformImplicitFlushForNewResource.get();
if (overrideNewResourceImplicitFlush != -1) {
useNewResourceImplicitFlush = overrideNewResourceImplicitFlush == 0 ? false : true;
}
useGpuIdleImplicitFlush = checkPlatformSupportsGpuIdleImplicitFlush();
int32_t overrideGpuIdleImplicitFlush = DebugManager.flags.PerformImplicitFlushForIdleGpu.get();
if (overrideGpuIdleImplicitFlush != -1) {
useGpuIdleImplicitFlush = overrideGpuIdleImplicitFlush == 0 ? false : true;
}
}
2021-11-06 09:42:54 +08:00
template <typename GfxFamily>
size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForStallingCommands(const DispatchFlags &dispatchFlags) const {
auto barrierTimestampPacketNodes = dispatchFlags.barrierTimestampPacketNodes;
if (barrierTimestampPacketNodes && barrierTimestampPacketNodes->peekNodes().size() > 0) {
2021-12-10 03:31:27 +08:00
return getCmdSizeForStallingPostSyncCommands();
2021-11-06 09:42:54 +08:00
} else {
return getCmdSizeForStallingNoPostSyncCommands();
}
}
2021-11-18 03:51:43 +08:00
template <typename GfxFamily>
inline void CommandStreamReceiverHw<GfxFamily>::programActivePartitionConfigFlushTask(LinearStream &csr) {
if (csrSizeRequestFlags.activePartitionsChanged) {
programActivePartitionConfig(csr);
}
}
2022-03-10 01:15:48 +08:00
template <typename GfxFamily>
bool CommandStreamReceiverHw<GfxFamily>::hasSharedHandles() {
if (!csrSizeRequestFlags.hasSharedHandles) {
for (const auto &allocation : this->getResidencyAllocations()) {
if (allocation->peekSharedHandle()) {
csrSizeRequestFlags.hasSharedHandles = true;
break;
}
}
}
return csrSizeRequestFlags.hasSharedHandles;
}
template <typename GfxFamily>
size_t CommandStreamReceiverHw<GfxFamily>::getCmdSizeForComputeMode() {
return EncodeComputeMode<GfxFamily>::getCmdSizeForComputeMode(this->peekHwInfo(), hasSharedHandles(), isRcs());
}
2019-03-26 18:59:46 +08:00
} // namespace NEO
