/* * Copyright (C) 2020 Intel Corporation * * SPDX-License-Identifier: MIT * */ #pragma once #include "shared/source/command_container/command_encoder.h" #include "shared/source/command_stream/linear_stream.h" #include "shared/source/device/device.h" #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/preamble.h" #include "shared/source/helpers/register_offsets.h" #include "shared/source/helpers/simd_helper.h" #include "shared/source/helpers/string.h" #include "shared/source/kernel/dispatch_kernel_encoder_interface.h" #include "opencl/source/helpers/hardware_commands_helper.h" #include namespace NEO { template uint32_t EncodeStates::copySamplerState(IndirectHeap *dsh, uint32_t samplerStateOffset, uint32_t samplerCount, uint32_t borderColorOffset, const void *fnDynamicStateHeap) { auto sizeSamplerState = sizeof(SAMPLER_STATE) * samplerCount; auto borderColorSize = samplerStateOffset - borderColorOffset; dsh->align(alignIndirectStatePointer); auto borderColorOffsetInDsh = static_cast(dsh->getUsed()); auto borderColor = dsh->getSpace(borderColorSize); memcpy_s(borderColor, borderColorSize, ptrOffset(fnDynamicStateHeap, borderColorOffset), borderColorSize); dsh->align(INTERFACE_DESCRIPTOR_DATA::SAMPLERSTATEPOINTER_ALIGN_SIZE); auto samplerStateOffsetInDsh = static_cast(dsh->getUsed()); auto dstSamplerState = reinterpret_cast(dsh->getSpace(sizeSamplerState)); auto srcSamplerState = reinterpret_cast(ptrOffset(fnDynamicStateHeap, samplerStateOffset)); SAMPLER_STATE state = {}; for (uint32_t i = 0; i < samplerCount; i++) { state = srcSamplerState[i]; state.setIndirectStatePointer(static_cast(borderColorOffsetInDsh)); dstSamplerState[i] = state; } return samplerStateOffsetInDsh; } template void EncodeMathMMIO::encodeMulRegVal(CommandContainer &container, uint32_t offset, uint32_t val, uint64_t dstAddress) { int logLws = 0; int i = val; while (val >> logLws) { logLws++; } EncodeSetMMIO::encodeREG(container, CS_GPR_R0, offset); EncodeSetMMIO::encodeIMM(container, CS_GPR_R1, 0); i = 0; while (i < logLws) { if (val & (1 << i)) { EncodeMath::addition(container, AluRegisters::R_1, AluRegisters::R_0, AluRegisters::R_2); EncodeSetMMIO::encodeREG(container, CS_GPR_R1, CS_GPR_R2); } EncodeMath::addition(container, AluRegisters::R_0, AluRegisters::R_0, AluRegisters::R_2); EncodeSetMMIO::encodeREG(container, CS_GPR_R0, CS_GPR_R2); i++; } EncodeStoreMMIO::encode(*container.getCommandStream(), CS_GPR_R1, dstAddress); } /* * Compute *firstOperand > secondOperand and store the result in * MI_PREDICATE_RESULT where firstOperand is an device memory address. * * To calculate the "greater than" operation in the device, * (secondOperand - *firstOperand) is used, and if the carry flag register is * set, then (*firstOperand) is greater than secondOperand. */ template void EncodeMathMMIO::encodeGreaterThanPredicate(CommandContainer &container, uint64_t firstOperand, uint32_t secondOperand) { EncodeSetMMIO::encodeMEM(container, CS_GPR_R0, firstOperand); EncodeSetMMIO::encodeIMM(container, CS_GPR_R1, secondOperand); /* CS_GPR_R* registers map to AluRegisters::R_* registers */ EncodeMath::greaterThan(container, AluRegisters::R_0, AluRegisters::R_1, AluRegisters::R_2); EncodeSetMMIO::encodeREG(container, CS_PREDICATE_RESULT, CS_GPR_R2); } /* * encodeAlu() performs operations that leave a state including the result of * an operation such as the carry flag, and the accu flag with subtraction and * addition result. * * Parameter "postOperationStateRegister" is the ALU register with the result * from the operation that the function caller is interested in obtaining. * * Parameter "finalResultRegister" is the final destination register where * data from "postOperationStateRegister" will be copied. */ template void EncodeMathMMIO::encodeAlu(MI_MATH_ALU_INST_INLINE *pAluParam, AluRegisters srcA, AluRegisters srcB, AluRegisters op, AluRegisters finalResultRegister, AluRegisters postOperationStateRegister) { MI_MATH_ALU_INST_INLINE aluParam; aluParam.DW0.Value = 0x0; aluParam.DW0.BitField.ALUOpcode = static_cast(AluRegisters::OPCODE_LOAD); aluParam.DW0.BitField.Operand1 = static_cast(AluRegisters::R_SRCA); aluParam.DW0.BitField.Operand2 = static_cast(srcA); *pAluParam = aluParam; pAluParam++; aluParam.DW0.Value = 0x0; aluParam.DW0.BitField.ALUOpcode = static_cast(AluRegisters::OPCODE_LOAD); aluParam.DW0.BitField.Operand1 = static_cast(AluRegisters::R_SRCB); aluParam.DW0.BitField.Operand2 = static_cast(srcB); *pAluParam = aluParam; pAluParam++; /* Order of operation: Operand1 Operand2 */ aluParam.DW0.Value = 0x0; aluParam.DW0.BitField.ALUOpcode = static_cast(op); aluParam.DW0.BitField.Operand1 = 0; aluParam.DW0.BitField.Operand2 = 0; *pAluParam = aluParam; pAluParam++; aluParam.DW0.Value = 0x0; aluParam.DW0.BitField.ALUOpcode = static_cast(AluRegisters::OPCODE_STORE); aluParam.DW0.BitField.Operand1 = static_cast(finalResultRegister); aluParam.DW0.BitField.Operand2 = static_cast(postOperationStateRegister); *pAluParam = aluParam; pAluParam++; } template uint32_t *EncodeMath::commandReserve(CommandContainer &container) { size_t size = sizeof(MI_MATH) + sizeof(MI_MATH_ALU_INST_INLINE) * NUM_ALU_INST_FOR_READ_MODIFY_WRITE; auto cmd = reinterpret_cast(container.getCommandStream()->getSpace(size)); MI_MATH mathBuffer; mathBuffer.DW0.Value = 0x0; mathBuffer.DW0.BitField.InstructionType = MI_MATH::COMMAND_TYPE_MI_COMMAND; mathBuffer.DW0.BitField.InstructionOpcode = MI_MATH::MI_COMMAND_OPCODE_MI_MATH; mathBuffer.DW0.BitField.DwordLength = NUM_ALU_INST_FOR_READ_MODIFY_WRITE - 1; *reinterpret_cast(cmd) = mathBuffer; cmd++; return cmd; } template void EncodeMathMMIO::encodeAluAdd(MI_MATH_ALU_INST_INLINE *pAluParam, AluRegisters firstOperandRegister, AluRegisters secondOperandRegister, AluRegisters finalResultRegister) { encodeAlu(pAluParam, firstOperandRegister, secondOperandRegister, AluRegisters::OPCODE_ADD, finalResultRegister, AluRegisters::R_ACCU); } template void EncodeMathMMIO::encodeAluSubStoreCarry(MI_MATH_ALU_INST_INLINE *pAluParam, AluRegisters regA, AluRegisters regB, AluRegisters finalResultRegister) { /* regB is subtracted from regA */ encodeAlu(pAluParam, regA, regB, AluRegisters::OPCODE_SUB, finalResultRegister, AluRegisters::R_CF); } /* * greaterThan() tests if firstOperandRegister is greater than * secondOperandRegister. */ template void EncodeMath::greaterThan(CommandContainer &container, AluRegisters firstOperandRegister, AluRegisters secondOperandRegister, AluRegisters finalResultRegister) { uint32_t *cmd = EncodeMath::commandReserve(container); /* firstOperandRegister will be subtracted from secondOperandRegister */ EncodeMathMMIO::encodeAluSubStoreCarry(reinterpret_cast(cmd), secondOperandRegister, firstOperandRegister, finalResultRegister); } template void EncodeMath::addition(CommandContainer &container, AluRegisters firstOperandRegister, AluRegisters secondOperandRegister, AluRegisters finalResultRegister) { uint32_t *cmd = EncodeMath::commandReserve(container); EncodeMathMMIO::encodeAluAdd(reinterpret_cast(cmd), firstOperandRegister, secondOperandRegister, finalResultRegister); } template void EncodeIndirectParams::setGroupCountIndirect(CommandContainer &container, const NEO::CrossThreadDataOffset offsets[3], void *crossThreadAddress) { for (int i = 0; i < 3; ++i) { if (NEO::isUndefinedOffset(offsets[i])) { continue; } EncodeStoreMMIO::encode(*container.getCommandStream(), GPUGPU_DISPATCHDIM[i], ptrOffset(reinterpret_cast(crossThreadAddress), offsets[i])); } } template void EncodeIndirectParams::setGlobalWorkSizeIndirect(CommandContainer &container, const NEO::CrossThreadDataOffset offsets[3], void *crossThreadAddress, const uint32_t *lws) { for (int i = 0; i < 3; ++i) { if (NEO::isUndefinedOffset(offsets[i])) { continue; } EncodeMathMMIO::encodeMulRegVal(container, GPUGPU_DISPATCHDIM[i], lws[i], ptrOffset(reinterpret_cast(crossThreadAddress), offsets[i])); } } template void EncodeSetMMIO::encodeIMM(CommandContainer &container, uint32_t offset, uint32_t data) { MI_LOAD_REGISTER_IMM cmd = Family::cmdInitLoadRegisterImm; cmd.setRegisterOffset(offset); cmd.setDataDword(data); auto buffer = container.getCommandStream()->getSpaceForCmd(); *buffer = cmd; } template void EncodeSetMMIO::encodeMEM(CommandContainer &container, uint32_t offset, uint64_t address) { MI_LOAD_REGISTER_MEM cmd = Family::cmdInitLoadRegisterMem; cmd.setRegisterAddress(offset); cmd.setMemoryAddress(address); auto buffer = container.getCommandStream()->getSpaceForCmd(); *buffer = cmd; } template void EncodeSetMMIO::encodeREG(CommandContainer &container, uint32_t dstOffset, uint32_t srcOffset) { MI_LOAD_REGISTER_REG cmd = Family::cmdInitLoadRegisterReg; cmd.setSourceRegisterAddress(srcOffset); cmd.setDestinationRegisterAddress(dstOffset); auto buffer = container.getCommandStream()->getSpaceForCmd(); *buffer = cmd; } template void EncodeStoreMMIO::encode(LinearStream &csr, uint32_t offset, uint64_t address) { MI_STORE_REGISTER_MEM cmd = Family::cmdInitStoreRegisterMem; cmd.setRegisterAddress(offset); cmd.setMemoryAddress(address); remapOffset(&cmd); auto buffer = csr.getSpaceForCmd(); *buffer = cmd; } template void EncodeSurfaceState::encodeBuffer(void *dst, uint64_t address, size_t size, uint32_t mocs, bool cpuCoherent) { auto ss = reinterpret_cast(dst); UNRECOVERABLE_IF(!isAligned(size)); SURFACE_STATE_BUFFER_LENGTH Length = {0}; Length.Length = static_cast(size - 1); ss->setWidth(Length.SurfaceState.Width + 1); ss->setHeight(Length.SurfaceState.Height + 1); ss->setDepth(Length.SurfaceState.Depth + 1); ss->setSurfaceType((address != 0) ? R_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_BUFFER : R_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_NULL); ss->setSurfaceFormat(SURFACE_FORMAT::SURFACE_FORMAT_RAW); ss->setSurfaceVerticalAlignment(R_SURFACE_STATE::SURFACE_VERTICAL_ALIGNMENT_VALIGN_4); ss->setSurfaceHorizontalAlignment(R_SURFACE_STATE::SURFACE_HORIZONTAL_ALIGNMENT_HALIGN_4); ss->setTileMode(R_SURFACE_STATE::TILE_MODE_LINEAR); ss->setVerticalLineStride(0); ss->setVerticalLineStrideOffset(0); ss->setMemoryObjectControlState(mocs); ss->setSurfaceBaseAddress(address); ss->setCoherencyType(cpuCoherent ? R_SURFACE_STATE::COHERENCY_TYPE_IA_COHERENT : R_SURFACE_STATE::COHERENCY_TYPE_GPU_COHERENT); ss->setAuxiliarySurfaceMode(AUXILIARY_SURFACE_MODE::AUXILIARY_SURFACE_MODE_AUX_NONE); } template void EncodeSurfaceState::encodeExtraBufferParams(GraphicsAllocation *allocation, GmmHelper *gmmHelper, void *memory, bool forceNonAuxMode, bool isReadOnlyArgument) { using RENDER_SURFACE_STATE = typename Family::RENDER_SURFACE_STATE; using AUXILIARY_SURFACE_MODE = typename RENDER_SURFACE_STATE::AUXILIARY_SURFACE_MODE; auto surfaceState = reinterpret_cast(memory); Gmm *gmm = allocation ? allocation->getDefaultGmm() : nullptr; if (gmm && gmm->isRenderCompressed && !forceNonAuxMode && GraphicsAllocation::AllocationType::BUFFER_COMPRESSED == allocation->getAllocationType()) { // Its expected to not program pitch/qpitch/baseAddress for Aux surface in CCS scenarios surfaceState->setCoherencyType(RENDER_SURFACE_STATE::COHERENCY_TYPE_GPU_COHERENT); surfaceState->setAuxiliarySurfaceMode(AUXILIARY_SURFACE_MODE::AUXILIARY_SURFACE_MODE_AUX_CCS_E); } if (DebugManager.flags.DisableCachingForStatefulBufferAccess.get()) { surfaceState->setMemoryObjectControlState(gmmHelper->getMOCS(GMM_RESOURCE_USAGE_OCL_BUFFER_CACHELINE_MISALIGNED)); } } template void *EncodeDispatchKernel::getInterfaceDescriptor(CommandContainer &container, uint32_t &iddOffset) { if (container.nextIddInBlock == container.getNumIddPerBlock()) { container.getIndirectHeap(HeapType::DYNAMIC_STATE)->align(HardwareCommandsHelper::alignInterfaceDescriptorData); container.setIddBlock(container.getHeapSpaceAllowGrow(HeapType::DYNAMIC_STATE, sizeof(INTERFACE_DESCRIPTOR_DATA) * container.getNumIddPerBlock())); container.nextIddInBlock = 0; EncodeMediaInterfaceDescriptorLoad::encode(container); } iddOffset = container.nextIddInBlock; auto interfaceDescriptorData = static_cast(container.getIddBlock()); return &interfaceDescriptorData[container.nextIddInBlock++]; } template void EncodeDispatchKernel::patchBindlessSurfaceStateOffsets(const size_t sshOffset, const KernelDescriptor &kernelDesc, uint8_t *crossThread) { auto &hwHelper = HwHelperHw::get(); for (const auto &argT : kernelDesc.payloadMappings.explicitArgs) { CrossThreadDataOffset bindless = undefined; SurfaceStateHeapOffset bindful = undefined; switch (argT.type) { case ArgDescriptor::ArgTPointer: { auto &arg = argT.as(); bindless = arg.bindless; bindful = arg.bindful; } break; case ArgDescriptor::ArgTImage: { auto &arg = argT.as(); bindless = arg.bindless; bindful = arg.bindful; } break; default: break; } if (NEO::isValidOffset(bindless)) { auto patchLocation = ptrOffset(crossThread, bindless); auto bindlessOffset = static_cast(sshOffset) + bindful; auto patchValue = hwHelper.getBindlessSurfaceExtendedMessageDescriptorValue(bindlessOffset); patchWithRequiredSize(patchLocation, sizeof(patchValue), patchValue); } } } template bool EncodeDispatchKernel::inlineDataProgrammingRequired(const KernelDescriptor &kernelDesc) { auto checkKernelForInlineData = true; if (DebugManager.flags.EnablePassInlineData.get() != -1) { checkKernelForInlineData = !!DebugManager.flags.EnablePassInlineData.get(); } if (checkKernelForInlineData) { return kernelDesc.kernelAttributes.flags.passInlineData; } return false; } template size_t EncodeStates::getAdjustStateComputeModeSize() { return 0; } template size_t EncodeIndirectParams::getCmdsSizeForIndirectParams() { return 3 * sizeof(typename Family::MI_LOAD_REGISTER_MEM); } template size_t EncodeIndirectParams::getCmdsSizeForSetGroupCountIndirect() { return 3 * (sizeof(MI_STORE_REGISTER_MEM)); } template size_t EncodeIndirectParams::getCmdsSizeForSetGroupSizeIndirect() { return 3 * (sizeof(MI_LOAD_REGISTER_REG) + sizeof(MI_LOAD_REGISTER_IMM) + sizeof(MI_MATH) + sizeof(MI_MATH_ALU_INST_INLINE) + sizeof(MI_STORE_REGISTER_MEM)); } template void EncodeSempahore::programMiSemaphoreWait(MI_SEMAPHORE_WAIT *cmd, uint64_t compareAddress, uint32_t compareData, COMPARE_OPERATION compareMode) { MI_SEMAPHORE_WAIT localCmd = Family::cmdInitMiSemaphoreWait; localCmd.setCompareOperation(compareMode); localCmd.setSemaphoreDataDword(compareData); localCmd.setSemaphoreGraphicsAddress(compareAddress); localCmd.setWaitMode(MI_SEMAPHORE_WAIT::WAIT_MODE::WAIT_MODE_POLLING_MODE); *cmd = localCmd; } template void EncodeSempahore::addMiSemaphoreWaitCommand(LinearStream &commandStream, uint64_t compareAddress, uint32_t compareData, COMPARE_OPERATION compareMode) { auto semaphoreCommand = commandStream.getSpaceForCmd(); programMiSemaphoreWait(semaphoreCommand, compareAddress, compareData, compareMode); } template size_t EncodeSempahore::getSizeMiSemaphoreWait() { return sizeof(MI_SEMAPHORE_WAIT); } template void EncodeAtomic::programMiAtomic(MI_ATOMIC *atomic, uint64_t writeAddress, ATOMIC_OPCODES opcode, DATA_SIZE dataSize) { MI_ATOMIC cmd = Family::cmdInitAtomic; cmd.setAtomicOpcode(opcode); cmd.setDataSize(dataSize); cmd.setMemoryAddress(static_cast(writeAddress & 0x0000FFFFFFFFULL)); cmd.setMemoryAddressHigh(static_cast(writeAddress >> 32)); *atomic = cmd; } template void EncodeBatchBufferStartOrEnd::programBatchBufferStart(LinearStream *commandStream, uint64_t address, bool secondLevel) { MI_BATCH_BUFFER_START cmd = Family::cmdInitBatchBufferStart; if (secondLevel) { cmd.setSecondLevelBatchBuffer(MI_BATCH_BUFFER_START::SECOND_LEVEL_BATCH_BUFFER_SECOND_LEVEL_BATCH); } cmd.setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT); cmd.setBatchBufferStartAddressGraphicsaddress472(address); auto buffer = commandStream->getSpaceForCmd(); *buffer = cmd; } template void EncodeBatchBufferStartOrEnd::programBatchBufferEnd(CommandContainer &container) { MI_BATCH_BUFFER_END cmd = Family::cmdInitBatchBufferEnd; auto buffer = container.getCommandStream()->getSpaceForCmd(); *buffer = cmd; } template void EncodeSurfaceState::getSshAlignedPointer(uintptr_t &ptr, size_t &offset) { auto sshAlignmentMask = getSurfaceBaseAddressAlignmentMask(); uintptr_t alignedPtr = ptr & sshAlignmentMask; offset = 0; if (ptr != alignedPtr) { offset = ptrDiff(ptr, alignedPtr); ptr = alignedPtr; } } template void EncodeMiFlushDW::programMiFlushDw(LinearStream &commandStream, uint64_t immediateDataGpuAddress, uint64_t immediateData, bool timeStampOperation, bool commandWithPostSync) { programMiFlushDwWA(commandStream); auto miFlushDwCmd = commandStream.getSpaceForCmd(); MI_FLUSH_DW miFlush = GfxFamily::cmdInitMiFlushDw; if (commandWithPostSync) { auto postSyncType = timeStampOperation ? MI_FLUSH_DW::POST_SYNC_OPERATION_WRITE_TIMESTAMP_REGISTER : MI_FLUSH_DW::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA_QWORD; miFlush.setPostSyncOperation(postSyncType); miFlush.setDestinationAddress(immediateDataGpuAddress); miFlush.setImmediateData(immediateData); } appendMiFlushDw(&miFlush); *miFlushDwCmd = miFlush; } template size_t EncodeMiFlushDW::getMiFlushDwCmdSizeForDataWrite() { return sizeof(typename GfxFamily::MI_FLUSH_DW) + EncodeMiFlushDW::getMiFlushDwWaSize(); } template void EncodeMemoryPrefetch::programMemoryPrefetch(LinearStream &commandStream, const GraphicsAllocation &graphicsAllocation, uint32_t size, const HardwareInfo &hwInfo) {} template size_t EncodeMemoryPrefetch::getSizeForMemoryPrefetch() { return 0; } } // namespace NEO