/*
 * Copyright (C) 2019-2020 Intel Corporation
 *
 * SPDX-License-Identifier: MIT
 *
 */

#pragma once

#include "shared/source/built_ins/built_ins.h"
#include "shared/source/built_ins/sip.h"
#include "shared/source/command_container/command_encoder.h"
#include "shared/source/command_stream/linear_stream.h"
#include "shared/source/command_stream/preemption.h"
#include "shared/source/device/device.h"
#include "shared/source/helpers/hw_helper.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/interlocked_max.h"
#include "shared/source/helpers/preamble.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/os_context.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"

#include "level_zero/core/source/cmdlist.h"
#include "level_zero/core/source/cmdlist_hw.h"
#include "level_zero/core/source/cmdqueue_hw.h"
#include "level_zero/core/source/device.h"
#include "level_zero/core/source/fence.h"
#include "level_zero/tools/source/metrics/metric.h"

#include <limits>
#include <thread>

namespace L0 {

template <GFXCORE_FAMILY gfxCoreFamily>
ze_result_t CommandQueueHw<gfxCoreFamily>::createFence(const ze_fence_desc_t *desc,
                                                       ze_fence_handle_t *phFence) {
    *phFence = Fence::create(this, desc);
    return ZE_RESULT_SUCCESS;
}

template <GFXCORE_FAMILY gfxCoreFamily>
ze_result_t CommandQueueHw<gfxCoreFamily>::destroy() {
    delete commandStream;
    buffers.destroy(this->getDevice()->getDriverHandle()->getMemoryManager());
    delete this;
    return ZE_RESULT_SUCCESS;
}

template <GFXCORE_FAMILY gfxCoreFamily>
ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
    uint32_t numCommandLists, ze_command_list_handle_t *phCommandLists, ze_fence_handle_t hFence, bool performMigration) {
    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
    using MI_BATCH_BUFFER_START = typename GfxFamily::MI_BATCH_BUFFER_START;
    using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END;

    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
    using POST_SYNC_OPERATION = typename PIPE_CONTROL::POST_SYNC_OPERATION;

    size_t spaceForResidency = 0;
    size_t preemptionSize = 0u;
    constexpr size_t residencyContainerSpaceForPreemption = 2;
    constexpr size_t residencyContainerSpaceForFence = 1;
    constexpr size_t residencyContainerSpaceForTagWrite = 1;

    NEO::Device *neoDevice = device->getNEODevice();
    NEO::PreemptionMode statePreemption = commandQueuePreemptionMode;
    auto devicePreemption = device->getDevicePreemptionMode();
    if (commandQueuePreemptionMode == NEO::PreemptionMode::Initial) {
        preemptionSize += NEO::PreemptionHelper::getRequiredCmdStreamSize<GfxFamily>(commandQueuePreemptionMode,
                                                                                     devicePreemption) +
                          NEO::PreemptionHelper::getRequiredPreambleSize<GfxFamily>(*neoDevice) +
                          NEO::PreemptionHelper::getRequiredStateSipCmdSize<GfxFamily>(*neoDevice);
        statePreemption = devicePreemption;
    }
    if (devicePreemption == NEO::PreemptionMode::MidThread) {
        spaceForResidency += residencyContainerSpaceForPreemption;
    }

    bool directSubmissionEnabled = csr->isDirectSubmissionEnabled();

    NEO::ResidencyContainer residencyContainer;
    L0::Fence *fence = nullptr;

    device->activateMetricGroups();

    size_t totalCmdBuffers = 0;
    for (auto i = 0u; i < numCommandLists; i++) {
        auto commandList = CommandList::fromHandle(phCommandLists[i]);

        totalCmdBuffers += commandList->commandContainer.getCmdBufferAllocations().size();
        spaceForResidency += commandList->commandContainer.getResidencyContainer().size();
        auto commandListPreemption = commandList->getCommandListPreemptionMode();
        if (statePreemption != commandListPreemption) {
            preemptionSize += sizeof(PIPE_CONTROL);
            preemptionSize += NEO::PreemptionHelper::getRequiredCmdStreamSize<GfxFamily>(commandListPreemption, statePreemption);
            statePreemption = commandListPreemption;
        }

        interlockedMax(commandQueuePerThreadScratchSize, commandList->getCommandListPerThreadScratchSize());
    }

    size_t linearStreamSizeEstimate = totalCmdBuffers * sizeof(MI_BATCH_BUFFER_START);

    if (directSubmissionEnabled) {
        linearStreamSizeEstimate += sizeof(MI_BATCH_BUFFER_START);
    } else {
        linearStreamSizeEstimate += sizeof(MI_BATCH_BUFFER_END);
    }

    if (hFence) {
        fence = Fence::fromHandle(hFence);
        spaceForResidency += residencyContainerSpaceForFence;
        linearStreamSizeEstimate += NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(device->getHwInfo());
    }

    spaceForResidency += residencyContainerSpaceForTagWrite;

    residencyContainer.reserve(spaceForResidency);

    auto scratchSpaceController = csr->getScratchSpaceController();
    bool gsbaStateDirty = false;
    bool frontEndStateDirty = false;
    handleScratchSpace(residencyContainer,
                       scratchSpaceController,
                       gsbaStateDirty, frontEndStateDirty);

    gsbaStateDirty |= !gsbaInit;
    frontEndStateDirty |= !frontEndInit;

    if (!gpgpuEnabled) {
        linearStreamSizeEstimate += estimatePipelineSelect();
    }

    if (frontEndStateDirty) {
        linearStreamSizeEstimate += estimateFrontEndCmdSize();
    }

    if (gsbaStateDirty) {
        linearStreamSizeEstimate += estimateStateBaseAddressCmdSize();
    }

    linearStreamSizeEstimate += NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForPipeControlWithPostSyncOperation(device->getHwInfo());

    linearStreamSizeEstimate += preemptionSize;

    size_t alignedSize = alignUp<size_t>(linearStreamSizeEstimate, minCmdBufferPtrAlign);
    size_t padding = alignedSize - linearStreamSizeEstimate;
    reserveLinearStreamSize(alignedSize);
    NEO::LinearStream child(commandStream->getSpace(alignedSize), alignedSize);

    if (!gpgpuEnabled) {
        programPipelineSelect(child);
    }
    if (frontEndStateDirty) {
        programFrontEnd(scratchSpaceController->getScratchPatchAddress(), child);
    }
    if (gsbaStateDirty) {
        programGeneralStateBaseAddress(scratchSpaceController->calculateNewGSH(), child);
    }

    if (commandQueuePreemptionMode == NEO::PreemptionMode::Initial) {
        NEO::PreemptionHelper::programCsrBaseAddress<GfxFamily>(child, *neoDevice, csr->getPreemptionAllocation());
        NEO::PreemptionHelper::programStateSip<GfxFamily>(child, *neoDevice);
        NEO::PreemptionHelper::programCmdStream<GfxFamily>(child,
                                                           devicePreemption,
                                                           commandQueuePreemptionMode,
                                                           csr->getPreemptionAllocation());
        commandQueuePreemptionMode = devicePreemption;
        statePreemption = commandQueuePreemptionMode;
    }

    if (devicePreemption == NEO::PreemptionMode::MidThread) {
        residencyContainer.push_back(csr->getPreemptionAllocation());
        auto sipIsa = neoDevice->getBuiltIns()->getSipKernel(NEO::SipKernelType::Csr, *neoDevice).getSipAllocation();
        residencyContainer.push_back(sipIsa);
    }

    for (auto i = 0u; i < numCommandLists; ++i) {
        auto commandList = CommandList::fromHandle(phCommandLists[i]);
        auto cmdBufferAllocations = commandList->commandContainer.getCmdBufferAllocations();
        auto cmdBufferCount = cmdBufferAllocations.size();

        auto commandListPreemption = commandList->getCommandListPreemptionMode();
        if (statePreemption != commandListPreemption) {
            NEO::MemorySynchronizationCommands<GfxFamily>::addPipeControl(child, false);
            NEO::PreemptionHelper::programCmdStream<GfxFamily>(child,
                                                               commandListPreemption,
                                                               statePreemption,
                                                               csr->getPreemptionAllocation());
            statePreemption = commandListPreemption;
        }

        for (size_t iter = 0; iter < cmdBufferCount; iter++) {
            auto allocation = cmdBufferAllocations[iter];
            NEO::EncodeBatchBufferStartOrEnd<GfxFamily>::programBatchBufferStart(&child, allocation->getGpuAddress(), true);
        }

        printfFunctionContainer.insert(printfFunctionContainer.end(),
                                       commandList->getPrintfFunctionContainer().begin(),
                                       commandList->getPrintfFunctionContainer().end());

        NEO::PageFaultManager *pageFaultManager = nullptr;
        if (performMigration) {
            pageFaultManager = device->getDriverHandle()->getMemoryManager()->getPageFaultManager();
            if (pageFaultManager == nullptr) {
                performMigration = false;
            }
        }

        for (auto alloc : commandList->commandContainer.getResidencyContainer()) {
            if (residencyContainer.end() ==
                std::find(residencyContainer.begin(), residencyContainer.end(), alloc)) {
                residencyContainer.push_back(alloc);

                if (performMigration) {
                    if (alloc &&
                        (alloc->getAllocationType() == NEO::GraphicsAllocation::AllocationType::SVM_GPU ||
                         alloc->getAllocationType() == NEO::GraphicsAllocation::AllocationType::SVM_CPU)) {
                        pageFaultManager->moveAllocationToGpuDomain(reinterpret_cast<void *>(alloc->getGpuAddress()));
                    }
                }
            }
        }
    }

    commandQueuePreemptionMode = statePreemption;

    if (hFence) {
        residencyContainer.push_back(&fence->getAllocation());
        NEO::MemorySynchronizationCommands<GfxFamily>::obtainPipeControlAndProgramPostSyncOperation(
            child, POST_SYNC_OPERATION::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA,
            fence->getGpuAddress(), Fence::STATE_SIGNALED, true, device->getHwInfo());
    }

    dispatchTaskCountWrite(child, true);
    residencyContainer.push_back(csr->getTagAllocation());
    void *endingCmd = nullptr;
    if (directSubmissionEnabled) {
        endingCmd = child.getSpace(0);
        NEO::EncodeBatchBufferStartOrEnd<GfxFamily>::programBatchBufferStart(&child, 0ull, false);
    } else {
        MI_BATCH_BUFFER_END cmd = GfxFamily::cmdInitBatchBufferEnd;
        auto buffer = child.getSpaceForCmd<MI_BATCH_BUFFER_END>();
        *(MI_BATCH_BUFFER_END *)buffer = cmd;
    }

    if (padding) {
        void *paddingPtr = child.getSpace(padding);
        memset(paddingPtr, 0, padding);
    }

    submitBatchBuffer(ptrDiff(child.getCpuBase(), commandStream->getCpuBase()), residencyContainer, endingCmd);

    this->taskCount = csr->peekTaskCount();

    csr->makeSurfacePackNonResident(residencyContainer);

    if (getSynchronousMode() == ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS) {
        this->synchronize(std::numeric_limits<uint32_t>::max());
    }

    return ZE_RESULT_SUCCESS;
}

template <GFXCORE_FAMILY gfxCoreFamily>
void CommandQueueHw<gfxCoreFamily>::programFrontEnd(uint64_t scratchAddress, NEO::LinearStream &commandStream) {
    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
    UNRECOVERABLE_IF(csr == nullptr);
    NEO::PreambleHelper<GfxFamily>::programVFEState(&commandStream,
                                                    device->getHwInfo(),
                                                    commandQueuePerThreadScratchSize,
                                                    scratchAddress,
                                                    device->getMaxNumHwThreads(),
                                                    csr->getOsContext().getEngineType());
    frontEndInit = true;
}

template <GFXCORE_FAMILY gfxCoreFamily>
size_t CommandQueueHw<gfxCoreFamily>::estimateFrontEndCmdSize() {
    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
    return NEO::PreambleHelper<GfxFamily>::getVFECommandsSize();
}

template <GFXCORE_FAMILY gfxCoreFamily>
size_t CommandQueueHw<gfxCoreFamily>::estimatePipelineSelect() {

    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
    return NEO::PreambleHelper<GfxFamily>::getCmdSizeForPipelineSelect(device->getHwInfo());
}

template <GFXCORE_FAMILY gfxCoreFamily>
void CommandQueueHw<gfxCoreFamily>::programPipelineSelect(NEO::LinearStream &commandStream) {
    NEO::PipelineSelectArgs args = {0, 0};
    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
    NEO::PreambleHelper<GfxFamily>::programPipelineSelect(&commandStream, args, device->getHwInfo());
    gpgpuEnabled = true;
}

template <GFXCORE_FAMILY gfxCoreFamily>
void CommandQueueHw<gfxCoreFamily>::dispatchTaskCountWrite(NEO::LinearStream &commandStream, bool flushDataCache) {
    using GfxFamily = typename NEO::GfxFamilyMapper<gfxCoreFamily>::GfxFamily;
    using PIPE_CONTROL = typename GfxFamily::PIPE_CONTROL;
    using POST_SYNC_OPERATION = typename PIPE_CONTROL::POST_SYNC_OPERATION;

    UNRECOVERABLE_IF(csr == nullptr);

    auto taskCountToWrite = csr->peekTaskCount() + 1;
    auto gpuAddress = static_cast<uint64_t>(csr->getTagAllocation()->getGpuAddress());

    NEO::MemorySynchronizationCommands<GfxFamily>::obtainPipeControlAndProgramPostSyncOperation(
        commandStream, POST_SYNC_OPERATION::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA,
        gpuAddress, taskCountToWrite, true, device->getHwInfo());
}
} // namespace L0