/*===================== begin_copyright_notice ==================================

Copyright (c) 2017 Intel Corporation

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


======================= end_copyright_notice ==================================*/

#include "Optimizer.h"
#include <fstream>
#include <sstream>
#include "G4_Opcode.h"
#include "Timer.h"
#include "G4Verifier.h"
#include <map>
#include "LVN.h"
#include "ifcvt.h"
#include <random>
#include <chrono>
#include "FlowGraph.h"
#include "SendFusion.h"

using namespace std;
using namespace vISA;

void Optimizer::LVN()
{
    // Run a simple LVN pass that replaces redundant
    // immediate loads in current BB. Also this pass
    // does not optimize operations like a
    // conventional VN pass because those require
    // more compile time, and are presumably already
    // done by FE generating VISA. This pass catches
    // redundancies that got introduced mainly by HW
    // conformity or due to VISA lowering.
    int numInstsRemoved = 0;
    Mem_Manager mem(1024);
    PointsToAnalysis p(kernel.Declares, kernel.fg.getNumBB());
    p.doPointsToAnalysis(kernel.fg);
    for(BB_LIST_ITER bb_it = kernel.fg.BBs.begin();
        bb_it != kernel.fg.BBs.end();
        bb_it++)
    {
        G4_BB* bb = (*bb_it);
        ::LVN lvn(fg, bb, mem, *fg.builder, p);

        lvn.doLVN();

        numInstsRemoved += lvn.getNumInstsRemoved();
    }

    if(kernel.getOption(vISA_OptReport))
    {
        std::ofstream optreport;
        getOptReportStream(optreport, kernel.getOptions());
        optreport << "===== LVN =====" << std::endl;
        optreport << "Number of instructions removed: " << numInstsRemoved << std::endl << std::endl;
        closeOptReportStream(optreport);
    }
}

// helper functions

static int getDstSubReg( G4_DstRegRegion *dst )
{
    int dstSubReg;
    if (dst->getBase()->isPhyReg())
    {
        dstSubReg = dst->getSubRegOff();
    }
    else
    {
        dstSubReg = dst->getSubRegOff() +
            static_cast<G4_RegVar*>(dst->getBase())->getPhyRegOff();
    }

    return dstSubReg;
}

static int getSrcSubReg( G4_Operand *src )
{
    MUST_BE_TRUE( src->isSrcRegRegion(), "expect Src Reg Region" );
    int srcSubReg;
    if (src->asSrcRegRegion()->getBase()->isPhyReg())
    {
        srcSubReg = src->asSrcRegRegion()->getSubRegOff();
    }
    else
    {
        srcSubReg = src->asSrcRegRegion()->getSubRegOff() +
            static_cast<G4_RegVar*>(src->asSrcRegRegion()->getBase())->getPhyRegOff();
    }
    return srcSubReg;
}

static void mergeBitVec( unsigned bitVec[2], G4_Operand* opnd,
                        unsigned left_bound, unsigned right_bound )
{
    unsigned opndBitVecL = opnd->getBitVecL(), opndBitVecH = opnd->getBitVecH();
    unsigned left_bound2 = opnd->getLeftBound(), right_bound2 = opnd->getRightBound();
    if( right_bound < left_bound2 || right_bound2 < left_bound ){
        return;
    }
    short dist = left_bound2 - left_bound;
    if( dist > 0 ){
        if( dist >= 32 ){
            unsigned lbit = opndBitVecL << (dist - 32);
            opndBitVecH |= lbit;
            opndBitVecL = 0;
        }
        else{
            unsigned lbit = opndBitVecL >> (32 - dist);
            opndBitVecH <<= dist;
            opndBitVecH |= lbit;
            opndBitVecL <<= dist;
        }
    }
    bitVec[0] |= opndBitVecL;
    bitVec[1] |= opndBitVecH;
}

//
// The routine traverses all BBs that can be reached from the entry of a subroutine (not
// traversing into nested subroutine calls). Mark retLoc[bb] = entryId (to associate bb
// with the subroutine entry. When two subroutines share code, we return the location of the
// subroutine that was previously traversed so that the two routines can then use
// the same location to save their return addresses.
//
unsigned Optimizer::determineReturnAddrLoc(unsigned entryId, unsigned* retLoc, G4_BB* bb)
{
    if (bb->isAlreadyTraversed(fg.getTraversalNum()))
        return retLoc[bb->getId()];
    bb->markTraversed(fg.getTraversalNum());

    if (retLoc[bb->getId()] != UNDEFINED_VAL)
        return retLoc[bb->getId()];
    else
    {
        retLoc[bb->getId()] = entryId;
        G4_INST* lastInst = bb->size()==0 ? NULL : bb->back();

        if (lastInst && lastInst->isReturn())
        {
            if (lastInst->getPredicate() == NULL)
                return entryId;
            else
                return determineReturnAddrLoc(entryId, retLoc, bb->fallThroughBB());
        }
        else if(lastInst && lastInst->isCall()) // skip nested subroutine calls
        {
            return determineReturnAddrLoc(entryId, retLoc, bb->BBAfterCall());
        }
        unsigned sharedId = entryId;
        for (BB_LIST_ITER it = bb->Succs.begin(); it != bb->Succs.end(); it++)
        {
            unsigned loc = determineReturnAddrLoc(entryId, retLoc, *it);
            if (loc != entryId)
            {
                while (retLoc[loc] != loc)  // find the root of subroutine loc
                    loc = retLoc[loc];      // follow the link to reach the root
                if (sharedId == entryId)
                {
                    sharedId = loc;
                }
                else if (sharedId != loc)
                {
                    //
                    // The current subroutine share code with two other subroutines, we
                    // force all three of them to use the same location by linking them
                    // togethers.
                    //
                    retLoc[loc] = sharedId;
                }
            }
        }
        return sharedId;
    }
}

//
// DFS to check if there is any conflict in subroutine return location
//
bool Optimizer::isSubRetLocConflict(G4_BB *bb, std::vector<unsigned> &usedLoc, unsigned stackTop)
{
    if (bb->isAlreadyTraversed(fg.getTraversalNum()))
        return false;
    bb->markTraversed(fg.getTraversalNum());

    G4_INST* lastInst = bb->size()==0 ? NULL : bb->back();
    if (lastInst && lastInst->isReturn())
    {
        if (lastInst->getPredicate() == NULL)
            return false;
        else
        {
            return isSubRetLocConflict( bb->fallThroughBB(), usedLoc, stackTop);
        }
    }
    else if(lastInst && lastInst->isCall())     // need to traverse to next level
    {
        unsigned curSubRetLoc = bb->getSubRetLoc();
        //
        // check conflict firstly
        //
        for (unsigned i=0; i<stackTop; i++)
            if (usedLoc[i] == curSubRetLoc)
                return true;
        //
        // then traverse all the subroutines and return BB
        //
        usedLoc[stackTop] = curSubRetLoc;
        unsigned afterCallId = bb->BBAfterCall()->getId();
        for (std::list<G4_BB*>::iterator it = bb->Succs.begin(); it != bb->Succs.end(); it++)
        {
            if ((*it)->getId() == afterCallId)
            {
                if (isSubRetLocConflict( bb->BBAfterCall(), usedLoc, stackTop))
                    return true;
            }
            else
            {
                G4_BB* subEntry = (*it);
                if (isSubRetLocConflict( subEntry, usedLoc, stackTop+1))
                    return true;
            }
        }
    }
    else
    {
        for (BB_LIST_ITER it = bb->Succs.begin(); it != bb->Succs.end(); it++)
            if (isSubRetLocConflict(*it, usedLoc, stackTop))
                return true;
    }

    return false;
}

void Optimizer::assignLocForReturnAddr()
{

    unsigned* retLoc = (unsigned*)mem.alloc(fg.getNumBB()*sizeof(unsigned));
    //
    // a data structure for doing a quick map[id] ---> block
    //
    G4_BB**  BBs = (G4_BB**)mem.alloc(fg.getNumBB()*sizeof(G4_BB*));
    for (BB_LIST_ITER it = fg.BBs.begin(); it != fg.BBs.end(); it++)
    {
        unsigned i = (*it)->getId();
        retLoc[i] = UNDEFINED_VAL;
        BBs[i] = (*it);                                                     // BBs are sorted by ID
    }

    //
    // Firstly, keep the original algorithm unchanged to mark the retLoc
    //
    std::list<G4_BB *> caller;                                          // just to accelerate the algorithm later

    for (unsigned i = 0; i < fg.getNumBB(); i++)
    {
        G4_BB* bb = BBs[i];
        if (bb->isEndWithCall() == false)
        {
            continue;
        }

#ifdef _DEBUG
        G4_INST *last = bb->empty() ? NULL : bb->back();
        MUST_BE_TRUE(last, ERROR_FLOWGRAPH);
#endif

        caller.push_back(bb);                   // record the  callers, just to accelerate the algorithm

        G4_BB* subEntry = bb->getCalleeInfo()->getInitBB();
        if (retLoc[subEntry->getId()] != UNDEFINED_VAL) // a loc has been assigned to the subroutine
        {
            // Need to setSubRetLoc if subEntry is part of another subRoutine because,
            // in the final phase, we use SubRetLoc != UNDEFINED_VAL to indicate
            // a block is an entry of a subroutine.
            subEntry->setSubRetLoc(retLoc[subEntry->getId()]);
        }
        else
        {
            fg.prepareTraversal();
            unsigned loc = determineReturnAddrLoc(subEntry->getId(),retLoc, subEntry);
            if (loc != subEntry->getId())
            {
                retLoc[subEntry->getId()] = loc;
            }
            subEntry->setSubRetLoc(loc);
            //
            // We do not merge indirect call here, because it will createt additional (bb->getSubRetLoc() != bb->getId())  cases that kill the share code detection
            //
        }

        // retBB is the exit basic block of callee, ie the block with return statement at end
        G4_BB* retBB = bb->getCalleeInfo()->getExitBB();

        if( retLoc[retBB->getId()] == UNDEFINED_VAL )
        {
            // retBB block was unreachable so retLoc element corresponding to that block was
            // left undefined
            retLoc[retBB->getId()] = subEntry->getSubRetLoc();
        }
    }
#ifdef DEBUG_VERBOSE_ON
    DEBUG_MSG(std::endl << "Before merge indirect call: " << std::endl);
    for (unsigned i = 0; i < fg.getNumBB(); i++)
        if (retLoc[i] == UNDEFINED_VAL) {
            DEBUG_MSG("BB" << i << ": X   ");
        } else {
            DEBUG_MSG("BB" << i << ": " << retLoc[i] << "   ");
        }
        DEBUG_MSG(std::endl);
#endif

        //
        // this final phase is needed. Consider the following scenario.  Sub2 shared code with both
        // Sub1 and Sub3. All three must use the same location to save return addresses. If we traverse
        // Sub1 then Sub3, retLoc[Sub1] and retLoc[Sub3] all point to their own roots.  As we traverse
        // Sub2, code sharing is detected, we need to this phase to make sure that Sub1 and Sub3 use the
        // same location.
        //
        for (unsigned i = 0; i < fg.getNumBB(); i++)
        {
            G4_BB* bb = BBs[i];
            if (bb->getSubRetLoc() != UNDEFINED_VAL)
            {
                if (bb->getSubRetLoc() != bb->getId())
                {
                    unsigned loc = bb->getId();
                    while (retLoc[loc] != loc)  // not root
                        loc = retLoc[loc];  // follow the link to reach the root
                }
            }
        }

        //
        // Merge the retLoc in indirect call cases
        //
        for (std::list<G4_BB*>::iterator it = caller.begin(); it != caller.end(); it++)
        {
            G4_BB *bb = *it;
            G4_INST *last = bb->empty() ? NULL : bb->back();
            MUST_BE_TRUE(last, ERROR_FLOWGRAPH);

            unsigned fallThroughId = bb->fallThroughBB() == NULL ? UNDEFINED_VAL : bb->fallThroughBB()->getId();
            if ((last && last->getPredicate() == NULL && bb->Succs.size() > 1) || (last && last->getPredicate() != NULL && bb->Succs.size() > 2))
            {
                //
                // merge all subroutines to the last one, it is a trick to conduct the conditional call by using last one instead of first one
                //
                unsigned masterEntryId = bb->Succs.back()->getId();
                //
                // find the root of the master subroutine
                //
                unsigned masterRetLoc = masterEntryId;
                while (retLoc[masterRetLoc] != masterRetLoc)
                    masterRetLoc = retLoc[masterRetLoc];
                //
                // check other subroutines in one vertex
                //
                for (std::list<G4_BB*>::iterator it1 = bb->Succs.begin(); it1 != bb->Succs.end(); it1++)
                {
                    G4_BB *subBB = *it1;
                    if (subBB->getId() != masterEntryId && subBB->getId() != fallThroughId)
                    {
                        //
                        // find the root of the current subroutine
                        //
                        unsigned loc = subBB->getId();
                        while (retLoc[loc] != loc)
                            loc = retLoc[loc];
                        //
                        // Merge: let all the items in retLoc with value loc pointing to masterRetLoc
                        // Suppose indirect call X calls subroutine A and B, indirect call Y calls B and C, and indirect call Z calls C and D.
                        // Before merge, the A~D will be assigned different return location. Suppose we process the callers in order X-->Z-->Y in the merge,
                        // if we just modified the return locations of one indirect call, we will fail to merge the return locations of A~D.
                        //
                        if (loc != masterRetLoc)
                        {
                            for (unsigned i = 0; i < fg.getNumBB(); i++)
                                if (retLoc[i] == loc)
                                    retLoc[i] = masterRetLoc;
                        }
                    }
                }
            }
        }

#ifdef DEBUG_VERBOSE_ON
        DEBUG_MSG(std::endl << "After merge indirect call: " << std::endl);
        for (unsigned i = 0; i < fg.getNumBB(); i++)
            if (retLoc[i] == UNDEFINED_VAL) {
                DEBUG_MSG("BB" << i << ": X   ");
            } else {
                DEBUG_MSG("BB" << i << ": " << retLoc[i] << "   ");
            }
            DEBUG_MSG(std::endl << std::endl);
#endif

            //
            //  Assign ret loc for subroutines firstly, and then check if it is wrong (due to circle in call graph).
            //
            for (unsigned i = 0; i < fg.getNumBB(); i++)
            {
                //
                // reset the return BB's retLoc
                //
                unsigned loc = i;
                if (retLoc[i] != UNDEFINED_VAL)
                {
                    while (retLoc[loc] != loc)
                        loc = retLoc[loc];
                    retLoc[i] = loc;
                    BBs[i]->setSubRetLoc(retLoc[loc]);
                }
            }

            for (std::list<G4_BB*>::iterator it = caller.begin(); it != caller.end(); it++)
            {
                //
                // set caller BB's retLoc
                //
                G4_BB *bb = *it;
#ifdef _DEBUG
                G4_INST *last = bb->empty() ? NULL : bb->back();
                MUST_BE_TRUE(last, ERROR_FLOWGRAPH);
#endif
                G4_BB *subBB = bb->getCalleeInfo()->getInitBB();
                //
                // 1: Must use retLoc here, because some subBB is also the caller of another subroutine, so the entry loc in BB may be changed in this step
                // 2: In some cases, the caller BB is also the entry BB. At this time, the associated entry BB ID will be overwritten. However, it will not impact the
                // conflict detection and return location assignment, since we only check the return BB and/or caller BB in these two moudles.
                //
                bb->setSubRetLoc(retLoc[subBB->getId()]);
            }

#ifdef _DEBUG
            for (unsigned i = 0; i < fg.getNumBB(); i++)
            {
                G4_BB* bb = BBs[i];
                if (bb->getSubRetLoc() != UNDEFINED_VAL)
                {
                    if (!bb->empty() && bb->front()->isLabel())
                    {
                        DEBUG_VERBOSE(((G4_Label*)bb->front()->getSrc(0))->getLabel()
                            << " assigned location " << bb->getSubRetLoc() << std::endl);
                    }
                }
            }
#endif

            //
            // detect the conflict (circle) at last
            //
            std::vector<unsigned> usedLoc(fg.getNumBB());
            unsigned stackTop = 0;
            for (std::list<G4_BB*>::iterator it = caller.begin(); it != caller.end(); it++)
            {
                G4_BB* bb = *it;
                MUST_BE_TRUE(bb->BBAfterCall() != NULL, ERROR_FLOWGRAPH);
                //
                // Must re-start the traversal from each caller, otherwise will lose some circle cases like TestRA_Call_1_1_3B, D, F, G, H
                //
                fg.prepareTraversal();

                usedLoc[stackTop] = bb->getSubRetLoc();
                unsigned afterCallId = bb->BBAfterCall()->getId();
                for (std::list<G4_BB*>::iterator it = bb->Succs.begin(); it != bb->Succs.end(); it++)
                {
                    G4_BB* subEntry = (*it);
                    if (subEntry->getId() == afterCallId)
                        continue;

                    if (isSubRetLocConflict( subEntry, usedLoc, stackTop+1))
                    {
                        MUST_BE_TRUE(false,
                            "ERROR: Fail to assign call-return variables due to cycle in call graph!");
                    }
                }
            }
}

void  Optimizer::insertCallReturnVar()
{
    for (BB_LIST_ITER it = fg.BBs.begin(); it != fg.BBs.end(); it++)
    {
        G4_BB *bb = *it;
        G4_INST *last = bb->empty() ? NULL : bb->back();
        if (last)
        {
            if (last->isCall())
            {
                insertSaveAddr(bb);
            }
            else
            {
                if (last->isReturn())
                {
                    // G4_BB_EXIT_TYPE is just a dummy BB, and the return will be the last
                    // inst in each of its predecessors
                    insertRestoreAddr(bb);
                }
            }
        }
    }
}

void  Optimizer::insertSaveAddr(G4_BB* bb)
{
    MUST_BE_TRUE(bb != NULL, ERROR_INTERNAL_ARGUMENT);
    MUST_BE_TRUE(bb->getSubRetLoc() != UNDEFINED_VAL,
        ERROR_FLOWGRAPH); // must have a assigned loc


    G4_INST *last = bb->back();
    MUST_BE_TRUE1(last->isCall(), last->getLineNo(),
        ERROR_FLOWGRAPH);
    if (last->getDst() == NULL)
    {
        unsigned loc = bb->getSubRetLoc();
        G4_Declare* dcl = getRetDecl(loc);

        last->setDest(builder.createDstRegRegion(Direct, dcl->getRegVar(), 0, 0, 1, Type_UD)); // RET__loc12<1>:ud

        last->setExecSize(2);
    }
}

void  Optimizer::insertRestoreAddr(G4_BB* bb)
{
    MUST_BE_TRUE(bb != NULL, ERROR_INTERNAL_ARGUMENT);

    G4_INST *last = bb->back();
    MUST_BE_TRUE1(last->isReturn(), last->getLineNo(),
        ERROR_FLOWGRAPH);
    if (last->getSrc(0) == NULL)
    {
        unsigned loc = bb->getSubRetLoc();
        G4_Declare* dcl = getRetDecl(loc);

        G4_SrcRegRegion* new_src = builder.createSrcRegRegion(Mod_src_undef,   // RET__loc12<0;2,1>:ud
            Direct,
            dcl->getRegVar(),
            0,
            0,
            builder.createRegionDesc(0,2,1),
            Type_UD);

        last->setSrc(new_src,0);
        last->setDest(builder.createNullDst( Type_UD ) );

        last->setExecSize(2);
    }
}

//
// determine if fall-through jump is needed
//
// also remove redundant jumps
//   if there is no predicate applied to a jump and its target is its fall though BB,
//   remove the jump instruction.

void Optimizer::insertFallThroughJump()
{

    for (BB_LIST_ITER it = fg.BBs.begin(); it != fg.BBs.end();)
    {
        G4_BB* bb = *it;
        BB_LIST_ITER next = ++it;
        //
        // determine if the current bb needs a fall through jump
        // check if the fall-through bb follows the current bb
        //
        G4_BB* fb = bb->fallThroughBB();
        if (fb && (next == fg.BBs.end() || // bb is the last bb
            fb != (*next)))
        {
            // This is bogus in SIMD CF, as bad things happen when you randomly insert jumps
            // in the middle of SIMD CF
        }
        else if (next != fg.BBs.end())
        {
            // do not remove a jmpi if it's the target of an indirect jmp
            // this makes the code more readable
            if (!(*next)->empty() && (*next)->front()->isLabel() &&
                !bb->empty() && bb->back()->opcode() == G4_jmpi &&
                bb->back()->getPredicate() == NULL &&
                !bb->back()->isIndirectJmpTarget() ) {
                    if ((*next)->front()->getSrc(0) == bb->back()->getSrc(0))
                    {
                        std::list<G4_INST*>::iterator it = bb->end();
                        it--;
                        bb->erase(it);
                    }
            }
        }
        it = next;
    }
}

//
// Check if any GRF/MRF is out-of-boundary
//
void Optimizer::chkRegBoundary()
{
    for (BB_LIST_ITER it = fg.BBs.begin(); it != fg.BBs.end();it++)
    {
        G4_BB* bb = *it;
        for (INST_LIST_ITER i = bb->begin(); i != bb->end(); i++)
        {
            G4_INST *inst = *i;
            if (!chkOpndBoundary(inst, inst->getDst())) {
                MUST_BE_TRUE(false, "region boundary check failure");
            }
            else
            {
                for (int src=0; src<G4_MAX_SRCS; src++)
                    if (!chkOpndBoundary(inst, inst->getSrc(src)))
                    {
                        MUST_BE_TRUE(false, "region boundary check failure");
                    }
            }
        }
    }
}

//
// Check if GRF/MRF (if we know the phy reg num) is out-of-boundary
// If the operand is not GRF/MRF, we return "true" without any check
//
bool Optimizer::chkOpndBoundary(G4_INST *inst, G4_Operand *opnd)
{
    if (opnd == NULL || (!opnd->isDstRegRegion() && !opnd->isSrcRegRegion()))
        return true;

    // we can not check indirect addressing now, leave it to address propagation
    if (opnd->isDstRegRegion() && opnd->asDstRegRegion()->getRegAccess() != Direct)
        return true;
    if (opnd->isSrcRegRegion() && opnd->asSrcRegRegion()->getRegAccess() != Direct)
        return true;

    return true;
}

void Optimizer::regAlloc()
{
    //
    // insert pseudo save/restore return address so that reg alloc
    // can assign registers to hold the return addresses
    //
    assignLocForReturnAddr();
    insertCallReturnVar();

    if (builder.getOption(vISA_DumpDotAll))
    {
        kernel.dumpDotFile("insertCallReturnVar");
    }

    fg.prepareTraversal();

    if (builder.getOption(vISA_DumpDotAll))
    {
        kernel.dumpDotFile("callReturn");
    }

    //
    // assign registers
    //
    int status = ::regAlloc(builder, builder.phyregpool, kernel);
    if (status != CM_SUCCESS)
    {
        RAFail = true;
    }
}

void Optimizer::countBankConflicts()
{
    std::list<G4_INST*> conflicts;
    unsigned int numLocals = 0, numGlobals = 0;
    bool isSKLPlus = ( getGenxPlatform() >= GENX_SKL ? true : false );

    for(BB_LIST_ITER bb_it = kernel.fg.BBs.begin();
        bb_it != kernel.fg.BBs.end();
        bb_it++)
    {
        G4_BB* curBB = (*bb_it);

        for(INST_LIST_ITER inst_it = curBB->begin();
            inst_it != curBB->end();
            inst_it++)
        {
            G4_INST* curInst = (*inst_it);
            G4_Operand* src0 = curInst->getSrc(0);
            G4_Operand* src1 = curInst->getSrc(1);
            G4_Operand* src2 = curInst->getSrc(2);

            if( ( isSKLPlus && src0 == NULL ) || src1 == NULL || src2 == NULL)
                continue;

            if( ( isSKLPlus && !src0->isSrcRegRegion() ) || !src1->isSrcRegRegion() || !src2->isSrcRegRegion())
                continue;

            if( (isSKLPlus && !src0->asSrcRegRegion()->getBase()->asRegVar()->getPhyReg()->isGreg()) ||
                !src1->asSrcRegRegion()->getBase()->asRegVar()->getPhyReg()->isGreg() ||
                !src1->asSrcRegRegion()->getBase()->asRegVar()->getPhyReg()->isGreg())
                continue;

            // We have a 3 src instruction with each src operand a GRF register region
            unsigned int src0grf = 0, src1grf = 0, src2grf = 0;

            if( isSKLPlus )
            {
                src0grf = src0->getBase()->asRegVar()->getPhyReg()->asGreg()->getRegNum() +
                    src0->asSrcRegRegion()->getRegOff();
            }
            src1grf = src1->getBase()->asRegVar()->getPhyReg()->asGreg()->getRegNum() +
                src1->asSrcRegRegion()->getRegOff();
            src2grf = src2->getBase()->asRegVar()->getPhyReg()->asGreg()->getRegNum() +
                src2->asSrcRegRegion()->getRegOff();

            bool isConflict = false;

            unsigned int src0partition = 0, src1partition = 0, src2partition = 0;

            if( isSKLPlus )
            {
                if(src0grf < 64)
                {
                    src0partition = 1;
                    if(src0grf%2 == 0)
                        src0partition = 0;
                }
                else
                {
                    src0partition = 3;
                    if(src0grf%2 == 0)
                        src0partition = 2;
                }
            }

            if(src1grf < 64)
            {
                src1partition = 1;
                if(src1grf%2 == 0)
                    src1partition = 0;
            }
            else
            {
                src1partition = 3;
                if(src1grf%2 == 0)
                    src1partition = 2;
            }

            if(src2grf < 64)
            {
                src2partition = 1;
                if(src2grf%2 == 0)
                    src2partition = 0;
            }
            else
            {
                src2partition = 3;
                if(src2grf%2 == 0)
                    src2partition = 2;
            }

            if( ( !isSKLPlus || src0partition == src1partition ) && src1partition == src2partition)
                isConflict = true;

            if(curInst->getExecSize() == 16)
                isConflict = true;

            if(isConflict == true)
            {
                conflicts.push_back(curInst);
                numBankConflicts++;

                auto isGlobal = [](G4_Operand* opnd, FlowGraph& fg)
                    -> bool {
                    if (fg.globalOpndHT.isOpndGlobal(opnd))
                        return true;
                    return false;
                };

                if( isSKLPlus )
                {
                    if(!isGlobal(curInst->getSrc(0), kernel.fg) &&
                        curInst->getSrc(0)->getTopDcl()->getRegVar()->getPhyReg())
                        numLocals++;
                    else
                        numGlobals++;
                }

                if(!isGlobal(curInst->getSrc(1), kernel.fg) &&
                    curInst->getSrc(1)->getTopDcl()->getRegVar()->getPhyReg())
                    numLocals++;
                else
                    numGlobals++;

                if(!isGlobal(curInst->getSrc(2), kernel.fg) &&
                    curInst->getSrc(2)->getTopDcl()->getRegVar()->getPhyReg())
                    numLocals++;
                else
                    numGlobals++;
            }
        }
    }

    if(numBankConflicts > 0)
    {
        std::ofstream optreport;
        getOptReportStream( optreport, builder.getOptions() );

        optreport << std::endl << std::endl;

        optreport << "===== Bank conflicts =====" << std::endl;
        optreport <<  "Found " << numBankConflicts << " conflicts (" << numLocals << " locals, " << numGlobals << " globals) in kernel: " << kernel.getName() << std::endl;
        for(std::list<G4_INST*>::iterator it = conflicts.begin();
            it != conflicts.end();
            it++)
        {
            G4_INST* i = (*it);
            i->emit( optreport );
            optreport << " // $" << i->getCISAOff() << ":#" << i->getLineNo() << std::endl;
        }

        optreport << std::endl << std::endl;

        closeOptReportStream( optreport );
    }
}

void Optimizer::insertDummyCompactInst()
{
    // Only for SKL+ and compaction is enabled.
    if (getGenxPlatform() < GENX_SKL || !builder.getOption(vISA_Compaction))
        return;

    // Insert mov (1) r0 r0 at the beginning of this kernel.
    G4_Declare *dcl = builder.getBuiltinR0();
    auto src = builder.createSrcRegRegion(Mod_src_undef, Direct,
        dcl->getRegVar(), 0, 0, builder.getRegionScalar(), Type_UD);
    auto dst = builder.createDstRegRegion(Direct, dcl->getRegVar(), 0, 0, 1, Type_UD);
    G4_INST *movInst = builder.createInternalInst(nullptr, G4_mov, nullptr, false,
        1, dst, src, nullptr, InstOpt_WriteEnable);

    auto bb = fg.getEntryBB();
    for (auto it = bb->begin(), ie = bb->end(); it != ie; ++it)
    {
        if ((*it)->opcode() != G4_label)
        {
            bb->insert(it, movInst);
            return;
        }
    }

    // The entry block is empty or only contains a label.
    bb->push_back(movInst);
}

void Optimizer::insertHashMovs()
{
    // As per request from IGC team, we want to conditionally insert
    // two mov instructions like following:
    //
    // send ... {EOT}
    // mov (16) null<1>:d        lo32 {NoMask}
    // mov (16) null<1>:d        hi32 {NoMask}
    //
    for(BB_LIST_ITER bb_it = kernel.fg.BBs.begin();
        bb_it != kernel.fg.BBs.end();
        bb_it++)
    {
        G4_BB* bb = (*bb_it);

        for(INST_LIST_ITER inst_it = bb->begin();
            inst_it != bb->end();
            inst_it++)
        {
            G4_INST* inst = (*inst_it);

            if(inst->isEOT())
            {
                // We have to insert new instructions after EOT.
                // Lexically, EOT could even be in the middle
                // of the program.
                G4_INST* lo;
                G4_INST* hi;
                uint64_t hashVal = builder.getOptions()->getuInt64Option(vISA_HashVal);
                lo = kernel.fg.builder->createInternalInst(
                    NULL,
                    G4_mov,
                    NULL,
                    false,
                    16,
                    kernel.fg.builder->createNullDst(Type_UD),
                    kernel.fg.builder->createImm((unsigned int)(hashVal & 0xffffffff), Type_UD),
                    NULL,
                    NULL,
                    InstOpt_WriteEnable);

                hi = kernel.fg.builder->createInternalInst(
                    NULL,
                    G4_mov,
                    NULL,
                    false,
                    16,
                    kernel.fg.builder->createNullDst(Type_UD),
                    kernel.fg.builder->createImm((unsigned int)((hashVal >> 32) & 0xffffffff), Type_UD),
                    NULL,
                    NULL,
                    InstOpt_WriteEnable);

                bb->push_back(lo);
                bb->push_back(hi);
                break;
            }
        }
    }
}

bool isLifetimeOpRemovalCandidate(G4_INST* inst)
{
    if( inst->isPseudoKill() || inst->isLifeTimeEnd() )
    {
        return true;
    }

    return false;

}

bool isPseudoUse(G4_INST* inst)
{
    if (inst->isPseudoUse())
    {
        return true;
    }
    return false;
}

void Optimizer::removeLifetimeOps()
{
    // Call this function after RA only.

    // Remove all pseudo_kill and lifetime.end
    // instructions.
    // Also remove pseudo_use instructions.
    for( BB_LIST_ITER bbs = kernel.fg.BBs.begin();
        bbs != fg.BBs.end();
        bbs++ )
    {
        G4_BB* bb = *bbs;        
        bb->erase(
            std::remove_if(bb->begin(), bb->end(),
            [](G4_INST* inst) { return inst->isPseudoKill() || inst->isLifeTimeEnd() || inst->isPseudoUse(); }),
            bb->end());
    }
}

void Optimizer::runPass(PassIndex Index)
{
    const PassInfo &PI = Passes[Index];

    // Do not execute.
    if (PI.Option != vISA_EnableAlways && !builder.getOption(PI.Option))
        return;

    std::string Name = PI.Name;

    if (builder.getOption(vISA_DumpDotAll))
        kernel.dumpDotFile(("before." + Name).c_str());

    if (PI.Timer != TIMER_NUM_TIMERS)
        startTimer(PI.Timer);

    // Execute pass.
    (this->*(PI.Pass))();

    if (PI.Timer != TIMER_NUM_TIMERS)
        stopTimer(PI.Timer);

    if (builder.getOption(vISA_DumpDotAll))
        kernel.dumpDotFile(("after." + Name).c_str());

#ifdef _DEBUG
    bool skipVerify = Index == PI_regAlloc && RAFail == true;
    if (!skipVerify)
    {
        verifyG4Kernel(kernel, Index, true, G4Verifier::VC_ASSERT);
    }
#endif
}

void Optimizer::initOptimizations()
{
#define INITIALIZE_PASS(Name, Option, Timer) \
    Passes[PI_##Name] = PassInfo(&Optimizer::Name, ""#Name, Option, Timer)

    // To initialize a pass, the member function name is the first argument.
    // This member function must return void and take no argument.
    //
    // The second argument is the corresponding option to enable this pass.
    // If it always runs then use vISA_EnableAlways.
    //
    // The third argument is the intended timer for this pass. If no timing
    // is necessary, then TIMER_NUM_TIMERS can be used.
    //
    INITIALIZE_PASS(cleanMessageHeader,      vISA_LocalCleanMessageHeader, TIMER_OPTIMIZER);
    INITIALIZE_PASS(renameRegister,          vISA_LocalRenameRegister,     TIMER_OPTIMIZER);
    INITIALIZE_PASS(newLocalDefHoisting,     vISA_LocalDefHoist,           TIMER_OPTIMIZER);
    INITIALIZE_PASS(newLocalCopyPropagation, vISA_LocalCopyProp,           TIMER_OPTIMIZER);
    INITIALIZE_PASS(sendFusion,              vISA_EnableSendFusion,        TIMER_OPTIMIZER);
    INITIALIZE_PASS(cselPeepHoleOpt,         vISA_enableCSEL,              TIMER_OPTIMIZER);
    INITIALIZE_PASS(optimizeLogicOperation,  vISA_EnableAlways,            TIMER_OPTIMIZER);
    INITIALIZE_PASS(HWConformityChk,         vISA_EnableAlways,            TIMER_HW_CONFORMITY);
    INITIALIZE_PASS(preRA_Schedule,          vISA_preRA_Schedule,          TIMER_PRERA_SCHEDULING);
    INITIALIZE_PASS(regAlloc,                vISA_EnableAlways,            TIMER_TOTAL_RA);
    INITIALIZE_PASS(removeLifetimeOps,       vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(countBankConflicts,      vISA_OptReport,               TIMER_MISC_OPTS);
    INITIALIZE_PASS(removeRedundMov,         vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(removeEmptyBlocks,       vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(insertFallThroughJump,   vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(reassignBlockIDs,        vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(evalAddrExp,             vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(FoldAddrImmediate,       vISA_FoldAddrImmed,           TIMER_MISC_OPTS);
    INITIALIZE_PASS(chkRegBoundary,          vISA_EnableAlways,            TIMER_NUM_TIMERS);
    INITIALIZE_PASS(localSchedule,           vISA_LocalScheduling,         TIMER_SCHEDULING);
    INITIALIZE_PASS(NoDD,                    vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(HWWorkaround,            vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(NoSrcDepSet,             vISA_EnableNoSrcDep,          TIMER_MISC_OPTS);
    INITIALIZE_PASS(insertInstLabels,        vISA_EnableAlways,            TIMER_NUM_TIMERS);
    INITIALIZE_PASS(insertHashMovs,          vISA_InsertHashMovs,          TIMER_NUM_TIMERS);
    INITIALIZE_PASS(insertDummyCompactInst,  vISA_InsertDummyCompactInst,  TIMER_NUM_TIMERS);
    INITIALIZE_PASS(mergeScalarInst,         vISA_MergeScalar,             TIMER_OPTIMIZER);
    INITIALIZE_PASS(lowerMadSequence,        vISA_EnableMACOpt,            TIMER_OPTIMIZER);
    INITIALIZE_PASS(LVN,                     vISA_LVN,                     TIMER_OPTIMIZER);
    INITIALIZE_PASS(ifCvt,                   vISA_ifCvt,                   TIMER_OPTIMIZER);
    INITIALIZE_PASS(dumpPayload,             vISA_dumpPayload,             TIMER_MISC_OPTS);
    INITIALIZE_PASS(normalizeRegion,         vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(checkBarrierUsage,       vISA_EnableAlways,            TIMER_MISC_OPTS);
    INITIALIZE_PASS(createR0Copy,            vISA_enablePreemption,        TIMER_MISC_OPTS);
    INITIALIZE_PASS(initializePayload,       vISA_InitPayload,             TIMER_NUM_TIMERS);
    INITIALIZE_PASS(cleanupBindless,         vISA_enableCleanupBindless,   TIMER_OPTIMIZER);
    INITIALIZE_PASS(countGRFUsage,           vISA_PrintRegUsage,           TIMER_MISC_OPTS);
    INITIALIZE_PASS(splitVariables,          vISA_EnableSplitVariables,    TIMER_MISC_OPTS);
    INITIALIZE_PASS(changeMoveType,          vISA_ChangeMoveType,          TIMER_MISC_OPTS);
    INITIALIZE_PASS(reRAPostSchedule,        vISA_ReRAPostSchedule,        TIMER_OPTIMIZER);
    INITIALIZE_PASS(accSubPostSchedule,      vISA_accSubstitution,         TIMER_OPTIMIZER);
    INITIALIZE_PASS(dce,                     vISA_EnableDCE,               TIMER_OPTIMIZER);
    INITIALIZE_PASS(reassociateConst,        vISA_reassociate,             TIMER_OPTIMIZER);
    INITIALIZE_PASS(split4GRFVars,           vISA_split4GRFVar,            TIMER_OPTIMIZER);
    INITIALIZE_PASS(loadThreadPayload,       vISA_loadThreadPayload,       TIMER_MISC_OPTS);

    // Verify all passes are initialized.
#ifdef _DEBUG
    for (unsigned i = 0; i < PI_NUM_PASSES; ++i)
    {
        ASSERT_USER(Passes[i].Pass, "uninitialized pass");
    }
#endif
}

void replaceAllSpilledRegions(G4_Kernel& kernel, G4_Declare* oldDcl, G4_Declare* newDcl)
{
    // Iterate fg and replace all references to oldDcl with newDcl.
    // This requires creating new operands since changing dcl in
    // existing operands is disallowed.
    for (auto bb : kernel.fg.BBs)
    {
        for (auto inst : *bb)
        {
            auto dst = inst->getDst();
            if (dst &&
                !dst->isNullReg())
            {
                if (dst->getTopDcl() == oldDcl)
                {
                    G4_DstRegRegion* newDstRgn = kernel.fg.builder->createDstRegRegion(
                        dst->getRegAccess(), newDcl->getRegVar(), dst->getRegOff(),
                        dst->getSubRegOff(), dst->getHorzStride(), dst->getType());
                    inst->setDest(newDstRgn);
                }
            }

            for (unsigned int i = 0; i < G4_MAX_SRCS; i++)
            {
                auto src = inst->getSrc(i);

                if (src &&
                    src->isSrcRegRegion())
                {
                    auto srcRgn = inst->getSrc(i)->asSrcRegRegion();

                    if (srcRgn->getTopDcl() == oldDcl)
                    {
                        G4_SrcRegRegion* newSrcRgn = kernel.fg.builder->createSrcRegRegion(
                            srcRgn->getModifier(), srcRgn->getRegAccess(), newDcl->getRegVar(),
                            srcRgn->getRegOff(), srcRgn->getSubRegOff(), srcRgn->getRegion(),
                            srcRgn->getType());
                        inst->setSrc(newSrcRgn, i);
                    }
                }
            }
        }
    }
}

void getPhyRegs(G4_Operand* opnd, unsigned int& start, unsigned int& end)
{
    // start/end are inclusive, ie both offsets are referenced by the variable
    start = 0;
    end = 0;

    auto b = opnd->getBase();
    if (opnd->isAddrExp())
    {
        b = opnd->asAddrExp()->getRegVar();
    }

    if (b && b->isRegVar())
    {
        auto r = b->asRegVar();
        if (r->getPhyReg() &&
            r->getPhyReg()->isGreg())
        {
            if (!opnd->isAddrExp())
            {
                start = opnd->getLinearizedStart();
                end = opnd->getLinearizedEnd();
            }
            else
            {
                r = r->getDeclare()->getRootDeclare()->getRegVar();
                auto phyReg = r->getPhyReg()->asGreg();
                auto subRegOff = r->getPhyRegOff();
                start = phyReg->getRegNum() * G4_GRF_REG_NBYTES;
                start += subRegOff * r->getDeclare()->getElemSize();
                end = start + r->getDeclare()->getRootDeclare()->getByteSize() - 1;
            }
        }
    }
}

void computeGlobalFreeGRFs(G4_Kernel& kernel)
{
    auto gtpin = kernel.getGTPinData();
    gtpin->clearFreeGlobalRegs();
    std::vector<bool> freeGRFs;
    freeGRFs.resize(kernel.getNumRegTotal() * G4_GRF_REG_NBYTES, true);
    unsigned int start = 0, end = 0;

    // Mark r0 as busy. Done explicitly because move from r0 is inserted
    // after reRA pass.
    for (unsigned int i = 0; i < G4_GRF_REG_NBYTES; i++)
    {
        freeGRFs[i] = false;
    }

    for (auto bb : kernel.fg.BBs)
    {
        for (auto inst : *bb)
        {
            auto dst = inst->getDst();
            if (dst &&
                dst->getBase() &&
                dst->getBase()->isRegVar())
            {
                getPhyRegs(dst, start, end);
                for (unsigned int i = start; i <= end; i++)
                {
                    freeGRFs[i] = false;
                }
            }

            for (unsigned int i = 0; i < G4_MAX_SRCS; i++)
            {
                auto src = inst->getSrc(i);

                if (!src)
                    continue;

                if (src->isSrcRegRegion() &&
                    src->asSrcRegRegion()->getBase()->isRegVar())
                {
                    getPhyRegs(src, start, end);
                    for (unsigned int i = start; i <= end; i++)
                    {
                        freeGRFs[i] = false;
                    }
                }
                else if (src->isAddrExp())
                {
                    getPhyRegs(src, start, end);
                    for (unsigned int i = start; i <= end; i++)
                    {
                        freeGRFs[i] = false;
                    }
                }
            }
        }
    }

    for (unsigned int i = 0; i < freeGRFs.size(); i++)
    {
        if (freeGRFs[i])
        {
            gtpin->addFreeGlobalReg(i);
        }
    }
}

typedef struct Assignment
{
    G4_Declare* dcl;
    AssignedReg reg;
    unsigned int GRFBaseOffset;
} Assignment;

void storeGRFAssignments(DECLARE_LIST& declares, std::vector<Assignment>& assignments)
{
    for (auto dcl : declares)
    {
        auto regVar = dcl->getRegVar();
        if (regVar &&
            regVar->isPhyRegAssigned() &&
            regVar->getPhyReg()->isGreg())
        {
            Assignment a;
            AssignedReg assignedPhyReg;
            assignedPhyReg.phyReg = regVar->getPhyReg();
            assignedPhyReg.subRegOff = regVar->getPhyRegOff();
            a.dcl = dcl;
            a.reg = assignedPhyReg;
            a.GRFBaseOffset = dcl->getGRFBaseOffset();
            assignments.push_back(a);

        }
    }
}

void restoreGRFAssignments(std::vector<Assignment>& assignments)
{
    for (auto& a : assignments)
    {
        auto regVar = a.dcl->getRegVar();
        regVar->setPhyReg(a.reg.phyReg, a.reg.subRegOff);
        a.dcl->setGRFBaseOffset(a.GRFBaseOffset);
    }
}

// simple heuristics to decide if it's profitable to do copy propagation for the move
// add more as necessary
bool Optimizer::isCopyPropProfitable(G4_INST* movInst) const
{
    assert(movInst->opcode() == G4_mov && "execpt move instruction");

    // if inst is a simd16 W/HF packing, we don't want to optimize it if
    // there are >=2 simd16 mad uses, since it will slow down the mad.
    // for gen9 additionally check for simd8 mad as it doesn't support strided regions
    auto dst = movInst->getDst();
    auto src0 = movInst->getSrc(0);
    auto hasStrideSource = dst->getHorzStride() == 1 &&
        src0->isSrcRegRegion() &&
        !(src0->asSrcRegRegion()->getRegion()->isContiguous(movInst->getExecSize()) ||
            src0->asSrcRegRegion()->getRegion()->isScalar());

    hasStrideSource &= movInst->getExecSize() == 16 || (!builder.hasAlign1Ternary() && movInst->getExecSize() == 8);

    auto hasNSIMD16or8MadUse = [](G4_INST* movInst, int N, bool checkSIMD8)
    {   
        int numMadUses = 0;
        for (auto iter = movInst->use_begin(), iterEnd = movInst->use_end(); iter != iterEnd; ++iter)
        {
            auto use = *iter;
            auto inst = use.first;
            if (inst->opcode() == G4_pseudo_mad && 
                (inst->getExecSize() == 16 || (checkSIMD8 && inst->getExecSize() == 8)))
            {
                ++numMadUses;
                if (numMadUses == N)
                {
                    return true;
                }
            }
        }
        return false;
    };

    if (hasStrideSource)
    {
        if (hasNSIMD16or8MadUse(movInst, 2, true))
        {
            return false;
        }
    }

    // another condition where copy prop may be not profitable:
    // mov is from HF to F and dst is used in simd16 mad.
    // copy propagating away the move results in mix mode mad which is bad for bank conflicts
    if (dst->getType() == Type_F && src0->getType() == Type_HF)
    {
        if (hasNSIMD16or8MadUse(movInst, 4, false))
        {
            return false;
        }
    }
    return true;
}

void Optimizer::reRAPostSchedule()
{
    std::vector<Assignment> assignments;
    auto oldRAType = kernel.getRAType();
    auto gtpin = kernel.getGTPinData();
    computeGlobalFreeGRFs(kernel);
    auto freeGRFsBeforeReRA = gtpin->getNumFreeGlobalRegs();

    storeGRFAssignments(kernel.Declares, assignments);
    // This pass is run for gtpin since they need re-allocation after
    // scheduling pass to get as many free registers for instrumentation.

    // Store dcls referenced by srcs of 3-src instructions. These will
    // not be participate in re-allocation because that could potentially
    // alter bank conflict profile which might affect kernel performance.
    std::set<G4_Declare*> threeSrcDcl;
    for (auto bb : kernel.fg.BBs)
    {
        for (auto inst : *bb)
        {
            if (inst->isMovAddr() &&
                inst->getDst()->getTopDcl()->getRegFile() == G4_RegFileKind::G4_GRF)
            {
                // inst's dst is a spilled address register from previous RA pass.
                // Addr/flags are spilled to GRF. When address regs are spilled to GRF,
                // points2 analysis will miss those entries since it expects only addr
                // reg when src is G4_AddrExp type. So we replace such spilled addr
                // operands with new address operands.
                G4_Declare* dstDcl = inst->getDst()->getTopDcl();
                auto name = builder.getNameString(fg.mem, 16, "ADDR%d", kernel.Declares.size());
                G4_Declare* newDstAddrDcl = kernel.fg.builder->createDeclareNoLookup(name,
                    G4_RegFileKind::G4_ADDRESS, dstDcl->getNumElems(),
                    dstDcl->getNumRows(), dstDcl->getElemType());
                replaceAllSpilledRegions(kernel, dstDcl, newDstAddrDcl);
            }

            if (inst->getNumSrc() == 3 && !inst->isSend())
            {
                for (auto i = 0; i < inst->getNumSrc(); i++)
                {
                    auto srcOpnd = inst->getSrc(i);
                    if (srcOpnd->isSrcRegRegion())
                    {
                        auto dcl = srcOpnd->getTopDcl();
                        threeSrcDcl.insert(dcl);
                    }
                }
            }
        }
    }

    for (auto dcl : kernel.Declares)
    {
        if (threeSrcDcl.find(dcl->getRootDeclare()) == threeSrcDcl.end())
        {
            dcl->prepareForRealloc(&kernel);
        }
        dcl->getRegVar()->forceAlign(G4_Align::Either);
    }

    kernel.fg.clearBBLRASummaries();

    gtpin->setRAPass(gtPinData::RAPass::ReRAPass);

    gtpin->removeUnmarkedInsts();

    // This time we want to try only first-fit and no localra
    regAlloc();

    removeLifetimeOps();

    computeGlobalFreeGRFs(kernel);

    if (RAFail ||
        gtpin->getNumFreeGlobalRegs() < freeGRFsBeforeReRA)
    {
        restoreGRFAssignments(assignments);
        kernel.setRAType(oldRAType);

        computeGlobalFreeGRFs(kernel);
    }
}

void Optimizer::accSubPostSchedule()
{
    if (!builder.doAccSub() || !builder.getOption(vISA_doAccSubAfterSchedule))
    {
        return;
    }

    // clean existing def use
    kernel.fg.globalOpndHT.clearHashTable();
    for (auto bb : kernel.fg.BBs)
    {
        for (auto inst : *bb)
        {
            inst->clearDef();
            inst->clearUse();
        }
    }
    kernel.fg.localDataFlowAnalysis();
    HWConformity hwConf(builder, kernel, mem);
    for (auto bb : kernel.fg.BBs)
    {
        hwConf.accSubstitution(bb);
    }
}

void* gtPinData::getFreeGRFInfo(unsigned int& size)
{
    // Here is agreed upon format for reporting free GRFs:
    //struct freeBytes
    //{
    //    unsigned short startByte;
    //    unsigned short numConsecutiveBytes;
    //};

    // Added magic 0xDEADD00D at start and
    // magic 0xDEADBEEF at the end of buffer
    // on request of gtpin team.
    //
    //struct freeGRFInfo
    //{
    //    unsigned short numItems;
    //
    //    freeBytes data[numItems];
    //};
    typedef struct freeBytes
    {
        unsigned short startByte;
        unsigned short numConsecutiveBytes;
    } freeBytes;

    typedef struct freeGRFInfo
    {
        unsigned int magicStart;
        unsigned int numItems;
        freeBytes* data;
    } freeGRFInfo;

    // Compute free register information using vector for efficiency,
    // then convert to POS for passing back to gtpin.
    std::vector<std::pair<unsigned short, unsigned short>> vecFreeBytes;

    for (auto byte : globalFreeRegs)
    {
        if (vecFreeBytes.size() > 0)
        {
            auto& lastFree = vecFreeBytes.back();
            if (byte == (lastFree.first + lastFree.second))
            {
                lastFree.second += 1;
            }
            else
            {
                vecFreeBytes.push_back(std::make_pair(byte, 1));
            }
        }
        else
        {
            vecFreeBytes.push_back(std::make_pair(byte, 1));
        }
    }

    // Now convert vector to POS
    unsigned int numItems = (unsigned int)vecFreeBytes.size();
    freeGRFInfo* buffer = (freeGRFInfo*)malloc(numItems * sizeof(freeBytes) + sizeof(unsigned int)
        + sizeof(unsigned int) + sizeof(unsigned int));
    if (buffer)
    {
        buffer->numItems = numItems;
        buffer->magicStart = 0xDEADD00D;
        memcpy_s((unsigned char*)buffer + sizeof(unsigned int) + sizeof(unsigned int),
            numItems * sizeof(freeBytes), vecFreeBytes.data(), numItems * sizeof(freeBytes));
        unsigned int magicEnd = 0xDEADBEEF;
        memcpy_s((unsigned char*)buffer + sizeof(unsigned int) + sizeof(unsigned int) + (numItems * sizeof(freeBytes)),
            sizeof(magicEnd), &magicEnd, sizeof(magicEnd));

        // numItems - unsigned int
        // magicStart - unsigned int
        // magicEnd - unsigned int
        // data - numItems * sizeof(freeBytes)
        size = sizeof(unsigned int) + sizeof(unsigned int) + sizeof(unsigned int) + (numItems * sizeof(freeBytes));
    }

    return (void*)buffer;
}

int Optimizer::optimization()
{
    // remove redundant message headers.
    runPass(PI_cleanMessageHeader);

    runPass(PI_sendFusion);

    // rename registers.
    runPass(PI_renameRegister);

    runPass(PI_newLocalDefHoisting);

    // remove redundant movs
    runPass(PI_newLocalCopyPropagation);

    // split 4GRF variables.
    runPass(PI_splitVariables);

    runPass(PI_mergeScalarInst);

    runPass(PI_cselPeepHoleOpt);

    // this must be run after copy prop cleans up the moves
    runPass(PI_cleanupBindless);

    runPass(PI_reassociateConst);

    runPass(PI_lowerMadSequence);

    // optimize logic operantions
    runPass(PI_optimizeLogicOperation);

    // Dead code elimination
    runPass(PI_dce);

    // HW conformity check
    runPass(PI_HWConformityChk);

    // Local Value Numbering
    runPass(PI_LVN);

    runPass(PI_split4GRFVars);

    // PreRA scheduling
    runPass(PI_preRA_Schedule);

    // perform register allocation
    runPass(PI_regAlloc);

    if (RAFail)
    {
        return CM_SPILL;
    }

    runPass(PI_removeLifetimeOps);

    runPass(PI_countBankConflicts);

    // some passes still rely on G4_Declares and their def-use even after RA,
    // and removeRedundantMove will break them since it deletes moves solely based on GRF assignment 
    // without maintaining def-use of G4_Declares.
    // so when these passes are active we have to defer removeRedundMov until after the passes
    // ToDo: study the perf impact of moving this post scheduling
    bool preserveVirtualDefUse = kernel.getOption(vISA_ReRAPostSchedule) || builder.doAccSub();
    if (!preserveVirtualDefUse)
    {
        runPass(PI_removeRedundMov);
    }

    //
    // if a fall-through BB does not immediately follow its predecessor
    // in the code layout, then insert a jump-to-fall-through in the predecessor
    //
    runPass(PI_insertFallThroughJump);

    // Run if-conversion to convert short if-blocks.
    runPass(PI_ifCvt);

    //
    // re-assign block ID so that we can use id to determine the ordering of
    // two blocks in the code layout
    //
    runPass(PI_reassignBlockIDs);

    runPass(PI_FoldAddrImmediate);

    //
    // Check if any GRF/MRF is out-of-boundary
    // FIXME: Now we can only check part of the out-of-boundary cases. We can not find out-of-boundary errors in:
    // (1). <post_dst>/<curr_dst> in send inst; (2). cross 256-bit bar case in compressed instruction
    //
    runPass(PI_chkRegBoundary);

    runPass(PI_localSchedule);

    runPass(PI_accSubPostSchedule);

    // NoDD optimization
    runPass(PI_NoDD);

    runPass(PI_reRAPostSchedule);

    if (preserveVirtualDefUse)
    {
        runPass(PI_removeRedundMov);
    }

    // Insert NoSrcDepSet flag to improve performance
    runPass(PI_NoSrcDepSet);

    // remove any placeholders blocks inserted to aid regalloc
    // run this pass after reRA pass otherwise CFG can become
    // invalid (funcInfo, calleeInfo may point to bad initBB).
    runPass(PI_removeEmptyBlocks);

    // HW workaround
    runPass(PI_HWWorkaround);

    runPass(PI_normalizeRegion);

    runPass(PI_changeMoveType);

    runPass(PI_countGRFUsage);

    runPass(PI_dumpPayload);

    // this must be the last step of the optimization so as to not violate
    // the CFG assumption
    runPass(PI_insertInstLabels);

    runPass(PI_insertHashMovs);

    if (builder.getOptions()->getTarget() == VISA_CM)
    {
        runPass(PI_checkBarrierUsage);
    }

    // Create a copy of R0 at the top of kernel.
    // This must be done after all other optimizer
    // passes, except initializePayload, to avoid
    // potential code ordering issue.
    runPass(PI_createR0Copy);

    runPass(PI_initializePayload);

    runPass(PI_loadThreadPayload);

    // Insert a dummy compact instruction if requested for SKL+
    runPass(PI_insertDummyCompactInst);

    return CM_SUCCESS;
}

//  When constructing CFG we have the assumption that a label must be the first
//  instruction in a bb.  During structure analysis, however, we may end up with a bb that
//  starts with multiple endifs if the bb is the target of multiple gotos that have been
//  converted to an if. Instead of creating a BB for each of the endif, we associate each endif with a label
//  and emit them only at the very end.
//
//  For break and continue, UIP must be the lable directly attached to the while
//  op. If not, create such a label
//
//  DO
//    IF
//      P =
//      CONT L1
//    ENDIF L1
//    IF
//      BREAK L2
//    ENDIF L1
//  L1
//  (P) WHILE
//  L2
//
//  will be transfered into
//
//  DO
//    IF
//      P =
//      Spill <- P
//      CONT L3         // UIP becomes L3
//    ENDIF L1
//    IF
//      BREAK L3        // UIP becomes L3
//    ENDIF L1
//  L1                  // existing label
//  P <- fill
//  L3                  // new label
//  (P) WHILE
//  L2
//
void Optimizer::insertInstLabels()
{
    for( BB_LIST_CITER iter = fg.BBs.cbegin(), bend = fg.BBs.cend(); iter != bend; ++iter )
    {
        G4_BB* bb = *iter;
        INST_LIST_ITER iter2 = bb->begin();
        INST_LIST_ITER iend  = bb->end();
        while( iter2 != iend )
        {
            INST_LIST_ITER currIter = iter2;
            ++iter2;

            G4_INST* inst = *currIter;
            if( inst->getInstLabel() != NULL )
            {
                G4_INST* labelInst = fg.createNewLabelInst( inst->getInstLabel(), inst->getLineNo(), inst->getCISAOff() );
                bb->insert( currIter, labelInst );
            }
        }
    }

    // Patch labels if necessary.
    for(auto iter = fg.BBs.begin(), iend = fg.BBs.end(); iter != iend; ++iter)
    {
        G4_BB *bb = *iter;
        if (bb->empty())
            continue;

        G4_INST *inst = bb->back();
        G4_opcode opc = inst->opcode();
        if (opc != G4_cont && opc != G4_break)
            continue;

        // The matching while BB.
         G4_BB *whileBB = nullptr;
         if (opc == G4_cont)
         {
             // The whileBB is the first successor bb, if this is continue.
             whileBB = bb->Succs.front();
         }
         else
         {
             // For break, the whileBB should be the physical predecessor of
             // break's first successor bb.
             BB_LIST_ITER iter = bb->Succs.begin();
             while (iter != bb->Succs.end())
             {
                 G4_BB * succBB = (*iter);
                 if (succBB->getPhysicalPred() &&
                     (!succBB->getPhysicalPred()->empty()) &&
                     (succBB->getPhysicalPred()->back()->opcode() == G4_while))
                 {
                     whileBB = succBB->getPhysicalPred();
                     break;
                 }
                 iter++;
             }
         }

         if (whileBB == nullptr ||
             whileBB->empty() ||
             whileBB->back()->opcode() != G4_while)
         {
             ASSERT_USER(false, "can not find while BB");
             continue;
         }

         // If while instruction is following the label, then no need
         // to insert a new uip label, just use the existing one.
         G4_InstCF *instCF = inst->asCFInst();
         auto whileIter = std::prev(whileBB->end());
         G4_INST *prevInst = *std::prev(whileIter);
         if (prevInst->isLabel())
         {
             instCF->setUip(prevInst->getLabel());
         }
         else
         {
             std::string NewUipName = instCF->getUipLabelStr();
             NewUipName += "_UIP";
             G4_Label *label = builder.createLabel(NewUipName, LABEL_BLOCK);
             instCF->setUip(label);

             G4_INST *newInst = fg.createNewLabelInst(label, inst->getLineNo(),
                                                      inst->getCISAOff());

             whileBB->insert(whileIter, newInst);
         }
    }
}

//Fold address register into address register offset, such that we can same one instruction
//that computes:
//add a0.0 a0.0 immed;
//mul dst r[a0.0, 0] src2
//-->
//mul dst r[a0.0, immed] src2

//The condition is that immed is in range [-512..511] and it is dividable by 32.
//This is a local OPT.
//For simplicity, only execsize 1 is considered.
//Since physical registers are alreasy assigned, we use the info directly here without check.

void Optimizer::reverseOffsetProp(
    AddrSubReg_Node addrRegInfo[8],
    int subReg,
    unsigned int srcNum,
    INST_LIST_ITER lastIter,
    INST_LIST_ITER iend
    )
{
    if( addrRegInfo[subReg].usedImmed && addrRegInfo[subReg].canUseImmed )
    {
        INST_LIST_ITER iter;
        G4_INST *inst;
        G4_Operand *inst_src;
        G4_DstRegRegion *inst_dst;
        for (iter = addrRegInfo[subReg].iter; iter != lastIter; ++iter)
        {
            if( iter == lastIter )
                break;
            inst = *iter;
            if( inst->isDead())
                continue;
            inst_dst = inst->getDst();
            if( inst_dst &&
                inst_dst->getRegAccess() != Direct )
            {
                int subReg1 = getDstSubReg( inst_dst );

                short currOff = inst_dst->getAddrImm();
                if( subReg1 == subReg )
                {
                    // create a new dst
                    G4_DstRegRegion tmpRgn( *inst_dst );
                    G4_DstRegRegion *newDst = &tmpRgn;
                    newDst->setImmAddrOff( short(currOff - addrRegInfo[subReg].immAddrOff) );
                    inst->setDest( builder.createDstRegRegion(*newDst) );
                }
            }
            for (unsigned i = 0; i < G4_Inst_Table[inst->opcode()].n_srcs; i++)
            {
                inst_src = inst->getSrc(i);
                if( inst_src && inst_src->isSrcRegRegion() &&
                    inst_src->asSrcRegRegion()->getRegAccess() != Direct )
                {
                    int subReg1 = getSrcSubReg( inst_src );

                    short currOff = inst_src->asSrcRegRegion()->getAddrImm();
                    if( subReg1 == subReg )
                    {
                        G4_SrcRegRegion tmpRgn(*inst_src->asSrcRegRegion());
                        G4_SrcRegRegion *newSrc = &tmpRgn;
                        newSrc->setImmAddrOff( short(currOff - addrRegInfo[subReg].immAddrOff) );
                        inst->setSrc( builder.createSrcRegRegion(*newSrc) , i );
                    }
                }
            }
        }
        // Immed has been propagated to srcs before this src in *ii, also reverse this
        if( srcNum > 0 )
        {
            inst = *lastIter;
            for (unsigned i = 0; i < srcNum; i++)
            {
                inst_src = inst->getSrc(i);
                if( inst_src && inst_src->isSrcRegRegion() &&
                    inst_src->asSrcRegRegion()->getRegAccess() != Direct )
                {
                    int subReg1 = getSrcSubReg( inst_src );

                    short currOff = inst_src->asSrcRegRegion()->getAddrImm();
                    if( subReg1 == subReg )
                    {
                        G4_SrcRegRegion tmpRgn(*inst_src->asSrcRegRegion());
                        G4_SrcRegRegion *newSrc = &tmpRgn;
                        newSrc->setImmAddrOff( short(currOff - addrRegInfo[subReg].immAddrOff) );
                        inst->setSrc( builder.createSrcRegRegion(*newSrc) , i );
                    }
                }
            }
        }
    }

    addrRegInfo[subReg].immAddrOff = 0;
    addrRegInfo[subReg].iter = iend;
    addrRegInfo[subReg].canRemoveInst = false;
    addrRegInfo[subReg].canUseImmed = false;
    addrRegInfo[subReg].usedImmed = false;
}

void Optimizer::FoldAddrImmediate()
{
    AddrSubReg_Node* addrRegInfo = new AddrSubReg_Node[getNumAddrRegisters()];
    BB_LIST_ITER ib, bend(fg.BBs.end());
    int dst_subReg = 0, src0_subReg = 0;
    G4_DstRegRegion *dst;
    G4_Operand *src0, *src1;
    unsigned num_srcs;

    for(ib = fg.BBs.begin(); ib != bend; ++ib)
    {
        G4_BB* bb = (*ib);
        INST_LIST_ITER ii, iend(bb->end());
        // reset address offset info
        for(unsigned i = 0; i < getNumAddrRegisters(); i++)
        {
            addrRegInfo[i].subReg = 0;
            addrRegInfo[i].immAddrOff = 0;
            addrRegInfo[i].iter = iend;
            addrRegInfo[i].canRemoveInst = false;
            addrRegInfo[i].canUseImmed = false;
            addrRegInfo[i].usedImmed = false;
        }
        for (ii = bb->begin(); ii != iend; ii++)
        {
            G4_INST *inst = *ii;
            if(inst->isDead())
            {
                continue;
            }
            num_srcs = G4_Inst_Table[inst->opcode()].n_srcs;
            dst = inst->getDst();
            if (dst)
            {
                dst_subReg = getDstSubReg( dst );
            }
            src0 = inst->getSrc(0);
            if( src0 && src0->isSrcRegRegion() )
            {
                src0_subReg = getSrcSubReg( src0 );
            }
            src1 = inst->getSrc(1);

            if (dst != NULL && dst->getRegAccess() == Direct &&
                src0 != NULL && src0->isSrcRegRegion() && src0->asSrcRegRegion()->getRegAccess() == Direct &&
                dst->isAreg() && dst->isA0() &&
                src0->isAreg() && src0->asSrcRegRegion()->isA0() &&
                src1==NULL)
            {
                continue;
            }

            if (inst->opcode() == G4_add && inst->getExecSize() == 1 &&
                inst->getPredicate() == NULL &&
                (src1->isImm() && !src1->isRelocImm()) &&
                dst && dst->getRegAccess() == Direct &&
                src0 && src0->isSrcRegRegion() && src0->asSrcRegRegion()->getRegAccess() == Direct &&
                dst->isAreg() && dst->isA0() &&
                src0->isAreg() && src0->asSrcRegRegion()->isA0() &&
                dst_subReg == src0_subReg )
            {
                // since there is use of a0.x here, we can not remove the former def of a0.x
                // reverse immed offset propagation
                reverseOffsetProp( addrRegInfo, dst_subReg, 0, ii, iend );

                int64_t offset = src1->asImm()->getImm();
                if( offset >= -512 && offset <= 511 && offset%0x20 == 0)
                {
                    // this kills the previous def on a0.x
                    if( addrRegInfo[dst_subReg].canRemoveInst && addrRegInfo[dst_subReg].iter != iend )
                    {
                        // mark dead
                        (*(addrRegInfo[dst_subReg].iter))->markDead();
                    }
                    addrRegInfo[dst_subReg].subReg = dst_subReg;
                    addrRegInfo[dst_subReg].immAddrOff = (short)offset;
                    addrRegInfo[dst_subReg].iter = ii;
                    addrRegInfo[dst_subReg].canRemoveInst = true;
                    addrRegInfo[dst_subReg].canUseImmed = true;
                    addrRegInfo[dst_subReg].usedImmed = false;
                }
            }
            else
            {
                G4_Operand *src;
                // if there is any direct use of addr reg, the ADD inst can not be removed
                for (unsigned i = 0; i < num_srcs; i++)
                {
                    src = inst->getSrc(i);
                    if( src && src->isSrcRegRegion() && src->asSrcRegRegion()->getRegAccess() == Direct &&
                        src->asSrcRegRegion()->isAreg() && src->asSrcRegRegion()->isA0())
                    {
                        // TODO: show if an inst is generated for spill code
                        // if there is no regVar for this srcRegion, the physical register is hard-wired in input or generated by spillCode.
                        // in this case, the subregister info is in the subRegOff of G4_SrcRegRegion
                        // this also applies to dst register
                        int subReg = getSrcSubReg( src );

                        // it is possible that several elements are used
                        int width, hstride, vstride, outerloop = 1;
                        width = src->asSrcRegRegion()->getRegion()->width;
                        hstride = src->asSrcRegRegion()->getRegion()->horzStride;
                        vstride = src->asSrcRegRegion()->getRegion()->vertStride;
                        if( vstride != 0 )
                        {
                            outerloop = inst->getExecSize() / vstride;
                        }

                        for( int k = 0; k < outerloop; k++ )
                        {
                            for( int j = 0; j < width; j++ )
                            {
                                int currSubreg = subReg + k * vstride + j * hstride;
                                // there may be inst whose src or dst addr immediate offset are already changed
                                // reverse the change
                                reverseOffsetProp( addrRegInfo, currSubreg, i, ii, iend );
                            }
                        }
                    }
                }
                // use of address register in index region
                for(unsigned i = 0; i < num_srcs; i++)
                {
                    src = inst->getSrc(i);
                    if( src && src->isSrcRegRegion() && src->asSrcRegRegion()->getRegAccess() != Direct)
                    {
                        int subReg = getSrcSubReg( src );

                        // if VxH is used and more than one sub registers are used in addressing
                        // do not fold the immediate even though they have the same immediate value
                        unsigned short vertStride = src->asSrcRegRegion()->getRegion()->vertStride;
                        if( vertStride == UNDEFINED_SHORT ||
                            (vertStride > 0 && (unsigned short)inst->getExecSize() / vertStride > 1 ))
                        {
                            int numSubReg = 0;
                            if( vertStride == UNDEFINED_SHORT )
                            {
                                numSubReg = inst->getExecSize()/src->asSrcRegRegion()->getRegion()->width;
                            }
                            else
                            {
                                numSubReg = 1; //inst->getExecSize()/vertStride;
                            }
                            for( int j = subReg; j < subReg + numSubReg; j++ )
                            {
                                reverseOffsetProp( addrRegInfo, j, i, ii, iend );
                            }
                        }
                        else
                        {
                            // we check the existing address reg imm offset.
                            short currOff = src->asSrcRegRegion()->getAddrImm();
                            if( addrRegInfo[subReg].canUseImmed )
                            {
                                if( currOff % 0x20 == 0 &&
                                    (currOff + addrRegInfo[subReg].immAddrOff) <= 511 &&
                                    (currOff + addrRegInfo[subReg].immAddrOff) >= -512)
                                {
                                    G4_SrcRegRegion tmpRgn(*src->asSrcRegRegion());
                                    G4_SrcRegRegion* newSrc = &tmpRgn;
                                    newSrc->setImmAddrOff( short(currOff + addrRegInfo[subReg].immAddrOff) );
                                    inst->setSrc( builder.createSrcRegRegion(*newSrc), i );

                                    addrRegInfo[subReg].usedImmed = true;
                                }
                                else
                                {
                                    // if the offset can not be folded into all uses of a0.0, reverse the former folding
                                    reverseOffsetProp( addrRegInfo, subReg, i, ii, iend );
                                }
                            }
                        }
                    }
                }
                if (dst)
                {
                    // make sure the addr reg is not redefined
                    // direct access to a0.x
                    if( dst->isAreg() && dst->getRegAccess() == Direct && dst->isA0() )
                    {
                        int width, hstride;
                        width = inst->getExecSize();
                        hstride = dst->getHorzStride();

                        for( int j = 0; j < width; j++ )
                        {
                            int currSubreg = dst_subReg + j * hstride;
                            // this kills the previous def on a0.x
                            if( addrRegInfo[currSubreg].iter != iend && addrRegInfo[currSubreg].canRemoveInst )
                            {
                                // mark dead
                                (*(addrRegInfo[currSubreg].iter))->markDead();
                            }
                            addrRegInfo[currSubreg].immAddrOff = 0;
                            addrRegInfo[currSubreg].iter = iend;
                            addrRegInfo[currSubreg].canRemoveInst = false;
                            addrRegInfo[currSubreg].canUseImmed = false;
                            addrRegInfo[currSubreg].usedImmed = false;
                        }
                    }
                    // check if dst is indirectly addressed
                    else if( dst->getRegAccess() != Direct )
                    {
                        short currOff = dst->getAddrImm();
                        if( addrRegInfo[dst_subReg].canUseImmed )
                        {
                            if( currOff % 0x20 == 0 &&
                                (currOff + addrRegInfo[dst_subReg].immAddrOff) <= 511 &&
                                (currOff + addrRegInfo[dst_subReg].immAddrOff) >= -512 )
                            {
                                // create a new dst
                                G4_DstRegRegion tmpRgn( *dst );
                                G4_DstRegRegion *newDst = &tmpRgn;
                                newDst->setImmAddrOff( short(currOff + addrRegInfo[dst_subReg].immAddrOff) );
                                inst->setDest( builder.createDstRegRegion( *newDst ) );
                                addrRegInfo[dst_subReg].usedImmed = true;
                            }
                            else
                            {
                                // if the offset can not be folded into all uses of a0.0, reverse the former folding
                                reverseOffsetProp( addrRegInfo, dst_subReg, 0, ii, iend );
                            }
                        }
                    }
                }
            }
        }

        // if a def lives out of this BB, we can not delete the defining inst
        for (unsigned i = 0; i < getNumAddrRegisters(); i++)
        {
            // reverse immed offset propagation
            reverseOffsetProp( addrRegInfo, i, 0, iend, iend );
        }
        // remove the ADD instructions that marked as dead
        for(ii = bb->begin(); ii != bb->end();)
        {
            G4_INST *inst = *ii;
            INST_LIST_ITER curr = ii++;
            if( inst->isDead() )
            {
                bb->erase(curr);
            }
        }
    }

    delete [] addrRegInfo;
}
G4_SrcModifier Optimizer::mergeModifier( G4_Operand *src, G4_Operand *use )
{
    if( ( src == NULL || !src->isSrcRegRegion() )  && use && use->isSrcRegRegion() ){
        return use->asSrcRegRegion()->getModifier();
    }else if( ( use == NULL || !use->isSrcRegRegion() ) && src && src->isSrcRegRegion() ){
        return src->asSrcRegRegion()->getModifier();
    }else if( src && src->isSrcRegRegion() && use && use->isSrcRegRegion() ){
        G4_SrcModifier mod1 = src->asSrcRegRegion()->getModifier(), mod2 = use->asSrcRegRegion()->getModifier();
        if( mod2 == Mod_Abs || mod2 == Mod_Minus_Abs ){
            return mod2;
        }else if( mod2 == Mod_src_undef ){
            return mod1;
        }else{
            // mod2 == Minus
            if( mod1 == Mod_Minus ){
                return Mod_src_undef;
            }else if( mod1 == Mod_Abs ){
                return Mod_Minus_Abs;
            }else if( mod1 == Mod_Minus_Abs ){
                return Mod_Abs;
            }else{
                return mod2;
            }
        }
    }else{
        return Mod_src_undef;
    }
}

// Prevent sinking in presence of lifetime.end for any src op.
// For eg,
// add V33, V32, 0x1
// ...
// lifetime.end V32 <-- This prevents sinking of add to mov
// ...
// pseudo_kill V34 <-- This prevents hoisting of V34 to add dst
// mov V34, V33
//
static bool checkLifetime(G4_INST *defInst, G4_INST *inst)
{
    // Check whether current instruction ends any src opnd of op
    if (inst->opcode() != G4_pseudo_lifetime_end)
        return true;

    G4_RegVar *Var = GetTopDclFromRegRegion(inst->getSrc(0))->getRegVar();
    // Check whether lifetime op corresponds to any operand of current inst.
    if (defInst->getPredicate())
    {
        G4_RegVar *opndVar = defInst->getPredicate()->asPredicate()->getBase()->asRegVar();
        if (opndVar == Var)
            return false;
    }
    if (defInst->getCondMod())
    {
        G4_RegVar *opndVar = defInst->getCondMod()->asCondMod()->getBase()->asRegVar();
        if (opndVar == Var)
            return false;
    }
    if (defInst->getDst() &&
        !defInst->getDst()->isNullReg())
    {
        G4_RegVar *opndVar = GetTopDclFromRegRegion(defInst->getDst())->getRegVar();
        if (opndVar == Var)
            return false;
    }
    for (unsigned int srcOpnd = 0; srcOpnd < G4_MAX_SRCS; srcOpnd++)
    {
        G4_Operand *src = defInst->getSrc(srcOpnd);
        if (src && src->isSrcRegRegion())
        {
            G4_RegVar *opndVar = GetTopDclFromRegRegion(src)->getRegVar();
            if (opndVar == Var)
                return false;
        }
    }

    return true;
}

//
// Sink definition towards its use.
//
// For example, without sinking the def instruction once, use cannot
// be hoisted, since there is a data dependency between the middle
// instruction and the last move.
//
// def: shr (1) V39(0,0)<1>:ud V38(0,0)<0;1,0>:d 0x4:w {Align1, Q1}
//      mov (8) V68(0,0)<1>:ud r0.0<8;8,1>:ud {Align1, NoMask}
// use: mov (1) V68(0,2)<1>:ud V39(0,0)<0;1,0>:ud {Align1, NoMask}
//
// after sinking, it becomes
//
//      mov (8) V68(0,0)<1>:ud r0.0<8;8,1>:ud {Align1, NoMask}
// def: shr (1) V39(0,0)<1>:ud V38(0,0)<0;1,0>:d 0x4:w {Align1, Q1}
// use: mov (1) V68(0,2)<1>:ud V39(0,0)<0;1,0>:ud {Align1, NoMask}
//
// which makes local def hoisting possible.
//
// The third argument 'other' points to the first instruction (upwards) that
// has data-dependency with the use instruction.
//
static bool canSink(G4_BB *bb, INST_LIST_RITER revIter, INST_LIST_RITER other)
{
    // The use instruction.
    G4_INST *inst = *revIter;

    // Currently we do not handle multiple definition for this optimization.
    if (inst->def_size() != 1)
        return false;

    // Find its def instruction.
    G4_INST *defInst = inst->def_back().first;

    ASSERT_USER(*other != defInst, "iterator points to def already");

    // Walk up to check if sinking is safe.
    INST_LIST_RITER it = other;

    while (*it != defInst)
    {
       if ((*it)->isWAWdep(defInst) ||
           (*it)->isRAWdep(defInst) ||
           (*it)->isWARdep(defInst))
           return false;

       if (!checkLifetime(defInst, *it))
           return false;

       // move towards to defInst.
       ++it;
    }

    // At this point, there is no data dependency and sinking is safe.
    //
    // We do sinking right here.
    //
    ASSERT_USER(*it == defInst, "iterator out of sync");

    // Both 'other' and 'it' are reverse iterators, and sinking is through
    // forward iterators. The fisrt base should not be decremented by 1,
    // otherwise, the instruction will be inserted before not after.
    bb->insert(other.base(), defInst);
    bb->erase(--it.base());

    return true;
}

static bool canHoist(FlowGraph &fg, G4_BB *bb, INST_LIST_RITER revIter)
{
    G4_INST *inst = *revIter;

    if (inst->isMixedMode() && fg.builder->getOption(vISA_DisableleHFOpt))
        return false;
    // Cannot hoist if this is not a move, or it is a global operand.
    if  (inst->opcode() != G4_mov ||
         fg.globalOpndHT.isOpndGlobal(inst->getSrc(0)) ||
         !inst->canHoist(bb->isInSimdFlow(), fg.builder->getOptions()))
    {
         return false;
    }

    // Do not do def-hoisting for seeting flags which is likely to increase flag
    // register pressure.
    if (auto Dst = inst->getDst())
    {
        G4_Declare *Dcl = Dst->getTopDcl();
        if (Dcl && Dcl->getRegFile() == G4_RegFileKind::G4_FLAG)
        {
            return false;
        }
    }

    G4_Operand *src = inst->getSrc(0);
    unsigned srcLB = src->getLeftBound();
    unsigned srcRB = src->getRightBound();
    unsigned bitVec[2] = {0, 0};

    // Now check each definition of src(0)
    for (auto I = inst->def_begin(), E = inst->def_end(); I != E; ++I)
    {
        ASSERT_USER(I->second == Opnd_src0, "invalid use-def chain");
        if (!inst->canHoistTo(I->first, bb->isInSimdFlow(), fg.builder))
            return false;

        // Further check data-dependency, that is, no other instruction
        // should have no WAR or WAW dependency.
        //
        //   defInst
        //
        //   other inst
        //
        //   inst <-- revIter
        //
        INST_LIST_RITER other = revIter;
        ++other;

        // Measure the distance in between
        unsigned distance = 0;

        // Walkup until hits its defining instruction.
        while (*other != I->first)
        {
            // FIXME: remove duplicate computations for multiple definitions.
            if (inst->isWAWdep(*other) || inst->isWARdep(*other))
            {
                break;
            }
            ++other;
            ++distance;
        }

        // Check the distance first, if this is too far then the following
        // sinking optimization is every expensive.
        #define MAX_DEF_HOIST_DIST 160
        if (distance > MAX_DEF_HOIST_DIST)
            return false;

        // There is a data dependency.
        if (*other != I->first)
        {
            // check if sinking is possible.
            if (!canSink(bb, revIter, other))
                return false;
        }

        // Check if this def-touched region is covered by the source region.
        mergeBitVec(bitVec, I->first->getDst(), srcLB, srcRB);
        if ((bitVec[0] & (~src->getBitVecL())) ||
            (bitVec[1] & (~src->getBitVecH())))
        {
            return false;
        }
    }

    // Check if the combined region touched by all defintions is exactly
    // the same as the source region.
    return (bitVec[0] == src->getBitVecL()) && (bitVec[1] == src->getBitVecH());
}

static G4_DstRegRegion *buildNewDstOperand(FlowGraph &fg, G4_INST *inst, G4_INST *defInst)
{
    G4_Operand *src = inst->getSrc(0);
    G4_DstRegRegion *dst = inst->getDst();

    G4_Type srcType = src->getType();
    G4_Type dstType = dst->getType();
    G4_DstRegRegion *dstRegion = dst;
    bool indirectDst = (dstRegion->getRegAccess() != Direct);
    unsigned char srcElSize = (unsigned char)G4_Type_Table[srcType].byteSize;

    G4_DstRegRegion *defDstRegion = defInst->getDst();
    G4_DstRegRegion *newDstOpnd = dst;

    unsigned char defDstElSize = (unsigned char)G4_Type_Table[defDstRegion->getType()].byteSize;
    G4_CmpRelation rel = src->compareOperand(defDstRegion);
    G4_Type defDstType = defDstRegion->getType();

    unsigned char dstElSize = (unsigned char)G4_Type_Table[dstType].byteSize;
    unsigned short dstHS = dst->getHorzStride();

    if (rel == Rel_gt || srcElSize != defDstElSize ||
        (defInst->getSaturate() && srcType != defDstType) || inst->isRawMov() ||
        (dstType != defDstType &&
         (IS_FTYPE(defDstType) || (IS_FTYPE(dstType) && defDstType != srcType))))
    {
        unsigned short regOff = 0, subRegOff = 0;
        if (rel == Rel_gt)
        {
            // compute new dst for defInst
            // get dst portion based on src region
            unsigned defDstLB = defDstRegion->getLeftBound();

            unsigned srcLB = src->getLeftBound();
            RegionDesc *srcRegionDesc = src->asSrcRegRegion()->getRegion();
            bool contRegion = srcRegionDesc->isContiguous(inst->getExecSize());

            uint32_t dist = defDstLB - srcLB, dstDist = 0, tempLen = 0;
            if (src->asSrcRegRegion()->isScalar() || contRegion)
            {
                // mov (1) V18(0,0)[1]:b 0x73:w [Align1]
                // mov (1) V18(0,1)[1]:b 0x61:w [Align1]
                // mov (1) V18(0,2)[1]:b 0x70:w [Align1]
                // mov (1) V18(0,3)[1]:b 0:w [Align1]
                // mov (1) V20(1,0)[1]:ud V18(0,0)[0;1,0]:ud [Align1]
                // length of subregoff part
                tempLen = dstRegion->getSubRegOff() * dstElSize + dist * dstHS;

                if (tempLen >= GENX_GRF_REG_SIZ)
                {
                    regOff = dst->getRegOff() + 1;
                    subRegOff = (unsigned short)((tempLen - GENX_GRF_REG_SIZ) / defDstElSize);
                }
                else
                {
                    regOff = dst->getRegOff();
                    subRegOff = (unsigned short)tempLen / defDstElSize;
                }
            }
            else
            {
                // mov (16) V18(0,0)[1]:b 0x73:w [Align1]
                // mov (16) V18(0,16)[1]:b 0x61:w [Align1]
                // mov (16) V18(1,0)[1]:b 0x70:w [Align1]
                // mov (16) V18(1,16)[1]:b 0:w [Align1]
                // mov (32) V20(1,0)[1]:b V18(0,0)[32;16,1]:b [Align1]
                // mov (32) V20(2,0)[1]:b V18(0,16)[32;16,1]:b [Align1]

                // Compute the linear index of the first element from defInst's dst
                // in useInst's src.
                //
                // mov <2> V50(0, 14)<1>:b 0xa:w                  <- defInst
                // mov <16> V51(0, 9)<2>:b V50(0, 0)<0; 16, 2>:b  <- useInst
                //
                // Starting from left bound difference, dist = 14.
                //
                // FirstEltIndex is 7 = 14 / 2. With this index, we can compute
                // the register offset and sub-register offset in useInst's dst.
                //
                // In the above example, there is only a single row. In general
                // there may have multiple rows in useInst's src region.
                //
                // (1) convert difference in number of elements.
                MUST_BE_TRUE(dist % srcElSize == 0, "unexpected difference");
                dist = dist / srcElSize;

                // (2) compute row and column index, by default a single row.
                unsigned rowIndex = 0, colIndex = dist;
                if (srcRegionDesc->vertStride > 0)
                {
                    rowIndex = dist / srcRegionDesc->vertStride;
                    colIndex = dist % srcRegionDesc->vertStride;
                }

                // (3) compute the final linear index.
                MUST_BE_TRUE(srcRegionDesc->horzStride == 0 || colIndex % srcRegionDesc->horzStride == 0, "invalid region");
                unsigned FirstEltIndex = rowIndex * srcRegionDesc->width +
                                         (srcRegionDesc->horzStride == 0 ? colIndex : (colIndex / srcRegionDesc->horzStride));

                // (4) compute the register and subregister offet in useInst's dst.
                dstDist = FirstEltIndex * dstElSize * dstHS;
                tempLen = dstDist + dst->getSubRegOff() * dstElSize;
                regOff = (unsigned short)(dst->getRegOff() +
                    tempLen / GENX_GRF_REG_SIZ);

                subRegOff = (unsigned short)(tempLen % GENX_GRF_REG_SIZ) / defDstElSize;
            }

            unsigned short defDstHS = defDstRegion->getHorzStride();
            newDstOpnd = fg.builder->createDstRegRegion(
                dst->getRegAccess(),
                dst->getBase(),
                indirectDst ? dst->getRegOff() : regOff,
                indirectDst ? dst->getSubRegOff() : subRegOff,
                dstHS * defDstHS, defDstRegion->getType());
            newDstOpnd->setImmAddrOff(dst->getAddrImm() +
                (indirectDst ? (short)tempLen : 0));
        }
        else
        {
            unsigned char scale = dstElSize / defDstElSize;

            // If instruction that gets def hoisted is just re-interpretation of bits,
            // doesn't do conversion, then I think original type of the defInst
            // should be used. This preserves the original behavior.
            //
            // mov (8) V57(0,0)[1]:hf V52(0,0)[8;8,1]:f [Align1, Q1] %21
            // mov (8) V58(0,0)[1]:w  V59(0,0)[8;8,1]:w [Align1, Q1] %22
            if (dst->getType() == src->getType())
            {
                newDstOpnd = fg.builder->createDstRegRegion(
                    dst->getRegAccess(),
                    dst->getBase(),
                    dst->getRegOff(),
                    indirectDst
                    ? dst->getSubRegOff()
                    : (scale == 0
                    ? dst->getSubRegOff() /
                    (defDstElSize / dstElSize)
                    : dst->getSubRegOff() * scale),
                    dstHS, defDstRegion->getType());
                newDstOpnd->setImmAddrOff(dst->getAddrImm());
            }
            else
            {
                newDstOpnd = fg.builder->createDstRegRegion(dst->getRegAccess(),
                    dst->getBase(),
                    dst->getRegOff(),
                    dst->getSubRegOff(), dstHS,
                    dst->getType());
                newDstOpnd->setImmAddrOff(dst->getAddrImm());

            }
        }
    }
    return newDstOpnd;
}

//
//  inst0:   op0  rx<2>(0), ry0, ry1
//  inst1:   op1  rx<2>(1), ry2, ry3
//  inst2:   mov  rz<1>(0), rx<0, 0>(8; 8, 1)
//
// ==>
//
//  inst0:   op0, rz<2>(0), ry0, ry1
//  inst1:   op1, rz<2>(1), ry2, ry3
//  inst2:   mov  rz<1>(0), rx<0, 0>(8; 8, 1)  (to be deleted)
//
// Def-use/use-def chains will be updated as follows:
//
// (0) all use-defs remain the same for inst0 and inst1.
//
// (1) remove all use-defs of inst2. (They must be from inst0 and inst1,
//     which is the pre-condition of doHoisting.)
//
// (2) remove all def-uses of inst0 and inst1 from dst.
//
// (3) remove all def-uses of inst2.
//
// (4) add new def-uses to inst0 and inst1.
//
static void doHoisting(FlowGraph &fg, G4_BB *bb, INST_LIST_RITER revIter)
{
    G4_INST *inst = *revIter;

    for (auto I = inst->def_begin(), E = inst->def_end(); I != E; ++I)
    {
        G4_INST *defInst = I->first;

        // Build a new dst operand for each def instruction.
        G4_DstRegRegion *newDst = buildNewDstOperand(fg, inst, defInst);

        // Update the defInst with a new operand and set attributes properly.
        if (inst->def_size() == 1)
        {
            defInst->setDest(newDst);
        }
        else
        {
            defInst->setDest(fg.builder->duplicateOperand(newDst)->asDstRegRegion());
        }

        // (4) for each def-use of inst, add it to defInst, if it is
        // an effective use.
        for (auto UI = inst->use_begin(), UE = inst->use_end(); UI != UE; ++UI)
        {
            G4_Operand *UseOpnd = UI->first->getOperand(UI->second);
            // this comparison is necessary, since uses of inst's dst may be
            // different from those from defInst's dst.
            G4_CmpRelation rel = defInst->getDst()->compareOperand(UseOpnd);
            if (rel != Rel_disjoint)
            {
                defInst->addDefUse(UI->first, UI->second);
            }
        }

        if (inst->getPredicate())
        {
            ASSERT_USER(inst->def_size() == 1, "multiple defs not implemented");
            // Remove existing definitions on defInst[opnd_pred].
            defInst->removeDefUse(Opnd_pred);

            defInst->setPredicate(inst->getPredicate());

            // (4) Transfer definitions of inst[opnd_pred] to definitions of
            // defInst[opnd_pred].
            inst->transferDef(defInst, Opnd_pred, Opnd_pred);
        }
        if (inst->getSrc(0)->asSrcRegRegion()->isScalar() &&
            inst->getExecSize() > 1)
        {
            defInst->setExecSize(defInst->getExecSize() * inst->getExecSize());
        }
        defInst->setSaturate(inst->getSaturate() || defInst->getSaturate());
        if (bb->isInSimdFlow())
        {
            // set writeEnable of dstInst to be off
            defInst->setOptions((defInst->getOption() & ~0xFFF000C) |
                                (inst->getMaskOption()));
        }
    }

    // (1), (2), (3) Remove all defs/uses and it is ready to be deleted.
    inst->removeAllDefs();
    inst->removeAllUses();
}

void Optimizer::newLocalDefHoisting()
{
    unsigned numDefHoisted = 0;
    for (auto bb : fg.BBs)
    {
        for (auto I = bb->rbegin(); I != bb->rend(); /* empty */)
        {
            if (canHoist(fg, bb, I))
            {
                doHoisting(fg, bb, I);
                ++numDefHoisted;

                // list::erase does not take a reverse_iterator.
                //
                // The base iterator is an iterator of the same type as the one
                // used to construct the reverse_iterator, but pointing to the
                // element next to the one that the reverse_iterator is currently
                // pointing to (a reverse_iterator has always an offset of -1
                // with respect to its base iterator).
                I = INST_LIST::reverse_iterator(bb->erase(--I.base()));
            }
            else
            {
                ++I;
            }
        }
    }

    if (builder.getOption(vISA_OptReport))
    {
        std::ofstream optReport;
        getOptReportStream(optReport, builder.getOptions());
        optReport << "             === Local Definition Hoisting Optimization ===\n";
        optReport << "Number of defs hoisted: " << numDefHoisted << "\n";
        closeOptReportStream(optReport);
    }
}

//  find a common (integer) type for constant folding.  The rules are:
//  -- both types must be int
//  -- Q and UQ are not folded
//  -- UD if one of the type is UD
//  -- D otherwise
//
//  returns Type_UNDEF if no appropriate type can be found
//
static G4_Type findConstFoldCommonType( G4_Type type1, G4_Type type2 )
{
    if (IS_TYPE_INT(type1) && IS_TYPE_INT(type2))
    {
        if (G4_Type_Table[type1].byteSize == 8 || G4_Type_Table[type2].byteSize == 8)
        {
            return Type_UNDEF;
        }
        if (type1 == Type_UD || type2 == Type_UD)
        {
            return Type_UD;
        }
        else
        {
            return Type_D;
        }
    }
    return Type_UNDEF;
}

// Do the following algebraic simplification:
// - mul v, src0, 0 ==> 0, commutative
// - and v, src0, 0 ==> 0, commutative
// - mul v, src0, 1 ==> src0, commutative
// - shl v, src0, 0 ==> src0
// - shr v, src0, 0 ==> src0
// - asr v, src0, 0 ==> src0
// - add v, src0, 0 ==> src0, commutative
void Optimizer::doSimplification(G4_INST *inst)
{
    // Just handle following commonly used ops for now.
    if (inst->opcode() != G4_mul && inst->opcode() != G4_and &&
        inst->opcode() != G4_add && inst->opcode() != G4_shl &&
        inst->opcode() != G4_shr && inst->opcode() != G4_asr &&
        inst->opcode() != G4_mov)
    {
        return;
    }


    // Perform 'mov' to 'movi' transform when it's a 'mov' of
    // - simd8
    // - it's a raw mov
    // - dst is within a single GRF.
    // - src uses VxH indirect access.
    // - src is within one GRF.
    // - indices to src are all within src.
    if (inst->opcode() == G4_mov && inst->getExecSize() == 8 &&
        inst->isRawMov() && inst->getDst() &&
        !inst->getDst()->asDstRegRegion()->isCrossGRFDst() &&
        inst->getSrc(0) && inst->getSrc(0)->isSrcRegRegion() &&
        inst->getSrc(0)->asSrcRegRegion()->isIndirect() &&
        inst->getSrc(0)->asSrcRegRegion()->getRegion()->isRegionWH() &&
        inst->getSrc(0)->asSrcRegRegion()->getRegion()->width == 1 &&
        // destination stride in bytes must be equal to the source element size
        // in bytes.
        getTypeSize(inst->getSrc(0)->getType()) ==
        getTypeSize(inst->getDst()->getType()) *
        inst->getDst()->asDstRegRegion()->getHorzStride()) {
        // Convert 'mov' to 'movi' if the following conditions are met.

        auto getSingleDefInst = [](G4_INST *UI,
                                   Gen4_Operand_Number OpndNum)
            -> G4_INST * {
            G4_INST *Def = nullptr;
            for (auto I = UI->def_begin(), E = UI->def_end(); I != E; ++I) {
                if (I->second != OpndNum)
                    continue;
                if (Def) {
                    // Not single defined, bail out
                    Def = nullptr;
                    break;
                }
                Def = I->first;
            }
            return Def;
        };

        unsigned SrcSizeInBytes = inst->getExecSize() *
                getTypeSize(inst->getSrc(0)->getType());
        if (SrcSizeInBytes == G4_GRF_REG_NBYTES/2 ||
            SrcSizeInBytes == G4_GRF_REG_NBYTES) {
            G4_INST *LEA = getSingleDefInst(inst, Opnd_src0);
            if (LEA && LEA->opcode() == G4_add &&
                LEA->getExecSize() == inst->getExecSize()) {
                G4_Operand *Op0 = LEA->getSrc(0);
                G4_Operand *Op1 = LEA->getSrc(1);
                G4_Declare *Dcl = nullptr;
                int Offset = 0;
                if (Op0->isAddrExp()) {
                    G4_AddrExp *AE = Op0->asAddrExp();
                    Dcl = AE->getRegVar()->getDeclare();
                    Offset = AE->getOffset();
                }
                if (Dcl && (Offset % SrcSizeInBytes) == 0 &&
                    Op1->isImm() && Op1->getType() == Type_UV) {
                    // Immeidates in 'uv' ensures each element is a
                    // byte-offset within half-GRF.
                    G4_SubReg_Align SubAlign = Sixteen_Word;
                    if (SrcSizeInBytes <= G4_GRF_REG_NBYTES/2)
                        SubAlign = Eight_Word;
                    inst->setOpcode(G4_movi);
                    if (Dcl->getAlign() == Either &&
                        Dcl->getSubRegAlign() != Sixteen_Word) {
                        Dcl->setSubRegAlign(SubAlign);
                    }
                    RegionDesc *rd = builder.createRegionDesc(8, 8, 1);
                    inst->getSrc(0)->asSrcRegRegion()->setRegion(rd);
                    // Set subreg alignment for the address variable.
                    Dcl =
                        LEA->getDst()->getBase()->asRegVar()->getDeclare();
                    assert(Dcl->getRegFile() == G4_ADDRESS &&
                           "Address variable is required.");
                    Dcl->setSubRegAlign(Eight_Word);
                }
            }
        }
    }

    auto isInteger = [](G4_Operand *opnd, int64_t val)
    {
        if (opnd && IS_TYPE_INT(opnd->getType()) && !opnd->isRelocImm())
        {
            return opnd->isImm() && opnd->asImm()->getInt() == val;
        }
        return false;
    };

    G4_Operand *src0 = inst->getSrc(0);
    G4_Operand *src1 = inst->getSrc(1);
    G4_Operand *newSrc = nullptr;
    if (inst->opcode() == G4_mul || inst->opcode() == G4_and)
    {
        if (isInteger(src1, 0))
        {
            inst->removeDefUse(Opnd_src0);
            newSrc = builder.createImm(0, Type_W);
        }
        else if (isInteger(src0, 0))
        {
            inst->removeDefUse(Opnd_src1);
            newSrc = builder.createImm(0, Type_W);
        }
        else if (inst->opcode() == G4_mul)
        {
            if (isInteger(src1, 1))
            {
                newSrc = src0;
            }
            else if (isInteger(src0, 1))
            {
                inst->swapDefUse();
                newSrc = src1;
            }
        }
    }
    else if (inst->opcode() == G4_shl || inst->opcode() == G4_shr ||
             inst->opcode() == G4_asr || inst->opcode() == G4_add)
    {
        if (isInteger(src1, 0))
        {
            newSrc = src0;
        }
        else if (inst->opcode() == G4_add && isInteger(src0, 0))
        {
            inst->swapDefUse();
            newSrc = src1;
        }
    }

    if (newSrc != nullptr)
    {
        inst->setOpcode(G4_mov);
        if (newSrc != src0)
        {
            inst->setSrc(newSrc, 0);
        }
        inst->setSrc(nullptr, 1);
    }
}

//
// Do very simple const only reassociation to fold const values
// e.g.,
// V2 = V1 + K1
// V3 = V2 + K2
// -->
// V3 = V1 + (K1 + K2)
// we only search one level (+ and +, * and *) for now as more complex reassociation should be taken
// care of by IGC earlier.
// also only do it for integer type for now
//
void Optimizer::reassociateConst()
{
    for (auto BB : fg.BBs)
    {
        for (auto iter = BB->begin(), iterEnd = BB->end(); iter != iterEnd; ++iter)
        {
            G4_INST* inst = *iter;
            if (inst->opcode() != G4_add && inst->opcode() != G4_mul)
            {
                continue;
            }
            auto isSrc1Const = [](G4_INST* inst)
            {
                if (!IS_INT(inst->getDst()->getType()))
                {
                    return false;
                }
                if (!inst->getSrc(0)->isImm() && inst->getSrc(1)->isImm())
                {
                    return true;
                }
                else if (inst->getSrc(0)->isImm() && !inst->getSrc(1)->isImm())
                {
                    auto tmp = inst->getSrc(0);
                    inst->setSrc(inst->getSrc(1), 0);
                    inst->setSrc(tmp, 1);
                    inst->swapDefUse(); // swap def/use for src0 and src1
                    return true;
                }
                return false;
            };
            if (!isSrc1Const(inst))
            {
                continue;
            }
            auto src0Def = inst->getSingleDef(Opnd_src0);
            if (!src0Def)
            {
                continue;
            }

            auto isGoodSrc0Def = [isSrc1Const](G4_INST* def, G4_INST* use)
            {
                assert(use->getSrc(0)->isSrcRegRegion() && "expect src0 to be src region");
                if (def->opcode() != use->opcode())
                {
                    return false;
                }
                if (def->getSaturate() || def->getPredicate() || def->getCondMod() ||
                    def->getMaskOffset() != use->getMaskOffset())
                {
                    return false;
                }
                if (!isSrc1Const(def))
                {
                    return false;
                }
                auto useSrc = use->getSrc(0)->asSrcRegRegion();
                if (useSrc->hasModifier() || getTypeSize(def->getDst()->getType()) != getTypeSize(useSrc->getType()) ||
                    def->getDst()->compareOperand(useSrc) != Rel_eq)
                {
                    // make sure def fully defines use and have the same integer type size (signed-ness should not matter)
                    return false;
                }
                if (def->getDst()->compareOperand(def->getSrc(0)) != Rel_disjoint)
                {
                    // can't sink source if def overwrites it
                    return false;
                }
                // additionally check for the use inst that dst type size is >= src type size
                // otherwise the first add may truncate upper bits due to overflow,
                // which makes reassociation unsafe
                if (getTypeSize(useSrc->getType()) < getTypeSize(use->getDst()->getType()))
                {
                    return false;
                }

                return true;
            };

            if (isGoodSrc0Def(src0Def, inst) && !chkFwdOutputHazard(src0Def, iter))
            {
                //std::cout << "reassociate: \n";
                //src0Def->dump();
                //inst->dump();
                G4_Imm* constOne = src0Def->getSrc(1)->asImm();
                G4_Imm* constTwo = inst->getSrc(1)->asImm();
                G4_Imm* resultImm = foldConstVal(constOne, constTwo, inst->opcode());

                if (resultImm)
                {
                    inst->setSrc(builder.duplicateOperand(src0Def->getSrc(0)), 0);
                    inst->setSrc(resultImm, 1);
                    inst->removeDefUse(Opnd_src0);
                    src0Def->copyDef(inst, Opnd_src0, Opnd_src0);
                    //ToDo: remove this when DCE pass is enabled
                    if (src0Def->use_size() == 0 && !fg.globalOpndHT.isOpndGlobal(src0Def->getDst()) &&
                        !src0Def->getDst()->isIndirect())
                    {
                        src0Def->markDead();
                        src0Def->removeAllDefs();
                    }
                    //std::cout << "--> new inst:\t";
                    //inst->dump();
                }
            }
        }
        BB->erase(std::remove_if(BB->begin(), BB->end(), [](G4_INST* inst) { return inst->isDead(); }),
            BB->end());
    }

}

// helper function to fold BinOp with two immediate operands
// supported opcodes are add/mul/shl/shr/asr with int type
// returns nullptr if one of the source is not imm
G4_Imm* Optimizer::foldConstVal(G4_Imm* const1, G4_Imm* const2, G4_opcode op)
{
    bool isNonQInt = IS_TYPE_INT(const1->getType()) && IS_TYPE_INT(const2->getType()) &&
        !IS_QTYPE(const1->getType()) && !IS_QTYPE(const2->getType());

    if (!isNonQInt)
    {
        return nullptr;
    }

    G4_Type src0T = const1->getType(), src1T = const2->getType(), resultType = src0T;

    if (op == G4_add || op == G4_mul || op == G4_and)
    {
        resultType = findConstFoldCommonType(src0T, src1T);
        if (resultType == Type_UNDEF)
        {
            return nullptr;
        }

        int64_t res;
        switch (op)
        {
        case G4_add:
            res = (int64_t)(const1->getInt()) + (int64_t)(const2->getInt());
            break;

        case G4_mul:
            res = (int64_t)(const1->getInt()) * (int64_t)(const2->getInt());
            break;

        case G4_and:
            res = (int64_t)(const1->getInt()) & (int64_t)(const2->getInt());
            break;

        default:
            return nullptr;
        }

        // result type is either D or UD
        // don't fold if the value overflows D/UD
        if (!G4_Imm::isInTypeRange(res, resultType))
        {
            return nullptr;
        }
        return builder.createImmWithLowerType(res, resultType);
    }
    else
    {
        uint32_t shift = const2->getInt() & 0x1f;

        if (op == G4_shl || op == G4_shr)
        {
            uint32_t value = (uint32_t)const1->getInt();
            // set result type to D/UD as it may overflow W. If the value fits the type will be lowered later
            // source type matters here since it affects sign extension
            resultType = IS_SIGNED_INT(resultType) ? Type_D : Type_UD;
            int64_t res = op == G4_shl ?
                ((int64_t)value) << shift :
                value >> shift;
            if (!G4_Imm::isInTypeRange(res, resultType))
            {
                return nullptr;
            }

            return builder.createImmWithLowerType(res, resultType);
        }

        if (op == G4_asr)
        {
            if (IS_SIGNED_INT(resultType))
            {
                int64_t value = const1->getInt();
                int64_t res = value >> shift;
                return builder.createImmWithLowerType(res, resultType);
            }
            else
            {
                uint64_t value = const1->getInt();
                uint64_t res = value >> shift;
                return builder.createImmWithLowerType(res, resultType);
            }
        }
    }
    return nullptr;
}



// Currently constant folding is done for the following code patterns:
// - add v, imm, imm
// - mul v, imm, imm
// - shl v, imm, imm
// - shr v, imm, imm
// - asr v, imm, imm
// Restrictions:
// - operand type cannot be float or Q/UQ
// - saturation is not allowed
void Optimizer::doConsFolding( G4_INST *inst )
{
    G4_Operand *src0, *src1;

    src0 = inst->getSrc(0);
    src1 = inst->getSrc(1);

    G4_Imm *newSrc = nullptr;

    if (src0 && src0->isImm() && !src0->isRelocImm() &&
        src1 && src1->isImm() && !src1->isRelocImm() &&
        inst->getSaturate() == false)
    {
        newSrc = foldConstVal(src0->asImm(), src1->asImm(), inst->opcode());
    }

    if (newSrc != NULL)
    {
        // change instruction into a MOV
        inst->setOpcode( G4_mov );
        inst->setSrc( newSrc, 0 );
        inst->setSrc( NULL, 1 );
    }
}

static void hoistUseInst(G4_BB *bb, G4_INST *inst , INST_LIST_ITER forwardIter, bool &canRemove)
{
    // check if we can move the use inst up.
    // currently we do not handle multiple use for this optimization
    G4_INST* useInst = inst->use_front().first;
    if( inst->hasOneUse() )
    {
        forwardIter--;
        INST_LIST_ITER backwardIter = forwardIter;
        INST_LIST_ITER instListEnd = bb->end();
        while( backwardIter != instListEnd &&
               *backwardIter != useInst )
        {
            backwardIter++;
        }

        INST_LIST_ITER useInstIter = backwardIter;
        backwardIter--;
        while( backwardIter != forwardIter )
        {
            if( useInst->isWAWdep(*backwardIter) ||
                useInst->isRAWdep(*backwardIter) ||
                useInst->isWARdep(*backwardIter) )
            {
                break;
            }
            backwardIter--;
        }
        if( backwardIter != forwardIter )
        {
            canRemove = false;
        }
        else
        {
            // hoisting
            backwardIter++;
            bb->insert( backwardIter, useInst );
            bb->erase( useInstIter );
        }
    }
    else
    {
        canRemove = false;
    }
}

// conver modifier(imm)
template<class T>
static
typename std::enable_if<std::is_floating_point<T>::value, T>::type
getImmValue(T imm, G4_SrcModifier modifier)
{
    switch (modifier)
    {
    case Mod_Minus:
        return -imm;
    case Mod_Abs:
        return std::abs(imm);
    case Mod_Minus_Abs:
        return -(std::abs(imm));
    case Mod_Not:
        MUST_BE_TRUE(false, "unexpected not modifier for floating types");
        return imm;
    default:
        return imm;
    }
}

template<class T>
static
typename std::enable_if<std::is_integral<T>::value, T>::type
getImmValue(T imm, G4_SrcModifier modifier)
{
    switch (modifier)
    {
    case Mod_Minus:
        return -imm;
    case Mod_Abs:
        return std::llabs(imm);
    case Mod_Minus_Abs:
        return -(std::llabs(imm));
    case Mod_Not:
        return ~imm;
    default:
        return imm;
    }
}

// Source operand of the MOV instruction is already known being able to be
// propagated into all its uses. But, due to the dependency issue, it cannot be
// propagated. Try to propagate the type if a narrower type could be used.
static bool propagateType(IR_Builder &Builder, G4_BB *BB, G4_INST *Mov, G4_INST::MovType MT) {
    // Only propagate type if a narrower type could be used.
    if (MT != G4_INST::ZExt && MT != G4_INST::SExt)
        return false;

    G4_DstRegRegion *Dst = Mov->getDst();
    if (Dst->isIndirect())
        return false;

    // Check all propagation types are the same.
    G4_Type PT = Type_UNDEF;
    for (auto UI = Mov->use_begin(), UE = Mov->use_end(); UI != UE; ++UI) {
        auto Use = UI->first;
        auto OpndNum = UI->second;
        auto Opnd = Use->getOperand(OpndNum);
        if (!Opnd->isSrcRegRegion())
          return false;
        if (Opnd->asSrcRegRegion()->isIndirect())
          return false;
        G4_Type PropType = Use->getPropType(OpndNum, MT, Mov);
        if (PropType == Type_UNDEF)
          return false;
        if (PT != Type_UNDEF && PT != PropType)
          return false;
        PT = PropType;
    }
    if (PT == Type_UNDEF)
        return false;
    // Create a new destination of MOV of the propagation type.
    unsigned NumElt = Mov->getExecSize();
    auto NewDcl = Builder.createTempVar(NumElt, PT, Either, Any);
    auto NewDst = Builder.Create_Dst_Opnd_From_Dcl(NewDcl, Dst->getHorzStride());
    Mov->setDest(NewDst);
    // Propagate type
    for (auto UI = Mov->use_begin(), UE = Mov->use_end(); UI != UE; ++UI) {
        auto Use = UI->first;
        auto OpndNum = UI->second;
        auto Opnd = Use->getOperand(OpndNum)->asSrcRegRegion();
        auto NewOpnd = Builder.Create_Src_Opnd_From_Dcl(NewDcl, Opnd->getRegion());
        Use->setSrc(NewOpnd, OpndNum - 1);
    }
    return true;
}

void Optimizer::newLocalCopyPropagation()
{
    BB_LIST_ITER ib, bend(fg.BBs.end());

    for (ib = fg.BBs.begin(); ib != bend; ++ib)
    {
        G4_BB* bb = *ib;

        bb->resetLocalId();

        INST_LIST_ITER ii = bb->begin(), iend(bb->end());
        while (ii != iend)
        {
            G4_INST *inst = *ii;
            G4_Operand *dst = inst->getDst();
            if (!dst)
            {
                ii++;
                continue;
            }

            doConsFolding(inst);
            doSimplification(inst);

            G4_INST::MovType MT = inst->canPropagate(fg.builder);
            // Skip super mov.
            if (MT == G4_INST::SuperMov) {
                ii++;
                continue;
            }

            if (!isCopyPropProfitable(inst))
            {
                ++ii;
                continue;
            }
            bool canRemove = true;

            // check if each use may be copy propagated.
            USE_EDGE_LIST_ITER iter, iend1(inst->use_end());
            for (iter = inst->use_begin(); iter != iend1; iter++)
            {
                G4_INST *useInst = iter->first;
                Gen4_Operand_Number opndNum = iter->second;

                if (inst->getDst()->isA0() && useInst->isSplitSend() && VISA_WA_CHECK(builder.getPWaTable(), WaSendSEnableIndirectMsgDesc))
                {
                    canRemove = false;
                    break;
                }

                if (!inst->canPropagateTo(useInst, opndNum, MT, bb->isInSimdFlow()))
                {
                    canRemove = false;
                    break;
                }

                // Make sure there is no lifetime.end for src0 of the move inst
                INST_LIST_ITER cpIter = ii;
                cpIter++;
                while (*cpIter != useInst)
                {
                    // Detect patterns like:
                    //
                    // mov A, B
                    // ...
                    // lifetime.end B
                    // op C, A, D
                    //
                    // Because of presence of lifetime.end B, copy propagation for inst
                    // mov A, B
                    // cannot be done

                    if ((*cpIter)->isLifeTimeEnd())
                    {
                        // Check whether lifetime end is for same opnd
                        G4_Declare* lifetimeEndTopDcl = GetTopDclFromRegRegion((*cpIter)->getSrc(0));
                        G4_Declare* curInstDstTopDcl = GetTopDclFromRegRegion((*ii)->getDst());

                        if (lifetimeEndTopDcl == curInstDstTopDcl)
                        {
                            canRemove = false;
                            break;
                        }
                    }

                    cpIter++;
                }
            }

            if (canRemove && inst->getSrc(0)->isSrcRegRegion())
            {
                // check for anti-dependencies for src0 of the move instruction
                bool def_use_in_between = false;

                G4_INST* lastUse = inst->use_front().first;
                for (USE_EDGE_LIST_ITER
                        iter = inst->use_begin(),
                        uend = inst->use_end(); iter != uend; ++iter)
                {
                    G4_INST* useInst = iter->first;
                    if (useInst->getLocalId() > lastUse->getLocalId())
                    {
                        lastUse = useInst;
                    }
                }

                INST_LIST_ITER forwardIter = ii;
                forwardIter++;
                INST_LIST_ITER instListEnd = bb->end();

                while (!def_use_in_between &&
                       forwardIter != instListEnd &&
                       *forwardIter != lastUse)
                {
                    if ((*forwardIter)->isWARdep(inst))
                    {
                        def_use_in_between = true;
                        break;
                    }
                    forwardIter++;
                }

                // check if hoisting is possible
                if (def_use_in_between)
                {
                    hoistUseInst(bb, inst, forwardIter, canRemove);
                }

                if (!canRemove) {
                    // Check whether the type could be propagated instead to demote
                    // type if possible.
                    propagateType(builder, bb, inst, MT);
                }
            }

            if (!canRemove)
            {
                ii++;
                continue;
            }

            G4_Operand *src = inst->getSrc(0);
            // do propagation
            for (iter = inst->use_begin(); iter != iend1; /* empty */)
            {
                G4_INST *useInst = (*iter).first;
                Gen4_Operand_Number opndNum = (*iter).second;
                G4_Operand *use = useInst->getOperand(opndNum);
                G4_Type propType = useInst->getPropType(opndNum, MT, inst);

                // replace use with def
                if (src->isImm())
                {
                    G4_Imm* newImm = builder.createImm(src->asImm()->getImm(),
                        propType);
                    G4_SrcModifier modifier = use->asSrcRegRegion()->getModifier();
                    if (modifier != Mod_src_undef)
                    {
                        if (IS_TYPE_FLOAT_ALL(propType))
                        {
                            if (propType == Type_DF)
                            {
                                double imm = getImmValue(newImm->getDouble(), modifier);
                                newImm = builder.createDFImm(imm);
                            }
                            else
                            {
                                float imm = getImmValue(newImm->getFloat(), modifier);
                                newImm = builder.createImm(imm);
                            }
                        }
                        else
                        {
                            int64_t imm = getImmValue(newImm->getImm(), modifier);
                            newImm = builder.createImm(imm, propType);
                        }
                    }
                    useInst->setSrc(newImm,
                                    opndNum - 1);
                }
                else
                {
                    if (use == NULL)
                    {
                        break;
                    }
                    G4_SrcModifier new_mod = mergeModifier( src, use );

                    unsigned use_elsize = G4_Type_Table[use->getType()].byteSize;
                    unsigned dstElSize = G4_Type_Table[inst->getDst()->getType()].byteSize;
                    RegionDesc *rd = src->asSrcRegRegion()->getRegion();
                    G4_Operand *new_src_opnd = NULL;
                    bool new_src = false;
                    unsigned char scale = 1, newExecSize = useInst->getExecSize();

                    // Compute the composed region if exists.
                    auto getComposedRegion = [=](unsigned dStride, unsigned ex1,
                                                 RegionDesc *rd1, unsigned ex2,
                                                 RegionDesc *rd2) -> RegionDesc *
                    {
                        // Easy cases.
                        if (rd1->isScalar())
                            return rd1;
                        else if (rd2->isScalar())
                            return rd2;
                        else if (dStride == 1 && rd1->isContiguous(ex1))
                            return rd2;
                        else if (dStride == 1 && rd2->isContiguous(ex2))
                            return rd1;

                        // rd1 and rd2 must be single strided. Use a non-zero
                        // invalid stride value as the initial value, which
                        // simplifies and unifies the checking.
                        uint16_t stride1 = 64;
                        if (rd1->isContiguous(ex1))
                            stride1 = 1;
                        else
                            rd1->isSingleNonUnitStride(ex1, stride1);

                        uint16_t stride2 = 64;
                        if (rd2->isContiguous(ex2))
                            stride2 = 1;
                        else
                            rd2->isSingleNonUnitStride(ex2, stride2);

                        // All are single strided; the composition is the product of strides.
                        if (stride1 * stride2 * dStride <= 32)
                            return builder.createRegionDesc((uint16_t)ex2, stride1 * stride2 * dStride, 1, 0);

                        // Should be unreachable, since the legality check
                        // before should reject cases that are difficult to do
                        // composition. Assert?
                        return nullptr;
                    };

                    if (MT == G4_INST::Trunc)
                    {
                        G4_DstRegRegion *dst = inst->getDst();
                        G4_SrcRegRegion *src0 = src->asSrcRegRegion();
                        unsigned typeSizeRatio = G4_Type_Table[src0->getType()].byteSize / G4_Type_Table[dst->getType()].byteSize;
                        unsigned numElt = src0->isScalar() ? 1 : inst->getExecSize() * typeSizeRatio;
                        // src0 region is guaranteed to be scalar/contiguous due to canPropagate() check earlier
                        RegionDesc* region = src0->isScalar() ?
                            builder.getRegionScalar() :
                            builder.createRegionDesc((uint16_t)inst->getExecSize()
                            * typeSizeRatio, inst->getExecSize(),
                            (uint16_t)typeSizeRatio);
                        if (src0->isIndirect())
                        {
                            new_src_opnd = builder.createIndirectSrc(new_mod,
                                src0->getBase(), src0->getRegOff(), src0->getSubRegOff() * typeSizeRatio,
                                region, propType, src0->getAddrImm());
                        }
                        else
                        {
                            G4_Declare* newDcl = builder.createTempVar(numElt, inst->getDst()->getType(), Either, Any);
                            newDcl->setAliasDeclare(src0->getBase()->asRegVar()->getDeclare(), 0);

                            new_src_opnd =
                                builder.createSrcRegRegion(new_mod, Direct,
                                newDcl->getRegVar(), src0->getRegOff(),
                                src0->getSubRegOff() * typeSizeRatio,
                                region, propType);
                        }
                        new_src = true;
                    }
                    else if (dstElSize < use_elsize)
                    {
                        // FIXME: How could this happen? Revisit later if
                        // NoMask is guaranteed.
                        // TODO!!! src is aligned to use type. -- should check this.
                        new_src = true;
                        scale = use_elsize/dstElSize;
                        unsigned short vs = rd->vertStride, wd = rd->width;
                        // packed word/byte
                        if (use->asSrcRegRegion()->isScalar())
                        {
                            rd = builder.getRegionScalar();
                        }
                        else if (inst->isComprInst() && vs == wd)
                        {
                            rd = builder.getRegionStride1();
                        }
                        else
                        {
                            rd = builder.createRegionDesc(vs/scale, wd/scale, 1);
                        }
                    }
                    else if (inst->getExecSize() < useInst->getExecSize() &&
                               rd && use->isSrcRegRegion())
                    {
                        unsigned dStride = inst->getDst()->getHorzStride();
                        RegionDesc *rd2 = use->asSrcRegRegion()->getRegion();
                        if (auto compRd = getComposedRegion(dStride, inst->getExecSize(),
                                                            rd, newExecSize, rd2))
                        {
                            new_src = true;
                            rd = compRd;
                        }
                    }

                    if (new_mod != Mod_src_undef || new_src)
                    {
                        // For truncation case, new src operand is already built.
                        if (MT != G4_INST::Trunc) {
                            new_src_opnd = builder.createSrcRegRegion(
                                new_mod,
                                src->asSrcRegRegion()->getRegAccess(),
                                src->asSrcRegRegion()->getBase(),
                                src->asSrcRegRegion()->getRegOff(),
                                src->asSrcRegRegion()->getSubRegOff()/scale,
                                rd,
                                propType);
                            if (src->asSrcRegRegion()->getRegAccess() != Direct)
                            {
                                new_src_opnd->asSrcRegRegion()->setImmAddrOff(src->asSrcRegRegion()->getAddrImm());
                            }
                        }
                    }
                    else if (inst->hasOneUse())
                    {
                        new_src_opnd = src;
                        new_src_opnd->asSrcRegRegion()->setModifier(new_mod);
                        new_src_opnd->asSrcRegRegion()->setType(propType);
                    }
                    else
                    {
                        new_src_opnd = builder.duplicateOperand(src);
                        new_src_opnd->asSrcRegRegion()->setModifier(new_mod);
                        new_src_opnd->asSrcRegRegion()->setType(propType);
                    }
                    useInst->setSrc(new_src_opnd, opndNum - 1);
                }

                iter = inst->eraseUse(iter);
                // due to truncation a (partial) def of the move may no longer be a def of the use
                inst->copyDef(useInst, Opnd_src0, opndNum, true);

                doConsFolding(useInst);
            }
            // remove decl corresponding to this def
            // TODO!!! what if there is some alias to this decl?
            // remove MOV inst

            // remove it from the use list of its deflists
            inst->removeDefUse(Opnd_src0);

            INST_LIST_ITER tmp = ii;
            ii++;
            bb->erase(tmp);
        }
    }
}

void Optimizer::cselPeepHoleOpt()
{

    if (!builder.hasCondModForTernary())
    {
        return;
    }
    BB_LIST_ITER ib, bend(fg.BBs.end());
    G4_Operand *cmpDst = NULL;
    G4_SrcRegRegion *cmpSrc0 = NULL;
    G4_Operand *cmpSrc1 = NULL;
    G4_Predicate *predCmp = NULL;
    for(ib = fg.BBs.begin(); ib != bend; ++ib)
    {
        G4_BB* bb = (*ib);
        INST_LIST_ITER ii;
        INST_LIST_ITER nextIter;
        INST_LIST_ITER iiEnd;
        if( ( bb->begin() == bb->end() ) ){
            continue;
        }

        ii = bb->begin();
        iiEnd = bb->end();

        if (ii == iiEnd)
        {
            continue;
        }
        nextIter = ii;

        do
        {
            ii = nextIter;
            ++nextIter;
            G4_INST *inst = *ii;
            G4_opcode op = inst->opcode();
            cmpDst = inst->getDst();
            bool nullDst = inst->hasNULLDst();
            predCmp = inst->getPredicate();
            /*
            csel doesn't have the same symantics for destination
            as cmp instruction
            */
            if((op != G4_cmp)           ||
                (cmpDst && !nullDst)    ||
                predCmp != NULL         ||
                inst->isDead())
                continue;

            cmpSrc0 = inst->getSrc(0)->asSrcRegRegion();
            cmpSrc1 = inst->getSrc(1)->asSrcRegRegion();

            G4_CondMod *cModifier = inst->getCondMod();

                // check if dst is global
            if( fg.globalOpndHT.isOpndGlobal( cModifier ) )
            {
                continue;
            }

            /*
            csel instruction implicitly compares src2 to 0
            only supports floats
            no predication
            */


            if(!cmpSrc1->isImm()                                                                            ||
                (cmpSrc1->asImm()->getImm() != 0 &&
                    (cmpSrc1->asImm()->getType() != Type_F || cmpSrc1->asImm()->getFloat()!= -0.0f))        ||
                cmpSrc0->getType() != Type_F                                                                ||
                cmpSrc0->isImm())
                continue;

            if (inst->getSrc(0)->isRelocImm() ||
                inst->getSrc(1)->isRelocImm())
            {
                continue;
            }

            // Only allow single strided regions.
            uint16_t src0Stride = 0;
            if (!cmpSrc0->getRegion()->isSingleStride(inst->getExecSize(), src0Stride))
                continue;

            /*
                Can do scan until use instruction to see if src0 is modified
                but I think for general case this will suffice. If we are
                not capturing opportunities can revisit.
            */

            if(inst->useEmpty())
                continue;

            int execSize = inst->getExecSize();
            if( execSize == 2)
                continue;

            USE_EDGE_LIST_ITER iter = inst->use_begin();
            USE_EDGE_LIST_ITER endUseList = inst->use_end();

            bool canOpt = true;
            int maxInstID = 0;

            for(; iter != endUseList; ++iter)
            {
                G4_INST* useInst = (*iter).first;

                if (useInst->getNumSrc() != 2)
                {
                    canOpt = false;
                    break;
                }
                //assumes instructions are created in order
                //through APIs. I think it's a safe assumption
                //since internally we are not creating select instructions.
                if(useInst->getId() >= maxInstID)
                    maxInstID = useInst->getId();

                G4_Operand *dstUse = useInst->getDst();
                G4_Operand *selSrc0 = useInst->getSrc(0);
                G4_Operand *selSrc1 = useInst->getSrc(1);

                if( useInst->opcode() != G4_sel     ||
                    selSrc0->isImm()                ||
                    selSrc1->isImm()                ||
                    selSrc0->getType() != Type_F    ||
                    selSrc1->getType() != Type_F    ||
                    dstUse->getType()   != Type_F   ||
                    //3-src restriction
                    !builder.isOpndAligned(dstUse, 16)  ||
                    !builder.isOpndAligned(selSrc0, 16) ||
                    !builder.isOpndAligned(selSrc1, 16))
                {
                    canOpt = false;
                    break;
                }

                //   if inst is NoMask use inst can be anything.
                //   if inst is not NoMask then useInst needs to be subset of inst.
                if(!(inst->getMaskOption() & InstOpt_WriteEnable))
                {
                    auto isInclusive = [](int lb1, int rb1, int lb2, int rb2)
                    {
                        return lb1 <= lb2 && rb1 >= rb2;
                    };
                    if (!isInclusive(inst->getMaskOffset(), inst->getMaskOffset() + inst->getExecSize(),
                        useInst->getMaskOffset(), useInst->getMaskOffset() + useInst->getExecSize()))
                    {
                        canOpt = false;
                        break;
                    }
                }

                uint8_t numPredDefs = 0;
                DEF_EDGE_LIST_ITER useIter = useInst->def_begin();
                DEF_EDGE_LIST_ITER iterEnd = useInst->def_end();

                //    Just in case some weird code is generated with partial writes to predicate
                for(; useIter != iterEnd; ++useIter)
                {
                    if((*useIter).second == Opnd_pred)
                        ++numPredDefs;

                    // Check whether pseudo_kill for dst exists between cmp and sel
                    // cmp.xx.fx.0 (8) ... src0 $0
                    // ...
                    // pseudo_kill dst
                    // (f0) sel (8) dst src1 src2
                    //
                    // These two cannot be merged because pseudo_kill is in between them

                    INST_LIST_ITER cselOptIt = ii;
                    cselOptIt++;
                    while((*cselOptIt) != useInst)
                    {
                        if((*cselOptIt)->isLifeTimeEnd())
                        {
                            if(GetTopDclFromRegRegion((*cselOptIt)->getDst()) ==
                                GetTopDclFromRegRegion(useInst->getDst()))
                            {
                                canOpt = false;
                                break;
                            }
                        }

                        cselOptIt++;
                    }
                }

                if(numPredDefs > 1)
                {
                    canOpt = false;
                    break;
                }
            }

            INST_LIST_ITER tempInstIter = nextIter;
            //explicit check that cmp sr0 is not over written or partially writen to
            //between cmp and sel.
            for( ;tempInstIter != iiEnd; ++tempInstIter)
            {
                G4_INST *tempInst = *tempInstIter;

                if(tempInst->getId() == maxInstID)
                    break;

                if(!tempInst->getDst())
                    continue;

                //also checks for indirect, will return inerference.
                G4_CmpRelation rel = tempInst->getDst()->compareOperand(cmpSrc0);
                if(rel != Rel_disjoint)
                {
                    canOpt = false;
                    break;
                }
            }

            if(canOpt)
            {
                for (auto iter = inst->use_begin(); iter != inst->use_end(); /*empty*/)
                {
                    G4_INST* useInst = (*iter).first;
                    G4_CondMod *mod = inst->getCondMod();
                    useInst->setOpcode(G4_csel);
                    useInst->setSrc(builder.duplicateOperand(inst->getSrc(0)), 2);
                    useInst->setCondMod((G4_CondMod*)builder.duplicateOperand(mod));
                    useInst->setPredicate(NULL);

                    G4_SrcRegRegion * opnd2 = useInst->getSrc(2)->asSrcRegRegion();

                    if(!opnd2->isScalar() && inst->getExecSize() > useInst->getExecSize())
                    {
                        //earlier check establishes that useInst mask is equivalent or subset
                        //sel instruction
                        /*
                            case which considering:
                            cmp (16)
                            sel (8)
                        */
                        if(useInst->getMaskOffset() != inst->getMaskOffset())
                        {
                            //check elsewhere gurantees this is float.
                            G4_Type type = opnd2->getType();
                            unsigned short typeSize = (unsigned short) G4_Type_Table[type].byteSize;
                            unsigned offset = opnd2->getRegOff()* G4_GRF_REG_NBYTES + opnd2->getSubRegOff()*typeSize;
                            offset += useInst->getExecSize() * src0Stride * typeSize;

                            opnd2->setRegOff(offset % G4_GRF_REG_NBYTES);
                            opnd2->setSubRegOff(offset / G4_GRF_REG_NBYTES);
                        }
                    }
                    //
                    // Modifying useDef links
                    //
                    // cmp.xx.f0.0 (8) ... src2 $0  <- inst (to be deleted)
                    // (f0) sel (8) dst src0 src1   <- useInst
                    // =>
                    // csel.xx.f0.0 (8) dst src0 src1 src2

                    // useInst's predicate becomes NULL.
                    iter = inst->eraseUse(iter);

                    // inst's src0 becomes useInst's src2.
                    inst->copyDef(useInst, Opnd_src0, Opnd_src2);
                }
                ASSERT_USER(inst->useEmpty(), "all predicate uses are removed.");
                inst->removeAllDefs();
                bb->erase( ii );
            }


        }while( nextIter != iiEnd );
    }
}

// helper function to convert
// and/or p3 p1 p2
// ==>
// (p1) sel t1 1 0
// (p2) sel t2 1 0
// and/or.nz.p3 t1 t2
// if the original inst is NoMask and Q1/H1, we do
// and/or p3 p1 p2
// ==>
// and/or (1) p3 p1 p2
// p3's type is uw for simd8/16 and ud for simd32
static void expandPseudoLogic(IR_Builder& builder,
                              G4_BB* bb,
                              INST_LIST_ITER& iter)

{
    G4_INST* inst = *iter;
    MUST_BE_TRUE(inst->isPseudoLogic(),
        "inst must be either pseudo_and/or/xor/not");
    INST_LIST_ITER newIter = iter;

    bool isFirstInst = iter == bb->begin();
    if (!isFirstInst)
    {
        --iter;
    }

    if (inst->isWriteEnableInst() &&
        (inst->getMaskOffset() == 0 || inst->getMaskOffset() == 16) &&
        // we can't do this for simd8 inst in simd16 kernels as it will overwrite upper flag bits
        (inst->getExecSize() > 8 || inst->getExecSize() == builder.kernel.getSimdSize()))
    {
        G4_opcode newOpcode = G4_illegal;

		if (inst->getMaskOffset() == 16)
		{
			MUST_BE_TRUE(inst->getExecSize() == 16, "Only support simd16 pseudo-logic instructions");
			// we have to use the upper flag bits (.1) instead
			MUST_BE_TRUE(inst->getSrc(0)->isSrcRegRegion() && inst->getSrc(0)->isFlag(),
				"expect src0 to be flag");
			inst->getSrc(0)->asSrcRegRegion()->setSubRegOff(1);
			if (inst->getSrc(1) != nullptr)
			{
				MUST_BE_TRUE(inst->getSrc(1)->isSrcRegRegion() && inst->getSrc(1)->isFlag(),
					"expect src1 to be flag");
				inst->getSrc(1)->asSrcRegRegion()->setSubRegOff(1);
			}
			inst->getDst()->setSubRegOff(1);
		}

        switch (inst->opcode())
        {
            case G4_pseudo_and:
                newOpcode = G4_and;
                break;
            case G4_pseudo_or:
                newOpcode = G4_or;
                break;
            case G4_pseudo_xor:
                newOpcode = G4_xor;
                break;
            case G4_pseudo_not:
                newOpcode = G4_not;
                break;
            default:
                MUST_BE_TRUE(false, "unexpected opcode for pseudo-logic instructions");
        }

        inst->setOpcode(newOpcode);
        inst->setExecSize(1);
    }
    else
    {
        uint8_t tmpSize = inst->getExecSize();
        auto LowerOpnd = [=, &builder](Gen4_Operand_Number opNum, G4_INST* &SI) -> G4_Operand *
        {
            G4_Operand *Opnd = inst->getOperand(opNum);
            if (Opnd)
            {
                auto src = Opnd->asSrcRegRegion();
                auto newDcl = builder.createTempVar(tmpSize, Type_UW, Either, Any);
                auto newDst = builder.createDstRegRegion(Direct, newDcl->getRegVar(), 0, 0, 1, Type_UW);
                auto newPred = builder.createPredicate(PredState_Plus, src->getBase(), src->getSubRegOff());
                auto newSel = builder.createInternalInst(newPred, G4_sel, nullptr, false, tmpSize, newDst,
                                                         builder.createImm(1, Type_UW),
                                                         builder.createImm(0, Type_UW),
                                                         inst->getOption(),
                                                         inst->getLineNo(),
                                                         inst->getCISAOff(),
                                                         inst->getSrcFilename());
                inst->transferDef(newSel, opNum, Gen4_Operand_Number::Opnd_pred);
                bb->insert(newIter, newSel);
                SI = newSel;
                return builder.Create_Src_Opnd_From_Dcl(newDcl, builder.getRegionStride1());
            }
            return Opnd;
        };

        G4_INST *Sel0 = nullptr;
        G4_Operand *logicSrc0 = LowerOpnd(Gen4_Operand_Number::Opnd_src0, Sel0);

        G4_INST *Sel1 = nullptr;
        G4_Operand *logicSrc1 = LowerOpnd(Gen4_Operand_Number::Opnd_src1, Sel1);

        if (logicSrc1  == nullptr)
        {
            MUST_BE_TRUE(inst->opcode() == G4_pseudo_not, "Must be a pseudo-not instruction");
            // for not P1 P0
            // we generate
            // (P0) sel V1 1 0
            // xor.P1 null V1 1
            // so that the upper bits would stay zero
            logicSrc1 = builder.createImm(1, Type_UW);
        }

        auto nullDst = builder.createNullDst(Type_UW);
        auto newCondMod = builder.createCondMod(Mod_nz, inst->getDst()->getBase()->asRegVar(), 0);
        G4_opcode newOpcode = G4_illegal;
        switch (inst->opcode())
        {
            case G4_pseudo_and:
                newOpcode = G4_and;
                break;
            case G4_pseudo_or:
                newOpcode = G4_or;
                break;
            case G4_pseudo_xor:
                newOpcode = G4_xor;
                break;
            case G4_pseudo_not:
                // see comment above
                newOpcode = G4_xor;
                break;
            default:
                MUST_BE_TRUE(false, "unexpected opcode for pseudo-logic instructions");
        }

        G4_INST* newLogicOp = builder.createInternalInst(
            NULL,
            newOpcode,
            newCondMod,
            false,
            tmpSize,
            nullDst,
            logicSrc0,
            logicSrc1,
            inst->getOption(),  // keep the original instruction emask
            inst->getLineNo(),
            inst->getCISAOff(),
            inst->getSrcFilename() );

        // Fix def-use
        if (Sel0 != nullptr)
        {
            Sel0->addDefUse(newLogicOp, Gen4_Operand_Number::Opnd_src0);
        }
        if (Sel1 != nullptr)
        {
            Sel1->addDefUse(newLogicOp, Gen4_Operand_Number::Opnd_src1);
        }
        inst->transferUse(newLogicOp);
        bb->insert(newIter, newLogicOp);
        bb->erase(newIter);
    }

    // iter either points to the start or the first expanded instruction.  Caller will advance it to the previous instruction
    if (isFirstInst)
    {
        iter = bb->begin();
    }
    else
    {
        ++iter;
    }
}

// mov(1) P0 Imm(NoMask)
// (P0) mov(esize) r[A0, 0] src0 src1
// == >
// smov(esize) r[A0, 0] src0 src1 Imm
//
// esize is either 8 or 16
bool Optimizer::createSmov(G4_BB *bb, G4_INST* flagMove, G4_INST* next_inst)
{
    if ((next_inst->getExecSize() != 8 && next_inst->getExecSize() != 16) ||
        next_inst->getPredicate() == NULL ||
        next_inst->getCondMod() != NULL ||
        next_inst->getSaturate() == true ||
        next_inst->getDst()->getRegAccess() == Direct ||
        getTypeSize(next_inst->getDst()->getType()) == 1 ||
        getTypeSize(next_inst->getSrc(0)->getType()) == 1 ||
        (getGenxPlatform() < GENX_SKL && getGenxPlatform() != GENX_BDW) ||
        getTypeSize(next_inst->getDst()->getType()) < getTypeSize(next_inst->getSrc(0)->getType()))
    {
        return false;
    }

    if (next_inst->getSrc(0)->isSrcRegRegion() &&
        next_inst->getSrc(0)->asSrcRegRegion()->getModifier() != Mod_src_undef)
    {
        return false;
    }

    if (flagMove->use_size() != 1 || flagMove->use_front().first != next_inst)
    {
        return false;
    }

    G4_CmpRelation rel = flagMove->getDst()->compareOperand(next_inst->getPredicate());
    if (rel != Rel_eq && !(rel == Rel_gt && next_inst->getMaskOffset() == 0))
    {
        return false;
    }

    if (builder.getOptions()->getTarget() == VISA_3D || bb->isInSimdFlow())
    {
        if (!flagMove->isWriteEnableInst())
        {
            return false;
        }
    }

    next_inst->setOpcode(G4_smov);
    next_inst->setSrc(flagMove->getSrc(0), 1);
    next_inst->setPredicate(nullptr);

    flagMove->removeUseOfInst();
    return true;
}

// Returns true if *iter has an use that is a cmp and we can fold that cmp
// into *iter as a conditional modifier. The cmp instruction is deleted as part of folding.
// Note that iter may be modified to point to the next inst if we decide to sync *iter to 
// where the cmp was to work around dependencies
bool Optimizer::foldCmpToCondMod(G4_BB* bb, INST_LIST_ITER& iter)
{
    // find a cmp that uses inst dst
    G4_INST* inst = *iter;
    G4_INST* cmpInst = nullptr;
    bool canFold = false;

    if (inst->getCondMod())
    {
        return false;
    }

    for (auto UI = inst->use_begin(), UE = inst->use_end(); UI != UE; ++UI)
    {
        cmpInst = (*UI).first;

        // cmp instruction must be of the form
        // cmp [<op> P0] null src 0
        // where src is singly defined by inst 
        if (cmpInst->opcode() == G4_cmp && cmpInst->getExecSize() == inst->getExecSize() &&
            cmpInst->hasNULLDst() && 
            cmpInst->getSrc(0)->isSrcRegRegion() && 
            cmpInst->getSrc(0)->asSrcRegRegion()->getModifier() == Mod_src_undef &&
            cmpInst->def_size() == 1 && !cmpInst->getPredicate() &&
            cmpInst->getSrc(1)->isImm() && cmpInst->getSrc(1)->asImm()->isZero())
        {
            canFold = true;
            break;
        }
    }

    if (!canFold)
    {
       return false;
    }

    auto cmpIter = std::find(iter, bb->end(), cmpInst);

    auto isSupportedCondMod = [](G4_CondModifier mod) 
    {
        return mod == Mod_g || mod == Mod_ge || mod == Mod_l || mod == Mod_le || 
            mod == Mod_e || mod == Mod_ne;
    };
    G4_CondModifier mod = cmpInst->getCondMod()->getMod();
    if (!isSupportedCondMod(mod))
    {
        return false;
    }

    if (builder.getOptions()->getTarget() == VISA_3D || bb->isInSimdFlow())
    {
        // Make sure masks of both instructions are same
        if (inst->getMaskOption() != cmpInst->getMaskOption())
        {
            return false;
        }
    }

    auto getUnsignedTy = [](G4_Type Ty)
    {
        switch (Ty)
        {
        case Type_D:
            return Type_UD;
        case Type_W:
            return Type_UW;
        case Type_B:
            return Type_UB;
        case Type_Q:
            return Type_UQ;
        case Type_V:
            return Type_UV;
        default:
            break;
        }
        return Ty;
    };

    G4_Type T1 = inst->getDst()->getType();
    G4_Type T2 = cmpInst->getSrc(0)->getType();
    if (getUnsignedTy(T1) != getUnsignedTy(T2))
    {
        return false;
    }
    // Skip if the dst needs saturating but it's used as different sign.
    if (inst->getSaturate() && T1 != T2)
    {
        return false;
    }

    if (chkBwdOutputHazard(iter, cmpIter))
    {
        return false;
    }

    auto isSafeToSink = [](INST_LIST_ITER defIter,
        INST_LIST_ITER beforeIter, int maxDist)
    {
        G4_INST *inst = *defIter;
        int dist = 0;
        for (auto it = std::next(defIter); it != beforeIter; ++it)
        {
            if (dist++ >= maxDist)
                return false;
            if (inst->isWAWdep(*it) || inst->isRAWdep(*it) ||
                inst->isWARdep(*it))
                return false;
            if (!checkLifetime(inst, *it))
                return false;
        }
        return true;
    };

    // Merge two instructions
    // If legal, use the cmp location as new insert position.
    bool sinkInst = false;

    if (inst->getDst()->compareOperand(cmpInst->getSrc(0)) == Rel_eq)
    {
        if (inst->use_size() == 1)
        {
            // see if we can replace dst with null
            if (inst->supportsNullDst(builder) && !fg.globalOpndHT.isOpndGlobal(inst->getDst()))
            {
                inst->setDest(builder.createDstRegRegion(
                    Direct,
                    builder.phyregpool.getNullReg(),
                    0,
                    0,
                    inst->getDst()->getHorzStride(),
                    inst->getDst()->getType()));
            }
            // Check if it is safe to sink inst right before cmp inst, which lowers flag pressure in general.
            const int MAX_DISTANCE = 20;
            sinkInst = isSafeToSink(iter, cmpIter, MAX_DISTANCE);
        }
        inst->setCondMod(cmpInst->getCondMod());
        inst->setOptions((inst->getOption() & ~InstOpt_Masks) | (cmpInst->getMaskOption()));
        // The sign of dst should follow its use instead of its
        // def. The later is meaningless from how hardware works.
        auto honorSignedness = [](G4_CondModifier mod) {
            switch (mod) {
            case Mod_g:
            case Mod_ge:
            case Mod_l:
            case Mod_le:
                return true;
            default:
                break;
            }
            return false;
        };
        if (honorSignedness(inst->getCondMod()->getMod()))
            inst->getDst()->setType(T2);

        // update def-use
        // since cmp is deleted, we have to
        // -- transfer cmp's use to inst
        // -- remove cmp from its definition's use list
        cmpInst->transferUse(inst, true);
        cmpInst->removeUseOfInst();
        if (!sinkInst)
        {
            bb->erase(cmpIter);
        }
        else
        {
            // Before and <- ii
            //        cmp <- next_iter
            // After  cmp <- ii
            //        and <- next
            std::iter_swap(iter, cmpIter);
            auto nextii = std::next(iter);
            bb->erase(iter);
            iter = nextii;
        }
        return true;
    }
    return false;
}

// Return true  : folding cmp and sel is performed
//        false : no folding is performed.
bool Optimizer::foldCmpSel(G4_BB *BB, G4_INST *selInst, INST_LIST_ITER &selInst_II)
{
    MUST_BE_TRUE( (selInst->opcode() == G4_sel &&
                   selInst->getPredicate() &&
                   selInst->getCondMod() == NULL),
                  "foldCmpSel: Inst should be a sel with predicate and without cmod." );
    G4_Predicate *pred = selInst->getPredicate();

    // global predicates are not eligible since folding the cmp removes the def
    if (fg.globalOpndHT.isOpndGlobal(pred))
    {
        return false;
    }

    // Needs to find cmp instruction that defines predicate of sel. The cmp is
    // not necessarily right before the sel. To be able to fold the cmp to the
    // sel, we have to check if the cmp can be moved right before the sel. It not,
    // no folding is performed.
    G4_INST *cmpInst = nullptr;
    for (auto DI = selInst->def_begin(), DE = selInst->def_end(); DI != DE; ++DI)
    {
        if ((*DI).second == Opnd_pred)
        {
            if (cmpInst)
            {   // only handle single def.
                cmpInst = nullptr;
                break;
            }
            cmpInst = (*DI).first;
            if (cmpInst->opcode() != G4_cmp)
            {
                cmpInst = nullptr;
                break;
            }
        }
    }
    if (!cmpInst)
    {
        // G4_cmp is not found, skip optimization.
        return false;
    }

    // Do a fast check first. Note that sel w/ cmod
    // does not allow predication. So, we will give up if cmp has predicate.
    bool isSubExecSize = (selInst->getExecSize() < cmpInst->getExecSize());
    bool isSameExecSize = (selInst->getExecSize() == cmpInst->getExecSize());
    if ((!isSameExecSize && !isSubExecSize) ||
        (cmpInst->getDst() && !cmpInst->hasNULLDst()) ||
        cmpInst->use_size() != 1 ||
        cmpInst->getPredicate() != nullptr)
    {
        return false;
    }

    // Check if two source operands have the same type and value.
    auto IsEqual = [](G4_Operand *opnd1, G4_Operand *opnd2)
    {
        if (opnd1->isImm() && opnd2->isImm())
            return opnd1->asImm()->isEqualTo(opnd2->asImm());
        if (opnd1->compareOperand(opnd2) != Rel_eq)
            return false;
        // footprint does not imply equality.
        // (1) region difference:  r10.0<1;4,2>:f vs r10.0<8;8,1>
        // (2) source modifier: r10.0<0;1,0>:f vs -r10.0<0;1,0>
        //
        if (opnd1->isSrcRegRegion() && opnd2->isSrcRegRegion())
            return opnd1->asSrcRegRegion()->sameSrcRegRegion(*opnd2->asSrcRegRegion());

        // Common cases should be covered.
        return false;
    };

    G4_Operand *sel_src0 = selInst->getSrc(0);
    G4_Operand *sel_src1 = selInst->getSrc(1);
    G4_Operand *cmp_src0 = cmpInst->getSrc(0);
    G4_Operand *cmp_src1 = cmpInst->getSrc(1);

    // Normalize SEL's predicate state to Plus.
    if (G4_PredState::PredState_Minus == pred->getState())
    {
        selInst->setSrc(sel_src0, 1);
        selInst->setSrc(sel_src1, 0);
        selInst->swapDefUse();
        pred->setState(G4_PredState::PredState_Plus);
        std::swap(sel_src0, sel_src1);
    }

    // source operands of SEL and CMP are reversed or not.
    bool reversed = false;
    G4_CondMod* condMod = cmpInst->getCondMod();

    auto canFold = [=, &reversed]() {
        G4_CmpRelation condRel = pred->asPredicate()->compareOperand(condMod);
        if (!(condRel == Rel_eq && isSameExecSize) &&
            !(condRel == Rel_lt && isSubExecSize))
            return false;

        if (!cmpInst->isWriteEnableInst() &&
            cmpInst->getMaskOption() != selInst->getMaskOption())
            return false;

        // P = cmp.gt A, B
        // C = (+P) sel A, B  => C = sel.gt A, B
        //
        // P = cmp.ne A, B
        // C = (+P) sel A, B  => C = sel.ne A, B
        //
        if (IsEqual(sel_src0, cmp_src0) && IsEqual(sel_src1, cmp_src1))
            return true;

        // Sel operands are reversed.
        // P = cmp.gt A, B
        // C = (+P) sel B, A  => C = sel.le B, A
        //
        // P = cmp.ne A, B
        // C = (+P) sel B, A  => C = sel.ne B, A
        //
        if (IsEqual(sel_src0, cmp_src1) && IsEqual(sel_src1, cmp_src0))
        {
            reversed = true;
            return true;
        }
        return false;
    };

    if (!canFold())
    {
        return false;
    }

    // check if cmpInst can be legally moved right before selInst;
    // if it cannot, we cannot fold cmp to sel!.
    INST_LIST_ITER cmpInst_II = selInst_II;
    while (cmpInst_II != BB->begin())
    {
        cmpInst_II--;
        if (*cmpInst_II == cmpInst)
        {
            break;
        }
    }

    // No local def (possible?)
    if (cmpInst_II == BB->begin())
    {
        return false;
    }

    // If cmpInst has no WAR harzard b/w cmpInst and selInst, cmpInst
    // can be moved right before selInst.
    if (chkFwdOutputHazard(cmpInst_II, selInst_II))
    {
        return false;
    }

    G4_CondModifier mod = condMod->getMod();
    if (reversed)
        mod = G4_CondMod::getReverseCondMod(mod);
    G4_CondMod* cmod = builder.createCondMod(mod, condMod->getBase(),
                                             condMod->getSubRegOff());
    selInst->setCondMod(cmod);
    selInst->setPredicate(nullptr);

    // update def-use
    // since cmp is deleted, we have to
    // -- remove def-use between cmp and sel
    // -- transfer cmp's remaining use to sel
    // -- remove cmp and its definitions' use
    selInst->removeDefUse(Opnd_pred);
    cmpInst->transferUse(selInst, true);
    cmpInst->removeUseOfInst();
    BB->erase(cmpInst_II);
    return true;
}

// try to fold a pseudo not instruction into its use(s)
// return true if successful
bool Optimizer::foldPseudoNot(G4_BB* bb, INST_LIST_ITER& iter)
{
    G4_INST* notInst = *iter;
    assert(notInst->opcode() == G4_pseudo_not && "expect not instruction");
    if (notInst->getPredicate() || notInst->getCondMod())
    {
        return false;
    }

    G4_DstRegRegion* dst = notInst->getDst();
    if (fg.globalOpndHT.isOpndGlobal(dst))
    {
        return false;
    }
    if (!notInst->getSrc(0)->isSrcRegRegion())
    {
        return false;
    }

    // unfortunately, def-use chain is not always properly maintained, so we have to skip opt
    // even if we can't find a use
    bool canFold = notInst->use_size() > 0;
    for (auto uses = notInst->use_begin(); uses != notInst->use_end(); ++uses)
    {
        auto&& use = *uses;
        G4_INST* useInst = use.first;
        Gen4_Operand_Number opndPos = use.second;
        if (!useInst->isLogic() || !G4_INST::isSrcNum(opndPos))
        {
            canFold = false;
            break;
        }

        // check the case where flag is partially used
        G4_SrcRegRegion* opnd = useInst->getSrc(G4_INST::getSrcNum(opndPos))->asSrcRegRegion();
        if (dst->compareOperand(opnd) != Rel_eq)
        {
            canFold = false;
            break;
        }
    }

    if (canFold)
    {
        G4_SrcRegRegion* origUse = notInst->getSrc(0)->asSrcRegRegion();
        if (notInst->getMaskOffset() == 16)
        {
            // Fold upper bits
            assert(notInst->getExecSize() == 16);
            origUse->setSubRegOff(1);
        }
        for (auto uses = notInst->use_begin(); uses != notInst->use_end(); ++uses)
        {
            auto use = *uses;
            G4_INST* useInst = use.first;
            Gen4_Operand_Number opndPos = use.second;
            G4_SrcRegRegion* opnd = useInst->getSrc(G4_INST::getSrcNum(opndPos))->asSrcRegRegion();
            int numNot = 1 + (origUse->getModifier() == Mod_Not ? 1 : 0) +
                (opnd->getModifier() == Mod_Not ? 1 : 0);
            G4_SrcModifier newModifier = numNot & 0x1 ? Mod_Not : Mod_src_undef;
            G4_SrcRegRegion* newSrc = builder.createSrcRegRegion(*origUse);
            newSrc->setModifier(newModifier);
            useInst->setSrc(newSrc, G4_INST::getSrcNum(opndPos));

        }

        for (auto defs = notInst->def_begin(); defs != notInst->def_end(); ++defs)
        {
            auto def = *defs;
            G4_INST* defInst = def.first;
            notInst->copyUsesTo(defInst, false);
        }
        notInst->removeAllDefs();
        notInst->removeAllUses();
        bb->erase(iter);
        return true;
    }
    return false;
}

/***
this function optmize the following cases:

case 1:
cmp.gt.P0 s0 s1
(P0) sel d s0 s1
==>
sel.gt.P0 d s0 s1

case 2:
and dst src0 src1   -- other OPs are also optimized: or, xor...
cmp.nz.P0 NULL dst 0
(P0) ...
==>
and.nz.P0 dst src0 src1
(P0) ...
add/addc instructions also handled in case 2. Few more cond
modifiers supported to such arithmetic instructions.

case 3:
cmp.l.P0 NULL src0 src1
cmp.l.P1 NULL src2 src3
and P2 P0 P1   -- other OPs are also optimized: or, xor...
==>
cmp.l.P2 NULL src0 src1
(P2)cmp.l.P2 NULL src2 src3

case 4:
mov (1) P0 Imm  (NoMask)
(P0) mov (8) r[A0, 0] src0 src1
==>
smov (8) r[A0, 0] src0 src1 Imm

case 5:
psuedo_not (1) P2 P1
and (1) P4 P3 P2
==>
and (1) P4 P3 ~P1
*/

void Optimizer::optimizeLogicOperation()
{
    BB_LIST_ITER ib, bend(fg.BBs.end());
    G4_Operand *dst = NULL;
    bool resetLocalIds =  false;
    bool doLogicOpt = builder.getOption(vISA_LocalFlagOpt);

    if (!doLogicOpt)
    {
        // we still need to expand the pseudo logic ops 
        for (auto bb : fg.BBs)
        {
            for (auto I = bb->begin(), E = bb->end(); I != E; ++I)
            {
                auto inst = *I;
                if (inst->isPseudoLogic())
                {
                    expandPseudoLogic(builder, bb, I);
                }
            }
        }
        return;
    }

    for(ib = fg.BBs.begin(); ib != bend; ++ib)
    {
        G4_BB* bb = (*ib);
        INST_LIST_ITER ii;
        if( ( bb->begin() == bb->end() ) ){
            continue;
        }
        resetLocalIds =  false;
        ii = bb->end();
        do
        {
            ii--;
            G4_INST *inst = *ii;
            G4_opcode op = inst->opcode();
            dst = inst->getDst();
            bool nullDst = inst->hasNULLDst();
            if ((!Opcode_can_use_cond_mod(op) && !Opcode_define_cond_mod(op) &&
                 !inst->isPseudoLogic()) ||
                !dst || nullDst)
            {
              continue;
            }

            INST_LIST_ITER next_iter = ii, pred_iter = ii;
            next_iter++;
            if( ii != bb->begin() )
            {
                pred_iter--;
            }

            if(!resetLocalIds)
            {
                bb->resetLocalId();
                resetLocalIds =  true;
            }

            // case 5
            if (inst->opcode() == G4_pseudo_not)
            {
                bool folded = foldPseudoNot(bb, ii);
                if (folded)
                {
                    ii = next_iter;
                    continue;
                }
            }

            // case 1
            if (pred_iter != ii && op == G4_sel && inst->getPredicate() && !inst->getCondMod())
            {
                if (!foldCmpSel(bb, inst, ii)) {
                    // Compare two operands with special simple checking on
                    // indirect access. That checking could be simplified as
                    // only dst/src of the same instruction are checked.
                    auto compareOperand =
                        [](G4_DstRegRegion *A, G4_Operand *B, unsigned ExecSize)
                        -> G4_CmpRelation {
                        G4_CmpRelation Res = A->compareOperand(B);
                        if (Res != Rel_interfere)
                            return Res;
                        if (A->getRegAccess() != IndirGRF ||
                            B->getRegAccess() != IndirGRF)
                            return Res;
                        if (A->getHorzStride() != 1)
                            return Res;
                        // Extra check if both are indirect register accesses.
                        G4_VarBase *BaseA = A->getBase();
                        G4_VarBase *BaseB = B->getBase();
                        if (!BaseA || !BaseB || BaseA != BaseB || !BaseA->isRegVar())
                            return Res;
                        if (!B->isSrcRegRegion())
                            return Res;
                        G4_SrcRegRegion *S = B->asSrcRegRegion();
                        if (!S->getRegion()->isContiguous(ExecSize))
                            return Res;
                        if (A->getRegOff() != S->getRegOff() ||
                            A->getSubRegOff() != S->getSubRegOff())
                            return Res;
                        if (A->getAddrImm() != S->getAddrImm())
                            return Res;
                        return Rel_eq;
                    };

                    unsigned ExSz = inst->getExecSize();
                    G4_DstRegRegion *Dst = inst->getDst();
                    G4_Operand *Src0 = inst->getSrc(0);
                    G4_Operand *Src1 = inst->getSrc(1);
                    int OpndIdx = -1;
                    if (compareOperand(Dst, Src0, ExSz) == Rel_eq &&
                        Src0->isSrcRegRegion() &&
                        Src0->asSrcRegRegion()->getModifier() == Mod_src_undef)
                        OpndIdx = 0;
                    else if (compareOperand(Dst, Src1, ExSz) == Rel_eq &&
                             Src1->isSrcRegRegion() &&
                             Src1->asSrcRegRegion()->getModifier() == Mod_src_undef)
                        OpndIdx = 1;
                    if (OpndIdx >= 2) {
                        // If dst is equal to one of operands of 'sel', that
                        // 'sel' could be transformed into a predicated 'mov',
                        // i.e.,
                        //
                        // transforms
                        //
                        //  (+p) sel dst, src0, src1
                        //
                        // into
                        //
                        //  (+p) mov dst, src0   if dst == src1
                        //
                        // or
                        //
                        //  (-p) mov dst, src1   if dst == src0
                        //
                        if (OpndIdx == 0) {
                            // Inverse predicate.
                            G4_Predicate *Pred = inst->getPredicate();
                            G4_PredState State = Pred->getState();
                            State = (State == PredState_Plus) ? PredState_Minus
                                                              : PredState_Plus;
                            Pred->setState(State);
                            // Swap source operands.
                            inst->setSrc(Src1, 0);
                            inst->setSrc(Src0, 1);
                            inst->swapDefUse();
                        }
                        inst->setOpcode(G4_mov);
                        inst->setSrc(nullptr, 1);
                        inst->removeDefUse(Opnd_src1);
                    }
                }
            }
            else if (builder.hasSmov() && inst->opcode() == G4_mov &&
                inst->getPredicate() == NULL &&
                inst->getCondMod()  == NULL &&
                inst->getExecSize() == 1 &&
                inst->getSrc(0)->isImm() &&
                inst->getDst()->isFlag() &&
                next_iter != bb->end() &&
                (*next_iter)->opcode() == G4_mov )
            {
                // case 4
                G4_INST *next_inst = *next_iter;
                if (createSmov(bb, inst, next_inst))
                {
                    bb->erase(ii);
                    ii = next_iter;
                }
                continue;
            }
            // List of instructions that can use a superset of cond modifiers
            // that logic instructions use
            //The sel instruction uses any conditional modifier internally and does not update the flag register if a conditional modifier is used.
            else if( inst->getPredicate() == NULL &&
                ( G4_Inst_Table[op].instType == InstTypeLogic ||
                inst->canSupportCondMod(builder) ) &&
                next_iter != bb->end() )
            {
                // FIXME: why this condition?
                if (op == G4_pseudo_mad && inst->getExecSize()==1)
                {
                    continue;
                }

                // case 2
                foldCmpToCondMod(bb, ii);
            }
            else if (inst->getPredicate() == NULL && inst->isPseudoLogic() )
            {
                bool merged = false;

                if (op == G4_pseudo_and || op == G4_pseudo_or)
                {
                    merged = foldPseudoAndOr(bb, ii);
                }

                // translate the psuedo op
                if (!merged)
                {
                    expandPseudoLogic(builder, bb, ii);
                }
            }
        }
        while( ii != bb->begin() );
    }
}

// see if we can fold a pseudo and/or instruction with previous cmp
// returns true if successful, and ii (inst-list-iter) is also updated
// to the next inst
bool Optimizer::foldPseudoAndOr(G4_BB* bb, INST_LIST_ITER& ii)
{
    // case 3

    // optimization should apply even when the dst of the pseudo-and/pseudo-or is global,
    // since we are just hoisting it up, and WAR/WAW checks should be performed as we search
    // for the src0 and src1 inst.

    G4_INST* inst = *ii;
    // look for def of srcs
    G4_Operand* src0 = inst->getSrc(0);
    G4_Operand* src1 = inst->getSrc(1);

    /*
    Old logic would scan from inst (and/or) up until it encountered
    def instructions that did full write.
    It would scan from def instruction down to inst looking for WAW, WAR
    conflicts between def instruction and intermediate instructions.
    Basically insuring that there were no partial writes in to flag registers
    use by the inst.

    The new code uses defInstList directly, and aborts if there are more then are
    two definitions. Which means there is more then one instruction writing to source.
    Disadvantage of that is that it is less precisise. For example if we are folding in to
    closest definition then before it was OK, but now will be disallowed.
    */
    int32_t maxSrc1 = 0;
    int32_t maxSrc2 = 0;
    G4_INST *defInstructions[2] = { nullptr, nullptr };
    G4_INST* src0DefInst = nullptr;
    G4_INST* src1DefInst = nullptr;

    //Picks the latest two instructions to compare.
    //local IDs are reset at the beginning of this function.
    if (inst->def_size() < 2)
    {
        return false;
    }
    //trying to find latest instructions that define src0 and src1
    for (auto I = inst->def_begin(), E= inst->def_end(); I != E; ++I)
    {
        G4_INST* srcInst = I->first;
        if ((I->second == Opnd_src0) && (srcInst->getLocalId() >= maxSrc1))
        {
            maxSrc1 = srcInst->getLocalId();
            defInstructions[0] = srcInst;
            src0DefInst = srcInst;
        }
        else if ((I->second == Opnd_src1) && (srcInst->getLocalId() >= maxSrc2))
        {
            maxSrc2 = srcInst->getLocalId();
            defInstructions[1] = srcInst;
            src1DefInst = srcInst;
        }
    }

    // Making sure that dst of pseudo instruction is not used or defined between pseudo instruction
    // and the first definition of the source
    if (defInstructions[0] && defInstructions[1])
    {
        // make defInst[0] the closer def to the pseudo-and/or
        if (maxSrc2 > maxSrc1)
        {
            std::swap(defInstructions[0], defInstructions[1]);
            std::swap(maxSrc1, maxSrc2);
        }
        //Doing backward scan until earlist src to make sure dst of and/or is not being written to
        //or being read
        /*
        handling case like in spmv_csr
        cmp.lt (M1, 1) P15 V40(0,0)<0;1,0> 0x10:w                                    /// $191
        cmp.lt (M1, 1) P16 V110(0,0)<0;1,0> V34(0,0)<0;1,0>                          /// $192
        and (M1, 1) P16 P16 P15                                                      /// $193
        */
        if (chkBwdOutputHazard(defInstructions[1], ii, defInstructions[0]))
        {
            return false;
        }
    }
    else
    {
        return false;
    }

    // check if the defInst can be folded into the pseudo and/or for a given source
    // folding is legal if
    // -- src is the only use of defInst
    // -- def completely defines the src
    // -- def inst does not have predicate
    // -- the defInst closer to the pseudo inst is a cmp without pred
    // -- def inst is not global operand
    // the last condition can be relaxed if defInst is the same as the inst's dst,
    // as they will be the same flag
    if (!(defInstructions[0]->opcode() == G4_cmp && defInstructions[0]->getPredicate() == nullptr))
    {
        return false;
    }

    auto checkSource = [this, inst](G4_INST* defInst, G4_Operand* src)
    {
        if (defInst == nullptr)
        {
            return false;
        }

        if (defInst->use_size() > 1)
        {
            return false;
        }
        G4_Operand* curr_dst = defInst->getCondMod() ?
            defInst->getCondMod() : (G4_Operand*) defInst->getDst();

        G4_CmpRelation rel = curr_dst->compareOperand(src);
        if (rel != Rel_eq ||
            (defInst->getPredicate() && defInst->opcode() != G4_sel))
        {
            return false;
        }

        if (fg.globalOpndHT.isOpndGlobal(curr_dst))
        {
            G4_DstRegRegion* dst = inst->getDst();
            if (dst->compareOperand(curr_dst) != Rel_eq)
            {
                return false;
            }
        }

        return true;
    };

    if (!checkSource(src0DefInst, src0) || !checkSource(src1DefInst, src1))
    {
        return false;
    }

    // do the case 3 optimization

    G4_PredState ps = (inst->opcode() == G4_pseudo_or) ? PredState_Minus : PredState_Plus;
    G4_INST *first_inst = defInstructions[1];
    G4_INST *second_inst = defInstructions[0];

    // unify subregister according to logic op dst
    G4_VarBase *curr_base = inst->getDst()->getBase()->asRegVar();
    unsigned short curr_subreg = 0;

    G4_Operand *first_def, *second_def;
    G4_VarBase *first_def_base, *second_def_base;
    int first_def_subreg, second_def_subreg;

    // change condmod and predicate of second inst

    if (first_inst->getCondMod())
    {
        first_def = first_inst->getCondMod();
        first_def_base = first_def->asCondMod()->getBase();
        first_def_subreg = first_def->asCondMod()->getSubRegOff();
    }
    else
    {
        first_def = first_inst->getDst();
        first_def_base = first_def->asDstRegRegion()->getBase()->asRegVar();
        first_def_subreg = 0;
    }

    if (second_inst->getCondMod())
    {
        second_def = second_inst->getCondMod();
        second_def_base = second_def->asCondMod()->getBase();
        second_def_subreg = second_def->asCondMod()->getSubRegOff();
    }
    else
    {
        second_def = second_inst->getDst();
        second_def_base = second_def->asDstRegRegion()->getBase()->asRegVar();
        second_def_subreg = 0;
    }

    bool change_flag = false;
    if (second_inst->getCondMod() &&
    	(second_def_base != curr_base || second_def_subreg != curr_subreg))
    {
        change_flag = true;
        G4_CondMod *new_condMod = builder.createCondMod(
            second_inst->getCondMod()->getMod(),
            curr_base,
            curr_subreg);

        second_inst->setCondMod(new_condMod);
    }
    // create a predicate

    G4_Predicate *new_pred = builder.createPredicate(
        ps,
        curr_base,
        curr_subreg);

    second_inst->setPredicate(new_pred);

    if (change_flag)
    {
        for (USE_EDGE_LIST_ITER iter = second_inst->use_begin();
            iter != second_inst->use_end();
            ++iter)
        {
            G4_INST *curr_inst = (*iter).first;
            if (curr_inst == inst)
            {
                continue;
            }
            if (curr_inst->getPredicate() &&
                (curr_inst->getPredicate()->getBase() != curr_base ||
                    curr_inst->getPredicate()->getSubRegOff() != curr_subreg) &&
                curr_inst->getPredicate()->compareOperand(second_def) == Rel_eq)
            {
                curr_inst->setPredicate((G4_Predicate*)builder.duplicateOperand(new_pred));
            }

            for (int k = 0; k < G4_Inst_Table[curr_inst->opcode()].n_srcs; k++)
            {
                G4_Operand *curr_src = curr_inst->getSrc(k);
                if (curr_src->isSrcRegRegion() && !(curr_inst->isMath() && k == 1 && curr_src->isNullReg()))
                {
                    if (curr_src->asSrcRegRegion()->compareOperand(second_def) == Rel_eq)
                    {
                        G4_SrcRegRegion *new_src_opnd = builder.createSrcRegRegion(
                            curr_src->asSrcRegRegion()->getModifier(),
                            curr_src->asSrcRegRegion()->getRegAccess(),
                            inst->getDst()->getBase(),
                            0,
                            0,
                            curr_src->asSrcRegRegion()->getRegion(),
                            curr_src->asSrcRegRegion()->getType());

                        curr_inst->setSrc(new_src_opnd, k);
                    }
                }
            }
        }
    }

    if (first_def_base != curr_base || first_def_subreg != curr_subreg)
    {
        if (first_inst->getCondMod() && first_def->isCondMod())
        {
            G4_CondMod *new_condMod = builder.createCondMod(
                first_inst->getCondMod()->getMod(),
                curr_base,
                curr_subreg);
            first_inst->setCondMod(new_condMod);
        }
        else
        {
            first_inst->setDest((G4_DstRegRegion*)builder.duplicateOperand(inst->getDst()));
        }
        for (USE_EDGE_LIST_ITER iter = first_inst->use_begin();
            iter != first_inst->use_end();
            ++iter)
        {
            G4_INST *curr_inst = (*iter).first;
            if (curr_inst == inst)
            {
                continue;
            }
            if (curr_inst->getPredicate() &&
                (curr_inst->getPredicate()->getBase() != curr_base ||
                    curr_inst->getPredicate()->getSubRegOff() != curr_subreg) &&
                curr_inst->getPredicate()->compareOperand(first_def) == Rel_eq)
            {
                curr_inst->setPredicate((G4_Predicate*)builder.duplicateOperand(new_pred));
            }

            for (int k = 0; k < G4_Inst_Table[curr_inst->opcode()].n_srcs; k++)
            {
                G4_Operand *curr_src = curr_inst->getSrc(k);
                if (curr_src->isSrcRegRegion() && !(curr_inst->isMath() && k == 1 && curr_src->isNullReg()))
                {
                    if (curr_src->asSrcRegRegion()->compareOperand(first_def) == Rel_eq)
                    {
                        G4_SrcRegRegion *new_src_opnd = builder.createSrcRegRegion(
                            curr_src->asSrcRegRegion()->getModifier(),
                            curr_src->asSrcRegRegion()->getRegAccess(),
                            curr_base,
                            0,
                            curr_subreg,
                            curr_src->asSrcRegRegion()->getRegion(),
                            curr_src->asSrcRegRegion()->getType());
                        curr_inst->setSrc(new_src_opnd, k);
                    }
                }
            }
        }
    }

    // update def-use
    // since inst (the pseudo-and/or) is deleted and the same flag is used for first
    // and second inst, we have to
    // -- transfer inst's use to the second_inst
    // -- add def-use between first_inst and second_inst
    // -- remove inst from first_inst and second_inst's use
    inst->transferUse(second_inst, true);
    first_inst->addDefUse(second_inst, Opnd_pred);
    inst->removeUseOfInst();

    INST_LIST_ITER new_iter = ii;
    ++ii;
    bb->erase(new_iter);
    return true;
}


    /***  The beginning of message header optimization  ***/

    /*
    * reuse the previous mrf which can save the redundant definitions.
    */
    void MSGTable::reusePreviousHeader(G4_INST *dest,
        G4_INST *source,
        G4_INST *mDot2,
        IR_Builder &builder)
    {
        if (dest==NULL) return;
        if (source != NULL)
        {
            dest->setDest( builder.duplicateOperand( source->getDst() ) );
        }
        else
        {
            short subRegOff = dest->getDst()->getSubRegOff();
            dest->setDest( builder.duplicateOperand( mDot2->getDst() ) );
            dest->getDst()->setSubRegOff( subRegOff );
        }
    }

    /*
    * insert a mov, from the previous mrf to current mrf
    * this is only required for the instruction,
    * whose payload size >1 so that we can't directly reuse
    * the previous mrf. keep a copy from the previous mrf,
    * so that we only need to update the fields that need to be changed.
    */
    void MSGTable::insertMovMRFInst(G4_INST *source_send,
        IR_Builder& builder,
        G4_BB *bb)
    {
        G4_INST *inst = NULL;
        INST_LIST_ITER pos;

        switch (first)
        {
        case MRF_FULL_REGISTER:
            inst = m;
            pos  = m_it;
            break;
        case MRF_X:
            inst = mDot0;
            pos  = mDot0_it;
            break;
        case MRF_Y:
            inst = mDot1;
            pos  = mDot1_it;
            break;
        case MRF_SIZE:
            inst = mDot2;
            pos  = mDot2_it;
            break;
        default:
            MUST_BE_TRUE(false, "did not catch the first MRF def instruction correctly");
            return;
        }

        // mov(8) m_new<1>, m_old<8;8:1>  {Align1}
        G4_Declare *srcDcl = source_send->getSrc(0)->getBase()->asRegVar()->getDeclare();
        G4_SrcRegRegion* newSrcOpnd = builder.Create_Src_Opnd_From_Dcl(srcDcl, builder.getRegionStride1());

        G4_INST* mov_mrf = builder.createInternalInst(
            NULL,
            G4_mov,
            NULL,
            false,
            m->getExecSize(),
            builder.duplicateOperand( m->getDst() ),
            newSrcOpnd, //builder.duplicateOperand( source->send->getSrc(0) ),
            NULL,
            m->getOption(), // to fix the SIMD control flow
            inst->getLineNo(),
            inst->getCISAOff(),
            inst->getSrcFilename() );
        bb->insert( pos, mov_mrf );

        // maintain def-use.
        //
        // (1) Uses. m is ready to be deleted.
        m->transferUse(mov_mrf);

        // (2) Defs
        // The defs should be from definitions for source->send's src(0).
        //
        // mov (8) V244(1,0)<1>:ud V88(0,0)<8;8,1>:ud {Align1, NoMask}
        // mov (8) V244(0,0)<1>:ud r0.0<8;8,1>:ud {Align1, NoMask}
        // mov (1) V244(0,2)<1>:ud 0x1f:ud {Align1, NoMask}
        // mov (1) V244(0,0)<1>:ud 0:uw {Align1, NoMask}
        // mov (1) V244(0,1)<1>:ud 0:uw {Align1, NoMask}
        // add (1) a0.0<1>:ud r1.1<0;1,0>:ud 0x40a8000:ud {Align1, NoMask}
        // send (8) null<1>:ud V244(0,0)<8;8,1>:ud a0.0<0;1,0>:ud {Align1, NoMask}  <- source_send
        // mov (8) V89(0,0)<1>:d V34_histogram(1,0)<8;8,1>:d {Align1, Q1}
        // mov (8) V246(1,0)<1>:ud V89(0,0)<8;8,1>:ud {Align1, NoMask}
        // mov (8) V246(0,0)<1>:ud V244(0,0)<8;8,1>:ud {Align1, NoMask} <-- mov_mrf
        // mov (8) V246(0,0)<1>:ud r0.0<8;8,1>:ud {Align1, NoMask}      <-- m
        //
        // There are more than one defs here.
        //
        // Note that
        //
        // mov (8) V244(1,0)<1>:ud V88(0,0)<8;8,1>:ud {Align1, NoMask}
        //
        // is a definition of send but not for mov_mrf. We enable checked
        // while copying defs.
        source_send->copyDef(mov_mrf, Opnd_src0, Opnd_src0, /*checked*/true);
    }

    /*
    * compare the two instructions
    * they must have:
    * the same instruction opcode, predicate, condmodifier, optionString
    * the same dst, and number of src args
    */
    bool Optimizer::isHeaderOptCandidate(G4_INST *dst, G4_INST *src)
    {
        if ( !dst   ||
            !src )
        {
            return true;
        }

        // Compare instructions
        if ( dst->opcode() != src->opcode()             ||
            dst->getOption() != src->getOption()       ||
            dst->getExecSize() != src->getExecSize()   ||
            dst->getPredicate() != src->getPredicate() ||
            dst->getCondMod() != src->getCondMod())
        {
            return false;
        }

        if ( dst->getNumSrc() != src->getNumSrc() )
        {
            return false;
        }

        // Compare destination args
        G4_Operand *dst_dest = dst->getDst();
        G4_Operand *src_dest = src->getDst();
        if ( !dst_dest   ||
            !src_dest   )
        {
            return false;
        }

        return true;
    }

    /*
    * compare the two instructions to see if they are redundent
    * they must have:
    * the same src args
    * the same def
    */
    bool Optimizer::isHeaderOptReuse(G4_INST *dst, G4_INST *src)
    {
        if ( !dst   &&
            !src )
        {
            return true;
        }
        if ( !dst   ||
            !src   )
        {
            return false;
        }

        for ( unsigned int i = 0; i < G4_MAX_SRCS; i++)
        {
            G4_Operand* opnd = dst->getSrc(i);

            if ( opnd != NULL && opnd->compareOperand( src->getSrc(i) ) != Rel_eq )
            {
                return false;
            }
        }

        // The same number of def instructions
        if (dst->def_size() != src->def_size())
        {
            return false;
        }

        if (dst->def_size() == 0)
        {
            return true; // both have no def at all
        }

        for (auto ii = dst->def_begin(); ii != dst->def_end(); ii++)
        {
            bool sameDef = false;
            for (auto jj = src->def_begin(); jj != src->def_end(); jj++)
            {
                if ((*ii).first == (*jj).first &&
                    (*ii).second == (*jj).second)
                {
                    sameDef = true;
                    break; // break the inner jj loop
                }
            }
            if (sameDef == false)
            {
                return false;
            }
        }
        return true;
    }

    bool Optimizer::headerOptValidityCheck(MSGTable *dest, MSGTable *source)
    {
        if ( !isHeaderOptCandidate(dest->a0Dot0, source->a0Dot0)   ||
            !isHeaderOptCandidate(dest->mDot0,  source->mDot0)    ||
            !isHeaderOptCandidate(dest->m,      source->m)        ||
            !isHeaderOptCandidate(dest->mDot1,  source->mDot1)    ||
            !isHeaderOptCandidate(dest->mDot2, source->mDot2) )
        {
            return false;
        }

        if ( dest->m )
        {
            if ( !(dest->m->hasOneUse() && dest->m->use_front().first == dest->send) )
            {
                return false;
            }
        }
        if ( dest->mDot0 )
        {
            if ( !(dest->mDot0->hasOneUse() && dest->mDot0->use_front().first == dest->send) )
            {
                return false;
            }
        }
        if ( dest->mDot1 )
        {
            if ( !(dest->mDot1->hasOneUse() && dest->mDot1->use_front().first == dest->send) )
            {
                return false;
            }
        }
        if ( dest->mDot2 )
        {
            if ( !(dest->mDot2->hasOneUse() && dest->mDot2->use_front().first == dest->send) )
            {
                return false;
            }
        }

        if ( dest->send                         &&
            dest->send->getSrc(0)              &&
            dest->send->getSrc(0)->getTopDcl() &&
            source->send                       &&
            source->send->getSrc(0)            &&
            source->send->getSrc(0)->getTopDcl() )
        {
            unsigned short dstSize, sourceSize;
            dstSize    = dest->send->getSrc(0)->getTopDcl()->getTotalElems() *
                dest->send->getSrc(0)->getTopDcl()->getElemSize();
            sourceSize = source->send->getSrc(0)->getTopDcl()->getTotalElems() *
                source->send->getSrc(0)->getTopDcl()->getElemSize();
            if ( dstSize != sourceSize )
            {
                return false;
            }
        }

        return true;
    }

    // a class to store all presently valid values, each value is an instruction
    // if the number of values exceeds the max allowed, the oldest is removed
    class InstValues
    {
        const int maxNumVal;
        std::list<G4_INST*> values;

    public:
        InstValues(int maxCount) : maxNumVal(maxCount) {}

        void addValue(G4_INST* inst)
        {
            if (values.size() == maxNumVal)
            {
                values.pop_front();
            }
            values.push_back(inst);
        }

        // delete all values that may be invalid after inst
        void deleteValue(G4_INST* inst)
        {
            if (inst->isOptBarrier())
            {
                values.clear();
            }

            G4_DstRegRegion* dst = inst->getDst();
            auto interferes = [dst](G4_INST* valInst)
            {
                G4_DstRegRegion* valDst = valInst->getDst();
                if (dst->compareOperand(valDst) != Rel_disjoint)
                {
                    return true;
                }
                for (int i = 0, numSrc = valInst->getNumSrc(); i < numSrc; ++i)
                {
                    G4_Operand* src = valInst->getSrc(i);
                    if (src != nullptr && dst->compareOperand(src) != Rel_disjoint)
                    {
                        return true;
                    }
                }
                return false;
            };
            if (dst != nullptr)
            {
                values.remove_if(interferes);
            }
        }

        G4_INST* findValue(G4_INST* inst)
        {
            for (auto valInst : values)
            {
                if (inst->opcode() != valInst->opcode() ||
                    inst->getExecSize() != valInst->getExecSize())
                {
                    continue;
                }
                // skip flags for now
                if ((inst->getPredicate() || valInst->getPredicate()) ||
                    (inst->getCondMod() || valInst->getCondMod()))
                {
                    continue;
                }
                // emask checks
                if (inst->getMaskOffset() != valInst->getMaskOffset() ||
                    inst->isWriteEnableInst() ^ valInst->isWriteEnableInst())
                {
                    continue;
                }
                // all source should be isomorphic (same type/shape)
                bool srcMatch = true;
                for (int i = 0, numSrc = inst->getNumSrc(); i < numSrc; ++i)
                {
                    G4_Operand* src = inst->getSrc(i);
                    G4_Operand* valSrc = valInst->getSrc(i);
                    if (src == nullptr || valSrc == nullptr || src->compareOperand(valSrc) != Rel_eq)
                    {
                        srcMatch = false;
                        break;
                    }
                }
                if (srcMatch)
                {
                    return valInst;
                }
            }
            return nullptr;
        }

        void clear()
        {
            values.clear();
        }
    };

    G4_Operand* Optimizer::updateSendsHeaderReuse(
        std::vector<std::vector<G4_INST*>> &instLookUpTable,
        std::vector<G4_INST*> &iVector)
    {
        int bSize = (int)iVector.size();
        for (auto & Cache : instLookUpTable)
        {
            if (Cache.size() == bSize)
            {
                bool match[8] = { false };
                bool anyMatch = false;
                for (int index = 0; index < bSize; ++index)
                {
                    G4_INST* cInst = Cache[index];
                    G4_INST* iInst = iVector[index];

                    // opcode check
                    if (cInst->opcode() != iInst->opcode() ||
                        cInst->getExecSize() != iInst->getExecSize())
                    {
                        continue;
                    }
                    // flag check
                    if (cInst->getPredicate() != iInst->getPredicate() ||
                        cInst->getCondMod() != iInst->getCondMod())
                    {
                        continue;
                    }
                    // emask check
                    if (cInst->getMaskOffset() != iInst->getMaskOffset() ||
                        cInst->isWriteEnableInst() ^ iInst->isWriteEnableInst())
                    {
                        continue;
                    }
                    // dst check
                    G4_DstRegRegion *cDstRgn = cInst->getDst();
                    G4_DstRegRegion *iDstRgn = iInst->getDst();
                    if (cDstRgn->getRegOff() != iDstRgn->getRegOff() ||
                        cDstRgn->getSubRegOff() != iDstRgn->getSubRegOff() ||
                        cDstRgn->getHorzStride() != iDstRgn->getHorzStride() ||
                        cDstRgn->getRegAccess() != iDstRgn->getRegAccess() ||
                        cDstRgn->getType() != iDstRgn->getType())
                    {
                        continue;
                    }

                    // all source should be isomorphic (same type/shape)
                    bool srcMatch = true;
                    for (int iSrc = 0, numSrc = cInst->getNumSrc(); iSrc < numSrc; ++iSrc)
                    {
                        G4_Operand* cOpnd = cInst->getSrc(iSrc);
                        G4_Operand* iOpnd = iInst->getSrc(iSrc);
                        if (cOpnd == nullptr || iOpnd == nullptr || cOpnd->compareOperand(iOpnd) != Rel_eq)
                        {
                            srcMatch = false;
                            break;
                        }
                    }
                    match[index] = srcMatch;
                    anyMatch |= srcMatch;
                }

                if (anyMatch)
                {
                    // at least partial match

                    for (int index = 0; index < bSize; ++index)
                    {
                        G4_INST* cInst = Cache[index];
                        G4_INST* iInst = iVector[index];

                        // mark it if there is match
                        if (match[index])
                        {
                            iInst->markDead();
                            continue;
                        }

                        // create new dst region to replace one in iVector[i]
                        G4_DstRegRegion *cDst = cInst->getDst();
                        G4_DstRegRegion *iDst = iInst->getDst();
                        G4_DstRegRegion *newDstRegion = builder.createDstRegRegion(Direct, cDst->getTopDcl()->getRegVar(),
                                iDst->getRegOff(), iDst->getSubRegOff(), iDst->getHorzStride(), iDst->getType());
                        iInst->setDest(newDstRegion);

                        // update the look-up table list
                        Cache[index] = iInst;
                    }
                    return Cache[0]->getDst();
                }
            }
        }
        return nullptr;
    }

    //
    // Perform value numbering on writes to the extended msg descriptor for bindless access of the form
    // op (1) a0.2<1>:ud src0 src1 src2 {NoMask}
    // and remove redundant instructions.  This is limited to within BB
    //
    void Optimizer::cleanupBindless()
    {
        // Perform send header cleanup for bindless sampler/surface
        for (auto bb : fg.BBs)
        {
            std::vector<std::vector<G4_INST*>> instLookUpTable;
            std::vector<G4_INST*> instVector;
            for (auto iter = bb->begin(), iterEnd = bb->end();
                iter != iterEnd; ++iter)
            {
                G4_INST *inst = *iter;
                G4_DstRegRegion *dst = inst->getDst();

                if (dst != nullptr && dst->getTopDcl() != nullptr &&
                    dst->getTopDcl()->getCapableOfReuse())
                {
                    // it is header definition instruction
                    instVector.push_back(inst);
                }

                if (inst->isSplitSend())
                {
                    G4_Operand *header = inst->getSrc(0);
                    G4_Operand *exDesc = inst->getSrc(3);

                    if (header->getTopDcl() && header->getTopDcl()->getCapableOfReuse() &&
                        exDesc->getTopDcl() && exDesc->getTopDcl()->getIsExDesc())
                    {

                        if (instVector.size() != 0)
                        {
                            // check if we can reuse cached values
                            G4_Operand *value = updateSendsHeaderReuse(instLookUpTable, instVector);
                            if (value == nullptr)
                            {
                                // no found, cache the header
                                instLookUpTable.push_back(instVector);
                            }
                            else
                            {
                                // update sends header src
                                G4_SrcRegRegion* newHeaderRgn = builder.createSrcRegRegion(
                                    Mod_src_undef, Direct, value->getBase(), 0, 0, builder.getRegionStride1(), Type_UD);
                                inst->setSrc(newHeaderRgn, 0);
                            }
                        }
                    }
                    // clear header def
                    instVector.clear();
                }
                else if (inst->isSend() || inst->isSendc())
                {
                    instVector.clear();
                }
            }
            bb->erase(
                std::remove_if(bb->begin(), bb->end(), [](G4_INST* inst) { return inst->isDead(); }),
                bb->end());
        }

        for (auto bb : fg.BBs)
        {
            InstValues values(4);
            for (auto iter = bb->begin(), iterEnd = bb->end();
                iter != iterEnd;)
            {
                G4_INST* inst = *iter;
                G4_DstRegRegion* dst = inst->getDst();
                if (dst != nullptr && dst->getTopDcl() != nullptr &&
                    dst->getTopDcl()->getIsExDesc())
                {
                    G4_INST* valInst = values.findValue(inst);
                    if (valInst != nullptr)
                    {
#ifdef DEBUG_VERBOSE_ON
                        std::cout << "can replace \n";
                        inst->emit(std::cout);
                        std::cout << "\n with \n";
                        valInst->emit(std::cout);
                        std::cout << "\n";
#endif
                        for (auto I = inst->use_begin(), E = inst->use_end(); I != E; ++I)
                        {
                            // each use is in the form of A0(0,0)<0;1,0>:ud,
                            // in src3 of a split send
                            G4_INST* useInst = I->first;
                            Gen4_Operand_Number num = I->second;
                            MUST_BE_TRUE(useInst->isSplitSend() && num == Opnd_src3,
                                "use inst must be a split send inst");
                            G4_SrcRegRegion* newExDesc = builder.createSrcRegRegion(
                                Mod_src_undef, Direct, valInst->getDst()->getBase(), 0, 0,
                                builder.getRegionScalar(), Type_UD);
                            useInst->setSrc(newExDesc, 3);
                        }
                        iter = bb->erase(iter);
                        continue;
                    }
                    else
                    {
#ifdef DEBUG_VERBOSE_ON
                        std::cout << "add new value:\n";
                        inst->emit(std::cout);
                        std::cout << "\n";
#endif
                        values.addValue(inst);
                    }
                }
                else
                {
                    values.deleteValue(inst);
                }
                ++iter;
            }
        }
    }

    /*
    * compare the two send and their defs
    * determine whether to remove the redundant mov inst
    * or reuse the previous MRF
    *
    * 1    mov (8) V152(0,0)<1>:ud r0.0<8;8,1>:ud {Align1, NoMask}
    * 2    mov (1) V152(0,2)<1>:ud 0x7000f:ud {Align1, NoMask}
    * 3    mov (1) V152(0,0)<1>:ud 0:uw {Align1, NoMask}
    * 4    mov (1) V152(0,1)<1>:ud 0:uw {Align1, NoMask}
    * 5    add (1) a0.0<1>:ud r1.0<0;1,0>:ud 0x2490000:ud {Align1, NoMask}
    * 6    send (8) V32_in(0,0)<1>:ud V152(0,0)<8;8,1>:ud a0.0<0;1,0>:ud {Align1, NoMask}
    *
    * 7    mov (8) V154(0,0)<1>:ud r0.0<8;8,1>:ud {Align1, NoMask}
    * 8    mov (1) V152(0,2)<1>:ud 0x1f:ud {Align1, NoMask}
    * 9    mov (1) V154(0,0)<1>:ud 0:uw {Align1, NoMask}
    * 10   mov (1) V154(0,1)<1>:ud 0:uw {Align1, NoMask}
    * 11   add (1) a0.0<1>:ud r1.1<0;1,0>:ud 0x2190000:ud {Align1, NoMask}
    * 12   send (8) V33(0,0)<1>:d V152(0,0)<8;8,1>:ud a0.0<0;1,0>:ud {Align1, NoMask}
    *
    * It is rather tricky to maintain def-use chains for this optimization.
    * The send instruction (Line 12) can reuse Inst[1, 3, 4] and need to
    * keep Inst[8, 11]. The defs for send at line 12 is Inst[1, 3, 4, 8, 11]
    * and the defs for send at line 6 is Inst[1, 2, 3, 4, 5].
    *
    * We take the following approach to maintain def-use.
    *
    * - Starting with initial valid defs, [7, 8, 9, 10, 11].
    *
    * - Each to be removed instruction transfers its use to a proper
    *   previous definition.
    *
    * - Each to be kept instruction remains, even there may have changes
    *   in its definition. For example, dst of move instruction at Line 8
    *   is changed from V154 to V152, but no def-use modification should
    *   be made for this instruction.
    *
    * - No def-use modification should be made to the final send, since
    *   all have been properly set.
    */
    void Optimizer::optMessageHeaders(MSGTableList& msgList,
        G4_BB* bb,
        DEFA0& myA0)
    {
        unsigned char redundancyCount = 0;
        bool isSameX, isSameY, isSameSize;
        bool replaceOldHeader = false;

        uint16_t payLoadSize;

        MSGTable *dest, *source;
        MSGTable_ITER iter=msgList.begin();

        if(iter == msgList.end())
        {
            return;
        }
        dest = *iter;   // dest is the front
        iter++;
        if(iter == msgList.end())
        {
            return;
        }
        source = *iter; // source is the cached one

        if (!headerOptValidityCheck(dest, source))
        {
            return;
        }

        if ( isHeaderOptReuse(dest->a0Dot0, myA0.pred)  &&
            !myA0.isA0Redef )
        {
            // Transfer uses of dstDot0 to myA0.pred. This removes uses from
            // dest->a0Dot0 and add to myA0.pred. dest->a0Dot0 to be deleted.
            dest->a0Dot0->transferUse(myA0.pred, /*keepExisting*/true);
            dest->a0Dot0->markDead();
        }

        payLoadSize = dest->send->getMsgDesc()->MessageLength();

        isSameX = isHeaderOptReuse(dest->mDot0, source->mDot0)      &&
            !source->isXRedef ;

        isSameY = isHeaderOptReuse(dest->mDot1, source->mDot1)      &&
            !source->isYRedef;

        isSameSize = isHeaderOptReuse(dest->mDot2, source->mDot2)   &&
            !source->isSizeRedef;

        if ( isSameX && dest->mDot0 )
        {
            redundancyCount++;
        }
        if ( isSameY && dest->mDot1 )
        {
            redundancyCount++;
        }
        if ( isSameSize && dest->mDot2 )
        {
            redundancyCount++;
        }

        if ( payLoadSize > 1                    &&
            redundancyCount < MESSAGE_HEADER_THRESHOLD )
        {
            return; // don't delete if redunant insts >=THRESHold
        };

        if ( payLoadSize > 1                    &&
            !(redundancyCount ==3              &&
            dest->send->getSrc(0)->compareOperand(source->send->getSrc(0))
            == Rel_eq ) )
        {
            dest->insertMovMRFInst(source->send, builder, bb);
            replaceOldHeader = true;
        }

        {   // always remove "mov(8) Mx<1>, r0.0<8;8,1>:ud{Align1}"
            dest->m->markDead();
            if ( !replaceOldHeader )
            {
                dest->m->transferUse(source->m, /*keepExisting*/true);
                dest->m=source->m;
            }
        }

        if ( isSameX && dest->mDot0 )
        {
            dest->mDot0->markDead();
            if ( !replaceOldHeader )
            {
                dest->mDot0->transferUse(source->mDot0, /*keepExisting*/true);
                dest->mDot0 = source->mDot0;
            }
        }
        else if ( payLoadSize==1 && dest->mDot0 )
        {
            dest->reusePreviousHeader(dest->mDot0, source->mDot0, source->mDot2, builder);
            if ( !replaceOldHeader )
            {
                source->mDot0 = dest->mDot0;
            }
        }

        if ( isSameY && dest->mDot1 )
        {
            dest->mDot1->markDead();
            if ( !replaceOldHeader )
            {
                dest->mDot1->transferUse(source->mDot1, /*keepExisting*/true);
                dest->mDot1 = source->mDot1;
            }
        }
        else if ( payLoadSize==1 && dest->mDot1 )
        {
            dest->reusePreviousHeader(dest->mDot1, source->mDot1, source->mDot2, builder);
            if ( !replaceOldHeader )
            {
                source->mDot1 = dest->mDot1;
            }
        }

        if ( isSameSize && dest->mDot2 )
        {
            dest->mDot2->markDead();
            if ( !replaceOldHeader )
            {
                dest->mDot2->transferUse(source->mDot2, /*keepExisting*/true);
                dest->mDot2 = source->mDot2;
            }
        }
        else if ( payLoadSize==1 && dest->mDot2 )
        {
            dest->reusePreviousHeader(dest->mDot2, source->mDot2, source->mDot2, builder);
            if ( !replaceOldHeader )
            {
                source->mDot2 = dest->mDot2;
            }
        }

        if ( payLoadSize==1 )
        {
            // Check this function's comments for why no def-use changes
            // should be made on resetting src(0).
            G4_Operand *src0 = source->send->getSrc(0);
            dest->send->setSrc(builder.duplicateOperand(src0), 0);
        }

        dest->opt = true;

        return;
    }

    bool Optimizer::isHeaderCachingCandidate(G4_INST *inst)
    {
        if (inst->isSend())
        {
            return true;
        }

        if ( inst->useEmpty() )
        {
            return false;
        }
        USE_EDGE_LIST_ITER iter;
        for( iter = inst->use_begin(); iter != inst->use_end(); ++iter )
        {
            if ((*iter).first->isSend())
            {
                G4_INST *send = (*iter).first;
                G4_Operand *header = send->getSrc(0);
                G4_DstRegRegion *dst = inst->getDst();

                //def to BuiltInA0 is part of header opt
                if (inst->getDst()                                            &&
                    inst->getDst()->getBase()                   &&
                    inst->getDst()->getBase()->isRegVar()       &&
                    inst->getDst()->getBase()->asRegVar() ==
                    builder.getBuiltinA0()->getRegVar()                         &&
                    inst->getDst()->getRegOff() == 0            &&
                    inst->getDst()->getSubRegOff() == 0         )
                {
                    return true;
                }

                // make sure that dst of the current inst is header, not payload
                // header is hard-coded to be 32 bytes
                if ( header->getTopDcl()    == dst->getTopDcl()         &&
                    dst->getLeftBound() >= header->getLeftBound()      &&
                    dst->getRightBound() <= header->getLeftBound() + GENX_GRF_REG_SIZ -1 )
                {
                    return true;
                }
                return false;
            }
        }

        return false;
    }

   /*
    * mark the below "and" instruction as redundant;
    * the "and" instruction is the header for a barrier:
    *   and (8) r1.0<1>:ud r0.2<0;1,0>:ud 0xf000000:ud {Align1, NoMask}
    *   send (1) null<1>:ud r1 0x3 0x2000004:ud{Align1}
    *   wait n0:ud {Align1}
    */
    void Optimizer::removeRedundantBarrierHeaders(G4_INST *sendInst,
                                          G4_SrcRegRegion* barrierSrc0,
                                          bool first)
    {
        bool barrier = false;
        G4_SrcRegRegion *src0 = NULL;
        if (!first) // skip the check as already done so for first barrier
        {
            barrier = isBarrierPattern(sendInst, src0);
        }
        if (barrier || first)
        {
            auto item = sendInst->def_begin();
            G4_INST *andInst = item->first;
            // delete all the uses of andInst
            // addInst and sendInst will have no def and no use
            andInst->removeAllUses();
            // sendInst.src0 (addInst.dst) will be replaced by barrierSend.src0
            // create a new G4_SrcRegRegion, which is a copy of barrierSend.src0
            G4_SrcRegRegion *src = builder.createSrcRegRegion(
                    Mod_src_undef,
                    Direct,
                    barrierSrc0->getTopDcl()->getRegVar(),
                    0,
                    0,
                    barrierSrc0->getRegion(),
                    barrierSrc0->getTopDcl()->getElemType() );
            sendInst->setSrc(src, 0);
            andInst->markDead();
        }
    }

    /*
    * pattern match of a code sequence for ISA_BARRIER:
    *   and (8) r1.0<1>:ud r0.2<0;1,0>:ud 0xf000000:ud {Align1, NoMask}
    *   send (1) null<1>:ud r1 0x3 0x2000004:ud{Align1}
    *   wait n0:ud {Align1}
    */
    bool Optimizer::isBarrierPattern(G4_INST *sendInst,
                                     G4_SrcRegRegion *& barrierSendSrc0)
    {
        /*
         * check G4_send
         */
        G4_SendMsgDescriptor *desc = sendInst->getMsgDesc();
        uint32_t descVal = desc->getDesc();
        if ((desc->getFuncId() == SFID_GATEWAY) &&
            (descVal == (0x1 << 25) + 0x4) && // 0x2000004
            (sendInst->def_size() == 1))
        {
            auto item = sendInst->def_begin();
            G4_INST *andInst = item->first; // getting "and" from send's def

           /*
            * check G4_and
            */
            if ((andInst) && (andInst->opcode() == G4_and) &&
                (item->second == Opnd_src0))
            {
                G4_Operand *src0 = andInst->getSrc(0);
                G4_Operand *src1 = andInst->getSrc(1);

                bool isSrc0 =
                    ((src0->isSrcRegRegion()) &&
                     (src0->asSrcRegRegion()->getBase()) &&
                     (src0->asSrcRegRegion()->getBase()->isRegVar()) &&
                     (src0->asSrcRegRegion()->getBase()->asRegVar() ==
                      builder.getBuiltinR0()->getRegVar()) &&
                     (src0->asSrcRegRegion()->getRegOff() == 0) &&
                     (src0->asSrcRegRegion()->getSubRegOff() == 2)); // r0.2

                bool isSrc1 = ((src1->isImm() && !src1->isRelocImm()) &&
                               (((src1->asImm()->getInt() == 0x0F000000) &&
                                 (getGenxPlatform() < GENX_SKL)) ||
                                ((src1->asImm()->getInt() == 0x8F000000) &&
                                 (getGenxPlatform() >= GENX_SKL))));

                if (isSrc0 && isSrc1 && sendInst->getSrc(0) &&
                    sendInst->getSrc(0)->isSrcRegRegion())
                {
                    barrierSendSrc0 = sendInst->getSrc(0)->asSrcRegRegion();
                    return true;
                }
            }
        }

        return false;
    }

    /*
     * add the the barrier header as the top instruction
     */
    void Optimizer::hoistBarrierHeaderToTop(G4_SrcRegRegion *barrierSendSrc0)
    {
        G4_Declare *dcl = barrierSendSrc0->getTopDcl();
        IR_Builder *mybuilder = &builder;

        // below code is copied from translateVISASyncInst() for ISA_BARRIER
        // all other dwords are ignored
        // and (8) r32.0:ud r0.2:ud 0x0F000000

        G4_SrcRegRegion* r0_src_opnd = builder.createSrcRegRegion(
            Mod_src_undef,
            Direct,
            mybuilder->getBuiltinR0()->getRegVar(),
            0,
            2,
            builder.getRegionScalar(),
            Type_UD);
        G4_DstRegRegion *mrf_dst1_opnd = builder.createDstRegRegion(
            Direct,
            dcl->getRegVar(),
            0,
            0,
            1,
            Type_UD);

        G4_Imm * g4Imm = NULL;

        //for SKL+ there are 5 bits for barrierID
        //5th bit is stored in bit 31 of second dword
        if(getGenxPlatform() < GENX_SKL)
        {
            g4Imm = builder.createImm( 0x0F000000, Type_UD );
        }
        else
        {
            g4Imm = builder.createImm( 0x8F000000, Type_UD );
        }

        // create inst
        G4_INST *andInst =
            builder.createInst(     NULL,
                G4_and,
                NULL,
                false,
                8,
                mrf_dst1_opnd,
                r0_src_opnd,
                g4Imm,
                InstOpt_WriteEnable,
                0 );

        // add "and" to the instruction list
        BB_LIST_ITER ib = fg.BBs.begin();
        G4_INST *inst = NULL;
        BB_LIST_ITER bend(fg.BBs.end());
        INST_LIST_ITER ii;
        // skip all the labels, find 1st inst and insert the barrier header
        for(ib = fg.BBs.begin(); ib != bend; ++ib)
        {
            G4_BB* bb = (*ib);
            ii = bb->begin();
            INST_LIST_ITER iend = bb->end();
            for (; ii != iend;ii++ )
            {
                inst = *ii;
                if (inst->opcode() != G4_label)
                {
                    bb->insert( ii, andInst );
                    return;
                }
            }
        }

    }

    /*
     * check whether there are new definitions in order to determine redundancy
     */
    bool Optimizer::chkNewDefBetweenSends(G4_INST *inst, MSGTableList& msgList, DEFA0& myA0)
    {
        bool isDef = false;
        msgList.unique();

        // check SIMD8 VxH region everytime
        if ( inst->getDst()                         &&
            inst->getDst()->isAddress() )
        {
            isDef = myA0.isA0Redef = true;
        }
        else
        {
            for (int i = 0; i < inst->getNumSrc(); i++)
            {
                if (inst->getSrc(i)                     &&
                    inst->getSrc(i)->isAddress() )
                {
                    isDef = myA0.isA0Redef = true;
                    break;
                }
            }
        }

        if ( msgList.size() < 2 )
        {
            return false;
        }
        MSGTable_ITER ii = msgList.begin();
        if(ii == msgList.end())
        {
            return false;
        }

        ii++;

        if(ii == msgList.end())
        {
            return false;
        }

        MSGTable* last = *(ii);
        if ( last == NULL       ||
            last->send == NULL )
        {
            return false;
        }
        G4_Operand *def = inst->getDst();
        if ( def == NULL)
        {
            return false;
        }
        if ( last->mDot0                                        &&
             ( def->compareOperand(last->mDot0->getSrc(0)) == Rel_eq ||
               ( last->mDot0->getSrc(1) && def->compareOperand(last->mDot0->getSrc(1)) == Rel_eq ) ||
                ( last->mDot0->getSrc(2) && def->compareOperand(last->mDot0->getSrc(2)) == Rel_eq ) ) )
        {
            isDef = last->isXRedef = true;
        }
        else if ( last->mDot1                                   &&
             ( def->compareOperand(last->mDot1->getSrc(0)) == Rel_eq ||
               ( last->mDot1->getSrc(1) && def->compareOperand(last->mDot1->getSrc(1)) == Rel_eq ) ||
               ( last->mDot1->getSrc(2) && def->compareOperand(last->mDot1->getSrc(2)) == Rel_eq ) ) )
        {
            isDef = last->isYRedef = true;
        }
        else if ( last->mDot2                                   &&
             ( def->compareOperand(last->mDot2->getSrc(0)) == Rel_eq  ||
               ( last->mDot2->getSrc(1) && def->compareOperand(last->mDot2->getSrc(1)) == Rel_eq ) ||
               ( last->mDot2->getSrc(2) && def->compareOperand(last->mDot2->getSrc(2)) == Rel_eq ) ) )
        {
             isDef = last->isSizeRedef = true;
        }
        else if ( last->m                                       &&
            ( def->compareOperand(last->m->getSrc(0)) == Rel_eq ||
              ( last->m->getSrc(1) && def->compareOperand(last->m->getSrc(1)) == Rel_eq ) ||
              ( last->m->getSrc(2) && def->compareOperand(last->m->getSrc(2)) == Rel_eq ) ) )
        {
            isDef = last->isR0Dot0Redef = true;
        }
        return isDef;

    }

    /*
    * Cache the send and its def into a table for optimization
    */
    void Optimizer::addEntryToMessageTable(G4_INST *inst,
        MSGTableList& msgList,
        G4_BB* bb,
        INST_LIST_ITER ii,
        DEFA0 &myA0)
    {
        MSGTable *item = msgList.front();
        if ( inst->isSend() )
        {
            item->send = inst;
            item->opt           = false;
            item->isR0Dot0Redef = false;
            item->isXRedef      = false;
            item->isYRedef      = false;
            item->isSizeRedef   = false;

            if (item->invalid)
            {
                msgList.pop_front();
            }
            else if (item->a0Dot0 != NULL &&   // only def a0.0
                 (item->m == NULL  || item->mDot2 == NULL))
            {
                if ( isHeaderOptCandidate(item->a0Dot0, myA0.pred) )
                {
                    if ( isHeaderOptReuse(item->a0Dot0, myA0.pred)  &&
                        !myA0.isA0Redef )
                    {
                        // Transfer uses of a0Dot0 to myA0.pred. This removes uses from
                        // a0Dot0 and add to myA0.pred. item->a0Dot0 to be deleted.
                        item->a0Dot0->transferUse(myA0.pred, /*keepExisting*/true);
                        item->a0Dot0->markDead();
                    }
                }
                msgList.pop_front();
            }
            else if (item->a0Dot0 &&  item->m && item->mDot2) // complete header def
            {

                msgList.unique();
                if (msgList.size() >= 2)
                {
                    optMessageHeaders(msgList, bb, myA0);
                    if (msgList.front()->opt                                    &&
                        msgList.front()->send->getMsgDesc()->MessageLength() == 1 )
                    {
                        // keep the oldest send for subsequent read operations
                        // but the instruction to define a0.0 needs to be latest
                        //msgList.back()->a0Dot0 = msgList.front()->a0Dot0;
                        msgList.pop_front(); // delete first element
                    }
                    else if (msgList.front()->opt                               &&
                        msgList.front()->send->getMsgDesc()->MessageLength() >= 1 )
                    {
                        // keep the latest send for subsequent write operations
                        msgList.pop_back();
                    }
                    else
                    {
                        msgList.pop_back();
                    }
                    myA0.isA0Redef      = false;
                }
            }
            else
            {
                // not an optimization candidate
                msgList.pop_front();
            }
        }
        else if ( inst->getDst()                                          &&
            inst->getDst()->getBase()                   &&
            inst->getDst()->getBase()->isRegVar()       &&
            inst->getDst()->getBase()->asRegVar() ==
            builder.getBuiltinA0()->getRegVar()                           &&
            inst->getDst()->getRegOff() == 0            &&
            inst->getDst()->getSubRegOff() == 0         )
        {
            // is builtInA0.0
            item->a0Dot0 = inst;
            item->a0Dot0_it = ii;

            if ( myA0.curr == NULL )
            {
                myA0.pred = NULL;
                myA0.isA0Redef      = false;
            }
            else if (!myA0.curr->isDead())
            {
                // only update the a0 def when we didn't remove it
                myA0.pred = myA0.curr;
                myA0.predIt = myA0.currIt;
            }
            myA0.currIt = ii;
            myA0.curr = inst;
        }
       else if ( inst->getSrc(0) )
       {
            G4_DstRegRegion *dst = inst->getDst();
            if (dst)
            {
                if (dst->getRegOff() == 0)
                {
                    // mov(8) m.0, builtInR0.0
                    G4_Operand *src = inst->getSrc(0);
                    if( dst->getSubRegOff() == 0          &&
                        inst->getExecSize() == 8                            &&
                        src                                                 &&
                        src->isSrcRegRegion()                               &&
                        src->asSrcRegRegion()->getBase()                    &&
                        src->asSrcRegRegion()->getBase()->isRegVar()        &&
                        src->asSrcRegRegion()->getBase()->asRegVar()  ==
                        builder.getBuiltinR0()->getRegVar()            &&
                        src->asSrcRegRegion()->getRegOff() == 0             &&
                        src->asSrcRegRegion()->getSubRegOff() == 0 )
                    {
                        if (item->first == MRF_UNDEF)
                            item->first = MRF_FULL_REGISTER;
                        item->m = inst;
                        item->m_it = ii;
                    }
                    // mov(1) m.0
                    else if(dst->getSubRegOff() == 0  &&
                        inst->getExecSize() == 1                        )
                    {
                        if (item->first == MRF_UNDEF)
                            item->first = MRF_X;
                        item->mDot0 = inst;
                        item->mDot0_it = ii;
                    }
                    // mov(1) m.1
                    else if(dst->getSubRegOff() == 1  &&
                        inst->getExecSize() == 1                        )
                    {
                        if (item->first == MRF_UNDEF)
                            item->first = MRF_Y;
                        item->mDot1 = inst;
                        item->mDot1_it = ii;
                    }
                    // mov(1) m0.2
                    else if(dst->getSubRegOff() == 2  &&
                        inst->getExecSize() == 1                        )
                    {
                        if (item->first == MRF_UNDEF)
                            item->first = MRF_SIZE;
                        item->mDot2 = inst;
                        item->mDot2_it = ii;
                    }
                    else
                    {
                        // unrecognized update to header
                        item->invalid = true;
                    }
                }
            }
        }

    }

    void Optimizer::messageHeaderReport(size_t ic_before,
        size_t ic_after,
        G4_Kernel& kernel)
    {
        if ( builder.getOption(vISA_OptReport))
        {
            std::ofstream optReport;
            getOptReportStream(optReport, builder.getOptions());
            optReport << "             === Message Header Optimization ===" <<endl;
            optReport<< fixed << endl;
            optReport<< kernel.getName() <<" is reduced from "
                <<ic_before <<" to " <<ic_after <<" instructions. "<<endl;
            if (((float)(ic_before)) != 0.0)
            {
            optReport<< setprecision(0)
                << (float)((ic_before-ic_after)*100)/(float)(ic_before)
                << "% instructions of this kernel are removed."
                <<endl;
            }
            optReport<<endl;
            closeOptReportStream(optReport);
        }
    }

    /**
    *  optimizer for removal of redundant message header instructions
    */
    void Optimizer::cleanMessageHeader()
    {
        MSGTableList msgList;
        BB_LIST_ITER ib, bend(fg.BBs.end());
        size_t ic_before = 0;
        size_t ic_after = 0;

        std::stack<MSGTable*> toDelete;

        bool isRedundantBarrier = false;
        G4_SrcRegRegion *barrierSendSrc0 = NULL;

        for(ib = fg.BBs.begin(); ib != bend; ++ib)
        {

            msgList.clear();
            MSGTable *newItem   = (MSGTable *)mem.alloc(sizeof(MSGTable));
            toDelete.push(newItem);
            memset(newItem, 0, sizeof(MSGTable));
            newItem->first      = MRF_UNDEF;

            msgList.push_front(newItem);
            G4_BB* bb = (*ib);
            INST_LIST_ITER ii = bb->begin();
            INST_LIST_ITER iend = bb->end();
            ic_before += bb->size();

            DEFA0 myA0;
            myA0.curr      = NULL;
            myA0.pred      = NULL;
            myA0.isA0Redef = false;

            for (; ii != iend; ii++ )
            {
                G4_INST *inst = *ii;
                if ( isHeaderCachingCandidate( inst ) )
                {
                    if ( inst->opcode() == G4_send && isRedundantBarrier )
                    {
                        removeRedundantBarrierHeaders(inst, barrierSendSrc0, false);
                    }
                    else if ( inst->opcode() == G4_send && !isRedundantBarrier )
                    {
                        isRedundantBarrier = isBarrierPattern(inst, barrierSendSrc0);
                        if (isRedundantBarrier)
                        {
                            removeRedundantBarrierHeaders(inst, barrierSendSrc0, true);
                        }
                    }

                    addEntryToMessageTable(inst, msgList, bb, ii, myA0);
                    if ( inst->isSend() )
                    {
                        MSGTable * item = (MSGTable *)mem.alloc(sizeof(MSGTable));
                        toDelete.push(item);
                        memset(item, 0, sizeof(MSGTable));
                        item->first  = MRF_UNDEF;
                        msgList.push_front(item);
                    }
                }
                else
                {
                    chkNewDefBetweenSends(inst, msgList, myA0);
                }
            }

            // Dead code elimination
            for(ii = bb->begin(); ii != bb->end();)
            {
                G4_INST *inst = *ii;
                INST_LIST_ITER curr = ii++;
                if( inst->isDead() )
                {
                    inst->removeUseOfInst();
                    bb->erase(curr);
                }
            }

            ic_after += bb->size();
        }

        messageHeaderReport(ic_before, ic_after, kernel);

        if (isRedundantBarrier)
        {
            hoistBarrierHeaderToTop(barrierSendSrc0);
        }

        //
        // Destroy STL iterators allocated using mem manager
        //
        while( toDelete.size() > 0 )
        {
            toDelete.top()->~MSGTable();
            toDelete.pop();
        }
        msgList.clear();
    }
    /***  The end of message header optimization  ***/

    void getOptReportStream(std::ofstream& reportStream, const Options *opt)
    {
        char optReportFileName[MAX_OPTION_STR_LENGTH+20];
        const char *asmFileName;
        opt->getOption(VISA_AsmFileName, asmFileName);
        SNPRINTF(optReportFileName, MAX_OPTION_STR_LENGTH, "%s_optreport.txt", asmFileName);
        reportStream.open(optReportFileName, ios::out | ios::app  );
        MUST_BE_TRUE(reportStream, "Fail to open " << optReportFileName);
    }

    void closeOptReportStream(std::ofstream& reportStream)
    {
        reportStream.close();
    }

    void Optimizer::sendFusion()
    {
        (void) doSendFusion(&fg, &mem);
    }

    // For a subroutine, insert a dummy move with {Switch} option immediately
    // before the first non-label instruction in BB. Otherwie, for a following
    // basic block, insert a dummy move before *any* instruction to ensure that
    // no instruction should be placed between the targe jip/uip label and its
    // associated instruction.
    void Optimizer::addSwitchOptionToBB(G4_BB* bb, bool isSubroutine)
    {
        auto instIter = bb->begin();
        if (isSubroutine)
        {
            for (auto instEnd = bb->end(); instIter != instEnd; ++instIter)
            {
                G4_INST* bbInst = *instIter;
                if (bbInst->isLabel())
                {
                    continue;
                }
                else
                {
                    break;
                }
            }
        }

        if (instIter != bb->end() && ((*instIter)->getOption() & InstOpt_Switch))
        {
            // this BB is already processed, skip
            return;
        }

        // mov (1) null<1>:ud r0.0<0;1,0>:ud {Switch}
        G4_DstRegRegion* movDst = builder.createNullDst(Type_UD);
        G4_SrcRegRegion* movSrc = builder.Create_Src_Opnd_From_Dcl(builder.getBuiltinR0(), builder.getRegionScalar());
        G4_INST* movInst = builder.createInternalInst(NULL, G4_mov, NULL, false, 1, movDst, movSrc, NULL, InstOpt_WriteEnable);
        movInst->setOptionOn(InstOpt_Switch);
        bb->insert(instIter, movInst);
    }

    void Optimizer::linePlaneWA(G4_INST* inst)
    {

        /*
        Putting it here instead of in HW confomrity because we need original src0 region
        in scheduler to calculate RB correctly. Otherwise setup moves for src0 get scheduled after instruction

        *  HW check #12: Check and correct the first operand for line instruction
        *  Actually it must be a replicated stream of 4 contiguous elements.
        *  That means <0;4,1> region. But in asm code it must be presented as
        *  replicated scalar - <0;1,0>.
        */
        if (inst->opcode() == G4_line || inst->opcode() == G4_pln)
        {
            G4_Operand *src = inst->getSrc(0);
            RegionDesc *rd = src->isSrcRegRegion() ? src->asSrcRegRegion()->getRegion() : NULL;
            MUST_BE_TRUE( rd != NULL, " Src0 of line inst is not regregion. " );
            if (rd->isScalar())
            {
                return;
            }
            MUST_BE_TRUE(( rd->vertStride == 0 || rd->vertStride == 4 ) && rd->width == 4,
                "Unexpected region for the first line operand." );

            // create a new rd for src0
            RegionDesc *new_rd = builder.getRegionScalar();
            src->asSrcRegRegion()->setRegion( new_rd );
        }
    }

	//
	// This inserts two dummy moves to clear flag dependencies before EOT:
	// mov(1) null:ud f0.0<0;1,0>:ud{ Align1, Q1, NoMask }
	// mov(1) null:ud f1.0<0;1,0>:ud{ Align1, Q1, NoMask }
	// This is done if f0/f1 is ever defined in a BB but not used in it, as we conservatively assume
	// that the flag may be undefined when the EOT is reached.
	// Note that USC only does this if EOT is inside control flow, i.e., EOT is an early exit
	//
	void Optimizer::clearARFDependencies()
	{
		auto flagToInt = [](G4_Areg* areg)
		{
			MUST_BE_TRUE(areg->isFlag(), "expect F0 or F1");
			return areg->getArchRegType() == AREG_F0 ? 0 : 1;
		};
		// see if F0 and F1 are ever defined but not used in the same BB
		bool unusedFlag[2]; // f0 and f1
		unusedFlag[0] = unusedFlag[1] = false;
		for (auto bb : fg.BBs)
		{
		    bool unusedFlagLocal[2]; // f0 and f1
		    unusedFlagLocal[0] = unusedFlagLocal[1] = false;

			for (auto inst : *bb)
			{
				// check predicate source
				if (inst->getPredicate())
				{
                    G4_VarBase* flag = inst->getPredicate()->getBase();
					if (flag->isRegVar())
					{
						G4_Areg* areg = (G4_Areg*) flag->asRegVar()->getPhyReg();
						unusedFlagLocal[flagToInt(areg)] = false;
					}
				}
				else
				{
					// check explicit source
					for (int i = 0; i < inst->getNumSrc(); ++i)
					{
						if (inst->getSrc(i) && inst->getSrc(i)->isSrcRegRegion() && inst->getSrc(i)->isFlag())
						{
							G4_SrcRegRegion* src = inst->getSrc(i)->asSrcRegRegion();
							if (src->getBase()->isRegVar())
							{
								G4_Areg* flag = (G4_Areg*)src->getBase()->asRegVar()->getPhyReg();
								unusedFlagLocal[flagToInt(flag)] = false;
							}
						}
					}
				}

				// check explicit dst
				if (inst->getDst() && inst->getDst()->isFlag())
				{
					// flag is an explicit dst
					G4_DstRegRegion* dst = inst->getDst();
					if (dst->getBase()->isRegVar())
					{
						G4_Areg* flag = (G4_Areg*)dst->getBase()->asRegVar()->getPhyReg();
						unusedFlagLocal[flagToInt(flag)] = true;
					}
				}
				// check cond mod
				else if (inst->getCondMod() && inst->getCondMod()->getBase())
				{
                    G4_VarBase* flag = inst->getCondMod()->getBase();
					if (flag->isRegVar())
					{
						G4_Areg* areg = (G4_Areg*)flag->asRegVar()->getPhyReg();
						unusedFlagLocal[flagToInt(areg)] = true;
					}
				}
			}

            if (unusedFlagLocal[0] &&
                unusedFlag[0] == false)
            {
                unusedFlag[0] = true;
            }

            if (unusedFlagLocal[1] &&
                unusedFlag[1] == false)
            {
                unusedFlag[1] = true;
            }

			if (unusedFlag[0] && unusedFlag[1])
			{
				break;
			}
		}

        if (unusedFlag[0] || unusedFlag[1])
        {
            for (auto bb : fg.BBs)
            {
                if (bb->size() == 0)
                {
                    return;
                }
                G4_INST* inst = bb->back();
                if (inst->isEOT())
                {
                    auto instIter = bb->end();
                    --instIter;
                    if (unusedFlag[0])
                    {
                        G4_SrcRegRegion* flagSrc = builder.createSrcRegRegion(Mod_src_undef,
                            Direct, builder.phyregpool.getF0Reg(), 0, 0,
                            builder.getRegionScalar(), Type_UD);
                        G4_DstRegRegion* nullDst = builder.createNullDst(Type_UD);
                        G4_INST* inst = builder.createInternalInst(NULL, G4_mov, NULL, false, 1, nullDst, flagSrc, NULL, InstOpt_WriteEnable);
                        bb->insert(instIter, inst);
                    }
                    if (unusedFlag[1])
                    {
                        G4_SrcRegRegion* flagSrc = builder.createSrcRegRegion(Mod_src_undef,
                            Direct, builder.phyregpool.getF1Reg(), 0, 0,
                            builder.getRegionScalar(), Type_UD);
                        G4_DstRegRegion* nullDst = builder.createNullDst(Type_UD);
                        G4_INST* inst = builder.createInternalInst(NULL, G4_mov, NULL, false, 1, nullDst, flagSrc, NULL, InstOpt_WriteEnable);
                        bb->insert(instIter, inst);
                    }
                }
            }
        }
    }

    // change the send src0 region to be consistent with assembler expectation
    // We do it here instead of HW conformity since they only affect binary encoding
    // ToDo: this should not be necessary anymore, should see if we can remove
    void Optimizer::fixSendSrcRegion(G4_INST* inst)
    {
        if (inst->isSend() && inst->getSrc(0) != NULL)
        {
            RegionDesc* newDesc = NULL;
            uint8_t execSize = inst->getExecSize();
            if (execSize == 1)
            {
                newDesc = builder.getRegionScalar();
            }
            else if (execSize > 8)
            {
                newDesc = builder.getRegionStride1();
            }
            else
            {
                newDesc = builder.createRegionDesc(execSize, execSize, 1);
            }
            inst->getSrc(0)->asSrcRegRegion()->setRegion(newDesc);
        }
    }

    /*
    some workaround for HW restrictions.  We apply them here so as not to affect optimizations, RA, and scheduling
    [DevBDW:A]: A goto instruction must not be followed by any instruction requiring register indirect access on source operands.
    [DevBDW:A]: A join instruction must not be followed by any instruction requiring register indirect access on source operands.
    [DevBDW]: A POW/FDIV operation must not be followed by an instruction that requires two destination registers.
    For BDW A-stepping, we need a WA to prevent the following instructions:
    Send r10  (read from data port to r10)
    Send null r10 (write dataport from r10)
    Insert a dummy mov from r10
    [BDW+]: call/return's execution size must be set to 16/32 so that the emask will be passed correctly to the callee
    [BDW,CHV,SKL]: A call instruction must be followed by an instruction that supports Switch.
                   When call takes a jump, the first instruction must have a Switch.
    */
    void Optimizer::HWWorkaround()
    {
        // Ensure the first instruction of a stack function has switch option.
        if (fg.getIsStackCallFunc() &&
            VISA_WA_CHECK(builder.getPWaTable(), WaThreadSwitchAfterCall) &&
            !builder.getOption(vISA_enablePreemption))
        {
            addSwitchOptionToBB(fg.getEntryBB(), true);
        }

        // set physical pred/succ as it's needed for the call WA
        fg.setPhysicalPredSucc();
        BB_LIST_ITER ib, bend(fg.BBs.end());
        for(ib = fg.BBs.begin(); ib != bend; ++ib)
        {
            G4_BB* bb = (*ib);
            INST_LIST_ITER ii = bb->begin();

            while (ii != bb->end())
            {
                G4_INST *inst = *ii;

                if (inst->isFence() && inst->getMsgDesc()->ResponseLength() > 0)
                {
                    // commit is enabled for the fence, need to generate a move after to make sure the fence is complete
                    // mov (8) r1.0<1>:ud r1.0<8;8,1>:ud {NoMask}
                    INST_LIST_ITER nextIter = ii;
                    nextIter++;
                    G4_DstRegRegion* dst = inst->getDst();
                    G4_Declare* fenceDcl = dst->getBase()->asRegVar()->getDeclare();
                    G4_DstRegRegion* movDst = builder.Create_Dst_Opnd_From_Dcl(fenceDcl, 1);
                    G4_SrcRegRegion* movSrc = builder.Create_Src_Opnd_From_Dcl(fenceDcl, builder.createRegionDesc(8,8,1));
                    G4_INST* movInst = builder.createInternalInst( NULL, G4_mov, NULL, false, 8, movDst, movSrc, NULL, InstOpt_WriteEnable);
                    bb->insert(nextIter, movInst);
                }

                if (inst->isCall() || inst->isFCall())
                {
                    if (VISA_WA_CHECK(builder.getPWaTable(), WaThreadSwitchAfterCall))
                    {
                        // WA:
                        // A call instruction must be followed by an instruction that supports Switch.
                        // When call takes a jump, the first instruction must have a Switch.
                        BB_LIST_ITER nextBBIter = ib;
                        ++nextBBIter;
                        if (nextBBIter != bend)
                        {
                            addSwitchOptionToBB(*nextBBIter, false);
                        }
                        // also do this for call target
                        addSwitchOptionToBB(bb->Succs.front(), true);
                    }
                }

                // we must set {Switch} if the instruction updates ARF with no scoreboard
                {
                    G4_DstRegRegion* dst = inst->getDst();
                    if (dst != nullptr && dst->getBase()->noScoreBoard())
                    {
                        inst->setOptionOn(InstOpt_Switch);
                    }
                }

                if (VISA_WA_CHECK(builder.getPWaTable(), WaDisableSendsPreemption) &&
                    inst->isSplitSend() &&
                    !(inst->getSrc(1)->isNullReg()))
                {
                    inst->setOptions(inst->getOption() | InstOpt_NoPreempt);
                }

                if (inst->isSend() && !inst->isNoPreemptInst() && builder.needsNoPreemptR2ForSend())
                {
                    G4_Operand *Src0 = inst->getSrc(0);
                    if (Src0 && Src0->isGreg())
                    {
                        unsigned LB = Src0->getLinearizedStart();
                        if (LB == 2 * GENX_GRF_REG_SIZ)
                        {
                            inst->setOptions(inst->getOption() | InstOpt_NoPreempt);
                        }
                    }
                }

                if (builder.hasFdivPowWA() &&
                    inst->isMath() &&
                    (inst->asMathInst()->getMathCtrl() == MATH_FDIV || inst->asMathInst()->getMathCtrl() == MATH_POW))
                {
                    INST_LIST_ITER nextIter = ii;
                    nextIter++;
                    if (nextIter == bb->end())
                    {
                        break;
                    }
                    // check next inst
                    G4_INST *nextInst = *nextIter;
                    if (!nextInst->isSend() && nextInst->getDst() && !nextInst->hasNULLDst() && nextInst->getDst()->crossGRF())
                    {
                        // insert a nop
                        G4_INST *nopInst = builder.createInternalInst(NULL, G4_nop, NULL, false, 1, NULL, NULL, NULL, inst->getOption());
                        bb->insert(nextIter, nopInst);
                    }
                }

                if (inst->isCall() || inst->isReturn())
                {
                    inst->setExecSize((unsigned char)kernel.getSimdSize());
                }

                // HW Workaround: for platforms without 64-bit regioning, change send src/dst type from QWord to DWord
                if (builder.no64bitRegioning() && inst->isSend())
                {
                    G4_DstRegRegion* dst = inst->getDst();
                    if (dst != NULL &&
                        G4_Type_Table[dst->getType()].byteSize == 8)
                    {
                        dst->setType(Type_D);
                    }

                    G4_Operand *src0 = inst->getSrc(0);
                    if (src0 != NULL &&
                        G4_Type_Table[src0->getType()].byteSize == 8)
                    {
                        src0->asSrcRegRegion()->setType(Type_D);
                    }

                    if (inst->isSplitSend())
                    {
                        G4_Operand *src1 = inst->getSrc(1);
                        if (src1 != NULL &&
                            G4_Type_Table[src1->getType()].byteSize == 8)
                        {
                            src1->asSrcRegRegion()->setType(Type_D);
                        }
                    }
                }

                if (inst->isEOT() && VISA_WA_CHECK(builder.getPWaTable(), WaClearTDRRegBeforeEOTForNonPS))
                {
                    // insert
                    // mov(8) tdr0:uw 0x0:uw {NoMask}
                    G4_DstRegRegion* tdrDst = builder.createDstRegRegion(Direct, builder.phyregpool.getTDRReg(),
                        0, 0, 1, Type_UW);
                    G4_Imm* src = builder.createImm(0, Type_UW);
                    G4_INST* movInst = builder.createInternalInst(nullptr, G4_mov, nullptr, false,
                        8, tdrDst, src, nullptr, InstOpt_WriteEnable | InstOpt_Switch);
                    bb->insert(ii, movInst);
                }


                if (VISA_WA_CHECK(builder.getPWaTable(), WaResetN0BeforeGatewayMessage) &&
                    inst->isSend() && inst->getMsgDesc()->isBarrierMsg())
                {
                    // mov (1) n0.0 0x0 {Switch}
                    G4_DstRegRegion* n0Dst = builder.createDstRegRegion(Direct,
                        builder.phyregpool.getN0Reg(), 0, 0, 1, Type_UD);
                    auto movInst = builder.createInternalInst(nullptr, G4_mov, nullptr, false, 1, n0Dst,
                        builder.createImm(0, Type_UD), nullptr, InstOpt_WriteEnable | InstOpt_Switch);
                    bb->insert(ii, movInst);
                }

                linePlaneWA(inst);
                fixSendSrcRegion(inst);
                ii++;
            }
        }

        if (VISA_WA_CHECK(builder.getPWaTable(), WaClearArfDependenciesBeforeEot))
        {
            clearARFDependencies();
        }
        if (VISA_WA_CHECK(builder.getPWaTable(), Wa_2201674230))
        {
            clearSendDependencies();
        }
    }

class NSDS {
    int GRF_BUCKET;
    int TOTAL_BUCKETS;

    struct InstrDescr;
    // Used to get the InstrDescr for a given instruction.
    // We use instr->local_id as the index into IDvec[]
    std::vector<InstrDescr *> IDvec;
    int idCnt;
    const int IDCNT_UNINIT = 0;
    const int IDCNT_INIT = 1;

    class Bucket;
    // The main data structure for the buckets
    std::vector<Bucket> bucketVec;

public:
    struct BucketDescr;
    typedef std::vector<BucketDescr> BDVec_t;
    typedef std::set<Bucket *> BSet_t;

private:
    enum RW {
        RW_UNINIT = 0,
        READ = 1 << 0,
        WRITE = 1 << 1,
        RW_MAX
    };

    // The IDs which are covered by other instructions
    std::set<InstrDescr *> coveredIDs;

    // This is the instruction descriptor
    struct InstrDescr {
        G4_INST *instr;
        // The buckets Read by this instruction and whether it is covered or not
        std::map<Bucket *, bool> bucketStatusR;
        // The buckets Written by this instr and whether it is covered or not
        std::map<Bucket *, bool> bucketStatusW;
        int coveredBucketsR;
        int coveredBucketsW;
        std::set<InstrDescr *> &coveredIDs;

        // Attach bucket to instruction descriptor (into bucketStatus{R,W})
        void addBucket(Bucket *bucket, RW rwType) {
            if (rwType & READ) {
                if (! bucketStatusR.count(bucket)) {
                    bucketStatusR.insert(std::pair<Bucket *, bool>(bucket, false));
                }
            }
            else if (rwType & WRITE) {
                if (! bucketStatusW.count(bucket)) {
                    bucketStatusW.insert(std::pair<Bucket *, bool>(bucket, false));
                }
            }
        }

        // Cover instructions reading BUCKET
        void coverR(Bucket *bucket) {
            // READ: If BUCKET has not been marked as covered, mark it now
            auto itR = bucketStatusR.find(bucket);
            if (itR != bucketStatusR.end() && itR->second == false) {
                coveredBucketsR++;
                itR->second = true;
            }

            // If there is at least one send instruction that reads at least
            // one of the buckets redy by this ID, collect it.
            if (coveredBucketsR) {
                coveredIDs.insert(this);
            }
        }

        // Cover instructions writing to BUCKET
        void coverW(Bucket *bucket) {
            // WRITE: If BUCKET has not been marked as covered, mark it now
            auto itW = bucketStatusW.find(bucket);
            if (itW != bucketStatusW.end() && itW->second == false) {
                coveredBucketsW++;
                itW->second = true;
            }

            // If there is at least one instruction that reads at least
            // one of the buckets written to by this ID, collect it.
            if (coveredBucketsW) {
                coveredIDs.insert(this);
            }
        }

        // Cover BUCKET for this instruction
        void coverRW(Bucket *bucket) {
            // READ: If BUCKET has not been marked as covered, mark it now
            auto itR = bucketStatusR.find(bucket);
            if (itR != bucketStatusR.end() && itR->second == false) {
                coveredBucketsR++;
                itR->second = true;
            }

            // WRITE: If BUCKET has not been marked as covered, mark it now
            auto itW = bucketStatusW.find(bucket);
            if (itW != bucketStatusW.end() && itW->second == false) {
                coveredBucketsW++;
                itW->second = true;
            }

            // If ID is fully covered from the read or write side, collect it.
            if ((coveredBucketsR && coveredBucketsR == bucketStatusR.size())
                || (coveredBucketsW && coveredBucketsW == bucketStatusW.size())) {
                coveredIDs.insert(this);
            }
        }

        InstrDescr(G4_INST *Instr, NSDS *nsds)
            : instr(Instr), coveredBucketsR(0), coveredBucketsW(0),
              coveredIDs(nsds->coveredIDs) {
            ;
        }

        void dump(void) {
            std::cerr << "  InstrDescr:\n";
            std::cerr << " ";
            instr->dump();

            std::cerr << "    BucketStatusR: ";
            for (auto pair : bucketStatusR) {
                Bucket *bucket = pair.first;
                bool covered = pair.second;
                std::cerr << bucket->getIndex()
                          << "[" << ((covered) ? "T" : "F") << "]"
                          << ", ";
            }
            std::cerr << "\n";

            std::cerr << "    BucketStatusW: ";
            for (auto pair : bucketStatusW) {
                Bucket *bucket = pair.first;
                bool covered = pair.second;
                std::cerr << bucket->getIndex()
                          << "[" << ((covered) ? "T" : "F") << "]"
                          << ", ";
            }
            std::cerr << "\n";

            std::cerr << "    coveredBucketsR: " << coveredBucketsR << "\n";
            std::cerr << "    coveredBucketsW: " << coveredBucketsW << "\n";
            std::cerr << "    ----\n";
        }
    };

    // Each bucket points to all instructions that read the register that
    // corresponds to this bucket.
    class Bucket {
        int index;                 // For debugging
        std::set<InstrDescr *> IDSet;
    public:
        Bucket(void) {
            index = 0;
        }
        void setIndex(int Index) { index = Index; }
        int getIndex(void) { return index; }
        std::set<InstrDescr *> &getIDSet(void) { return IDSet; }

        void addID(InstrDescr *ID) {
            IDSet.insert(ID);
        }
        void removeID(InstrDescr *ID) {
            IDSet.erase(ID);
        }

        // Cover instructions that read this bucket
        void coverReadIDs(CISA_SHARED_FUNCTION_ID SFID = SFID_NULL) {
            // Go through all the instructions that read this instruction.
            // Notify the instruction that one of its registers is covered
            for (InstrDescr *ID : IDSet) {
                assert(ID->instr->isSend());
                // Only sends with the same SFID can be covered
                if (SFID == SFID_NULL
                    || ID->instr->getMsgDesc()->getFuncId() == SFID) {
                    ID->coverR(this);
                }
            }
        }

        // Cover instructions that write this bucket
        void coverWrittenIDs(CISA_SHARED_FUNCTION_ID SFID = SFID_NULL) {
            // Go through all the instructions that read this instruction.
            // Notify the instruction that one of its registers is covered
            for (InstrDescr *ID : IDSet) {
                assert(ID->instr->isSend());
                // Only sends with the same SFID can be covered
                if (SFID == SFID_NULL
                    || ID->instr->getMsgDesc()->getFuncId() == SFID) {
                    ID->coverW(this);
                }
            }
        }

        // This bucket is reported as being written to.
        // Therefore kill the instructions that read this BD.
        // Each instruction may be reading from other buckets too.
        // Remove it from all those buckets.
        void kill(void) {
            // Temporarily cache the IDs to be removed
            std::set<InstrDescr *> removeIDs = IDSet;
            for (InstrDescr *ID : removeIDs) {
                // Remove ID from all the buckets it is reading
                for (auto &bucketStatus
                         : {ID->bucketStatusR, ID->bucketStatusW}) {
                    for (auto pair : bucketStatus) {
                        Bucket *B = pair.first;
                        B->removeID(ID);
                    }
                }
            }
        }

        // Return TRUE if no IDs are connected to this bucket
        int empty(void) {
            return IDSet.empty();
        }

        void dump(void) {
            std::cerr << "Bucket: " << index << "\n";
            for (InstrDescr *ID : IDSet) {
                ID->dump();
            }
        }
    };

    // Parse INSTR's OPND and generate the Bucket Descriptors.
    // Insert them into BDVec.
    void getBucketsForOperand(G4_INST* inst, RW rwType, G4_Operand* opnd,
                              BDVec_t& BDVec) {
        if (opnd->isLabel() || opnd->isImm()) {
            return;
        }
#define UNINIT_BUCKET -1
        int startingBucket = UNINIT_BUCKET;
        G4_VarBase* base = opnd->getBase();
        bool canSpanMultipleBuckets = false;

        assert(base && "If no base, then the operand is not touched by instr.");

        // If a register allocated regvar, then get the physical register
        G4_VarBase *phyReg = (base->isRegVar()) ?
            base->asRegVar()->getPhyReg() : base;

        switch (phyReg->getKind()) {
        case G4_VarBase::VK_phyGReg:
            // Handle GRFs - fast path
            startingBucket = opnd->getLinearizedStart()/G4_GRF_REG_NBYTES;
            canSpanMultipleBuckets = true;
            break;
        case G4_VarBase::VK_phyAReg:
            break;
        default:
            assert(0 && "Bad kind");
            break;
        }

        // Create one or more buckets and push them into the vector
        if (startingBucket != UNINIT_BUCKET) {
            if (canSpanMultipleBuckets) {
                unsigned int divisor;
                int baseBucket;
                assert(base->isGreg());
                divisor = G4_GRF_REG_NBYTES;
                baseBucket = GRF_BUCKET;
                int endingBucket = baseBucket
                    + opnd->getLinearizedEnd() / divisor;
                MUST_BE_TRUE(endingBucket >= startingBucket,
                             "Ending bucket less than starting bucket");
                int numBuckets = endingBucket - startingBucket + 1;
                for (int j = startingBucket;
                     j < (startingBucket + numBuckets); j++ ) {
                    BDVec.push_back(BucketDescr(j, rwType));
                }
            }
            else {
                BDVec.push_back(BucketDescr(startingBucket, rwType));
            }
        }
    }


    // Add INST insto the bucketVec
void genBucket(G4_INST *send, Bucket *bucket, RW rwType) {
        assert(send->isSend());
        // 1. Create Instruction Descriptor for SEND, or reuse existing one
        InstrDescr *ID;
        if (! send->getLocalId()) {
            ID = new InstrDescr(send, this);
            send->setLocalId(idCnt);
            IDvec[idCnt] = ID;
            idCnt++;
        } else {
            ID = IDvec[send->getLocalId()];
        }
        // 2. Add the bucket read by INST to the Instr Descriptor
        ID->addBucket(bucket, rwType);
        // 3. Insert the ID into the bucket
        bucket->addID(ID);
    }

    void IDvecInit(size_t bbSize) {
        assert(IDvec.size() == 0);
        IDvec.resize(bbSize + IDCNT_INIT);
        idCnt = IDCNT_INIT;
    }

    void IDvecFree(void) {
        for (InstrDescr *ID : IDvec) {
            if (ID) {
                delete ID;
            }
        }
        IDvec.clear();
        idCnt = IDCNT_INIT;
    }


public:

    // Debug print
    void dump(void) {
        for (size_t i = 0, e = bucketVec.size(); i != e; ++i) {
            Bucket *bucket = &bucketVec[i];
            if (bucket->empty()) {
                continue;
            }
            bucket->dump();
        }
    }

    struct BucketDescr {
        int bucketIdx;
        RW type;
        BucketDescr(int BucketIdx, RW rwType)
            : bucketIdx(BucketIdx), type(rwType) { ; }
    };

    // We pack all information we have collected for the instruction
    struct BucketDescrBox {
        BucketDescrBox() : hasIndirW(false) {
            BDVec.reserve(16);
        }
        // The vector of buckets read/writen by the instruction
        BDVec_t BDVec;
        // The instruction has an indirect write
        bool hasIndirW;
    };

    // Given an inst with physical register assignment,
    // return all bucket descriptors that the physical register can map
    // to. This requires taking in to account exec size, data
    // type, and whether inst is a send
    BucketDescrBox getBucketDescrs(G4_INST* inst, BucketDescrBox &instrBuckets) {
        // Iterate over all WRITTEN operands and create buckets.
        for (Gen4_Operand_Number opndNum : {Opnd_dst}) {
            G4_Operand *opnd = inst->getOperand(opndNum);
            if (! opnd || ! opnd->getBase()) {
                continue;
            }
            getBucketsForOperand(inst, WRITE, opnd, instrBuckets.BDVec);
            instrBuckets.hasIndirW |= opnd->asDstRegRegion()->isIndirect();
        }
        for (Gen4_Operand_Number opndNum
                 : {Opnd_src0, Opnd_src1, Opnd_src2, Opnd_src3}) {
            G4_Operand *opnd = inst->getOperand(opndNum);
            if (! opnd || ! opnd->getBase()) {
                continue;
            }
            getBucketsForOperand(inst, READ, opnd, instrBuckets.BDVec);
        }
        return instrBuckets;
    }

    // Kill the write buckets of the bucket descriptors
    void killWriteBuckets(BucketDescrBox &instrBuckets) {
        // An indirect write kills all active buckets.
        if (instrBuckets.hasIndirW) {
            // FIXME: make this faster by keeping track of active buckets
            for (Bucket &bucket : bucketVec) {
                bucket.kill();
            }
        } else {
            for (const BucketDescr &BD : instrBuckets.BDVec) {
                if (BD.type & WRITE) {
                    Bucket *bucket = &bucketVec[BD.bucketIdx];
                    bucket->kill();
                }
            }
        }
    }

    // // Kill the buckets that instr reasd from
    // void killReadBuckets(BucketDescrBox &instrBuckets) {
    //     for (const BucketDescr &BD : instrBuckets.BDVec) {
    //         if (BD.type & READ) {
    //             Bucket *bucket = &bucketVec[BD.bucketIdx];
    //             bucket->kill();
    //         }
    //     }
    // }

    // Kill the buckets of this INSTR only
    void killInstrBuckets(G4_INST *instr) {
        int IDidx = instr->getLocalId();
        assert(IDidx);
        InstrDescr *ID = IDvec[IDidx];
        // Go through the buckets related to this INSTR and
        // remove this ID from them all.
        for (auto &bucketStatus : {ID->bucketStatusR, ID->bucketStatusW}) {
            for (const auto &pair : bucketStatus) {
                Bucket *bucket = pair.first;
                bucket->removeID(ID);
            }
        }
        // We no longer carry an InstrDescr
        instr->setLocalId(IDCNT_UNINIT);
    }

    // Create new buckets for the bucket descriptors of the instruction
    void genReadBuckets(G4_INST *instr, const BucketDescrBox &instrBuckets) {
        for (const auto &BD : instrBuckets.BDVec) {
            if (BD.type & READ) {
                Bucket *bucket = &bucketVec[BD.bucketIdx];
                genBucket(instr, bucket, READ);
            }
        }
    }

    // Create new bucket for any registers written by the instruction
    void genWriteBuckets(G4_INST *instr, const BucketDescrBox &instrBuckets) {
        for (const auto &BD : instrBuckets.BDVec) {
            if (BD.type & WRITE) {
                Bucket *bucket = &bucketVec[BD.bucketIdx];
                genBucket(instr, bucket, WRITE);
            }
        }
    }

    // Cover the instructions that read the values read by instrBuckets
    void coverInstrReadingBucketsReadBy(const BucketDescrBox &instrBuckets,
                                        CISA_SHARED_FUNCTION_ID SFID) {
        for (const auto &BD : instrBuckets.BDVec) {
            if (BD.type == READ) {
                bucketVec[BD.bucketIdx].coverReadIDs(SFID);
            }
        }
    }

    // Cover the instructions that read the values read by instrBuckets
    void coverInstrWritingBucketsReadBy(const BucketDescrBox &instrBuckets) {
        for (const auto &BD : instrBuckets.BDVec) {
            if (BD.type == READ) {
                bucketVec[BD.bucketIdx].coverWrittenIDs();
            }
        }
    }

    // Return a vector of all fully covered instructions and clear the
    // set where these are held.
    std::vector<G4_INST *> getCoveredInstrsAndClear(void) {
        std::vector<G4_INST *> instrs;
        for (InstrDescr *ID : coveredIDs) {
            instrs.push_back(ID->instr);
        }
        coveredIDs.clear();
        return instrs;
    }

    NSDS(const Options *options, G4_BB *bb) {
        int totalGRFNum = options->getuInt32Option(vISA_TotalGRFNum);
        int TOTAL_BUCKETS = 0;
        GRF_BUCKET = TOTAL_BUCKETS;
        TOTAL_BUCKETS += totalGRFNum;

        bucketVec.resize(TOTAL_BUCKETS);
        // If not in debugging mode don't bother initializing ids
#if (defined(_DEBUG) || defined(_INTERNAL))
        for (size_t i = 0, e = bucketVec.size(); i != e; ++i) {
            bucketVec[i].setIndex((int)i);
        }
#endif
        IDvecInit(bb->size());
    };

    ~NSDS(void) {
        IDvecFree();
    }

    void verify(G4_BB *bb, G4_INST *fromInstr, G4_INST *toInstr) {
        NSDS::BucketDescrBox FromInstrBuckets;
        getBucketDescrs(fromInstr, FromInstrBuckets);

        bool foundFrom = false;
        for (auto instr : *bb) {
            if (instr != fromInstr && !foundFrom) {
                continue;
            }
            foundFrom = true;
            if (instr == fromInstr) {
                continue;
            }
            if (foundFrom && instr == toInstr) {
                break;
            }


            NSDS::BucketDescrBox InstrInstrBuckets;
            getBucketDescrs(instr, InstrInstrBuckets);
            assert(!InstrInstrBuckets.hasIndirW);
            for (const BucketDescr &BD : InstrInstrBuckets.BDVec) {
                if (BD.type & WRITE) {
                    // Look for this BD in FromInstrBuckets
                    for (const BucketDescr &FromBD : FromInstrBuckets.BDVec) {
                        if (FromBD.bucketIdx == BD.bucketIdx) {
                            assert(0);
                        }
                    }
                }
            }


        }
    }
};

    // Scan the code and insert the NoSrcDepSet flag if possible to remove some
    // scoreboard delays. The NoSrcDepSet flag allows the instruction that
    // follows to ignore the scoreboard dependence check for the registers read
    // and issue right away.
    //
    // Scenario 1: A send without the NoSrcDepSet can cover all sends with a
    //             RAR dependence against the registers they read.
    //    ... = send R1 SFID0 {NoSrcDepSet} (covered by the send below)
    //    ... = send R1 SFID2      -        (different SFID)
    //    ... = send R1 SFID0 {NoSrcDepSet} (covered by the send below)
    //    ... = send R1 SFID0
    //     ...
    //    ... = R1
    //     R1 = ...
    //
    // Scenario 2: A RAW dependence between an instruction and the destination
    //             register of a send allows it to be marked {NoSrcDepSet}
    //    send R1, Rx  SFID0 {NoSrcDepSet}
    //    R2 = send R1 SFID0 {NoSrcDepSet} (covered by RAW dependence)
    //    ... = R2
    //    R1 = ...
    //
    // Scenario 3: A covered send can also cover other sends of the same SFID
    //             if the sources of these sends are only accessed after the
    //             instruction that covered the first send
    //    send R2 SFID0 {NoSrcDepSet}
    //    send R1 SIFD0
    //    ...
    //    ... = R1
    //    R2 = ...

    // We can only set the flag if the registers read by the send are all read
    // by following sends before any write to that register.
    //
    // We visit the instructions top-down in a single pass.
    // For each instruction we collect the register buckets read and written.
    //
    // If the instruction is a send, we link it with the corresponding buckets
    // that are read by it.
    // Each instruction is wrapped in an IstrDescr object which keeps track of
    // the bucket status: that is whether they are covered or not by succeeding
    // instructions that read those registers.
    //
    //    InstrDescr
    //    +--------+ +--------+
    //    | sendA  | | sendB  |
    //    |r0,r1,r2| |r2,r3,r4|  bucketStatusR,W
    //    | T, F, T| | F, F, F| (covered flag T/F)
    //    +--------+ +--------+
    //      /  |  \  /   |   \
    // +----+----+----+----+----+....+----+----+----+...
    // | r0 | r1 | r2 | r3 | r4 |    |r128| p0 | p1 |
    // +----+----+----+----+----+....+----+----+----+...
    // Bucket                                           bucketVec
    //
    // For each register read we check the bucket tha that corresponds to it.
    // We iterate over all the sends connected to it and mark that bucket
    // as covered. If the send instruction has all its buckets covered,
    // then it is legal to mark it with the NoSrcDepSet flag.
    // The marked sends are removed from the bucket array.
    //
    // Upon an instruction that writes to a register, we access the
    // corresponding bucket in bucketVec, we go through all the instructions
    // that access it and we remove those instructions from the data structure.
    // For example upon an instruction r2=... we will remove both sendA and
    // sendB from the data structure.
    //

    // The entry point for the code that adds the NoSrcDepSet flag to
    // specific send isntructions for better performance.
    // For a description of the algorithm check the comments of the NSDS class.
    void Optimizer::NoSrcDepSet(void)
    {
        #define NSDS_RAW        // Enable NoSrcDepSet for RAW
        // Early return if invalid platform
        if (! builder.noSrcDepSetAllowedPlatform()
            || ! builder.getOption(vISA_EnableNoSrcDep)) {
            return;
        }
        #if (defined(_DEBUG) || defined(_INTERNAL))
        int dbgCnt = 0;
        #endif
        for (auto bb : fg.BBs) {
            NSDS nsds(builder.getOptions(), bb);
            for (auto instr : *bb) {
                if (instr->isLabel()) {
                    continue;
                }
                instr->setLocalId(0); // We are storing the IDs in it

                NSDS::BucketDescrBox instrBuckets;
                nsds.getBucketDescrs(instr, instrBuckets);

                // Cover any registers that the current instruction reads
                // To be safe, predicated instrs can kill but cannot cover.
                if (! instr->getPredicate()) {
                    // This is for Scenario 1.
                    // A send without the nosrcdepset flag can cover other sends
                    // that read the same register
                    if (instr->isSend()) {
                        if (builder.getOption(vISA_EnableNoSrcDepScen1)) {
                            auto SFID = instr->getMsgDesc()->getFuncId();
                            nsds.coverInstrReadingBucketsReadBy(instrBuckets, SFID);
                        }
                    }
                    // This is for Scenario 2.
                    // A non-send instruction reading from Rx
                    // can cover sends that write to Rx
                    else {
                        if (builder.getOption(vISA_EnableNoSrcDepScen2)) {
                            nsds.coverInstrWritingBucketsReadBy(instrBuckets);
                        }
                    }

                    // Check if all registers read by the instruction cover
                    // a send in NSDS. If so, this instruction gets marked.
                    for (G4_INST *coveredSend : nsds.getCoveredInstrsAndClear()) {
                        assert(coveredSend->isSend());
                        #if (defined(_DEBUG) || defined(_INTERNAL))
                        // Disable marking the instruction
                        int stopAt = builder.getOptions()->getuInt32Option(vISA_stopNoSrcDepSetAt);
                        if (stopAt == UINT_MAX || dbgCnt++ < abs(stopAt))
                        #endif
                            // MARK the instruction
                            coveredSend->setOptionOn(InstOpt_NoSrcDepSet);
                        // Kill the buckets associated to this instruction
                        nsds.killInstrBuckets(coveredSend);

                        // Verifier for NSDS
                        // #define NSDS_VERIFY
                        #ifdef NSDS_VERIFY
                        nsds.verify(bb, coveredSend, instr);
                        #endif
                    }
                }

                // Kill any bucket this instruction writes to.
                nsds.killWriteBuckets(instrBuckets);

                // Populate the buckets read by this send
                if (instr->isSend()) {
                    // RAR
                    nsds.genReadBuckets(instr, instrBuckets);
                    #ifdef NSDS_RAW
                    // RAW
                    nsds.genWriteBuckets(instr, instrBuckets);
                    #endif
                }
                #if (defined(_DEBUG) || defined(_INTERNAL))
                if (builder.getOptions()->getOption(vISA_DumpNoSrcDep)) {
                    nsds.dump();
                }
                #endif
            }
        }
    }

    //
    // rewrite source regions to satisfy various HW requirements.  This pass will not modify the instrppuctions otherwise
    // -- rewrite <1;1,0> to <2;2,1> when possible (exec size > 1, width is not used to cross GRF)
    //    as HW doesn't allow <1;1,0> to be co-issued
    //
    void Optimizer::normalizeRegion()
    {
        for (auto bb : fg.BBs)
        {
            for (auto inst : *bb)
            {
                if (inst->isCall() || inst->isReturn())
                {
                    // Do not rewrite region for call or return,
                    // as the effective execution size is 2.
                    continue;
                }

                if (inst->getExecSize() == 1)
                {
                    // Replace: mov (1) r64.0<4>:df  r3.0<0;1,0>:df
                    // with:    mov (1) r64.0<1>:df  r3.0<0;1,0>:df
                    // otherwise, will get incorrect results for HSW, HW mode
                    G4_Operand* dst = inst->getDst();
                    if (dst != NULL &&
                        dst->asDstRegRegion()->getHorzStride() > 1 &&
                        G4_Type_Table[dst->getType()].byteSize == 8)
                    {
                        dst->asDstRegRegion()->setHorzStride(1);
                    }
                }
                else
                {
                    for (int i = 0; i < inst->getNumSrc(); ++i)
                    {
                        G4_Operand* src = inst->getSrc(i);
                        // Only rewrite direct regions.
                        if (src && src->isSrcRegRegion() && src->asSrcRegRegion()->getRegAccess() == Direct)
                        {
                            G4_SrcRegRegion* srcRegion = src->asSrcRegRegion();
                            if (srcRegion->getRegion()->isContiguous(inst->getExecSize()))
                            {
                                srcRegion->rewriteContiguousRegion(builder, i);
                            }
                            else if (inst->isAlign1Ternary(builder))
                            {
                                // special checks for 3src inst with single non-unit stride region
                                // rewrite it as <s*2;s>
                                uint16_t stride = 0;
                                if (srcRegion->getRegion()->isSingleNonUnitStride(inst->getExecSize(), stride))
                                {
                                    MUST_BE_TRUE(stride <= 4, "illegal stride for align1 ternary region");
                                    srcRegion->setRegion(kernel.fg.builder->createRegionDesc(stride * 2, 2, stride));
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    void Optimizer::countGRFUsage()
    {
        unsigned int maxGRFNum = builder.getOptions()->getuInt32Option(vISA_TotalGRFNum);
        int count = 0;
        bool *GRFUse = (bool *) builder.mem.alloc(sizeof(bool) * maxGRFNum);
        for (unsigned int i = 0; i < maxGRFNum; ++i)
        {
            GRFUse[i] = false;
        }
        for (auto dcl : kernel.Declares)
        {
            if (dcl->getRegVar()->isGreg())
            {
                int GRFStart = dcl->getRegVar()->getPhyReg()->asGreg()->getRegNum();
                int numRows = dcl->getNumRows();
                MUST_BE_TRUE(GRFStart >= 0 && (GRFStart + numRows) <= (int)maxGRFNum,
                    "illegal GRF assignment");
                for (int i = GRFStart; i < GRFStart + numRows; ++i)
                {
                    GRFUse[i] = true;
                }
            }
        }

        for (unsigned int i = 0; i < maxGRFNum; ++i)
        {
            if (GRFUse[i])
            {
                count++;
            }
        }
        fg.builder->getJitInfo()->numGRFUsed = count;
        RELEASE_MSG("\tKernel " << kernel.getOrigCMName() << " : " << count << " registers\n");
    }

    //
    //  Dump the input payload to start of scratch space.
    //  this is strictly for debugging and we do not care if this gets overwritten by
    //  other usage of the scratch space (private memory, spill, etc.)
    //
    void Optimizer::dumpPayload()
    {
        int inputEnd = 0;
        for (int i = 0, numInput = kernel.fg.builder->getInputCount(); i < numInput; ++i)
        {
            input_info_t* input_info = kernel.fg.builder->getInputArg(i);
            if (inputEnd < input_info->size + input_info->offset)
            {
                inputEnd = input_info->size + input_info->offset;
            }
        }

        G4_BB* bb = kernel.fg.getEntryBB();
        // iter points to the first non-label inst
        auto iter = bb->begin(), bbEnd = bb->end();
        while (iter != bbEnd)
        {
            if (!(*iter)->isLabel())
            {
                break;
            }
            ++iter;
        }

        int regOffset = (inputEnd + GENX_GRF_REG_SIZ - 1) / GENX_GRF_REG_SIZ;

        static const unsigned SCRATCH_MSG_DESC_CATEGORY = 18;
        static const unsigned SCRATCH_MSG_DESC_OPERATION_MODE = 17;
        static const unsigned SCRATCH_MSG_DESC_CHANNEL_MODE = 16;
        static const unsigned SCRATCH_MSG_INVALIDATE_AFTER_READ = 15;
        static const unsigned SCRATCH_MSG_DESC_BLOCK_SIZE = 12;

        // write 8 GRFs at a time
        int msgSize = 8;
        for (int i = 1; i < regOffset; i += msgSize)
        {
            uint16_t extFuncCtrl = 0;
            // both scratch and block read use DC
            CISA_SHARED_FUNCTION_ID funcID = SFID_DP_DC;

            uint32_t headerPresent = 0x80000;
            uint32_t msgDescImm = headerPresent;
            uint32_t msgLength = 1;
            uint32_t blocksizeEncoding = 0x3; // 8 GRF
            msgDescImm |= (msgLength << getSendMsgLengthBitOffset());
            msgDescImm |= (1 << SCRATCH_MSG_DESC_CATEGORY);
            msgDescImm |= (1 << SCRATCH_MSG_DESC_CHANNEL_MODE);
            msgDescImm |= (1 << SCRATCH_MSG_DESC_OPERATION_MODE);

            msgDescImm |= (blocksizeEncoding << SCRATCH_MSG_DESC_BLOCK_SIZE);
            msgDescImm |= i;

            G4_SendMsgDescriptor* desc = kernel.fg.builder->createSendMsgDesc(
                msgDescImm, 0, 1, funcID, false, msgSize, extFuncCtrl, false, true, nullptr, nullptr);
            RegionDesc* region = kernel.fg.builder->getRegionStride1();
            G4_SrcRegRegion* headerOpnd = kernel.fg.builder->Create_Src_Opnd_From_Dcl(kernel.fg.builder->getBuiltinR0(), region);
            G4_Declare* tempDcl = builder.createHardwiredDeclare(msgSize * 8, Type_UD, i, 0);
            G4_SrcRegRegion* srcOpnd = kernel.fg.builder->Create_Src_Opnd_From_Dcl(tempDcl, region);
            G4_DstRegRegion* dstOpnd = kernel.fg.builder->createNullDst(Type_UD);

            G4_INST* sendInst = kernel.fg.builder->createSplitSendInst(
                nullptr, G4_sends, 16, dstOpnd, headerOpnd, srcOpnd,
                kernel.fg.builder->createImm(msgDescImm, Type_UD), InstOpt_WriteEnable, desc, nullptr, 0);
            bb->insert(iter, sendInst);
        }
    }

    void Optimizer::checkBarrierUsage()
    {
        builder.getJitInfo()->usesBarrier = false;
        for (auto bb : fg.BBs)
        {
            for (auto inst : *bb)
            {
                if (inst->isSend() && inst->getMsgDesc()->isBarrierMsg())
                {
                    builder.getJitInfo()->usesBarrier = true;
                    return;
                }
            }
        }
    }

    // Create a copy of R0 at top of kernel,
    // to support midthread preemption.
    // This must be called before all other optimizer
    // passes, except PI_initializePayload.
    void Optimizer::createR0Copy()
    {
        if (!builder.getIsKernel())
        {
            return;
        }
        bool needSwitch = (fg.getIsStackCallFunc() && VISA_WA_CHECK(builder.getPWaTable(), WaThreadSwitchAfterCall));

        // Skip copying of ``copy of R0'' if it's never assigned, a case where
        // ``copy of R0'' is never used. As EOT always use ``copy of R0'', that
        // case only happens for synthetic tests where no practical code is
        // generated.
        if (!builder.getBuiltinR0()->getRegVar()->isPhyRegAssigned() && !needSwitch)
            return;

        G4_Declare *R0Dcl = builder.getRealR0();
        G4_SrcRegRegion* R0Opnd = builder.createSrcRegRegion(
            Mod_src_undef,
            Direct,
            R0Dcl->getRegVar(),
            0,
            0,
            builder.getRegionStride1(),
            Type_UD);

        G4_DstRegRegion* R0CopyOpnd = builder.createDstRegRegion(
            Direct,
            builder.getBuiltinR0()->getRegVar(),
            0,
            0,
            1,
            Type_UD);
        R0CopyOpnd->computePReg();

        unsigned int options = InstOpt_WriteEnable;
        if (needSwitch)
            options |= InstOpt_Switch;
        G4_INST *movInst =
            builder.createInternalInst(NULL,
            G4_mov,
            NULL,
            false,
            8,
            R0CopyOpnd,
            R0Opnd,
            NULL,
            options);

        BB_LIST_ITER ib = kernel.fg.BBs.begin();
        G4_INST *inst = NULL;
        BB_LIST_ITER bend(kernel.fg.BBs.end());
        INST_LIST_ITER ii;

        for (ib = kernel.fg.BBs.begin(); ib != bend; ++ib)
        {
            G4_BB* bb = (*ib);
            ii = bb->begin();
            INST_LIST_ITER iend = bb->end();
            for (; ii != iend; ii++)
            {
                inst = *ii;
                if (inst->opcode() != G4_label)
                {
                    bb->insert(ii, movInst);
                    return;
                }
            }
        }
    }

    void Optimizer::initializePayload()
    {
        if (!kernel.fg.builder->getIsKernel())
        {
            return;
        }

        unsigned int inputEnd = 32;
        unsigned int inputCount = kernel.fg.builder->getInputCount();
        for (unsigned int id = 0; id < inputCount; id++)
        {
            input_info_t* input_info = kernel.fg.builder->getInputArg(id);
            if (inputEnd < (unsigned)(input_info->size + input_info->offset))
            {
                inputEnd = input_info->size + input_info->offset;
            }
        }

        int reg_offset = (inputEnd + 31) / 32;
        //bytes whithin grf that need to be initialized
        int remainder_bytes = 32 - (inputEnd % 32);
        //number of full grfs that need to be initialized
        int num_reg_init = 128 - reg_offset;
        //beginning execution size for byte remainder initialization
        uint16_t exec_size = 16;
        //offset within GRF from which to start to initialize
        int sub_offset = (inputEnd % 32);
        G4_BB* bb = kernel.fg.getEntryBB();

        // iter points to the first non-label inst
        auto iter = bb->begin(), bbEnd = bb->end();
        while (iter != bbEnd)
        {
            if (!(*iter)->isLabel())
            {
                break;
            }
            ++iter;
        }

        //inits builk of GRFS, two at a time
        for (int i = 0; i < num_reg_init - (num_reg_init % 2); i += 2)
        {
            int reg_init = reg_offset + i;
            G4_Declare* tempDcl = builder.createHardwiredDeclare(16, Type_UD, reg_init, 0);
            G4_DstRegRegion* dst = builder.createDstRegRegion(Direct, tempDcl->getRegVar(), 0, 0, 1, Type_UD);
            G4_Imm * src0 = builder.createImm(0, Type_UD);
            G4_INST* initInst = builder.createInternalInst(NULL, G4_mov, NULL, false, 16, dst, src0, NULL, InstOpt_WriteEnable);
            bb->insert(iter, initInst);
        }

        //init last register if bulk of GRFs was odd
        if ((num_reg_init % 2))
        {
            G4_Declare* tempDcl = builder.createHardwiredDeclare(8, Type_UD, 128 - (num_reg_init % 2), 0);
            G4_DstRegRegion* dst = builder.createDstRegRegion(Direct, tempDcl->getRegVar(), 0, 0, 1, Type_UD);
            G4_Imm * src0 = builder.createImm(0, Type_UD);
            G4_INST* spIncInst = builder.createInternalInst(NULL, G4_mov, NULL, false, 8, dst, src0, NULL, InstOpt_WriteEnable);
            G4_BB* bb = kernel.fg.getEntryBB();
            bb->insert(iter, spIncInst);
        }

        //inits remainder GRF
        //loops untill all bytes within GRF are initialized
        //on each iteration goes down by execution size
        //There was a small bug if inputEnd offset is GRF aligned it would think all of
        //last payload register is the "remainder" and will initialize it.
        while (remainder_bytes && (inputEnd % 32))
        {
            for (int i = 0; i < remainder_bytes - (remainder_bytes % exec_size); i += exec_size, sub_offset += exec_size)
            {
                G4_Declare* tempDcl = builder.createHardwiredDeclare(exec_size, Type_UB, reg_offset - 1, sub_offset);
                G4_DstRegRegion* dst = builder.createDstRegRegion(Direct, tempDcl->getRegVar(), 0, 0, 1, Type_UB);

                G4_Declare* tempDclSrc = builder.createHardwiredDeclare(1, Type_UD, 126, 0);
                G4_SrcRegRegion *src0 = builder.createSrcRegRegion(Mod_src_undef, Direct, tempDclSrc->getRegVar(), 0, 0, builder.getRegionScalar(), Type_UB);

                G4_INST* initInst = builder.createInternalInst(NULL, G4_mov, NULL, false, (uint8_t)exec_size, dst, src0, NULL, InstOpt_WriteEnable);
                bb->insert(iter, initInst);
            }
            //caluclates bytes that remain to be initialized
            remainder_bytes = remainder_bytes % exec_size;
            //next lowest execution size
            exec_size = (uint16_t)max(1, exec_size / 2);
        }

        //Initializaing Flag register
        {
            G4_Declare* tmpFlagDcl = builder.createTempFlag(2);
            tmpFlagDcl->getRegVar()->setPhyReg(builder.phyregpool.getF0Reg(), 0);
            G4_DstRegRegion *tempPredVar = builder.createDstRegRegion(Direct, tmpFlagDcl->getRegVar(), 0, 0, 1, Type_UD);
            G4_INST *predInst = builder.createInternalInst(NULL, G4_mov, NULL, false, 1,
                tempPredVar, builder.createImm(0, Type_UW), NULL,
                InstOpt_WriteEnable, 0, 0, 0);
            bb = kernel.fg.getEntryBB();
            bb->insert(iter, predInst);
        }

        {
            G4_Declare* tmpFlagDcl = builder.createTempFlag(2);
            tmpFlagDcl->getRegVar()->setPhyReg(builder.phyregpool.getF1Reg(), 0);
            G4_DstRegRegion *tempPredVar = builder.createDstRegRegion(Direct, tmpFlagDcl->getRegVar(), 0, 0, 1, Type_UD);
            G4_INST *predInst = builder.createInternalInst(NULL, G4_mov, NULL, false, 1,
                tempPredVar, builder.createImm(0, Type_UW), NULL,
                InstOpt_WriteEnable, 0, 0, 0);
            bb = kernel.fg.getEntryBB();
            bb->insert(iter, predInst);
        }

        {
            G4_Declare* tmpAddrDcl = builder.createDeclareNoLookup("initAddr_temp", G4_ADDRESS, 8, 1, Type_UW);
            tmpAddrDcl->getRegVar()->setPhyReg(builder.phyregpool.getAddrReg(), 0);
            G4_DstRegRegion *tempAddrVar = builder.createDstRegRegion(Direct, tmpAddrDcl->getRegVar(), 0, 0, 1, Type_UW);
            G4_INST *addrInst = builder.createInternalInst(NULL, G4_mov, NULL, false, 8,
                tempAddrVar, builder.createImm(0, Type_UW), NULL,
                InstOpt_WriteEnable, 0, 0, 0);
            bb = kernel.fg.getEntryBB();
            bb->insert(iter, addrInst);
        }

    }

    void Optimizer::loadThreadPayload()
    {
    }

    /*
    * Check if there is WAR/WAW dependency between end inst and the preceding instruction
    */
    bool Optimizer::chkFwdOutputHazard(INST_LIST_ITER& startIter, INST_LIST_ITER& endIter)
    {
        G4_INST *startInst = *startIter;

        INST_LIST_ITER forwardIter = startIter;
        forwardIter++;
        while( forwardIter != endIter )
        {
            if( (*forwardIter)->isWAWdep(startInst) ||
                (*forwardIter)->isWARdep(startInst) )
            {
                break;
            }
            forwardIter++;
        }

        if( forwardIter != endIter )
        {
            return true;
        }
        else
        {
            return false;
        }
    }

    // check if startInst has any WAR/WAW conflicts with subsequent insts up till endIter (excluding endIter)
    // precondition: startInst must be before endIter in the same BB
    // this is used to sink an inst (or its sources) to the endIter location
    bool Optimizer::chkFwdOutputHazard(G4_INST* startInst, INST_LIST_ITER endIter)
    {
        INST_LIST_ITER backIter = std::prev(endIter, 1);
        while (*backIter != startInst)
        {
            G4_INST* inst = *backIter;
            if (inst->isWARdep(startInst) || inst->isWAWdep(startInst))
            {
                return true;
            }
            --backIter;
        }
        return false;
    }

    bool Optimizer::chkBwdOutputHazard(INST_LIST_ITER& startIter, INST_LIST_ITER& endIter)
    {
        G4_INST *endInst = *endIter;

        INST_LIST_ITER backwardIter = endIter;
        backwardIter--;
        while( backwardIter != startIter )
        {
            if( endInst->isWAWdep(*backwardIter) ||
                endInst->isWARdep(*backwardIter) )
            {
                break;
            }
            backwardIter--;
        }

        if( backwardIter != startIter )
        {
            return true;
        }
        else
        {
            return false;
        }
    }

    bool Optimizer::chkBwdOutputHazard(G4_INST* startInst, INST_LIST_ITER& endIter)
    {
        G4_INST *endInst = *endIter;

        INST_LIST_ITER backwardIter = endIter;
        backwardIter--;
        while( (*backwardIter)->getId() != startInst->getId() )
        {
            if( endInst->isWAWdep(*backwardIter) ||
                /*
                    Makes sure there is not WAR conflict between this instruction and instruction preceding it:
                    ... grf1(use preceding inst)
                    grf1 <---- def this inst
                */
                endInst->isWARdep(*backwardIter) )
            {
                break;
            }
            backwardIter--;
        }

        if( (*backwardIter)->getId() != startInst->getId() )
        {
            return true;
        }
        else
        {
            return false;
        }
    }

    /*
    Skips WAW check for the skipInst
    */
    bool Optimizer::chkBwdOutputHazard(G4_INST* startInst, INST_LIST_ITER& endIter, G4_INST *skipInst)
    {
        G4_INST *endInst = *endIter;

        INST_LIST_ITER backwardIter = endIter;
        backwardIter--;
        while ((*backwardIter)->getId() != startInst->getId())
        {
            if (skipInst->getId() == (*backwardIter)->getId())
            {
                --backwardIter;
                continue;
            }

            /*
            Makes sure there is not WAR conflict between this instruction and instruction preceding it:
            ... grf1(use preceding inst)
            grf1 <---- def this inst
            */
            if (endInst->isWARdep(*backwardIter) ||
                endInst->isWAWdep(*backwardIter))
            {
                break;
            }
            --backwardIter;
        }

        if ((*backwardIter)->getId() != startInst->getId())
        {
            return true;
        }
        else
        {
            return false;
        }
    }

    // This function performs the following renaming to enable further optimization opportunities:
    //
    // op   v3, v1, v2
    // mov  v4, v3
    // mov  v5, v3
    // mov  v6, v3
    // ======>
    // op   v3, v1, v2
    // mov  v4, v3
    // mov  v5, v4
    // mov  v6, v4
    void Optimizer::renameRegister()
    {
        const int MAX_REG_RENAME_DIST = 250;
        const int MAX_REG_RENAME_SIZE = 2;

        BB_LIST_ITER ib, bend(fg.BBs.end());

        for(ib = fg.BBs.begin(); ib != bend; ++ib)
        {
            G4_BB* bb = (*ib);

            bb->resetLocalId();

            INST_LIST_ITER ii = bb->begin(), iend(bb->end());
            while ( ii != iend )
            {
                G4_INST *inst = *ii;

                if( !inst->isRawMov() ||
                    inst->getPredicate() ||
                    inst->getScratch() ||
                    inst->def_size() != 1 ||
                    !inst->canHoist(bb->isInSimdFlow(), fg.builder->getOptions()) )
                {
                    ii++;
                    continue;
                }

                G4_Operand *src = inst->getSrc(0);
                G4_DstRegRegion *dst = inst->getDst();

                G4_Declare *srcDcl = src->isRegRegion() ? GetTopDclFromRegRegion(src) : NULL;
                G4_Declare *dstDcl = GetTopDclFromRegRegion(dst);

                // If this move is between two different register files, then
                // do not do register renaming.
                if (srcDcl && dstDcl && srcDcl->getRegFile() != dstDcl->getRegFile())
                {
                    ++ii;
                    continue;
                }

                G4_INST *defInst = inst->def_front().first;
                G4_Declare *defDstDcl = GetTopDclFromRegRegion(defInst->getDst());
                if( ( dstDcl && dstDcl->getAddressed() ) ||
                    ( defDstDcl && defDstDcl->getAddressed() ) )
                {
                    ii++;
                    continue;
                }

                G4_DstRegRegion* defDstRegion = defInst->getDst();
                if (defInst->getScratch() ||
                    src->compareOperand( defDstRegion ) != Rel_eq )
                {
                    ii++;
                    continue;
                }

                unsigned int instMaskOption = inst->getMaskOption();
                bool canRename = true;

                if (defInst->use_size() == 1)
                {
                    ii++;
                    continue;
                }

                int32_t sizeRatio = dstDcl->getByteSize() / srcDcl->getByteSize();

                G4_INST* lastUse = defInst->use_front().first;
                for (auto iter = defInst->use_begin(), E = defInst->use_end(); iter != E; ++iter)
                {
                    G4_INST* useInst = (*iter).first;

                    if( useInst == inst ||
                        ( (useInst->getLocalId() - inst->getLocalId()) > MAX_REG_RENAME_DIST &&
                           sizeRatio > MAX_REG_RENAME_SIZE ) )
                    {
                        continue;
                    }

                    /*
                        it incorrectly renames in this case, because it doesn't consider
                        defInst mask.
                        BEFORE:
                        <TR><TD ALIGN="LEFT"><FONT color="black">0:mov (1) V44(0,0)[1]:d V46(0,0)[0;1,0]:d [Align1, NoMask] %11</FONT></TD></TR>
                        <TR><TD ALIGN="LEFT"><FONT color="black">0:mov (8) V48(0,0)[1]:d V44(0,0)[0;1,0]:d [Align1, Q1] %12</FONT></TD></TR>
                        <TR><TD ALIGN="LEFT"><FONT color="black">0:mov (8) V56(0,0)[1]:d V44(0,0)[0;1,0]:d [Align1, Q1] %22</FONT></TD></TR>

                        AFTER:
                        <TR><TD ALIGN="LEFT"><FONT color="black">0:mov (1) V44(0,0)[1]:d V46(0,0)[0;1,0]:d [Align1, NoMask] %11</FONT></TD></TR>
                        <TR><TD ALIGN="LEFT"><FONT color="black">0:mov (8) V48(0,0)[1]:d V44(0,0)[0;1,0]:d [Align1, Q1] %12</FONT></TD></TR>
                        <TR><TD ALIGN="LEFT"><FONT color="black">0:mov (8) V56(0,0)[1]:d V48(0,0)[0;1,0]:d [Align1, Q1] %22</FONT></TD></TR>

                        Fix: BB in SIMD control flow && (inst not NoMask) !(instMask == defFask == useMask)

                        Disallow replication?
                    */
                    if( useInst->getLocalId() < inst->getLocalId() ||
                        !useInst->isRawMov() ||
                        inst->getExecSize() != useInst->getExecSize() ||
                        (useInst->getSrc(0))->compareOperand( defDstRegion ) != Rel_eq ||
                        useInst->def_size() > 1 ||
                        (!(inst->isWriteEnableInst()) &&
                        useInst->getMaskOption() != instMaskOption) ||
                        //fix described above
                        (bb->isInSimdFlow() &&
                        !inst->isWriteEnableInst() &&
                        !(inst->getExecSize() == defInst->getExecSize() &&
                                inst->getExecSize() == useInst->getExecSize())
                        )
                      )
                    {
                        canRename = false;
                        break;
                    }

                    if( useInst->getLocalId() > lastUse->getLocalId() )
                    {
                        lastUse = useInst;
                    }
                }

                for (auto iter = defInst->use_begin(), E = defInst->use_end(); iter != E; ++iter)
                {
                    G4_INST* useInst = (*iter).first;
                    useInst->setScratch(true);
                }

                if( !canRename )
                {
                    ii++;
                    continue;
                }

                INST_LIST_ITER forwardIter = ii;
                forwardIter++;
                while( canRename &&
                       *forwardIter != lastUse &&
                       ( ((*forwardIter)->getLocalId() - inst->getLocalId()) <= MAX_REG_RENAME_DIST ||
                          sizeRatio <= MAX_REG_RENAME_SIZE ) )
                {
                    if( (*forwardIter)->isWAWdep( inst ) )
                    {
                        canRename = false;
                        break;
                    }
                    forwardIter++;
                }

                if( !canRename )
                {
                    ii++;
                    continue;
                }

                for (auto useIter = defInst->use_begin(); useIter != defInst->use_end(); /*empty*/)
                {
                    G4_INST* useInst = (*useIter).first;

                    if( useInst == inst ||
                        ( (useInst->getLocalId() - inst->getLocalId()) > MAX_REG_RENAME_DIST &&
                          sizeRatio > MAX_REG_RENAME_SIZE ) )
                    {
                        useIter++;
                        continue;
                    }

                    G4_Operand *useSrc = useInst->getSrc(0);
                    unsigned char execSize = useInst->getExecSize();
                    unsigned short dstHS = dst->getHorzStride();
                    RegionDesc *newSrcRd;


                    if( useSrc->asSrcRegRegion()->isScalar() )
                    {
                        newSrcRd = builder.getRegionScalar();
                    }
                    else
                    {
                        unsigned tExecSize = (execSize > 8) ? 8 : execSize;
                        if (RegionDesc::isLegal(tExecSize * dstHS, execSize, dstHS))
                        {
                            newSrcRd = builder.createRegionDesc( (uint16_t)tExecSize*dstHS, execSize, dstHS );
                        }
                        else
                        {
                            // Skip this use. TODO: normalize this region.
                            ++useIter;
                            continue;
                        }
                    }

                    G4_SrcRegRegion *newSrcOpnd = builder.createSrcRegRegion(
                        Mod_src_undef,
                        dst->getRegAccess(),
                        dst->getBase(),
                        dst->getRegOff(),
                        dst->getSubRegOff(),
                        newSrcRd,
                        useSrc->getType() );
                    if (dst->getRegAccess() != Direct)
                    {
                        newSrcOpnd->asSrcRegRegion()->setImmAddrOff( dst->getAddrImm() );
                    }

                    useInst->getSrc(0)->~G4_Operand();
                    useInst->setSrc( newSrcOpnd, 0 );

                    // Maintain def-use for this change:
                    // - remove this use from defInst
                    // - add a new use to inst
                    useIter = defInst->eraseUse(useIter);
                    inst->addDefUse(useInst, Opnd_src0);
                }

                ii++;
            }
        }
    }

    // use by mergeScalar
#define OPND_PATTERN_ENUM(DO) \
    DO(UNKNOWN) \
    DO(IDENTICAL) \
    DO(CONTIGUOUS) \
    DO(DISJOINT)

enum OPND_PATTERN
{
    OPND_PATTERN_ENUM(MAKE_ENUM)
};

static const char* patternNames[] =
{
    OPND_PATTERN_ENUM(STRINGIFY)
};

#define MAX_BUNDLE_SIZE 16
#define MAX_NUM_SRC 3

struct BUNDLE_INFO
{
    int size;
    int sizeLimit;
    G4_BB* bb;
    INST_LIST_ITER startIter;
    G4_INST* inst[MAX_BUNDLE_SIZE];
    OPND_PATTERN dstPattern;
    OPND_PATTERN srcPattern[MAX_NUM_SRC];

    BUNDLE_INFO(G4_BB* instBB, INST_LIST_ITER& instPos, int limit) : bb(instBB), sizeLimit(limit)
    {

        inst[0] = *instPos;
        startIter = instPos;
        dstPattern = OPND_PATTERN::UNKNOWN;
        for (int i = 0; i < MAX_NUM_SRC; i++)
        {
            srcPattern[i] = OPND_PATTERN::UNKNOWN;
        }
        size = 1;
    }

    void *operator new(size_t sz, Mem_Manager& m) {return m.alloc(sz);}

    void appendInst(G4_INST* lastInst)
    {
        MUST_BE_TRUE(size < MAX_BUNDLE_SIZE, "max bundle size exceeded");
        inst[size++] = lastInst;
    }

    void deleteLastInst()
    {
        assert(size > 0 && "empty bundle");
        inst[--size] = nullptr;
    }

    bool canMergeDst(G4_DstRegRegion* dst);
    bool canMergeSource(G4_Operand* src, int srcPos);
    bool canMerge(G4_INST* inst);

    bool doMerge(IR_Builder& builder,
        std::unordered_set<G4_Declare*>& modifiedDcl,
        std::vector<G4_Declare*>& newInputs);

    void print(std::ostream& output) const
    {
        output << "Bundle:\n";
        output << "Dst pattern:\t" << patternNames[dstPattern] << "\n";
        output << "Src Pattern:\t";
        for (int i = 0; i < inst[0]->getNumSrc(); ++i)
        {
            output << patternNames[srcPattern[i]] << " ";
        }
        output << "\n";
        for (int i = 0; i < size; ++i)
        {
            inst[i]->emit(output);
            output << "\n";
        }
    }

};

// This will create a new dcl or reuse the existing one if available.
// The returned dcl will have
//    its Reg's byte address == input's
//    its type == srcType
//    itss numElem == bundleSize.
static G4_Declare* getInputDeclare(IR_Builder& builder,
                                   std::vector<G4_Declare*>& declares,
                                   G4_Declare* input,
                                   G4_Type eltType,
                                   int bundleSize,
                                   int firstEltOffset)
{
    MUST_BE_TRUE(input->isInput() && input->getAliasDeclare() == NULL, "Expect root input variable");
    for (auto dcl : declares)
    {
        MUST_BE_TRUE(dcl->isInput() && dcl->getAliasDeclare() == NULL, "declare must be root input variable");
        if (dcl->getRegVar()->getByteAddr() == input->getRegVar()->getByteAddr() &&
            dcl->getElemType() == eltType &&
            dcl->getTotalElems() >= bundleSize)
        {
            return dcl;
        }
    }

    // Now, create a new dcl.
    // As the new dcl's offset is in unit of element type, we will convert offset in bytes to
    // offset in elementType.  As the previous check guarantees that the byte offset must be
    // multiple of elementType, here just add additional checks to make sure this is the case.
    uint32_t offset = input->getRegVar()->getPhyRegOff() * G4_Type_Table[input->getElemType()].byteSize + firstEltOffset;
    uint32_t eltBytes = G4_Type_Table[eltType].byteSize;
    MUST_BE_TRUE((offset % eltBytes) == 0, "Offset shoule be mutiple of element size");
    offset = offset / eltBytes;
    char *name = builder.getNameString(builder.mem, 16, "InputR%d.%d", input->getRegVar()->getPhyReg()->asGreg()->getRegNum(), offset);
    G4_Declare* newInputDcl = builder.createDeclareNoLookup(name, G4_INPUT, (uint16_t) bundleSize, 1,
        eltType);
    newInputDcl->getRegVar()->setPhyReg(input->getRegVar()->getPhyReg(), offset);
    declares.push_back(newInputDcl);
    return newInputDcl;
}

//#define DEBUG_VERBOSE_ON
//
// merge all instructions in the bundle into a single one, by modifying the first instruction
// in the bundle and delete the rest
// returns true if merge is successful
// merge can fail if a source/dst appears in multiple bundles and has a conflicing alignment
// (this only happens when the operand is in DISJOINT pattern)
//
bool BUNDLE_INFO::doMerge(IR_Builder& builder,
                          std::unordered_set<G4_Declare*>& modifiedDcl,
                          std::vector<G4_Declare*>& newInputs)
{
    if (size == 1)
    {
        return false;
    }

    if (size == 3)
    {
        for (int pos = 0, numSrc = inst[0]->getNumSrc(); pos < numSrc; ++pos)
        {
            if (srcPattern[pos] == OPND_PATTERN::CONTIGUOUS)
            {
                // since we are rounding up esize to 4, we have to make sure the source is
                // not out of bound. If it is we merge the first two inst instead
                G4_SrcRegRegion* lastSrc = inst[size - 1]->getSrc(pos)->asSrcRegRegion();
                if (lastSrc->getLeftBound() + G4_Type_Table[lastSrc->getType()].byteSize >=
                    lastSrc->getTopDcl()->getByteSize())
                {
                    deleteLastInst();
                    break;
                }
            }
        }
    }

    if (!builder.hasAlign1Ternary() && size == 2 && inst[0]->getNumSrc() == 3)
    {
        return false;
    }

    // for distinct patterns (both src and dst), we need to check the variable is only used once in the entire bundle,
    // so that the new variable created for the operand does not conflict with another.
    // this is to prevent coalescing two variables with conflicting life range, e.g.,
    //  add (M1_NM, 1) V66(0,0)<1> V66(0,0)<0;1,0> 0x1:d                             /// $36
    //  add (M1_NM, 1) V67(0,0)<1> V65(0,0)<0;1,0> 0x1:d                             /// $37
    //  or make a variable with two root aliases:
    //  mul (M1_NM, 1) V186(0,0)<1> V184(0,0)<0;1,0> V185(0,0)<0; 1, 0>                /// $206
    //  mul (M1_NM, 1) V187(0,0)<1> V182(0,0)<0;1,0> V182(0,0)<0; 1, 0>                /// $207
    std::set<G4_Declare*> uniqueDeclares;

    // check if merging is legal
    if (dstPattern == OPND_PATTERN::DISJOINT)
    {
        G4_Declare* rootDcl = inst[0]->getDst()->getTopDcl()->getAliasDeclare();
        for (int instId = 0; instId < size; ++instId)
        {
            G4_DstRegRegion* dst = inst[instId]->getDst();
            G4_Declare* dcl = dst->getTopDcl();
            if (dcl->getAliasDeclare() != rootDcl)
            {
                // all dsts in the bundle should have the same base variable (or NULL)
                // this is to avoid the situation where {V1, V2} is in one bundle and {V1, V3}
                // is in another bundle, and we can't optimize both since V2 and V3 may be both live
                return false;
            }
            if (dcl->getAliasDeclare() != NULL)
            {
                if (dcl->getAliasOffset() !=
                    G4_Type_Table[dst->getType()].byteSize * instId)
                {
                    return false;
                }
            }
            uniqueDeclares.insert(dcl);
        }
    }
    for (int i = 0; i < inst[0]->getNumSrc(); ++i)
    {
        if (srcPattern[i] == OPND_PATTERN::DISJOINT)
        {
            G4_Declare* rootDcl = inst[0]->getSrc(i)->getTopDcl()->getAliasDeclare();
            for (int instId = 0; instId < size; ++instId)
            {
                G4_Operand* src = inst[instId]->getSrc(i);
                G4_Declare* dcl = src->getTopDcl();
                if (dcl->getAliasDeclare() != rootDcl)
                {
                    // all srcs in the bundle, if aliased, should have the same alias variable
                    // see comments from above for dst
                    return false;
                }
                if (dcl->getAliasDeclare() != NULL)
                {
                    if (dcl->getAliasOffset() !=
                        G4_Type_Table[src->getType()].byteSize * instId)
                    {
                        return false;
                    }
                }
                if (uniqueDeclares.find(dcl) != uniqueDeclares.end())
                {
                    return false;
                }
                uniqueDeclares.insert(dcl);
            }
        }
    }

    uint8_t execSize = (uint8_t) Round_Up_Pow2(size);
    G4_INST* newInst = inst[0]; //reuse inst[0] as the new merged inst

    // at this point merge will definitely succeed
    // handle merged dst
    if (dstPattern == OPND_PATTERN::DISJOINT)
    {
        G4_Type dstType = newInst->getDst()->getType();
        // create a new declare with bundle size and have the original dsts alias to it
        G4_Declare* newDcl = NULL;
        if (newInst->getDst()->getTopDcl()->getAliasDeclare() != NULL)
        {
            newDcl = newInst->getDst()->getTopDcl()->getAliasDeclare();
        }
        else
        {
            newDcl = builder.createTempVar(execSize, dstType, Either, Any, "Merged");
        }
        for (int i = 0; i < size; ++i)
        {
            G4_Declare* dstDcl = inst[i]->getDst()->getTopDcl();
            if (dstDcl->getAliasDeclare() == NULL)
            {
                dstDcl->setAliasDeclare(newDcl, i * G4_Type_Table[dstType].byteSize);
                modifiedDcl.insert(dstDcl);
#ifdef DEBUG_VERBOSE_ON
                cout << "Dcl " << dstDcl->getName() << "--> (" << newDcl->getName() << ", "
                    << i * G4_Type_Table[dstType].byteSize << ")\n";
#endif
            }
        }
        G4_DstRegRegion* newDst = builder.createDstRegRegion(Direct, newDcl->getRegVar(), 0, 0, 1, dstType);
        newInst->setDest(newDst);
    }
    else
    {
        MUST_BE_TRUE(dstPattern == OPND_PATTERN::CONTIGUOUS, "unexpected dst pattern");
    }

    for (int i = 0; i < newInst->getNumSrc(); ++i)
    {
        if (srcPattern[i] == OPND_PATTERN::DISJOINT)
        {
            G4_Operand* oldSrc = newInst->getSrc(i);
            G4_Type srcType = oldSrc->getType();
            // create a new declare with bundle size and have the original dsts alias to it
            G4_Declare* newDcl = NULL;
            if (oldSrc->getTopDcl()->getAliasDeclare() != NULL)
            {
                newDcl = oldSrc->getTopDcl()->getAliasDeclare();
            }
            else
            {
                newDcl = builder.createTempVar(execSize, srcType, Either, Any, "Merged");
            }
            for (int j = 0; j < size; ++j)
            {
                MUST_BE_TRUE(inst[j]->getSrc(i)->isSrcRegRegion(), "Src must be a region");
                G4_SrcRegRegion* src = inst[j]->getSrc(i)->asSrcRegRegion();
                G4_Declare* srcDcl = src->getTopDcl();
                if (srcDcl->getAliasDeclare() == NULL)
                {
                    srcDcl->setAliasDeclare(newDcl, j * G4_Type_Table[srcType].byteSize);
                    modifiedDcl.insert(srcDcl);
#ifdef DEBUG_VERBOSE_ON
                    cout << "Dcl " << srcDcl->getName() << "--> (" << newDcl->getName() << ", "
                        << i * G4_Type_Table[srcType].byteSize << ")\n";
#endif
                }
            }
            G4_SrcRegRegion* newSrc = builder.createSrcRegRegion(
                oldSrc->asSrcRegRegion()->getModifier(), Direct,
                newDcl->getRegVar(), 0, 0, builder.getRegionStride1(), srcType);
            newInst->setSrc(newSrc, i);
        }
        else if (srcPattern[i] == OPND_PATTERN::CONTIGUOUS)
        {
            // update region
            G4_SrcRegRegion* src = newInst->getSrc(i)->asSrcRegRegion();
            G4_Declare* rootDcl = src->getTopDcl()->getRootDeclare();
            if (rootDcl->isInput())
            {
                //check if the existing input is good enough
                bool sameInput = true;
                for (int instId = 1; instId < size; ++instId)
                {
                    auto dcl = inst[instId]->getSrc(i)->getTopDcl()->getRootDeclare();
                    if (rootDcl != dcl)
                    {
                        sameInput = false;
                        break;
                    }
                }

                if (sameInput)
                {
                    src->setRegion(builder.getRegionStride1());
                }
                else
                {
                    // we need a new input variable that covers all inputs used in the bundle
                    // Since rootDcl may have a different type than the operands', need to use
                    // operand's type instead of rootDcl, so we pass srcType into getInputDeclare().
                    G4_Type srcType = newInst->getSrc(i)->getType();
                    int firstEltOffset = src->getRegOff() * GENX_GRF_REG_SIZ + src->getSubRegOff() * G4_Type_Table[srcType].byteSize;
                    G4_Declare* newInputDcl = getInputDeclare(builder, newInputs, rootDcl, srcType, size, firstEltOffset);
                    src = builder.createSrcRegRegion(
                        src->getModifier(), Direct, newInputDcl->getRegVar(),
                        0, 0, builder.getRegionStride1(), src->getType());
                    newInst->setSrc(src, i);
                }
            }
            else
            {
                src->setRegion(builder.getRegionStride1());
            }
        }
        else
        {
            MUST_BE_TRUE(srcPattern[i] == OPND_PATTERN::IDENTICAL, "unexpected source pattern");
        }
    }

    newInst->setExecSize(execSize);

    auto iter = startIter;
    ++iter;
    for (int i = 1; i < size; ++i)
    {
        G4_INST* instToDelete = *iter;
        instToDelete->transferUse(newInst, true);
        for (int srcNum = 0, numSrc = instToDelete->getNumSrc(); srcNum < numSrc; ++srcNum)
        {
            Gen4_Operand_Number opndPos = instToDelete->getSrcOperandNum(srcNum);
            instToDelete->transferDef( newInst, opndPos, opndPos );
        }

        iter = bb->erase(iter);
    }

#ifdef DEBUG_VERBOSE_ON
    cout << "new inst after merging:\n";
    newInst->emit(cout);
    cout << "\n";
    newInst->emitDefUse(cout);
    cout << "\n";
#endif

    return true;
}


//
// check to see if offset1 + type size == offset2
// if so and pattern is either unknown or contiguous, return true
// if pattern is unknown also change it to contiguous
//
// also check to see if distance between offset2 and origOffset
// (offset of the first instruction found in current bundle)
// exceeds GRF size. If so returns false.
//
//
static bool checkContiguous(unsigned offset1, unsigned offset2, G4_Type type, OPND_PATTERN& pattern, unsigned origOffset)
{
    if (origOffset + GENX_GRF_REG_SIZ <= offset2)
        return false;
    if (offset1 + G4_Type_Table[type].byteSize == offset2)
    {
        switch (pattern)
        {
        case OPND_PATTERN::UNKNOWN:
            pattern = OPND_PATTERN::CONTIGUOUS;
            return true;
        case OPND_PATTERN::CONTIGUOUS:
            return true;
        default:
            return false;
        }
    }
    return false;
}

// check if this instruction is eligbile to be merged into a vector instruction
// the conditions are:
// -- arithmetic, logic, mov, math, or pseudo_mad instructions
// -- simd1, NoMask, no predicates or conditional modifier
// -- dst must be direct GRF whose declare is a single element without alias
// -- all sources must be either direct GRF or immediates
static bool isMergeCandidate(G4_INST* inst, const IR_Builder& builder, bool isInSimdFlow)
{
    if (inst->isMixedMode() && builder.getOption(vISA_DisableleHFOpt))
        return false;

    if (!inst->isArithmetic() && !inst->isLogic() && !inst->isMath() && !inst->isMov() &&
        inst->opcode() != G4_pseudo_mad)
    {
        return false;
    }

    if (inst->getExecSize() != 1 ||
        (!inst->isWriteEnableInst() && isInSimdFlow))
    {
        return false;
    }

    if (inst->getPredicate() || inst->getCondMod())
    {
        return false;
    }

    MUST_BE_TRUE(inst->getDst() != NULL, "dst must not be NULL");
    if (inst->getDst()->isIndirect())
    {
        return false;
    }

    if (builder.no64bitRegioning() && getTypeSize(inst->getDst()->getType()) == 8)
    {
        // applying merge scalar makes code quality worse since we need to insert moves
        // to compensate for the lack of 64-bit regions later
        return false;
    }

    G4_VarBase *dstBase = inst->getDst()->getBase();
    G4_Declare* dstDcl = dstBase->isRegVar() ? dstBase->asRegVar()->getDeclare() : nullptr;
    if (dstDcl != nullptr &&
        (dstDcl->getAliasDeclare() != nullptr ||
         dstDcl->getTotalElems() != 1 ||
         G4_Type_Table[inst->getDst()->getType()].byteSize != dstDcl->getElemSize() ||
         !dstDcl->useGRF()))
    {
        return false;
    }

    if (dstDcl != nullptr && (dstDcl->isOutput() || dstDcl->isInput()))
    {
        return false;
    }

    for (int i = 0; i < inst->getNumSrc(); ++i)
    {
        G4_Operand* src = inst->getSrc(i);
        if (src->isSrcRegRegion())
        {
            if (src->asSrcRegRegion()->isIndirect())
            {
                return false;
            }
            G4_Declare* srcDcl = src->getTopDcl();
            if (srcDcl == nullptr)
            {
                return false;
            }

            if (!srcDcl->useGRF() /* || srcDcl->getTotalElems() != 1 */)
            {
                return false;
            }
            // can't do opt if source decl type is inconsistent with its use
            if (G4_Type_Table[srcDcl->getElemType()].byteSize != G4_Type_Table[src->getType()].byteSize)
            {
                return false;
            }
        }
        else if (src->isImm() && !src->isRelocImm())
        {
            // ok
        }
        else
        {
            return false;
        }
    }

    return true;
}

// returns true if dcl is a scalar root variable that is naturally aligned
// Note that this also rules out input
static bool isScalarNaturalAlignedVar(G4_Declare* dcl)
{
    return dcl->getTotalElems() == 1 && dcl->getAliasDeclare() == NULL &&
        dcl->getByteAlignment() == G4_Type_Table[dcl->getElemType()].byteSize && !dcl->isInput();
}

//
// Check if the dst can be combined into the bundle
// There are 2 scenarios:
// CONTIGUOUS: the dsts together form a contiguous region
// DISJOINT: all dsts are distinct scalar, naturally aligned root variables
//
bool BUNDLE_INFO::canMergeDst(G4_DstRegRegion* dst)
{
    // src must be either Imm or SrcRegRegion

    G4_DstRegRegion* firstDst = inst[0]->getDst();
    G4_DstRegRegion* prevDst = inst[size-1]->getDst();
    if (prevDst->getType() != dst->getType())
    {
        return false;
    }

    G4_Declare* prevDstDcl = prevDst->getTopDcl();
    G4_Declare* dstDcl = dst->getTopDcl();

    if ((dstDcl == nullptr) || (prevDstDcl == nullptr))
    {
        return false;
    }

    if (prevDstDcl == dstDcl)
    {
        if (dstPattern == OPND_PATTERN::DISJOINT)
        {
            return false;
        }

        if (!checkContiguous(prevDst->getLeftBound(), dst->getLeftBound(),
            dst->getType(), dstPattern, firstDst->getLeftBound()))
        {
            return false;
        }
    }
    else if (prevDstDcl->isInput() && dstDcl->isInput())
    {
        unsigned firstDstGRFOffset = firstDst->getLeftBound() + firstDst->getTopDcl()->getRegVar()->getByteAddr();
        unsigned prevDstGRFOffset = prevDst->getLeftBound() + prevDstDcl->getRegVar()->getByteAddr();
        unsigned dstGRFOffset = dst->getLeftBound() + dstDcl->getRegVar()->getByteAddr();
        if (!checkContiguous(prevDstGRFOffset, dstGRFOffset, dst->getType(), dstPattern, firstDstGRFOffset))
        {
            return false;
        }
    }
    else
    {
        switch (dstPattern)
        {
        case OPND_PATTERN::UNKNOWN:
            //allow if both sources are size 1 root variables with no alignment
            if (isScalarNaturalAlignedVar(prevDstDcl) && isScalarNaturalAlignedVar(dstDcl))
            {
                dstPattern = OPND_PATTERN::DISJOINT;
            }
            else
            {
                return false;
            }
            break;
        case OPND_PATTERN::DISJOINT:
            if (!isScalarNaturalAlignedVar(dstDcl))
            {
                return false;
            }
            //also check to see if dst is the same as any other previous dst
            for (int i = 0; i < size - 1; i++)
            {
                G4_INST* bundleInst = inst[i];
                if (dstDcl == bundleInst->getDst()->getTopDcl())
                {
                    return false;
                }
            }
            break;
        default:
            return false;
        }
    }

    // additionally check if dst has a WAR dependence with one of the source in earlier instructions
    // it may seem that WAR dependencies should not interfere with vectorization since src is always read
    // before the dst; however, the problem is that if src and dst in different instructions refer to
    // the same variable we can't get them to align properly.
    for (int i = 0; i < size; i++)
    {
        for (int srcPos = 0; srcPos < inst[i]->getNumSrc(); ++srcPos)
        {
            G4_Operand* src = inst[i]->getSrc(srcPos);
            // since both are scalar, check is as simple as comparing the dcl
            if (src->getTopDcl() != NULL && src->getTopDcl() == dst->getTopDcl())
            {
                return false;
            }
        }
    }

    return true;
}

//
// Check if this src at srcPos can be combined into the bundle
// There are 3 scenarios:
// IDENTICAL: all sources in the bundle are the same scalar variable
// CONTIGUOUS: the sources together form a contiguous region
// DISJOINT: all sources are distinct scalar, naturally aligned root variables
//
bool BUNDLE_INFO::canMergeSource(G4_Operand* src, int srcPos)
{
    // src must be either Imm or SrcRegRegion

    MUST_BE_TRUE(srcPos < MAX_NUM_SRC, "invalid srcPos");

    if (src->isRelocImm())
    {
        return false;
    }

    if ((inst[0]->isMath() && inst[0]->asMathInst()->isOneSrcMath()) && srcPos == 1)
    {
        // null source is always legal
        srcPattern[srcPos] = OPND_PATTERN::IDENTICAL;
        return true;
    }

    G4_Operand* firstSrc = inst[0]->getSrc(srcPos);
    G4_Operand* prevSrc = inst[size-1]->getSrc(srcPos);
    if (prevSrc->getType() != src->getType())
    {
        return false;
    }
    if (prevSrc->isImm())
    {
        // ToDo: can add packed vector support later
        if (!src->isImm() ||
            prevSrc->asImm()->getImm() != src->asImm()->getImm())
        {
            return false;
        }
        srcPattern[srcPos] = OPND_PATTERN::IDENTICAL;
    }
    else
    {
        if (!src->isSrcRegRegion())
        {
            return false;
        }
        G4_SrcRegRegion* prevSrcRegion = prevSrc->asSrcRegRegion();
        G4_SrcRegRegion* srcRegion = src->asSrcRegRegion();
        if (prevSrcRegion->getModifier() != srcRegion->getModifier())
        {
            return false;
        }

        G4_Declare* prevSrcDcl = prevSrc->getTopDcl();
        G4_Declare* srcDcl = src->getTopDcl();
        if (prevSrcDcl == srcDcl)
        {
            if (srcPattern[srcPos] == OPND_PATTERN::DISJOINT)
            {
                return false;
            }

            if (prevSrc->getLeftBound() == src->getLeftBound())
            {
                // the case where we have identical source for each instruction in the bundle
                switch (srcPattern[srcPos])
                {
                case OPND_PATTERN::UNKNOWN:
                    srcPattern[srcPos] = OPND_PATTERN::IDENTICAL;
                    break;
                case OPND_PATTERN::IDENTICAL:
                    // do nothing
                    break;
                default:
                    return false;
                }
            }
            else if (!checkContiguous(prevSrc->getLeftBound(), src->getLeftBound(),
                src->getType(), srcPattern[srcPos], firstSrc->getLeftBound()))
            {
                return false;
            }
        }
        else if (prevSrcDcl->isInput() && srcDcl->isInput())
        {
            unsigned firstSrcGRFOffset = firstSrc->getLeftBound() + firstSrc->getTopDcl()->getRegVar()->getByteAddr();
            unsigned prevSrcGRFOffset = prevSrc->getLeftBound() + prevSrcDcl->getRegVar()->getByteAddr();
            unsigned srcGRFOffset = src->getLeftBound() + srcDcl->getRegVar()->getByteAddr();
            if (!checkContiguous(prevSrcGRFOffset, srcGRFOffset, src->getType(), srcPattern[srcPos], firstSrcGRFOffset))
            {
                return false;
            }
            else if (prevSrcGRFOffset / GENX_GRF_REG_SIZ != srcGRFOffset / 32)
            {
                // resulting input would cross GRF boundary, and our RA does not like it one bit
                return false;
            }

            if (inst[0]->getNumSrc() == 3)
            {
                // for 3src inst, we can't merge if inst0's src is not oword-aligned
                if ((prevSrcGRFOffset & 0xF) != 0)
                {
                    return false;
                }
            }
        }
        else
        {
            switch (srcPattern[srcPos])
            {
            case OPND_PATTERN::UNKNOWN:
                //allow if both sources are size 1 root variables with no alignment
                if (isScalarNaturalAlignedVar(prevSrcDcl) && isScalarNaturalAlignedVar(srcDcl))
                {
                    srcPattern[srcPos] = OPND_PATTERN::DISJOINT;
                }
                else
                {
                    return false;
                }
                break;
            case OPND_PATTERN::DISJOINT:
                if (!isScalarNaturalAlignedVar(srcDcl))
                {
                    return false;
                }
                //also check to see if src is the same as any other previous sources
                for (int i = 0; i < size - 1; i++)
                {
                    G4_INST* bundleInst = inst[i];
                    if (srcDcl == bundleInst->getSrc(srcPos)->getTopDcl())
                    {
                        return false;
                    }
                }
                break;
            default:
                return false;
            }
        }
    }

    if (src->isSrcRegRegion())
    {
        // we additionally have to check if src has a RAW dependency with one of the
        // previous dst in the bundle
        for (int i = 0; i < size; i++)
        {
            G4_DstRegRegion* dst = inst[i]->getDst();
            // since both are scalar, check is as simple as comparing the dcl
            if (src->getTopDcl() == dst->getTopDcl())
            {
                return false;
            }
        }
    }

    return true;
}


//
// check if inst can be successfully appended to the bundle
// For this to be successful:
// -- inst must itself be a merge candidate (checked by caller)
// -- inst must have same opcode/dst modifier/src modifier as all other instructions in the bundle
// -- dst and src operand for the inst must form one of the legal patterns with the instructions in the bundle
//
bool BUNDLE_INFO::canMerge(G4_INST* inst)
{
    G4_INST* firstInst = this->inst[0];
    if (firstInst->opcode() != inst->opcode())
    {
        return false;
    }

    if (inst->isMath())
    {
        G4_MathOp firstMathOp = MATH_RESERVED;
        if (firstInst->isMath())
        {
            firstMathOp = firstInst->asMathInst()->getMathCtrl();
        }

        if (firstMathOp != inst->asMathInst()->getMathCtrl())
        {
            return false;
        }
    }

    if (firstInst->getSaturate() != inst->getSaturate())
    {
        return false;
    }

    if (!canMergeDst(inst->getDst()))
    {
        return false;
    }


    for (int i = 0; i < inst->getNumSrc(); ++i)
    {
        if (!canMergeSource(inst->getSrc(i), i))
        {
            return false;
        }
    }

    // append instruction to bundle
    appendInst(inst);
    return true;

}

//
// iter is advanced to the next instruction not belonging to the handle
//
static void findInstructionToMerge(BUNDLE_INFO* bundle, INST_LIST_ITER& iter, const INST_LIST_ITER& lastInst, const IR_Builder& builder, bool isInSimdFlow)
{

    for (; iter != lastInst && bundle->size < bundle->sizeLimit; ++iter)
    {
        G4_INST* nextInst = *iter;
        if (!isMergeCandidate(nextInst, builder, isInSimdFlow))
        {
            break;
        }

        if (!bundle->canMerge(nextInst))
        {
            break;
        }
    }

#ifdef DEBUG_VERBOSE_ON
    if (bundle->size > 1)
    {
        cout << "Find vectorization candidates\n";
        bundle->print(std::cout);
    }
#endif

}

//
// recompute bounds for declares in the given unordered_set
// this only affects DstRegRegion and SrcRegRegion
// If the operand is global, we have to update the global operand table as well
// as the bounds may have changed
//
void Optimizer::recomputeBound(std::unordered_set<G4_Declare*>& declares)
{

    for (auto bb : fg.BBs)
    {
        for (auto ii = bb->begin(), iiEnd = bb->end(); ii != iiEnd; ++ii)
        {
            G4_INST* inst = *ii;
            if (inst->getDst() != NULL)
            {
                G4_DstRegRegion* dst = inst->getDst();
                if (dst->getTopDcl() != NULL && declares.find(dst->getTopDcl()) != declares.end())
                {
                    bool isGlobal = builder.kernel.fg.globalOpndHT.isOpndGlobal(dst);
                    dst->computeLeftBound();
                    inst->computeRightBound(dst);
                    if (isGlobal)
                    {
                        builder.kernel.fg.globalOpndHT.addGlobalOpnd(dst);
                    }
                }
            }
            for (int i = 0, numSrc = inst->getNumSrc(); i < numSrc; ++i)
            {
                if (inst->getSrc(i) != NULL && inst->getSrc(i)->isSrcRegRegion())
                {
                    G4_SrcRegRegion* src = inst->getSrc(i)->asSrcRegRegion();
                    if (src->getTopDcl() != NULL && declares.find(src->getTopDcl()) != declares.end())
                    {
                        bool isGlobal = builder.kernel.fg.globalOpndHT.isOpndGlobal(src);
                        src->computeLeftBound();
                        inst->computeRightBound(src);
                        if (isGlobal)
                        {
                            builder.kernel.fg.globalOpndHT.addGlobalOpnd(src);
                        }
                    }
                }
            }
        }
    }
}

//
// Given a sequence of simd1 instructions (max 4), try to merge them into a single instruction
// e.g.,
// mul (1) r0.4<1>:f r0.0<0;1,0>:f r6.5<0;1,0>:f {NoMask}
// mul (1) r0.5<1>:f r0.1<0;1,0>:f r6.5<0;1,0>:f {NoMask}
// mul (1) r0.6<1>:f r0.2<0;1,0>:f r6.5<0;1,0>:f {NoMask}
// mul (1) r0.7<1>:f r0.3<0;1,0>:f r6.5<0;1,0>:f {NoMask}
// becomes
// mul (4) r0.4<1>:f r0.0<1;1,0>:f r6.5<0;1,0>:f {NoMask}
// A bunch of conditions have to be satisified; check BUNDLE_INFO for more details.
// This is only performed for 3D input as CM is very unlikely to benefit from this
// (put another way, if this succeeds for CM our FE is doing something wrong)
//
void Optimizer::mergeScalarInst()
{

    int bundleSizeLimit = MAX_BUNDLE_SIZE;
    if (builder.getOptions()->getTarget() == VISA_3D)
    {
        bundleSizeLimit = 4;
    }

    Mem_Manager mergeManager(1024);
    // set of declares that have been changed to alias to another declare
    std::unordered_set<G4_Declare*> modifiedDcl;
    std::vector<G4_Declare*> newInputs;

    //stats
    int numBundles = 0;
    int numDeletedInst = 0;

    for (G4_BB* bb : fg.BBs)
    {
        std::vector<BUNDLE_INFO*> bundles;
        INST_LIST_ITER ii = bb->begin(), iiEnd = bb->end();
        while (ii != iiEnd)
        {
            G4_INST *inst = *ii;
            auto nextIter = ii;
            ++nextIter;
            if (nextIter != iiEnd && isMergeCandidate(inst, builder, bb->isInSimdFlow()))
            {
                BUNDLE_INFO* bundle = new (mergeManager) BUNDLE_INFO(bb, ii, bundleSizeLimit);
                findInstructionToMerge(bundle, nextIter, bb->end(), builder, bb->isInSimdFlow());
                if (bundle->size > 1)
                {
                    bundles.push_back(bundle);
                }
                else
                {
                    bundle->~BUNDLE_INFO();
                }
                ii = nextIter;
            }
           else
            {
                ++ii;
            }
        }

        for (auto bundle : bundles)
        {
            bool success = bundle->doMerge(builder, modifiedDcl, newInputs);
            if (success)
            {
                numBundles++;
                numDeletedInst += bundle->size - 1;
            }
            bundle->~BUNDLE_INFO();
        }
    }

    // we have to reset the bound for all operands whose declares have been modified
    recomputeBound(modifiedDcl);

    if (builder.getOption(vISA_OptReport))
    {
        std::ofstream optReport;
        getOptReportStream(optReport, builder.getOptions());
        optReport << "             === Merge Scalar Optimization ===\n";
        optReport << "Number of optimized bundles:\t" << numBundles << "\n";
        optReport << "Number of instructions saved:\t" << numDeletedInst << "\n";
        closeOptReportStream(optReport);
    }
}

static bool isMad(G4_INST *I) { return I->opcode() == G4_pseudo_mad; }

static inline bool isWBTypeAndNotNull(G4_Operand *opnd)
{
    // Requires opnd to be not null.
    return opnd && (IS_BTYPE(opnd->getType()) || IS_WTYPE(opnd->getType()));
}

namespace {

/// Class to describe a mad sequence that can be turned into a sequence of MAC
/// instructions.
class MadSequenceInfo {
public:
    // IR builder object.
    IR_Builder &builder;

    // The basic block being examined.
    G4_BB *bb;

    enum MadSeqKind { MK_unknown, MK_isSafe, MK_isNotSafe};

private:
    // Flag indicates if ACC transformation is safe.
    MadSeqKind kind;

    // The single definition that defines the first mad's src2. If there are
    // multiple defintions, it is nullptr.
    G4_INST *src2Def;

    // The sequence of mad instruction to be examined.
    std::vector<G4_INST *> madSequence;

    // The use chain of the last mad. This use chain ended with an instruction
    // that has *B/*W type, and the chain length is limited by a predefined
    // constant.
    std::vector<G4_INST *> lastMadUserChain;

public:
    MadSequenceInfo(IR_Builder &builder, G4_BB *bb)
        : builder(builder), bb(bb), kind(MK_unknown), src2Def(nullptr) {}

    bool isSafe() const { return kind == MK_isSafe; }
    bool isNotSafe() const { return kind == MK_isNotSafe; }
    void setNotSafe() { kind = MK_isNotSafe ; }
    void setSafe() { kind = MK_isSafe; }

    G4_INST *getSrc2Def() { return src2Def; }
    G4_INST *getFirstMad() const { return madSequence.front(); }
    G4_INST *getLastMad() const { return madSequence.back(); }
    void appendMad(INST_LIST_ITER begin, INST_LIST_ITER end)
    {
        madSequence.insert(madSequence.end(), begin, end);
    }
    typedef std::vector<G4_INST *>::iterator mad_iter;
    mad_iter mad_begin() { return madSequence.begin(); }
    mad_iter mad_end() { return madSequence.end(); }

    void appendUser(G4_INST *inst) { lastMadUserChain.push_back(inst); }
    G4_INST *getLastUser() { return lastMadUserChain.back(); }

    void reset()
    {
        kind = MK_unknown;
        src2Def = nullptr;
        madSequence.clear();
        lastMadUserChain.clear();
    }

    // Collect all candidate instructions and perform minimal checks.
    INST_LIST_ITER populateCandidates(INST_LIST_ITER iter);

    // Make changes to all the candidates collected. This is the only
    // function that makes changes to the IR.
    void processCandidates();

private:
    void populateUserChain(G4_INST *defInst, int level);
    void populateSrc2Def();

    // Check whether this mad sequence can be turned into a MAC sequence.
    bool checkMadSequence();

    // Check whether the user chain blocks this transformation or not.
    bool checkUserChain();

    // Check if other instructions between defInst and useInst are also updating
    // ACC registers, which may block this transformation.
    bool checkACCDependency(G4_INST *defInst, G4_INST *useInst);

    // The common type for accumulator operands.
    G4_Type getCommonACCType()
    {
        G4_Type T = Type_UNDEF;

        // If there is no user chain to check. Use the last mad's destionation
        // operand type as the common type. Otherwise, use the last user's
        // destionation type.
        if (lastMadUserChain.empty())
            T = getLastMad()->getDst()->getType();
        else
            T = getLastUser()->getDst()->getType();

        return IS_FTYPE(T) ? Type_F : (IS_SIGNED_INT(T) ? Type_W : Type_UW);
    }
};

class AccRestriction {
    unsigned encoding;

public:
    // Accumulator Restriction Kind:
    enum Accumulator_RK {
      ARK_NoRestriction = 0x01,              // No restrictions.
      ARK_NoAccess = 0x02,                   // No accumulator access, implicit or explicit.
      ARK_NoSourceOperand = 0x04,            // Source operands cannot be accumulators.
      ARK_NoModifier = 0x08,                 // Source modifier is not allowed if source is an accumulator.
      ARK_NoExplicitSource = 0x010,          // Accumulator is an implicit source and thus cannot be an explicit source operand.
      ARK_NoDst = 0x20,                      // Accumulator cannot be destination, implicit or explicit.
      ARK_AccWrEnRequired = 0x40,            // AccWrEn is required. The accumulator is an implicit
                                             // destination and thus cannot be an explicit destination operand.
      ARK_NoIntegerSource = 0x80,            // Integer source operands cannot be accumulators.
      ARK_NoExplicitSrcAllowAccWrEn = 0x100, // No explicit accumulator access because this is a three-source instruction.
                                             // AccWrEn is allowed for implicitly updating the accumulator.
      ARK_NoBothSrcAndDst = 0x200            // An accumulator can be a source or destination operand but not both.
    };

    AccRestriction(unsigned val) : encoding(val) {}

    bool useAccAsSrc(G4_SrcRegRegion *opnd, bool isExplicit = true,
                     bool isAlreadyDst = false) const
    {
        if (!opnd)
            return false;

        if (encoding & ARK_NoAccess)
            return false;

        if (encoding & ARK_NoRestriction)
            return true;

        if (encoding & ARK_NoSourceOperand)
            return false;

        if (encoding & ARK_NoIntegerSource)
            return !IS_TYPE_INT(opnd->getType());

        if (encoding & ARK_NoExplicitSource)
            return !isExplicit;

        if (encoding & ARK_NoExplicitSrcAllowAccWrEn)
            return false;

        if (encoding & ARK_NoBothSrcAndDst)
            return !isAlreadyDst;

        if (encoding & ARK_NoModifier)
            return opnd->getModifier() == Mod_src_undef;

        return true;
    }

    bool useAccAsDst(G4_DstRegRegion *opnd, bool isExplicit = true,
                     bool isAlreadySrc = false) const
    {
        if (!opnd)
            return false;

        if (encoding & ARK_NoAccess)
            return false;

        if (encoding & ARK_NoRestriction)
            return true;

        if (encoding & ARK_NoDst)
            return false;

        if (encoding & ARK_AccWrEnRequired)
            return !isExplicit;

        if (encoding & ARK_NoBothSrcAndDst)
            return !isAlreadySrc;

        return true;
    }

    static AccRestriction getRestrictionKind(G4_INST *inst);
};

} // namespace

AccRestriction AccRestriction::getRestrictionKind(G4_INST *inst)
{
    switch (inst->opcode()) {
    default:
        break;
    case G4_add:
    case G4_asr:
    case G4_avg:
    case G4_csel:
    case G4_frc:
    case G4_sel:
    case G4_shr:
    case G4_smov:
        return ARK_NoRestriction;
    case G4_addc:
    case G4_subb:
        return ARK_AccWrEnRequired;
    case G4_and:
    case G4_not:
    case G4_or:
    case G4_xor:
        return ARK_NoModifier;
    case G4_cmp:
    case G4_cmpn:
    case G4_lzd:
        return ARK_NoRestriction;
    case G4_dp2:
    case G4_dp3:
    case G4_dp4:
    case G4_dph:
    case G4_line:
    case G4_movi:
    case G4_pln:
    case G4_sad2:
    case G4_sada2:
        return ARK_NoSourceOperand;
    case G4_lrp:
    case G4_mac:
        return ARK_NoExplicitSource;
    case G4_madm:
        return ARK_NoExplicitSrcAllowAccWrEn;
    case G4_mach:
        return ARK_NoExplicitSource | ARK_AccWrEnRequired;
    case G4_mov:
        return ARK_NoBothSrcAndDst;
    case G4_mul:
        return ARK_NoIntegerSource;
    case G4_rndd:
    case G4_rnde:
    case G4_rndu:
    case G4_rndz:
        return ARK_NoRestriction;
    case G4_shl:
        return ARK_NoDst;
    }

    return ARK_NoAccess;
}

/// Check this pseudo-mad's dst operand. Returns false if there is anything
/// blocking acc's usage.
static bool checkMadDst(G4_INST *inst, IR_Builder &builder)
{
    G4_DstRegRegion *dst = inst->getDst();
    if (!dst || builder.kernel.fg.globalOpndHT.isOpndGlobal(dst))
        return false;

    if (dst->getRegAccess() != Direct)
        return false;

    // Only acc0 is available for w/uw destination.
    // FIXME: This acc type size is only for simd 16.
    unsigned Sz = G4_Type_Table[Type_W].byteSize;
    Sz *= dst->getHorzStride() * inst->getExecSize();
    return Sz <= G4_GRF_REG_NBYTES;
}

// Check whether this mad sequence can be turned into a MAC sequence.
bool MadSequenceInfo::checkMadSequence()
{
    unsigned int maskOffset = getFirstMad()->getMaskOffset();

    // First check each individual mad.
    for (auto inst : madSequence)
    {
        ASSERT_USER(isMad(inst), "not a mad");

        // Only for simd 16. TODO: support simd8.
        if (inst->getExecSize() != 16)
            return false;

        if (inst->getMaskOffset() != maskOffset)
            return false;

        // Do not handle predicate yet.
        if (inst->getPredicate() != nullptr)
            return false;

        // Do not handle cond modifier yet.
        if (inst->getCondMod() != nullptr)
            return false;

        if (!checkMadDst(inst, builder))
            return false;

        G4_Operand *src0 = inst->getSrc(0);
        G4_Operand *src1 = inst->getSrc(1);
        G4_Operand *src2 = inst->getSrc(2);

        if (!src0 || !src1 || !src2)
            return false;

        if (IS_FTYPE(src0->getType()) && IS_FTYPE(src1->getType()) &&
            IS_FTYPE(src2->getType()))
        {
            // ok
        }
        else if ((!IS_BTYPE(src0->getType()) && !IS_WTYPE(src0->getType())) ||
            (!IS_BTYPE(src1->getType()) && !IS_WTYPE(src1->getType())))
        {
            // Only when src0 and src1 are of Byte/Word types.
            return false;
        }
        else if (!IS_BTYPE(src2->getType()) && !IS_WTYPE(src2->getType()) &&
                !IS_DTYPE(src2->getType()))
        {
            // Only when src2 is of Byte/Word/DWord types.
            return false;
        }
           
        // If there is a modifier for src2, or src2 is accessed somewhere
        // indirectly then we will not generate a MAC.
        if (!src2->isSrcRegRegion())
            return false;

        if (src2->asSrcRegRegion()->getModifier() != Mod_src_undef ||
            src2->asSrcRegRegion()->getRegAccess() != Direct ||
            (src2->getTopDcl() && src2->getTopDcl()->getAddressed()))
            return false;
    }

    // Now check instructions in pairs.
    G4_INST *defInst = getSrc2Def();
    G4_INST *lastMad = getLastMad();
    for (auto I = mad_begin(); defInst != lastMad; ++I)
    {
        G4_INST *useInst = *I;
        G4_Operand *dst = defInst->getDst();
        G4_Operand *src2 = useInst->getSrc(2);
        ASSERT_USER(dst && dst->isDstRegRegion(), "invalid dst");
        ASSERT_USER(src2 && src2->isSrcRegRegion(), "invalid src2");
        if (dst->compareOperand(src2) != Rel_eq)
            return false;

        // Move the next pair.
        defInst = useInst;
    }

    return true;
}

bool MadSequenceInfo::checkACCDependency(G4_INST *defInst, G4_INST *useInst)
{
    auto iter = std::find(bb->begin(), bb->end(), defInst);
    ASSERT_USER(iter != bb->end(), "no instruction found?");

    for (++iter; (*iter) != useInst; ++iter) {
        if ((*iter)->defAcc() || (*iter)->useAcc() || (*iter)->mayExpandToAccMacro(builder))
            return false;
    }
    return true;
}

// Check whether this user chain is safe to use ACC.
bool MadSequenceInfo::checkUserChain()
{
    // skip if there is no user to be analyzed.
    if (lastMadUserChain.empty())
        return true;

    G4_INST *defInst = getLastMad();
    G4_INST *useInst = defInst->use_back().first;

    while (true)
    {
        // TODO: enable simd8.
        if (useInst->getExecSize() != 16)
            return false;

        // Only when used as a source operand.
        Gen4_Operand_Number opndNum = defInst->use_back().second;
        if (!G4_INST::isSrcNum(opndNum))
            return false;

        G4_Operand *useOpnd = useInst->getSrc(G4_INST::getSrcNum(opndNum));
        if (!useOpnd || !useOpnd->isSrcRegRegion())
            return false;

        if (useOpnd->asSrcRegRegion()->isIndirect())
            return false;

        // check other source type.
        for (int i = 0, e = useInst->getNumSrc(); i < e; ++i)
        {
            G4_Operand *otherSrc = useInst->getSrc(i);
            if (otherSrc == useOpnd)
                continue;

            if (!isWBTypeAndNotNull(otherSrc))
                return false;
        }

        bool isLastUser = (useInst == getLastUser());

        // The last user does not use ACC as its dst.
        AccRestriction AR = AccRestriction::getRestrictionKind(useInst);
        if (!AR.useAccAsSrc(useOpnd->asSrcRegRegion(), true, !isLastUser))
            return false;

        // Now check between defInst and useInst, no ACC will be written by
        // other instructions.
        if (!checkACCDependency(defInst, useInst))
            return false;

        if (isLastUser)
            // No extra check for the last user.
            break;

        // This is not the last user. We check its dst too.
        G4_Operand *useDst = useInst->getDst();
        if (!useDst)
            return false;

        // check type, no support for *Q types yet.
        if (!IS_DTYPE(useDst->getType()))
            return false;

        // For each inner user, need to use ACC as its explicit dst.
        if (!AR.useAccAsDst(useDst->asDstRegRegion(), true, true))
            return false;

        if (defInst->getDst()->compareOperand(useOpnd) != Rel_eq)
            return false;

        // move to next pair.
        defInst = useInst;
        useInst = defInst->use_back().first;
    }

    // nothing wrong.
    return true;
}

void MadSequenceInfo::populateSrc2Def()
{
    G4_INST *firstMad = getFirstMad();
    ASSERT_USER(firstMad && isMad(firstMad), "invalid mad");

    src2Def = firstMad->getSingleDef(Opnd_src2, true);
    if (src2Def == nullptr)
        // Cannot find a single definition.
        return setNotSafe();

    // Check it right here.
    // Only support splats or simd16 initialization.
    if (src2Def->getExecSize() != 16 && src2Def->getExecSize() != 1)
    {
        return setNotSafe();
    }

    G4_Operand *Dst = src2Def->getDst();
    if (!Dst || builder.kernel.fg.globalOpndHT.isOpndGlobal(Dst))
        return setNotSafe();

    if (Dst->asDstRegRegion()->getRegAccess() != Direct)
        return setNotSafe();

    if (src2Def->getPredicate() || src2Def->getSaturate() ||
        !src2Def->hasOneUse())
        return setNotSafe();

    if (IS_DTYPE(src2Def->getExecType()))
    {
        // since we use <1>:w region for our acc temp, due to alignment requirements
        // we can't allow dword source types
        return setNotSafe();
    }

    // Check if there is any ACC dependency.
    if (!checkACCDependency(src2Def, firstMad))
        return setNotSafe();

    // Check restrictions on compression to ensure that changing the destination
    // type will not change the source region meaning due to instruction
    // compression.
    //
    // If both instructions are compressed or both non-compressed then it is
    // safe. Otherwise, check whether source regions are compression invariants.
    if (src2Def->isComprInst() ^ firstMad->isComprInst())
    {
        auto checkCompression = [](G4_INST *inst)
        {
            for (int i = 0; i < inst->getNumSrc(); ++i)
            {
                G4_Operand *opnd = inst->getSrc(i);
                if (!opnd || !opnd->isSrcRegRegion())
                    continue;
                if (!inst->isComprInvariantSrcRegion(opnd->asSrcRegRegion(), i))
                    return false;
                if (IS_DTYPE(opnd->getType()))
                    return false;
            }
            return true;
        };

        if (src2Def->isComprInst())
        {
            if (!checkCompression(src2Def))
               return setNotSafe();
        }
        else
        {
            if (!checkCompression(firstMad))
               return setNotSafe();
        }
    }

    AccRestriction AR = AccRestriction::getRestrictionKind(src2Def);
    if (!AR.useAccAsDst(Dst->asDstRegRegion()))
        return setNotSafe();
}

void MadSequenceInfo::populateUserChain(G4_INST *defInst, int level)
{
    // Failed.
    if (level <= 0)
        return setNotSafe();

    // Only for a single use.
    if (!defInst->hasOneUse())
        return setNotSafe();

    // Only when used as a source operand.
    Gen4_Operand_Number opndNum = defInst->use_back().second;
    if (!G4_INST::isSrcNum(opndNum))
        return setNotSafe();

    G4_INST *useInst = defInst->use_back().first;
    G4_Operand *useDst = useInst->getDst();

    // Should support a null dst?
    if (useDst == nullptr)
        return setNotSafe();

    appendUser(useInst);

    // If this user has *W/*B types then candidate found and stop the search.
    if (IS_BTYPE(useDst->getType()) || IS_WTYPE(useDst->getType()))
        return;

    // Search to the next level.
    return populateUserChain(useInst, level - 1);
}

// Currently, we assume that MAD instructions are back-to-back. This avoids
// dependency checking among mad and non-mad instructions.
//
// TODO: hoist or sink instructions.
//
INST_LIST_ITER MadSequenceInfo::populateCandidates(INST_LIST_ITER iter)
{
    // Find the first pseudo-mad instruction.
    iter = std::find_if(iter, bb->end(), isMad);

    // No mad found
    if (iter == bb->end())
        return iter;

    // Find the first non-mad instruction following this sequence.
    auto end = std::find_if_not(iter, bb->end(), isMad);

    ASSERT_USER(iter != end, "out of sync");
    appendMad(iter, end);

    // Checking the first mad's src2.
    populateSrc2Def();

    // If the mad sequence has *W/*B types then it is safe to use ACC regardless
    // of the destionation operand type in its use.
    // ..
    // mac (16) acc0.0<1>:w r10.7<0;1,0>:w r62.16<16;16,1>:ub {Align1, H1}
    // mac (16) acc0.0<1>:w r11.6<0;1,0>:w r63.0<16;16,1>:ub {Align1, H1}
    // mac (16) r24.0<1>:w r11.7<0;1,0>:w r63.16<16;16,1>:ub {Align1, H1}
    // add (16) r14.0<1>:d r14.0<8;8,1>:d r24.0<8;8,1>:w {Align1, H1}
    //
    // could be generated.
    G4_Type MadDstType = getLastMad()->getDst()->getType();
    if (IS_DTYPE(MadDstType))
    {
        // Populate the user chain up to some predetermined level.
        const int level = 4;
        populateUserChain(getLastMad(), level);
    }

    // We have gathered all candidates for this optimization. Now we make
    // comprehensive checks on the mad sequence and the user chain.
    if (isNotSafe())
        return end;

    if (!checkMadSequence())
        return end;

    if (!checkUserChain())
        return end;

    // everything is OK, preceed to do transformation.
    setSafe();
    return end;
}

// Makes changes to the mad sequence and its users.
void MadSequenceInfo::processCandidates()
{
    ASSERT_USER(isSafe(), "not safe for ACC");
    ASSERT_USER(getSrc2Def(), "null src");
    ASSERT_USER(!madSequence.empty(), "no mad");

    // In this function we replace all src2 to implicit ACC registers and
    // update operands in its use chain.
    G4_Type AdjustedType = getCommonACCType();

    // Fix src2Def
    G4_INST *src2Def = getSrc2Def();
    {
        // change dst of the last MAD
        G4_DstRegRegion *accDstOpnd = builder.createDstRegRegion(
            Direct, builder.phyregpool.getAcc0Reg(), 0, 0, 1, AdjustedType);
        src2Def->setDest(accDstOpnd);

        // Convert splat.
        if (src2Def->getExecSize() == 1)
        {
            src2Def->setExecSize(getFirstMad()->getExecSize());
        }
    }

    // update use-chain
    if (!lastMadUserChain.empty())
    {
        G4_INST *defInst = getLastMad();
        G4_INST *useInst = defInst->use_back().first;
        Gen4_Operand_Number opndNum = defInst->use_back().second;
        ASSERT_USER(defInst->hasOneUse(), "bad candidate");

        while (true)
        {
            RegionDesc *rd = builder.getRegionStride1();
            auto mod = useInst->getOperand(opndNum)->asSrcRegRegion()->getModifier();
            G4_SrcRegRegion *accSrcOpnd = builder.createSrcRegRegion(
                mod, Direct, builder.phyregpool.getAcc0Reg(), 0, 0, rd, AdjustedType);
            useInst->setSrc(accSrcOpnd, G4_INST::getSrcNum(opndNum));

            // The last use, only update source, and exit.
            if (useInst == getLastUser())
                break;

            // Also update the destination.
            G4_DstRegRegion *accDstOpnd = builder.createDstRegRegion(
                Direct, builder.phyregpool.getAcc0Reg(), 0, 0, 1, AdjustedType);
            useInst->setDest(accDstOpnd);

            // move to the next pair.
            defInst = useInst;
            useInst = defInst->use_back().first;
            opndNum = defInst->use_back().second;
            ASSERT_USER(defInst->hasOneUse(), "bad candidate");
        }
    }

    // update mad sequence
    for (auto I = mad_begin(), E = mad_end(); I != E; ++I) {
        G4_INST *inst = *I;
        ASSERT_USER(isMad(inst), "not a mad");

        RegionDesc *rd = builder.getRegionStride1();
        G4_SrcRegRegion *accSrcOpnd = builder.createSrcRegRegion(
            Mod_src_undef, Direct, builder.phyregpool.getAcc0Reg(), 0, 0, rd,
            AdjustedType);

        inst->setImplAccSrc(accSrcOpnd);
        inst->setSrc(nullptr, 2);
        // For the last mad, if it has *B/*W type, then no user will be modified
        // and do not change its destination operand. Otherwise, use acc as the
        // destination.
        if (getLastMad() != inst || !lastMadUserChain.empty())
        {
            G4_DstRegRegion *accDstOpnd = builder.createDstRegRegion(
                Direct, builder.phyregpool.getAcc0Reg(), 0, 0, 1, AdjustedType);
            inst->setDest(accDstOpnd);
        }
        inst->setOpcode(G4_mac);
        inst->fixMACSrc2DefUse();
    }
}

// Do any kind of proprocessing in this basic block to help MAC transformation.
// Returns false if we can easily detect this optimization is not possible.
// Otherwise, returns true.
static bool preprocessMadInBlock(IR_Builder &builder, G4_BB *bb)
{
    bool hasMad = false;
    for (auto inst : *bb)
    {
        if (isMad(inst))
        {
            hasMad = true;
            HWConformity::tryEliminateMadSrcModifier(builder, inst);
        }
    }

    // nothing to do if there is no mad.
    return hasMad;
}

//
// mul (16) V48(0,0)<1>:d r0.1<0;1,0>:w V42_in(7,2)<16;16,1>:ub {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r1.0<0;1,0>:w V42_in(7,1)<16;16,1>:ub V48(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.2<0;1,0>:w V42_in(7,3)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.3<0;1,0>:w V42_in(8,1)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.4<0;1,0>:w V42_in(8,2)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.5<0;1,0>:w V42_in(8,3)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.6<0;1,0>:w V42_in(9,1)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.7<0;1,0>:w V42_in(9,2)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// psuedo_mad (16) V51_tempConvolve(0,0)<1>:d r0.8<0;1,0>:w V42_in(9,3)<16;16,1>:ub V51_tempConvolve(0,0)<16;16,1>:d {Align1, H1}
// add (16) V51_tempConvolve(0,0)<1>:d V51_tempConvolve(0,0)<16;16,1>:d 0x4000:w {Align1, H1}
// shr.sat (16) V52(0,0)<1>:ub V51_tempConvolve(0,0)<16;16,1>:d 0xf:w {Align1, H1}
//
void Optimizer::lowerMadSequence()
{

    // Only enable CM for now.
    if (builder.getOptions()->getTarget() != VISA_CM)
        return;

    for (G4_BB *bb : fg.BBs)
    {
        // Preprocess this basic block. If no mad sequence found then skip to
        // the next basic block right away.
        if (!preprocessMadInBlock(builder, bb))
            continue;

        // Object to gather information for ACC optimization.
        MadSequenceInfo madInfo(builder, bb);

        auto iter = bb->begin();
        while (iter != bb->end())
        {
            // Returns an iterator to the next non-mad instruction after the mad
            // sequence. It is safe to insert/delete instructions before it.
            iter = madInfo.populateCandidates(iter);

            // Perform transformation. The resulted IR may still need to be
            // fixed by HWConformity, e.g. the dst may still have *B type.
            if (madInfo.isSafe())
                madInfo.processCandidates();

            // Cleanup immediate results, whether change has been made or not.
            madInfo.reset();
        }
    }
}

void Optimizer::ifCvt()
{
    runIfCvt(fg);
}

namespace {

enum SplitBounds : unsigned {
    LoLBound = 0,
    LoRBound = 63,
    HiLBound = 64,
    HiRBound = 127
};

bool isCandidateDecl(G4_Declare *Dcl)
{
    G4_Declare *RootDcl = Dcl->getRootDeclare();
    if (RootDcl->getRegFile() != G4_GRF)
        return false;

    // Only split 4GRF variables. We should be able to split > 4GRF variables,
    // but this should have been done in FE.
    if (RootDcl->getByteSize() != 4 * GENX_GRF_REG_SIZ)
        return false;

    if (RootDcl->getAddressed())
        return false;

    return true;
}

// Associated declarations for splitting.
struct DclMapInfo {
    // The low part of the splitted variable.
    G4_Declare *DclLow;

    // The high part of the splitted variable.
    G4_Declare *DclHigh;

    // Aliases of the low part. Created if needed for different types.
    std::vector<G4_Declare *> AliasLow;

    // Aliases of the high part. Created if needed for different types.
    std::vector<G4_Declare *> AliasHigh;

    DclMapInfo(G4_Declare *Lo, G4_Declare *Hi) : DclLow(Lo), DclHigh(Hi) {}

    // Return an appropriate declaration/alias for low or high part.
    G4_Declare *getDcl(IR_Builder &Builder, G4_Type Ty, bool IsLow)
    {
        return IsLow ? getDcl(Builder, Ty, DclLow, AliasLow)
                     : getDcl(Builder, Ty, DclHigh, AliasHigh);
    }

private:
    G4_Declare *getDcl(IR_Builder &Builder, G4_Type Ty, G4_Declare *RootDcl,
                       std::vector<G4_Declare *> &Aliases)
    {
        if (Ty == RootDcl->getElemType())
            return RootDcl;

        for (auto AL : Aliases)
        {
            if (Ty == AL->getElemType())
                return AL;
        }

        // Create such an alias if it does not exist yet.
        unsigned NElts = RootDcl->getByteSize() / G4_Type_Table[Ty].byteSize;
        auto Alias = Builder.createTempVar(NElts, Ty, Either, Any,
            (std::string(RootDcl->getName()) + "_" + G4_Type_Table[Ty].str).c_str(), false);
        Alias->setAliasDeclare(RootDcl, 0);
        Aliases.push_back(Alias);
        return Alias;
    }
};

}  // namespace

//
// mov (M1, 8) V45(0,0)<1>:q V42(0,0)<8;8,1>:d                                      /// $7
// mov (M3, 8) V45(2,0)<1>:q V42(1,0)<8;8,1>:d                                      /// $8
// shl (M1, 8) V46(0,0)<1>:q V45(0,0)<8;8,1>:q 0x2:q                                /// $9
// shl (M3, 8) V46(2,0)<1>:q V45(2,0)<8;8,1>:q 0x2:q                                /// $10
//
// into
//
// mov (M1, 8) V45L(0,0)<1>:q V42(0,0)<8;8,1>:d                                      /// $7
// mov (M3, 8) V45H(0,0)<1>:q V42(1,0)<8;8,1>:d                                      /// $8
// shl (M1, 8) V46(0,0)<1>:q V45L(0,0)<8;8,1>:q 0x2:q                                /// $9
// shl (M3, 8) V46(2,0)<1>:q V45H(0,0)<8;8,1>:q 0x2:q                                /// $10
//
void Optimizer::splitVariables()
{
    if (builder.getOptions()->getTarget() != VISA_3D)
    {
        return;
    }

    if (builder.getOption(vISA_Debug))
    {
        return;
    }

    // Only for simd16 and simd32.
    if (kernel.getSimdSize() == 8)
    {
        return;
    }

    // All declarations in this map are candidates for variable splitting.
    std::map<const G4_Declare *, DclMapInfo *> DclMap;

    // All instructions to be updated, and the flag indicates if this
    // instruction defines low part or ont.
    std::vector<std::pair<G4_INST *, bool>> InstsToUpdate;

    for (G4_BB *bb : fg.BBs)
    {
        for (G4_INST *inst : *bb)
        {
            // Only for variables defined by non-send instructions.
            if (inst->opcode() == G4_label || inst->isSend())
                continue;

            // TODO: Allow global variables.
            auto Dst = inst->getDst();
            if (!Dst || Dst->getHorzStride() != 1 ||
                fg.globalOpndHT.isOpndGlobal(Dst))
                continue;
            auto Dcl = Dst->getTopDcl();
            if (!Dcl || !isCandidateDecl(Dcl))
                continue;

            unsigned LBound = Dst->getLeftBound();
            unsigned RBound = Dst->getRightBound();
            if ((LBound == LoLBound && RBound == LoRBound) ||
                (LBound == HiLBound && RBound == HiRBound))
            {
                // OK, only defines low or high part.
                // Bound constraints imply that def and uses are in two grfs.
            }
            else
            {
                continue;
            }

            if (inst->useEmpty())
            {
                continue;
            }

            // Check all uses.
            bool DoSplitting = true;
            for (auto UI = inst->use_begin(), UE = inst->use_end(); UI != UE; ++UI)
            {
                G4_Operand *Op = UI->first->getOperand(UI->second);
                if (Op && Op->compareOperand(Dst) != Rel_eq)
                {
                    DoSplitting = false;
                    break;
                }

                // The def-use chain is broken in the following code:
                //
                // mov (8, M1_NM) V33(0,0):df 0  <- not def
                // mov (8, M3_NM) V33(2,0):df 0  <- not def
                // mov (8, M1) V33(0,0):df 1     <- def
                // mov (8, M3_NM) V33(2,0):df 1  <- def
                //
                // use of V33(0,0) and V33(2,0)
                //
                // FIXME: remove this if def-use chain is accurate.
                if (UI->first->isWriteEnableInst() && !inst->isWriteEnableInst())
                {
                    DoSplitting = false;
                    break;
                }

                // Only allow single definition:
                // mov (8, M1_NM) V33:q 0
                // mov (8, M1)    V33:q 1
                // add (8, M1_NM  V34:q V33:q V32:q
                //
                if (UI->first->getSingleDef(UI->second) != inst)
                {
                    DoSplitting = false;
                    break;
                }
            }

            if (!DoSplitting)
                continue;

            G4_Type Ty = Dcl->getElemType();
            auto Iter = DclMap.find(Dcl);
            if (Iter == DclMap.end())
            {
                unsigned NElts = Dcl->getTotalElems();
                auto DclLow = builder.createTempVar(NElts / 2, Ty, Either, Sixteen_Word, "Lo");
                auto DclHi = builder.createTempVar(NElts / 2, Ty, Either, Sixteen_Word, "Hi");
                DclMap[Dcl] = new DclMapInfo(DclLow, DclHi);
            }
            bool IsLow = LBound == LoLBound;
            InstsToUpdate.push_back(std::make_pair(inst, IsLow));
        }
    }

    // Iterate instructions that define hi or low parts. Update their defs and uses.
    for (auto IPair : InstsToUpdate)
    {
        G4_INST *Inst = IPair.first;
        bool IsLow = IPair.second;

        // Update Inst's Dst.
        {
            G4_DstRegRegion *Dst = Inst->getDst();
            G4_Type Ty = Dst->getType();
            auto NewDcl = DclMap[Dst->getTopDcl()]->getDcl(builder, Ty, IsLow);
            auto NewDst = builder.createDstRegRegion(Direct, NewDcl->getRegVar(), 0, 0, 1, Ty);
            Inst->setDest(NewDst);
        }

        // Update Inst's uses.
        for (auto UI = Inst->use_begin(), UE = Inst->use_end(); UI != UE; ++UI)
        {
            G4_INST *UseInst = UI->first;
            G4_SrcRegRegion *UseOpnd = UseInst->getOperand(UI->second)->asSrcRegRegion();
            G4_Type UseTy = UseOpnd->getType();
            auto NewUseDcl = DclMap[UseOpnd->getTopDcl()]->getDcl(builder, UseTy, IsLow);
            auto NewUseOpnd = builder.createSrcRegRegion(
                UseOpnd->getModifier(), UseOpnd->getRegAccess(),
                NewUseDcl->getRegVar(), 0, 0, UseOpnd->getRegion(), UseTy);
            UseInst->setSrc(NewUseOpnd, G4_INST::getSrcNum(UI->second));
        }
    }

    // Cleanup.
    for (auto DI : DclMap)
    {
        delete DI.second;
    }
}

//
// replacement of the above that can handle global variables
// basically we split any 4GRF variables (they typically result from 
// simd16 64-bit vars) into two half if
// -- they are not address taken or used in send
// -- none of the operands cross from the 2nd to the 3rd GRF
// This is intended to give RA more freedom as the split variables do
// not have to be allocated contiguously.
// Note that this invalidates existing def-use chains
//
void Optimizer::split4GRFVars()
{
    std::unordered_set<G4_Declare*> varToSplit;
    std::vector<G4_Declare*> varToSplitOrdering;
    // map each split candidate to their replacement split variables
    std::unordered_map<const G4_Declare *, DclMapInfo *> DclMap;

    if (builder.getOptions()->getTarget() != VISA_3D)
    {
        return;
    }

    if (builder.getOption(vISA_Debug))
    {
        return;
    }

    // Only for simd16 and simd32.
    if (kernel.getSimdSize() == 8)
    {
        return;
    }

    // first scan the decl list
    for (auto dcl : kernel.Declares)
    {
        if (dcl->getAliasDeclare() == nullptr)
        {
            if (isCandidateDecl(dcl))
            {
                if (varToSplit.find(dcl) == varToSplit.end())
                {
                    varToSplitOrdering.push_back(dcl);
                }
                   
                varToSplit.emplace(dcl);
            }
        }
        else
        {
            // strictly speaking this condition is not necesary, but having
            // no aliases that could point into a middle of the split candidate
            // makes replacing the split var much easier. By construction the root 
            // must appear before its alias decls
            uint32_t offset = 0;
            G4_Declare* rootDcl = dcl->getRootDeclare(offset);
            if (offset != 0 && isCandidateDecl(rootDcl))
            {
                varToSplit.erase(rootDcl);
            }
        }
    }

    if (varToSplit.empty())
    {
        // early exit if there are no split candidate
        return;
    }

    // first pass is to make sure the validity of all split candidates
    for (auto bb : kernel.fg.BBs)
    {
        for (auto inst : *bb)
        {
            auto removeCandidate = [&varToSplit](G4_Declare* dcl)
            {
                if (dcl)
                {
                    dcl = dcl->getRootDeclare();
                    varToSplit.erase(dcl);
                }
            };

            if (inst->isSend())
            {
                removeCandidate(inst->getDst()->getTopDcl());
                removeCandidate(inst->getSrc(0)->getTopDcl());
                if (inst->isSplitSend())
                {
                    removeCandidate(inst->getSrc(1)->getTopDcl());
                }
            }
            else
            {
                auto cross2GRF = [](G4_Operand* opnd)
                {
                    uint32_t lb = opnd->getLeftBound();
                    uint32_t rb = opnd->getRightBound();
                    return (lb < 2 * GENX_GRF_REG_SIZ) && (rb > 2 * GENX_GRF_REG_SIZ);
                };
                // check and remove decls with operands that cross 2GRF boundary
                if (inst->getDst())
                {
                    G4_Declare* dstDcl = inst->getDst()->getTopDcl();
                    if (dstDcl && cross2GRF(inst->getDst()))
                    {
                        removeCandidate(dstDcl);
                    }
                }
                for (int i = 0, numSrc = inst->getNumSrc(); i < numSrc; ++i)
                {
                    G4_Operand* src = inst->getSrc(i);
                    if (src && src->getTopDcl() && cross2GRF(src))
                    {
                        removeCandidate(src->getTopDcl());
                    }
                }
            }
        }
    }

    if (varToSplit.empty())
    {
        // early exit if there are no split candidate
        return;
    }

    // create Lo/Hi for each variable being split
    for (auto splitDcl : varToSplitOrdering)
    {
        // varToSplitOrdering may have extra elements since we never delete any inserted dcl from it.
        if (varToSplit.find(splitDcl) == varToSplit.end())
            continue;
        G4_Type Ty = splitDcl->getElemType();
        unsigned NElts = splitDcl->getTotalElems();
        std::string varName(splitDcl->getName());
        auto DclLow = builder.createTempVar(NElts / 2, Ty, Either, Sixteen_Word, 
            (varName + "Lo").c_str(), false);
        auto DclHi = builder.createTempVar(NElts / 2, Ty, Either, Sixteen_Word,
            (varName + "Hi").c_str(), false);
        DclMap[splitDcl] = new DclMapInfo(DclLow, DclHi);
        //std::cerr << "split " << splitDcl->getName() << " into (" <<
        //   DclLow->getName() << ", " << DclHi->getName() << ")\n";
    }

    // second pass actually does the replacement
    for (auto bb : kernel.fg.BBs)
    {
        for (auto inst : *bb)
        {
            bool changed = false;
            auto dst = inst->getDst();
            if (dst && dst->getTopDcl())
            {
                G4_Declare* dstRootDcl = dst->getTopDcl()->getRootDeclare();
                if (DclMap.count(dstRootDcl))
                {
                    bool isLow = dst->getLeftBound() < 2 * GENX_GRF_REG_SIZ;
                    auto NewDcl = DclMap[dstRootDcl]->getDcl(builder, dst->getType(), isLow);
                    auto NewDst = builder.createDstRegRegion(Direct, NewDcl->getRegVar(), 
                        dst->getRegOff() - (isLow ? 0 : 2), dst->getSubRegOff(), 
                        dst->getHorzStride(), dst->getType(), dst->getAccRegSel());
                    inst->setDest(NewDst);
                    changed = true;
                }
            }

            for (int i = 0, numSrc = inst->getNumSrc(); i < numSrc; ++i)
            { 
                G4_Operand* src = inst->getSrc(i);
                if (src && src->getTopDcl())
                {
                    G4_SrcRegRegion* srcRegion = src->asSrcRegRegion();
                    G4_Declare* srcRootDcl = src->getTopDcl()->getRootDeclare();;
                    if (DclMap.count(srcRootDcl))
                    {
                        bool isLow = src->getLeftBound() < 2 * GENX_GRF_REG_SIZ;
                        auto NewSrcDcl = DclMap[srcRootDcl]->getDcl(builder, src->getType(), isLow);
                        auto NewSrc = builder.createSrcRegRegion(
                            srcRegion->getModifier(), src->getRegAccess(),
                            NewSrcDcl->getRegVar(), srcRegion->getRegOff() - (isLow ? 0 : 2), 
                            srcRegion->getSubRegOff(), srcRegion->getRegion(), src->getType(), src->getAccRegSel());
                        inst->setSrc(NewSrc, i);
                        changed = true;
                    }
                }
            }
        }
    }

    for (auto DI : DclMap)
    {
        delete DI.second;
    }
}

//
// for some platforms int has half throughout compared to float,
// so for copy moves we should change their type
// from D/UD to F or W/UW to HF when possible
//
void Optimizer::changeMoveType()
{
    if (!builder.favorFloatMov())
    {
        return;
    }

    for (auto bb : fg.BBs)
    {
        for (auto inst : *bb)
        {

            if (inst->opcode() != G4_mov)
            {
                continue;
            }
            // copy move means dst and src has identical bits (implies same type width), and there are no
            // sat/conditional modifier as well as src modifier
            // ToDo: we should probably change isRawMov() to include mov UD D
            auto src0 = inst->getSrc(0);
            G4_Type dstTy = inst->getDst()->getType();
            G4_Type src0Ty = src0->getType();
            bool hasNoModifier = !inst->getSaturate() && !inst->getCondMod() &&
               (src0->isImm() || (src0->isSrcRegRegion() && src0->asSrcRegRegion()->getModifier() == Mod_src_undef));

            // it may be unsafe to change the move type for acc as it has higher precision
            auto changeType = [this](G4_INST* movInst, G4_Type newTy)
            {
                movInst->getDst()->setType(newTy);
                auto src0 = movInst->getSrc(0);
                if (src0->isImm())
                {
                    uint32_t mask = getTypeSize(newTy) == 4 ? 0xFFFFFFFF : 0xFFFF;
                    movInst->setSrc(fg.builder->createImm(src0->asImm()->getImm() & mask, newTy), 0);
                }
                else
                {
                    movInst->getSrc(0)->asSrcRegRegion()->setType(newTy);
                }
            };

            if (inst->getDst()->isGreg() && hasNoModifier)
            {
                if (src0->isGreg())
                {
                    bool isIntCopyMove = IS_TYPE_INT(dstTy) && IS_TYPE_INT(src0Ty) &&
                        getTypeSize(dstTy) == getTypeSize(src0Ty);
                    if (isIntCopyMove)
                    {
                        if (dstTy == Type_D || dstTy == Type_UD)
                        {
                            changeType(inst, Type_F);
                        }
                        else if (dstTy == Type_W || dstTy == Type_UW)
                        {
                            changeType(inst, Type_HF);
                        }
                    }
                }
                else if (src0->isImm())
                {
                    // allow sext and zext imm moves
                    int64_t immVal = src0->asImm()->getImm();
                    bool isIntImmMove = IS_TYPE_INT(dstTy) && IS_TYPE_INT(src0Ty) &&
                        G4_Imm::isInTypeRange(immVal, dstTy);
                    if (isIntImmMove)
                    {
                        if (dstTy == Type_D || dstTy == Type_UD)
                        {
                            changeType(inst, Type_F);
                        }
                        else if (dstTy == Type_W || dstTy == Type_UW)
                        {
                            changeType(inst, Type_HF);
                        }
                    }
                }
            }
        }
    }
}


// NoDD
namespace {
    // This describes the part of the GRF accessed.
    struct AccessMask {
        uint64_t mask;
        // int from;
        // int to;
        void setMask(int From, int To) {
            mask = (((uint64_t) 1L << (To + 1L)) - 1L)
                & ~(((uint64_t) 1L << From) - 1L);
        }
        AccessMask(void) {
            mask = 0;
        }
        AccessMask(int From, int To) {
            setMask(From, To);
        }
        AccessMask(G4_Operand *opnd, int bucket) {
            int linearStart = opnd->getLinearizedStart();
            int linearEnd = opnd->getLinearizedEnd();
            int buLinearStart = bucket * G4_GRF_REG_NBYTES;
            int buLinearEnd = buLinearStart + G4_GRF_REG_NBYTES - 1;

            // If the OPND is accessing more than 1 GRF
            // get the addresses that fit within the bucket
            int fittedLinearStart = std::max(linearStart, buLinearStart);
            int fittedLinearEnd = std::min(linearEnd, buLinearEnd);

            int from = fittedLinearStart - buLinearStart;
            assert(from >= 0);
            int to = fittedLinearEnd - buLinearStart;
            setMask(from, to);
            assert(to < G4_GRF_REG_NBYTES);
        }

        // Return TRUE if this mask and AM have some overlap
        bool hasOverlapWith(const AccessMask &AM) const {
            // if (AM.from >= from) {
            //     return AM.from <= to;
            // } else {
            //     return AM.to >= from;
            // }
            return (mask & AM.mask) != 0;
        }
        // Return TRUE if this mask fully covers the AM mask
        bool fullyCovers(const AccessMask &AM) const {
            // return (from <= AM.from && to >= AM.to);
            return (mask | AM.mask) == mask;
        }
        // Return TRUE if the mask covers the full GRF
        bool coversFullGRF(void) const {
            // return (to > from
            //         && (from % G4_GRF_REG_NBYTES) == 0
            //         && (to & G4_GRF_REG_NBYTES) == 0);
            return mask == ((uint64_t) 1 << G4_GRF_REG_NBYTES) - (uint64_t) 1;
        }
        void setBits(const AccessMask &AM) {
            mask |= AM.mask;
        }
        void clearBits(void) {
            mask = 0;
        }
        void clearBits(const AccessMask &AM) {
            mask &= ~AM.mask;
        }
        // Return true if all bits set in AM are also set in this mask
        bool areBitsSet(const AccessMask &AM) const {
            return (AM.mask & mask) == AM.mask;
        }
        // Debug print
        void dump(void) const {
            for (int i = 0; i < G4_GRF_REG_NBYTES; ++i) {
                if (((uint64_t) 1 << i) & mask)
                    std::cerr << "1";
                else
                    std::cerr << "0";
            }
            fprintf(stderr, "    0x%lX\n", (unsigned long)mask);
        }
    };




    enum RW {
        RW_UNINIT = 0,
        READ = 1 << 0,
        WRITE = 1 << 1,
        RW_MAX
    };


    // Points to the bucket
    struct BucketDescr {
        int bucketIdx;
        RW type;
        AccessMask mask;
        bool isARF;
        G4_Type dataType;

        BucketDescr(int BI, G4_Type DT, RW RWT, AccessMask AM, bool IsARF)
            : bucketIdx(BI), dataType(DT), type(RWT), mask(AM), isARF(IsARF)
        {
            assert(dataType != Type_UNDEF && "Bad data type ?");
        }
        bool sameReg(const BucketDescr &BD) const {
            return bucketIdx == BD.bucketIdx;
        }
        bool sameDataType(const BucketDescr &BD) const {
            return dataType == BD.dataType;
        }
        bool hasOverlapWith(const BucketDescr &BD) const {
            return sameReg(BD) && mask.hasOverlapWith(BD.mask);
        }
        bool accessesARF(void) const {
            return isARF;
        }
    };

    typedef std::vector<BucketDescr> BDVec_t;

    // Parse INSTR's OPND and generate the Bucket Descriptors.
    // Insert them into BDVec.
    void getBucketsForOperand(G4_INST* inst, RW rwType, G4_Operand* opnd,
                              BDVec_t& BDVec) {
        if (opnd->isLabel() || opnd->isImm()) {
            return;
        }
#define UNINIT_BUCKET -1
        int startingBucket = UNINIT_BUCKET;
        G4_VarBase* base = opnd->getBase();
        bool canSpanMultipleBuckets = false;
        bool isARF = false;

        assert(base && "If no base, then the operand is not touched by instr.");

        // If a register allocated regvar, then get the physical register
        G4_VarBase *phyReg = (base->isRegVar()) ?
            base->asRegVar()->getPhyReg() : base;

        switch (phyReg->getKind()) {
        case G4_VarBase::VK_phyGReg:
            // Handle GRFs - fast path
            startingBucket = opnd->getLinearizedStart()/G4_GRF_REG_NBYTES;
            canSpanMultipleBuckets = true;
            break;
        case G4_VarBase::VK_phyAReg:
            isARF = true;
            break;
        default:
            assert(0 && "Bad kind");
            break;
        }

        // Create one or more buckets and push them into the vector
        if (startingBucket != UNINIT_BUCKET) {
            if (canSpanMultipleBuckets) {
                assert(! isARF);
                unsigned int divisor;
                int baseBucket;
                assert(base->isGreg());
                divisor = G4_GRF_REG_NBYTES;
                baseBucket = 0;
                int endingBucket = baseBucket
                    + opnd->getLinearizedEnd() / divisor;
                MUST_BE_TRUE(endingBucket >= startingBucket,
                             "Ending bucket less than starting bucket");
                int numBuckets = endingBucket - startingBucket + 1;
                for (int bucket_j = startingBucket;
                     bucket_j < (startingBucket + numBuckets); bucket_j++ ) {
                    AccessMask AM(opnd, bucket_j);
                    BDVec.push_back(BucketDescr(bucket_j, opnd->getType(),
                                                rwType, AM, isARF));
                }
            }
            else {
                AccessMask AM(opnd, startingBucket);
                BDVec.push_back(BucketDescr(startingBucket, opnd->getType(),
                                            rwType, AM, isARF));
            }
        }
    }

    // We pack all information we have collected for the instruction
    struct BucketDescrWrapper {
        BucketDescrWrapper() : hasIndirW(false) {
            BDVec.reserve(16);
        }
        // The vector of buckets read/writen by the instruction
        BDVec_t BDVec;
        // The instruction has an indirect write
        bool hasIndirW;
        // Return TRUE if there is a RAW dependence between this and prevBDW
        // even without any overlap: e.g:
        // math (8) r86.0<1>:hf r86.0<8;8,1>:hf null:ud
        // math (8) r86.8<1>:hf r86.8<8;8,1>:hf null:ud
        // Although there is no a real data movement between
        // r86.0<1>:hf -> r86.8<8;8,1>:hf, adding noDDclr and noDDChck would
        // lead to a hang
        bool hasRAW(const BucketDescrWrapper &prevBDW) const {
            for (const auto &prevBD : prevBDW.BDVec) {
                if (prevBD.type == WRITE) {
                    for (const auto &currBD : BDVec) {
                        if (currBD.type == READ) {
                            // if (currBD.hasOverlapWith(prevBD)) {
                            if (currBD.sameReg(prevBD)) {
                                return true;
                            }
                        }
                    }
                }
            }
            return false;
        }

        // Return TRUE if there is a subreg WAW dependence between
        // this and prevBDW
        // This is TRUE only if the data types match.
        bool hasSubregWAW(const BucketDescrWrapper &prevBDW) const {
            bool noWAW = false;
            for (const auto &prevBD : prevBDW.BDVec) {
                if (prevBD.type == WRITE) {
                    for (const auto &currBD : BDVec) {
                        if (currBD.type == WRITE) {
                            if (currBD.sameReg(prevBD)) {
                                if (! currBD.hasOverlapWith(prevBD)
                                    && currBD.sameDataType(prevBD)) {
                                    noWAW = true;
                                } else {
                                    return false;
                                }
                            }
                        }
                    }
                }
            }
            return noWAW;
        }

        // Return TRUE if writing to more than 1 GRF
        bool touchesManyGRFs(void) const {
            std::set<int> bucketsTouched;
            for (const auto &BD : BDVec) {
                if (BD.type == WRITE) {
                    bucketsTouched.insert(BD.bucketIdx);
                }
            }
            return bucketsTouched.size() > 1;
        }

        // Return TRUE if we are accessing an ARF
        bool accessesARF(void) const {
            for (const auto &BD : BDVec) {
                if (BD.accessesARF()) {
                    return true;
                }
            }
            return false;
        }
    };

    // Given an inst with physical register assignment,
    // return all bucket descriptors that the physical register can map
    // to. This requires taking in to account exec size, data
    // type, and whether inst is a send
    BucketDescrWrapper getBucketDescrsForInst(G4_INST* inst,
                                              BucketDescrWrapper &instrBuckets) {
        // Iterate over all WRITTEN operands and create buckets.
        for (Gen4_Operand_Number opndNum : {Opnd_dst}) {
            G4_Operand *opnd = inst->getOperand(opndNum);
            if (! opnd || ! opnd->getBase()) {
                continue;
            }
            getBucketsForOperand(inst, WRITE, opnd, instrBuckets.BDVec);
            instrBuckets.hasIndirW |= opnd->asDstRegRegion()->isIndirect();
        }
        for (Gen4_Operand_Number opndNum
                 : {Opnd_src0, Opnd_src1, Opnd_src2, Opnd_src3}) {
            G4_Operand *opnd = inst->getOperand(opndNum);
            if (! opnd || ! opnd->getBase()) {
                continue;
            }
            getBucketsForOperand(inst, READ, opnd, instrBuckets.BDVec);
        }
        return instrBuckets;
    }


    bool mustAddNoDD(const BucketDescrWrapper &prevBDW,
                     const BucketDescrWrapper &currBDW) {
        if (currBDW.hasRAW(prevBDW)) {
            return false;
        }
        // This also confirms that the data types are equal
        if (currBDW.hasSubregWAW(prevBDW)) {
            return true;
        }
        return false;
    }

    static bool areOnSamePipeline(G4_INST *prevInstr, G4_INST *currInstr) {
        // Forbid different opcodes
        if (prevInstr->opcode() != currInstr->opcode()) {
            return false;
        }
        // Math check type
        if (prevInstr->isMath() && currInstr->isMath()) {
            auto prevMathTy = prevInstr->asMathInst()->getMathCtrl();
            auto currMathTy = currInstr->asMathInst()->getMathCtrl();
            return prevMathTy == currMathTy;
        }

        return true;
    }

}

// Scan the code and insert NoDDclr and NoDDchk flags to avoid unnecessary
// latencies when accessing sub-registers.
// This happens because the scoreboard works with a GRF granularity.
// Therefore a sub-reg is considered to access the whole GRF and dependencies
// are over-estimated.
//
// NoDDclr: Do not mark the destination registers in the scoreboard
// NoDDchk: Do not check the scoreboard for the source registers
//
// Example:
//     A: add(8) R1.0 = ...
//     B: add(8) R1.8 = ...
//    Instruction B will wait until A has written back to the RF because
//    R1.0 and R1.8 alias in the scoreboard
//   Solution:
//     A: add(8) R1.0 = ... {NoDDclr}
//     B: add(8) R1.8 = ... {NoDDchk}
//    Now B will issue right after A.
//
static bool tryToAddNoDD(G4_INST *currInstr, G4_INST *prevInstr,
                         BucketDescrWrapper &prevBDW,
                         BucketDescrWrapper &currBDW,
                         const Options *options) {
    // 1. To be safe, predicated instrs can kill but cannot cover.
    // 2. Must be on same pipeline
    // 3. If subreg WAW and no RAW dep, insert NoDD flags
    if (prevInstr
        && ! prevBDW.touchesManyGRFs()
        && ! prevBDW.hasIndirW
        && ! prevInstr->isSend()
        && ! currInstr->isSend()
        && ! currInstr->getPredicate()
        && ! prevInstr->getPredicate()
        && ! prevBDW.accessesARF()
        && ! currBDW.accessesARF()
        && areOnSamePipeline(prevInstr, currInstr)
        && mustAddNoDD(prevBDW, currBDW)) {

        // Tag the instructions with the NoDD flag
        prevInstr->setOptionOn(InstOpt_NoDDClr);
        currInstr->setOptionOn(InstOpt_NoDDChk);

        // Sanity check
        G4_Operand *dst = prevInstr->getOperand(Opnd_dst);
        G4_Operand *src = currInstr->getOperand(Opnd_src0);
        if (dst && src)
        {
            G4_VarBase *dstBase = dst->getBase();
            G4_VarBase *srcBase = src->getBase();
            if (dstBase && srcBase && dstBase->isRegVar() && srcBase->isRegVar())
            {
                G4_VarBase *dstReg = (dstBase->isRegVar()) ?
                    dstBase->asRegVar()->getPhyReg() : dstBase;
                G4_VarBase *srcReg = (srcBase->isRegVar()) ?
                    srcBase->asRegVar()->getPhyReg() : srcBase;
                if (srcReg && srcReg->isGreg() && dstReg && dstReg->isGreg())
                {
                    auto dstPhyReg = dstReg->asGreg();
                    int dstNum = dstPhyReg->getRegNum();
                    auto srcPhyReg = srcReg->asGreg();
                    int srcNum = srcPhyReg->getRegNum();
                    assert(dstNum != srcNum);
                }
            }
        }
        // DEBUGGING: dump NoDD pairs into a file
        if (options->getOption(vISA_DebugNoDD)) {
            fstream ofile("C:\\Intel\\nodd.txt",
                          std::fstream::out | std::fstream::app);
            ofile << *prevInstr << std::endl;
            ofile << *currInstr << std::endl << std::endl;
        }
        return true;
    }
    return false;
}

// NOTE: It is currently disabled (vISA_EnableNoDD == false).
void Optimizer::NoDD(void) {
    const Options *options = builder.getOptions();
    if (! builder.noDDAllowedPlatform()
        || ! options->getOption(vISA_EnableNoDD)) {
        return;
    }
    const int MAX_LOOK_BACK = options->getuInt32Option(vISA_NoDDLookBack);

    for (auto bb : fg.BBs) {
        // Keep the last few instructions
        std::vector<G4_INST *> prevInstrs;
        std::vector<BucketDescrWrapper> prevBDWs;
        // G4_INST *prevInstr = nullptr;
        // BucketDescrWrapper prevBDW;
        for (auto currInstr : *bb) {
            if (currInstr->isLabel()) {
                continue;
            }
            BucketDescrWrapper currBDW;
            getBucketDescrsForInst(currInstr, currBDW);

            // Try to insert NoDD in MAX_LOOK_BACK previous intructions
            bool succ = false;
            int prevIdx = 0;
            while (! succ && prevIdx < (int)prevInstrs.size()) {
                G4_INST *prevInstr = prevInstrs[prevIdx];
                BucketDescrWrapper &prevBDW = prevBDWs[prevIdx];
                succ = tryToAddNoDD(currInstr, prevInstr,
                                    prevBDW, currBDW, options);
                prevIdx++;
            }
            prevInstrs.insert(prevInstrs.begin(), currInstr);
            prevBDWs.insert(prevBDWs.begin(), currBDW);
            if (prevInstrs.size() == MAX_LOOK_BACK) {
                prevInstrs.pop_back();
                prevBDWs.pop_back();
            }
        }
    }
}

static bool isDeadInst(FlowGraph& fg, G4_INST* Inst)
{
    if (Inst->isMov() || Inst->isLogic() || Inst->isCompare() ||
        Inst->isArithmetic() || Inst->isVector()) {

        // Check side-effects.
        // - no global
        // - no indirect
        // - not physically assigned (including ARF)
        auto checkOpnd = [&](G4_Operand *Opnd) {
            if (Opnd == nullptr || Opnd->isNullReg())
                return true;
            if (fg.globalOpndHT.isOpndGlobal(Opnd))
                return false;
            if (Opnd->isDstRegRegion() && Opnd->asDstRegRegion()->isIndirect())
                return false;
            G4_VarBase *Base = Opnd->getBase();
            if (!Base->isRegVar())
                return false;
            if (Base->asRegVar()->isPhyRegAssigned())
                return false;
            return true;
        };

        // Should have no use.
        if (Inst->use_size() > 0)
            return false;

        // Skip instructions with special attributes.
        if (Inst->isYieldInst() || Inst->isBreakPointInst())
            return false;

        // Check defs. Assuming acc operands are all locally defined
        // and def-use are correctly maintained.
        if (!checkOpnd(Inst->getDst()) ||
            !checkOpnd(Inst->getCondMod()))
            return false;

        // At this point, this instruction is dead.
        return true;
    }

    // By default it is not dead.
    return false;
}

void Optimizer::dce()
{
    for (auto bb : fg.BBs) {
        for (auto I = bb->rbegin(), E = bb->rend(); I != E; ++I) {
            G4_INST* Inst = *I;
            if (isDeadInst(fg, Inst)) {
                Inst->removeAllDefs();
                Inst->markDead();
            }
        }
        bb->erase(
            std::remove_if(bb->begin(), bb->end(), [](G4_INST* Inst) { return Inst->isDead(); }),
            bb->end());
    }
}

static bool retires(G4_Operand* Opnd, G4_INST* SI)
{
    assert(Opnd && Opnd->isGreg());
    unsigned LB = Opnd->getLinearizedStart() / GENX_GRF_REG_SIZ;
    unsigned RB = Opnd->getLinearizedEnd() / GENX_GRF_REG_SIZ;

    auto overlaps = [=](G4_Operand* A) {
        if (A == nullptr || A->isNullReg() || !A->isGreg())
            return false;
        unsigned LB1 = A->getLinearizedStart() / GENX_GRF_REG_SIZ;
        unsigned RB1 = A->getLinearizedEnd() / GENX_GRF_REG_SIZ;
        return (RB >= LB1 && RB1 >= LB);
    };

    // RAW or WAW
    if (overlaps(SI->getDst()))
        return true;

    if (Opnd->isSrcRegRegion())
        return false;

    // WAR.
    if (overlaps(SI->getSrc(0)))
        return true;
    if (SI->isSplitSend() && overlaps(SI->getSrc(1)))
        return true;

    // Do not retire this send.
    return false;
}

// Emit a self-move to retire this send.
static G4_INST* emitRetiringMov(IR_Builder& builder, G4_BB* BB, G4_INST* SI,
                                INST_LIST_ITER InsertBefore)
{
    assert(SI && SI->isSend());
    G4_Operand* Src0 = SI->getSrc(0);

    unsigned RegNum = Src0->getLinearizedStart() / GENX_GRF_REG_SIZ;
    G4_Declare* Dcl = builder.createTempVar(16, Type_F, Either, Any);
    Dcl->setGRFBaseOffset(RegNum * G4_GRF_REG_NBYTES);
    Dcl->getRegVar()->setPhyReg(builder.phyregpool.getGreg(RegNum), 0);

    G4_DstRegRegion* MovDst = builder.createDstRegRegion(
        Direct, Dcl->getRegVar(), 0, 0, 1, Type_F);
    G4_SrcRegRegion* MovSrc = builder.createSrcRegRegion(
        Mod_src_undef, Direct, Dcl->getRegVar(), 0, 0,
        builder.getRegionStride1(), Type_F);
    G4_INST* MovInst = builder.createInternalInst(
        nullptr, G4_mov, nullptr, false, 8, MovDst, MovSrc, nullptr,
        InstOpt_M0 | InstOpt_WriteEnable);
    BB->insert(InsertBefore, MovInst);
    return MovInst;
}

// Use this instruction to retire live sends.
static void retireSends(std::vector<G4_INST*>& LiveSends, G4_INST* Inst)
{
    if (LiveSends.empty())
        return;

    // Predicated instructions may not retire a send.
    if (Inst->getPredicate() != nullptr && Inst->opcode() != G4_sel)
        return;

    // Collect operands for dependency checking.
    std::vector<G4_Operand*> Opnds;
    if (G4_DstRegRegion* Dst = Inst->getDst()) {
        if (!Dst->isNullReg() && !Dst->isIndirect() && Dst->isGreg())
            Opnds.push_back(Dst);
    }
    for (int i = 0; i < Inst->getNumSrc(); ++i) {
        G4_Operand* Opnd = Inst->getSrc(i);
        if (Opnd == nullptr || !Opnd->isSrcRegRegion() || Opnd->isNullReg())
            continue;
        G4_SrcRegRegion* Src = Opnd->asSrcRegRegion();
        if (!Src->isIndirect() && Src->isGreg())
            Opnds.push_back(Opnd);
    }

    // WRA, RAW or WAW dependency retires a live send.
    bool Changed = false;
    for (auto Opnd : Opnds) {
        for (auto& SI : LiveSends) {
            if (SI && retires(Opnd, SI)) {
                SI = nullptr;
                Changed = true;
            }
        }
    }
    // Remove nullptr values when there are changes.
    if (Changed) {
        auto Iter = std::remove(LiveSends.begin(), LiveSends.end(), (G4_INST*)0);
        LiveSends.erase(Iter, LiveSends.end());
    }
}

// Limit the number of live sends and clear all sends at the end of a block.
void Optimizer::clearSendDependencies()
{
    for (auto BB : fg.BBs) {
        // Live send instructions. This vector will only have MAX_SENDS
        // or less instructions.
        const unsigned MAX_SENDS = 3;
        std::vector<G4_INST*> LiveSends;

        for (auto I = BB->begin(); I != BB->end(); /*empty*/) {
            auto CurI = I++;
            G4_INST* Inst = *CurI;

            // Try to retire live sends.
            retireSends(LiveSends, Inst);
            if (!Inst->isSend())
                continue;

            // This is a send.
            if (LiveSends.size() >= MAX_SENDS) {
                // OK, too many live sends. Retire the earliest live send.
                G4_INST* SI = LiveSends.front();
                G4_INST* MovInst = emitRetiringMov(builder, BB, SI, CurI);
                retireSends(LiveSends, MovInst);
                assert(LiveSends.size() < MAX_SENDS);
            }

            // If this is EOT and send queue is not full, then nothing to do.
            // Otherwise a new send becomes live.
            if (Inst->isEOT())
                LiveSends.clear();
            else
                LiveSends.push_back(Inst);
        }

        // Retire remainig live sends in this block, if any.
        for (auto SI : LiveSends) {
            assert(SI && SI->isSend());
            auto InsertBefore = BB->end();
            G4_INST* LastInst = BB->back();
            if (LastInst->isFlowControl())
                InsertBefore = std::prev(InsertBefore);
            emitRetiringMov(builder, BB, SI, InsertBefore);
        }
    }
}
