/*========================== begin_copyright_notice ============================

Copyright (C) 2017-2021 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

//
/// GenXLowering
/// ------------
///
/// GenXLowering is a function pass that lowers certain LLVM IR instructions
/// that the rest of the GenX backend cannot deal with, or to implement peephole
/// optimizations.
///
/// It also performs a few other tasks:
///
/// 1. It implements add sinking for a variable index in a region/element
///    access. This ensures that, in a sequence of operations to calculate a
///    variable index for a region/element access, any add constant is sunk to
///    the end, such that it can become a constant offset in an indirect
///    operand, and give GenXAddressCommoning more chance to common up address
///    calculations.
///
/// 2. It splits struct values where possible, by splitting all struct phi nodes
///    before running the main pass, then removing an extractvalue by using the
///    corresponding insertvalue's input instead. Any struct value used as an
///    arg or return value still remains, and needs to be dealt with by register
///    allocation.
///
/// 3. It widens some byte vector operations to short vector.
///
///    Gen has restrictions on byte operands. The jitter copes with that, but
///    sometimes it needs to do even-odd splitting, which can lead to suboptimal
///    code if cmps and predicates are involved.
///    Here we attempt to pick up the common cases by converting a byte
///    operation to short.
///
///    Note that we might end up with the extends being baled into the
///    instruction anyway, resulting in a byte operation in vISA.
///
/// 4. Certain uses of shufflevector are lowered:
///
///    a. a splat (copy of one element across a vector);
///    b. a boolean slice (extract of a subvector) becomes rdpredregion;
///    c. a boolean unslice (insert subvector) becomes wrpredregion.
///    d. non-boolean shufflevector is lowered to sequence of rd/wrregions
///
///    The only one case of shufflevector allowed is shufflevector of
///    predicate and undef with replicated mask.
///
/// 5. A Trunc is lowered to a bitcast then a region/element read with a stride.
///    GenXCoalescing will coalesce the bitcast, and possibly bale in the region
///    read, so this will hopefully save an instruction or two.
///
/// 6. Certain floating point comparison instructions are lowered.
///
/// **IR restriction**: LLVM IR instructions not supported after this pass:
///
/// * insertelement
/// * extractelement
/// * trunc
/// * zext/sext/uitofp from (vector of) i1
/// * select on vector of i1
/// * ``llvm.uadd.with.overflow`` (the other
///   overflowing arithmetic intrinsics are not allowed by the GenX backend
///   anyway.)
///
///
/// **IR restriction**: all gather/scatter/atomic must have the width supported
/// by the hardware target.
///
/// **IR restriction**: rdpredregion intrinsic (which is generated by this pass
/// from certain cases of shufflevector, and represents a use of part of a
/// predicate) can only be used in select, wrregion, wrpredpredregion.
///
/// **IR restriction**: wrpredregion intrinsic (which is generated by this pass
/// from certain cases of shufflevector, and represents the write of part of a
/// predicate) must have a compare as its "new value" input.
///
/// **IR restriction**: No phi node of struct type after this pass. This is only
/// a general rule; subsequent passes have been known to reintroduce them so
/// GenXLiveness has another go at splitting them up.
///
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "GENX_LOWERING"

#include "GenX.h"
#include "GenXGotoJoin.h"
#include "GenXIntrinsics.h"
#include "GenXModule.h"
#include "GenXRegion.h"
#include "GenXSubtarget.h"
#include "GenXTargetMachine.h"
#include "GenXUtil.h"
#include "GenXVisa.h"
#include "IGC/common/debug/DebugMacros.hpp"
#include "visa_igc_common_header.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"

#include "llvmWrapper/IR/DerivedTypes.h"
#include "llvmWrapper/IR/Instructions.h"
#include "llvmWrapper/Support/TypeSize.h"

#include <algorithm>
#include <iterator>
#include <numeric>
#include "Probe/Assertion.h"

using namespace llvm;
using namespace genx;

static cl::opt<bool>
    EnableGenXByteWidening("enable-genx-byte-widening", cl::init(true),
                           cl::Hidden, cl::desc("Enable GenX byte widening."));
namespace {

// Diagnostic for lowering problems
class DiagnosticInfoLowering : public DiagnosticInfo {
private:
  std::string Description;
  StringRef Filename;
  unsigned Line;
  unsigned Col;
  static int KindID;
  static int getKindID() {
    if (KindID == 0)
      KindID = llvm::getNextAvailablePluginDiagnosticKind();
    return KindID;
  }

public:
  DiagnosticInfoLowering(Instruction *Inst, const Twine &Desc,
                         DiagnosticSeverity Severity = DS_Error);
  void print(DiagnosticPrinter &DP) const override;
  static bool classof(const DiagnosticInfo *DI) {
    return DI->getKind() == getKindID();
  }
};
int DiagnosticInfoLowering::KindID = 0;

DiagnosticInfoLowering::DiagnosticInfoLowering(Instruction *Inst,
                                               const Twine &Desc,
                                               DiagnosticSeverity Severity)
    : DiagnosticInfo(getKindID(), Severity), Line(0), Col(0) {
  auto DL = Inst->getDebugLoc();
  if (DL) {
    Filename = DL->getFilename();
    Line = DL.getLine();
    Col = DL.getCol();
  }
  std::string str;
  llvm::raw_string_ostream(str) << *Inst;

  Description =
      (Twine("GenXLowering failed for instruction ") + str + ": " + Desc).str();
}

void DiagnosticInfoLowering::print(DiagnosticPrinter &DP) const {
  std::string Loc((Twine(!Filename.empty() ? Filename : "<unknown>") + ":" +
                   Twine(Line) + (!Col ? Twine() : Twine(":") + Twine(Col)) +
                   ": ")
                      .str());
  DP << Loc << Description;
}

// GenXLowering : legalize execution widths and GRF crossing
class GenXLowering : public FunctionPass {
  DominatorTree *DT = nullptr;
  const GenXSubtarget *ST = nullptr;
  SmallVector<Instruction *, 8> ToErase;
  const DataLayout *DL = nullptr;

public:
  static char ID;
  explicit GenXLowering() : FunctionPass(ID), DT(nullptr) {}
  virtual StringRef getPassName() const { return "GenX lowering"; }
  void getAnalysisUsage(AnalysisUsage &AU) const;
  bool runOnFunction(Function &F);

private:
  bool translateSLMOWord(CallInst* CI, unsigned IID);
  bool splitGatherScatter(CallInst *CI, unsigned IID);
  bool processTwoAddressOpnd(CallInst *CI);
  bool processInst(Instruction *Inst);
  bool lowerRdRegion(Instruction *Inst);
  bool lowerWrRegion(Instruction *Inst);
  bool lowerRdPredRegion(Instruction *Inst);
  bool lowerWrPredRegion(Instruction *Inst);
  bool lowerInsertElement(Instruction *Inst);
  bool lowerExtractElement(Instruction *Inst);
  Value *scaleInsertExtractElementIndex(Value *IdxVal, Type *ElTy,
                                        Instruction *InsertBefore);
  bool lowerTrunc(Instruction *Inst);
  bool lowerCast(Instruction *Inst);
  bool lowerBoolScalarSelect(SelectInst *SI);
  bool lowerBoolVectorSelect(SelectInst *SI);
  bool lowerBoolShuffle(ShuffleVectorInst *Inst);
  bool lowerBoolSplat(ShuffleVectorInst *SI, Value *In, unsigned Idx);
  bool lowerShuffle(ShuffleVectorInst *Inst);
  void lowerShuffleSplat(ShuffleVectorInst *SI,
                         ShuffleVectorAnalyzer::SplatInfo Splat);
  bool lowerShuffleToSelect(ShuffleVectorInst *Inst);
  void lowerShuffleToMove(ShuffleVectorInst *SI);
  bool lowerShr(Instruction *Inst);
  bool lowerExtractValue(ExtractValueInst *Inst);
  bool lowerInsertValue(InsertValueInst *Inst);
  bool lowerUAddWithOverflow(CallInst *CI);
  bool lowerCtpop(CallInst *CI);
  bool lowerFCmpInst(FCmpInst *Inst);
  bool lowerUnorderedFCmpInst(FCmpInst *Inst);
  bool lowerMinMax(CallInst *CI, unsigned IntrinsicID);
  bool lowerSqrt(CallInst *CI);
  bool widenByteOp(Instruction *Inst);
  bool lowerLoadStore(Instruction *Inst);
  bool lowerGenXMul(CallInst *CI, unsigned IntrinsicID);
  bool lowerGenXMulSat(CallInst *CI, unsigned IntrinsicID);
  bool lowerMul64(Instruction *Inst);
  bool lowerLzd(Instruction *Inst);
  bool lowerTrap(CallInst *CI);
  bool lowerLLVMMaskedLoad(CallInst *CI);
  bool lowerLLVMMaskedStore(CallInst* CI);
  bool lowerLLVMMaskedGather(CallInst *CI);
  bool lowerLLVMMaskedScatter(CallInst *CI);
  bool generatePredicatedWrrForNewLoad(CallInst *CI);
};

} // end namespace

char GenXLowering::ID = 0;
namespace llvm {
void initializeGenXLoweringPass(PassRegistry &);
}
INITIALIZE_PASS_BEGIN(GenXLowering, "GenXLowering", "GenXLowering", false,
                      false)
INITIALIZE_PASS_END(GenXLowering, "GenXLowering", "GenXLowering", false, false)

FunctionPass *llvm::createGenXLoweringPass() {
  initializeGenXLoweringPass(*PassRegistry::getPassRegistry());
  return new GenXLowering;
}

void GenXLowering::getAnalysisUsage(AnalysisUsage &AU) const {
  AU.addRequired<TargetPassConfig>();
  AU.addPreserved<DominatorTreeWrapperPass>();
  AU.addPreserved<LoopInfoWrapperPass>();
  AU.addPreserved<GenXModule>();
}

/***********************************************************************
 * GenXLowering::runOnFunction : process one function to
 *    lower instructions as required for GenX backend.
 *
 * This does a postordered depth first traversal of the CFG,
 * processing instructions within a basic block in reverse, to
 * ensure that we see a def after its uses (ignoring phi node uses).
 * This helps peephole optimizations which generally want to be
 * approached from the top down. For example, add sinking in the index
 * of an indirect region/element wants to see the trunc before the trunc
 * is lowered to a bitcast and an element access.
 */
bool GenXLowering::runOnFunction(Function &F) {
  LLVM_DEBUG(dbgs() << "GenXLowering started\n");
  auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
  DT = DTWP ? &DTWP->getDomTree() : nullptr;
  ST = &getAnalysis<TargetPassConfig>()
            .getTM<GenXTargetMachine>()
            .getGenXSubtarget();
  DL = &F.getParent()->getDataLayout();
  // First split any phi nodes with struct type.
  splitStructPhis(&F);
  // Create a list of basic blocks in the order we want to process them, before
  // we start the lowering. This is because lowering can split a basic block.
  SmallVector<BasicBlock *, 8> BBs;
  for (auto i = po_begin(&F.getEntryBlock()), e = po_end(&F.getEntryBlock());
       i != e; ++i)
    BBs.push_back(*i);
  // Process each basic block.
  for (auto i = BBs.begin(), e = BBs.end(); i != e; ++i) {
    BasicBlock *BB = *i;
    // The effect of this loop is that we process the instructions in reverse
    // order, and we re-process anything inserted before the instruction
    // being processed.
    for (Instruction *Inst = BB->getTerminator();;) {
      processInst(Inst);
      BasicBlock *Parent = Inst->getParent();
      if (Inst != &Parent->front())
        Inst = Inst->getPrevNode();
      else {
        if (Parent == BB)
          break;
        // We have reached the start of the basic block, but it is a different
        // basic block to BB, so lowering must have split a BB. Just go back to
        // the end of the previous one.
        Inst = Parent->getPrevNode()->getTerminator();
      }
    }
  }
  // Erase the instructions that we saved in ToErase.
  for (SmallVectorImpl<Instruction *>::iterator i = ToErase.begin(),
                                                e = ToErase.end();
       i != e; ++i)
    (*i)->eraseFromParent();
  ToErase.clear();
  return true;
}

// Optimize two address operands if any.
//
// An instruction with a two address opernd should be predicated. If predicate
// is a constant splat, then the old value will be over-written. In this case,
// replace the old value with undef which allows more optimizations to kick in.
//
bool GenXLowering::processTwoAddressOpnd(CallInst *CI) {
  int OpNum = getTwoAddressOperandNum(CI);
  // Skip write regions whose OpNum is 0.
  if (OpNum > 0) {
    Type *Ty = CI->getArgOperand(OpNum)->getType();
    IGC_ASSERT_MESSAGE(Ty == CI->getType(), "two address op type out of sync");

    for (unsigned i = 0; i < CI->getNumArgOperands(); ++i) {
      auto Op = dyn_cast<Constant>(CI->getArgOperand(i));
      // Check if the predicate operand is all true.
      if (Op && Op->getType()->getScalarSizeInBits() == 1) {
        if (Op->getType()->isVectorTy())
          Op = Op->getSplatValue();
        if (Op && Op->isOneValue()) {
          CI->setOperand(OpNum, UndefValue::get(Ty));
          return true;
        }
        return false;
      }
    }
  }

  return false;
}

// Check whether given intrinsic is new load
// without predicate and old value arguments.
static bool isNewLoadInst(CallInst *Inst) {
  unsigned IID = GenXIntrinsic::getGenXIntrinsicID(Inst);
  switch (IID) {
  case GenXIntrinsic::genx_gather4_scaled2:
  case GenXIntrinsic::genx_gather_scaled2:
  case GenXIntrinsic::genx_gather4_masked_scaled2:
  case GenXIntrinsic::genx_gather_masked_scaled2:
    return true;
  default:
    return false;
  }
}

// Find single wrregion user of load instruction.
// Returns nullptr on failure.
static CallInst *getLoadWrregion(CallInst *Inst) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(Inst), "Expected new load intrinsics");
  if (Inst->getNumUses() != 1)
    return nullptr;

  auto *WrR = dyn_cast<CallInst>(Inst->user_back());
  if (!WrR)
    return nullptr;
  return GenXIntrinsic::isWrRegion(WrR) ? WrR : nullptr;
}

// Find single select user of load instruction.
// Returns nullptr on failure.
// TODO: maybe just lower every select to wrregion in lowerSelect?
static SelectInst *getLoadSelect(CallInst *Inst) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(Inst), "Expected new load intrinsics");
  if (Inst->getNumUses() != 1)
    return nullptr;

  auto *SI = dyn_cast<SelectInst>(Inst->user_back());
  if (!SI)
    return nullptr;
  // TODO: handle inverted selects.
  // Need to regenerate mask in this case.
  if (SI->getTrueValue() != Inst)
    return nullptr;
  return SI;
}

// Generate predicate for wrregion of splitted load.
// Returns new predicate.
static Value *generatePredicateForLoadWrregion(
    Value *OldPred, unsigned Offset, unsigned Width, unsigned NumChannels,
    Instruction *InsertBefore, const DebugLoc &DL, const Twine &Name) {

  if (Constant *C = dyn_cast<Constant>(OldPred)) {
    if (isa<ConstantInt>(OldPred))
      return OldPred;

    if (C->isAllOnesValue())
      return ConstantInt::get(Type::getInt1Ty(OldPred->getContext()), 1);
  }

  Value *Pred = OldPred;
  // If old predicate is result of rdpredregion or shufflevector then
  // we can reuse their predicate and offset to avoid double read of predicate.
  if (GenXIntrinsic::getGenXIntrinsicID(OldPred) == GenXIntrinsic::genx_rdpredregion) {
    auto *OldPredInst = cast<CallInst>(OldPred);
    Offset += cast<ConstantInt>(OldPredInst->getArgOperand(1))->getZExtValue();
    Pred = OldPredInst->getArgOperand(0);
  } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(OldPred)) {
    Offset +=
        ShuffleVectorAnalyzer::getReplicatedSliceDescriptor(SVI).InitialOffset;
    Pred = SVI->getOperand(0);
  }

  // Replicate mask across channels.
  SmallVector<Constant *, 16> NewMaskVals(Width);
  IGC_ASSERT(NumChannels);
  unsigned ChannelWidth = Width / NumChannels;
  Type *Int32Ty = IntegerType::getInt32Ty(Pred->getContext());
  for (unsigned i = 0; i < NumChannels; ++i)
    std::generate_n(NewMaskVals.begin() + ChannelWidth * i, ChannelWidth,
                    [Int32Ty, Offset]() mutable {
                      return ConstantInt::get(Int32Ty, Offset++);
                    });
  Constant *NewMask = ConstantVector::get(NewMaskVals);

  Value *Undef = UndefValue::get(Pred->getType());
  auto *Res = new ShuffleVectorInst(Pred, Undef, NewMask, Name, InsertBefore);
  Res->setDebugLoc(DL);
  return Res;
}

static Value *generatePredicatedWrregion(Value *OldVal, Value *NewVal,
                                         Value *Pred, unsigned Offset,
                                         Instruction *InsertBefore,
                                         const llvm::Twine &Name = "") {
  Type *NewValType = NewVal->getType();

  Region WrR(NewValType);
  WrR.Mask = Pred;
  WrR.Offset = Offset;
  return WrR.createWrRegion(OldVal, NewVal, Name, InsertBefore,
                            InsertBefore->getDebugLoc());
}


// Generate partial write for result of splitted 1-channel load instruction.
// Initially we could have following sequence for illegal load (on gather_scaled example):
//   res = gather_scaled <32>
//   mask = rdpredregion <32> pred, offset
//   newV = wrregion <32> oldV, res, wroffset, mask
// After splitting we want to get as less extra code as possible.
// To achieve this we generate following pattern:
// bale {
//   res1 = gather_scaled <16>
//   mask1 = rdpredregion <16> pred, offset
//   partialV = wrregion <16> oldV, res1, mask1
// }
// bale {
//   res2 = gather_scaled <16>
//   mask2 = rdpredregion <16> pred, offset + 16
//   newV = wrregion <16> partialV, res2, wroffset + 16 * elemsize, mask2
// }
// Bale markers show how this will be baled later.
static Value *generate1ChannelWrrregion(Value *Target, unsigned InitialOffset,
                                        CallInst *Load, Value *OldPred,
                                        unsigned SplitNum,
                                        Instruction *InsertBefore) {
  const DebugLoc &DL = Load->getDebugLoc();
  Type *LoadType = Load->getType();
  unsigned LoadWidth = cast<VectorType>(LoadType)->getNumElements();

  Value *Pred =
      generatePredicateForLoadWrregion(OldPred, LoadWidth * SplitNum, LoadWidth,
                                       1, InsertBefore, DL, "load1.pred.split");
  Region WrR(LoadType);
  WrR.Mask = Pred;
  WrR.Offset = InitialOffset +
               LoadWidth * SplitNum * (LoadType->getScalarSizeInBits() / 8);
  return WrR.createWrRegion(Target, Load, "load1.join", InsertBefore, DL);
}

// Generate partial write for result of splitted N-channel load.
// For channelled loads we need to also shuffle result of splitted
// instructions to write back them to destination in expected order.
// Temporary splits should always be predicated in case of atomics
// because latter load and store at the same time.
// Example for gather4_scaled (with two channels enabled). Before:
//   res = gather4_scaled <32> RG
//   mask = rdpredregion <64> pred, offset ; mask is replicated across channels
//   newV = wrregion <64> oldV, res, wroffset, mask
// After:
// bale {
//   res1temp = gather4_scaled <16> RG ; create temporary (unnecessary in case of non-atomics)
//   splitmask1 = rdpredregion <32> pred, offset ; replicated
//   res1 = wrregion <32> undef, res1temp, 0, splitmask1
// }
// bale {
//   res1R = rdregion <16> res1, 0
//   mask1R = rdpredregion <16> pred, offset ; same for all channels
//   partialVR = wrregion <16> oldV, res1R, wroffset, mask1R
// }
// bale {
//   res1G = rdregion <16> res1, 16 * elemsize
//   mask1G = rdpredregion <16> pred, offset
//   partialV = wrregion <16> partialVR, res1G, wroffset + 32 * elemsize, mask1G
// }
// bale {
//   res2temp = gather4_scaled <16> RG ; second temporary
//   splitmask2 = rdpredregion <32> pred, offset + 16
//   res2 = wrregion <32> undef, res2temp, 0, splitmask2
// }
// bale {
//   res2R = rdregion <16> res2, 0
//   mask2R = rdpredregion <16> pred, offset + 16
//   newVR = wrregion <16> partialV, res2R, wroffset + 16 * elemsize, mask2R
// }
// bale {
//   res2G = rdregion <16> res2, 16 * elemsize
//   mask2G = rdpredregion <16> pred, offset + 16
//   newV = wrregion <16> newVR, res2G, wroffset + 48 * elemsize, mask2G
// }
// As it can be noticed, splitting of channeled loads is quite expensive.
// We should hope that later passes (like region collapsing) can optimize it
// by analyzing how resulting value was assembled.
static Value *generateNChannelWrregion(Value *Target, unsigned InitialOffset,
                                       CallInst *Load, Value *OldPred,
                                       unsigned SplitNum, unsigned NumSplits,
                                       unsigned NumChannels,
                                       Instruction *InsertBefore) {
  const DebugLoc &DL = Load->getDebugLoc();
  Type *LoadType = Load->getType();
  unsigned LoadWidth = cast<VectorType>(LoadType)->getNumElements();
  IGC_ASSERT(NumChannels);
  unsigned ChannelWidth = LoadWidth / NumChannels;
  unsigned MaskOffset = ChannelWidth * SplitNum;

  // Generate temporary for load.
  Value *Pred = generatePredicateForLoadWrregion(
      OldPred, MaskOffset, LoadWidth, NumChannels, InsertBefore, DL, "loadN.pred.split");
  Region WrR(LoadType);
  WrR.Mask = Pred;
  Value *SplitRes = WrR.createWrRegion(UndefValue::get(LoadType), Load,
                                       "loadN.split", InsertBefore, DL);

  // Generate shuffle writes to the target.
  unsigned ElemByteSize = LoadType->getScalarSizeInBits() / 8;
  Type *ShuffleType =
      IGCLLVM::FixedVectorType::get(LoadType->getScalarType(), ChannelWidth);
  Region ChannelRdR(ShuffleType);
  Region ChannelWrR(ShuffleType);
  Value *ResChannel = nullptr;
  for (unsigned i = 0; i < NumChannels; ++i) {
    ChannelRdR.Offset = ChannelWidth * i * ElemByteSize;
    ResChannel = ChannelRdR.createRdRegion(SplitRes, "loadN.channel.read.join",
                                           InsertBefore, DL);
    Pred = generatePredicateForLoadWrregion(OldPred, MaskOffset, ChannelWidth,
                                            1, InsertBefore, DL,
                                            "loadN.channel.pred.join");
    ChannelWrR.Offset =
        InitialOffset +
        (ChannelWidth * SplitNum + ChannelWidth * NumSplits * i) * ElemByteSize;
    ChannelWrR.Mask = Pred;
    Target = ChannelWrR.createWrRegion(Target, ResChannel, "loadN.channel.join",
                                       InsertBefore, DL);
  }
  return Target;
}

// Try to infere predicate for load inst which will be used
// in splitting:
// - Check for mask operand first
// - Check wrr or select user
//
// If wrr or select user was found then return it's predicate
// because exactly this predicate must be used
// during splittig.
Value *getPredicateForLoadSplitting(CallInst *Load) {
  IGC_ASSERT(Load);
  Value *LoadPred = getMaskOperand(Load);
  CallInst *LoadWrr = getLoadWrregion(Load);
  SelectInst *SI = getLoadSelect(Load);

  if (!LoadWrr && !SI) {
    // No suitable user for mask inference and no predicate in load, so
    // it's not preidcated
    if (!LoadPred ||
        (isa<Constant>(LoadPred) && cast<Constant>(LoadPred)->isAllOnesValue()))
      return ConstantInt::get(IntegerType::getInt1Ty(Load->getContext()), 1);

    return LoadPred;
  }

  // Try to infer the mask from users.
  if (LoadWrr) {
    // If we found wrregion user, then use its predicate for splitted
    // instructions.
    return LoadWrr->getArgOperand(
        GenXIntrinsic::GenXRegion::PredicateOperandNum);
  }
  // Else it's select, get it's predicate
  return SI->getCondition();
}

// Get target for wrregions of splitted load.
// Returns tuple consisted of:
//  1. Target for wrregions
//  2. Predicate
//  3. Initial offset of target
//  4. Instruction to replace later
static std::tuple<Value *, Value *, unsigned, Instruction *>
getLoadTarget(CallInst *Load, const GenXSubtarget *ST) {
  Value *LoadPred = getPredicateForLoadSplitting(Load);
  CallInst *LoadWrr = getLoadWrregion(Load);
  SelectInst *SI = getLoadSelect(Load);

  if (LoadWrr) {
    // If wrregion can be represented as raw operand, we can reuse its target and offset.
    if (genx::isValueRegionOKForRaw(LoadWrr, true /* IsWrite */, ST)) {
      // TODO: mark wrregion to be erased once issue with ToErase and
      // iteration order will be resolved.
      Value *Target =
          LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
      Value *Offset =
          LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::WrIndexOperandNum);
      unsigned InitialOffset = cast<ConstantInt>(Offset)->getZExtValue();
      return {Target, LoadPred, InitialOffset, LoadWrr};
    }
  } else if (SI) {
    Value *Target = SI->getFalseValue();
    return {Target, LoadPred, 0, SI};
  }

  // Create new target for load.
  Value *Target = UndefValue::get(Load->getType());
  return {Target, LoadPred, 0, Load};
}

/***********************************************************************
 * splitGatherScatter : lower gather/scatter/atomic to the width support
 * by the hardware platform.
 *
 * This performs two functions:
 *
 * 1. If the operation is wider than what hardware can support, splits it
 *    into the legal width.
 *
 * 2. For typed gather4/scatter4, when r or both v and r are zero, replace
 *    with undef so that they are not encoded in the vISA instruction and the
 *    message skips them.
 */
bool GenXLowering::splitGatherScatter(CallInst *CI, unsigned IID) {
  enum {
    MASK_IDX = 0,
    PRED_IDX = 1,
    SURF_IDX = 2,
    U_IDX = 3,
    DATA_IDX = 6,
    NONEED = 11
  };

  unsigned MaskIdx = NONEED;
  unsigned PredIdx = NONEED;
  unsigned AddrIdx = NONEED;
  unsigned DataIdx = NONEED;
  unsigned AtomicSrcIdx = NONEED;
  bool IsTyped = false;
  int AtomicNumSrc = (-1); // -1 means not-an-atomic
  unsigned NumVectorElements = 1;

  switch (IID) {
  case GenXIntrinsic::genx_typed_atomic_add:
  case GenXIntrinsic::genx_typed_atomic_and:
  case GenXIntrinsic::genx_typed_atomic_fmax:
  case GenXIntrinsic::genx_typed_atomic_fmin:
  case GenXIntrinsic::genx_typed_atomic_imax:
  case GenXIntrinsic::genx_typed_atomic_imin:
  case GenXIntrinsic::genx_typed_atomic_max:
  case GenXIntrinsic::genx_typed_atomic_min:
  case GenXIntrinsic::genx_typed_atomic_or:
  case GenXIntrinsic::genx_typed_atomic_sub:
  case GenXIntrinsic::genx_typed_atomic_xchg:
  case GenXIntrinsic::genx_typed_atomic_xor:
    AtomicSrcIdx = 2;
    PredIdx = 0;
    AddrIdx = 3;
    IsTyped = true;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_typed_atomic_dec:
  case GenXIntrinsic::genx_typed_atomic_inc:
    PredIdx = 0;
    AddrIdx = 2;
    IsTyped = true;
    AtomicNumSrc = 0;
    break;
  case GenXIntrinsic::genx_typed_atomic_cmpxchg:
  case GenXIntrinsic::genx_typed_atomic_fcmpwr:
    AtomicSrcIdx = 2;
    PredIdx = 0;
    AddrIdx = 4;
    IsTyped = true;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_scatter4_typed:
  case GenXIntrinsic::genx_gather4_typed:
    DataIdx = DATA_IDX;
    MaskIdx = MASK_IDX;
    PredIdx = PRED_IDX;
    AddrIdx = U_IDX;
    IsTyped = true;
    break;
  case GenXIntrinsic::genx_scatter4_scaled:
  case GenXIntrinsic::genx_gather4_scaled:
    DataIdx = 6;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 5;
    break;
  case GenXIntrinsic::genx_gather4_scaled2:
    MaskIdx = 0;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_gather4_masked_scaled2:
    PredIdx = 5;
    MaskIdx = 0;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_svm_scatter4_scaled:
  case GenXIntrinsic::genx_svm_gather4_scaled:
    DataIdx = 5;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_scatter_scaled:
  case GenXIntrinsic::genx_gather_scaled:
    DataIdx = 6;
    PredIdx = 0;
    AddrIdx = 5;
    break;
  case GenXIntrinsic::genx_gather_scaled2:
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_gather_masked_scaled2:
    PredIdx = 5;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_svm_scatter:
  case GenXIntrinsic::genx_svm_gather:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 2;
    break;
  case GenXIntrinsic::genx_svm_atomic_dec:
  case GenXIntrinsic::genx_svm_atomic_inc:
    DataIdx = 2;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicNumSrc = 0;
  case GenXIntrinsic::genx_svm_atomic_add:
  case GenXIntrinsic::genx_svm_atomic_and:
  case GenXIntrinsic::genx_svm_atomic_fmax:
  case GenXIntrinsic::genx_svm_atomic_fmin:
  case GenXIntrinsic::genx_svm_atomic_imax:
  case GenXIntrinsic::genx_svm_atomic_imin:
  case GenXIntrinsic::genx_svm_atomic_max:
  case GenXIntrinsic::genx_svm_atomic_min:
  case GenXIntrinsic::genx_svm_atomic_or:
  case GenXIntrinsic::genx_svm_atomic_sub:
  case GenXIntrinsic::genx_svm_atomic_xchg:
  case GenXIntrinsic::genx_svm_atomic_xor:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicSrcIdx = 2;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_svm_atomic_cmpxchg:
  case GenXIntrinsic::genx_svm_atomic_fcmpwr:
    DataIdx = 4;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicSrcIdx = 2;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_dword_atomic_add:
  case GenXIntrinsic::genx_dword_atomic_and:
  case GenXIntrinsic::genx_dword_atomic_fmax:
  case GenXIntrinsic::genx_dword_atomic_fmin:
  case GenXIntrinsic::genx_dword_atomic_imax:
  case GenXIntrinsic::genx_dword_atomic_imin:
  case GenXIntrinsic::genx_dword_atomic_max:
  case GenXIntrinsic::genx_dword_atomic_min:
  case GenXIntrinsic::genx_dword_atomic_or:
  case GenXIntrinsic::genx_dword_atomic_sub:
  case GenXIntrinsic::genx_dword_atomic_xchg:
  case GenXIntrinsic::genx_dword_atomic_xor:
    DataIdx = 4;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicSrcIdx = 3;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_dword_atomic_cmpxchg:
  case GenXIntrinsic::genx_dword_atomic_fcmpwr:
    DataIdx = 5;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicSrcIdx = 3;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_dword_atomic_dec:
  case GenXIntrinsic::genx_dword_atomic_inc:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicNumSrc = 0;
    break;

  default:
    return false;
  }

  // nulling unused inputs for typed gather/scatter/atomic
  if (IsTyped) {
    Constant *V = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 1));
    Constant *R = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 2));
    // Only continue when R is known to be zero.
    if (R && R->isNullValue()) {
      CI->setOperand(AddrIdx + 2, UndefValue::get(R->getType()));
      if (V && V->isNullValue())
        CI->setOperand(AddrIdx + 1, UndefValue::get(V->getType()));
    }
    // check if LOD is zero for atomic
    if (AtomicNumSrc >= 0) {
      Constant *LOD = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 3));
      if (LOD && LOD->isNullValue())
        CI->setOperand(AddrIdx + 3, UndefValue::get(LOD->getType()));
    }
  }
  // Deduce intrinsic width: check predicate if exists, then check address vector.
  unsigned WidthOperand;
  if (PredIdx != NONEED)
    WidthOperand = PredIdx;
  else if (AddrIdx != NONEED)
    WidthOperand = AddrIdx;
  else
    IGC_ASSERT_EXIT_MESSAGE(0, "Cannot infer execution width of intrinsic (checked pred and addr operands)");
  auto Width = cast<VectorType>(CI->getArgOperand(WidthOperand)->getType())
                   ->getNumElements();
  unsigned TargetWidth = IsTyped ? 8 : 16;
  if (Width <= TargetWidth)
    return false;
  IGC_ASSERT(TargetWidth);
  IGC_ASSERT((Width % TargetWidth) == 0);
  auto NumSplits = Width / TargetWidth;
  IGC_ASSERT(NumSplits == 2 || NumSplits == 4);
  unsigned NumChannels = NumVectorElements;
  if (MaskIdx != NONEED) {
    NumChannels = (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))
                      ->getZExtValue();
    NumChannels = (NumChannels & 1) + ((NumChannels & 2) >> 1) +
                  ((NumChannels & 4) >> 2) + ((NumChannels & 8) >> 3);
  }

  unsigned NumBlks = 1;
  if (IID == GenXIntrinsic::genx_svm_scatter ||
      IID == GenXIntrinsic::genx_svm_gather) {
    NumBlks = (unsigned)cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
    NumBlks = (1 << NumBlks);
    auto ElmSz = CI->getArgOperand(DataIdx)->getType()->getScalarSizeInBits() / 8;
    if (ElmSz == 1 && NumBlks < 4)
      NumBlks = 4;
    else if (ElmSz == 2 && NumBlks < 2)
      NumBlks = 2;
  }
  const DebugLoc &DL = CI->getDebugLoc();
  Value *NewResult = nullptr;
  if (CI->getType() && CI->getType()->isVectorTy() &&
      cast<VectorType>(CI->getType())->getNumElements() >=
          Width * NumChannels * NumBlks) {
    if (DataIdx != NONEED)
      NewResult = CI->getArgOperand(DataIdx);
    else
      NewResult = UndefValue::get(CI->getType());
  }

  bool IsNewLoad = isNewLoadInst(CI);
  Value *LoadPred = nullptr;
  unsigned InitialOffset = 0;
  Instruction *InstToReplace = CI;
  if (IsNewLoad)
    std::tie(NewResult, LoadPred, InitialOffset, InstToReplace) =
        getLoadTarget(CI, ST);

  for (unsigned i = 0; i < NumSplits; ++i) {
    SmallVector<Value *, 8> Args;
    // initialize the args with the old values
    for (unsigned ArgI = 0; ArgI < CI->getNumArgOperands(); ++ArgI)
      Args.push_back(CI->getArgOperand(ArgI));
    // Predicate
    if (PredIdx != NONEED) {
      Value *V = CI->getArgOperand(PredIdx);
      if (auto C = dyn_cast<Constant>(V))
        Args[PredIdx] = getConstantSubvector(C, i * TargetWidth, TargetWidth);
      else
        Args[PredIdx] = Region::createRdPredRegion(
          V, i * TargetWidth, TargetWidth, "predsplit", CI, DL);
    }
    // address source
    unsigned NumAddrs = 1;
    if (IsTyped)
      NumAddrs = (AtomicNumSrc >= 0) ? 4 : 3;
    for (unsigned AddrI = 0; AddrI < NumAddrs; ++AddrI) {
      Value *V = CI->getArgOperand(AddrIdx + AddrI);
      Region R(V);
      R.Width = R.NumElements = TargetWidth;
      R.Offset = i * TargetWidth * V->getType()->getScalarSizeInBits()/8; // in bytes
      Args[AddrIdx + AddrI] = R.createRdRegion(V, "addrsplit", CI, DL);
    }
    // data source
    // We need to construct a new vector with 8 elements per enabled
    // color.
    if (DataIdx != NONEED) {
      Value *V = CI->getArgOperand(DataIdx);
      auto DataTy = IGCLLVM::FixedVectorType::get(
          V->getType()->getScalarType(), TargetWidth * NumChannels * NumBlks);
      auto ElmSz = V->getType()->getScalarSizeInBits() / 8;
      Value *NewVec = UndefValue::get(DataTy);
      if (!isa<UndefValue>(V)) {
        for (unsigned Channel = 0; Channel < NumChannels; ++Channel) {
          Region RdR(V);
          RdR.Width = RdR.NumElements = TargetWidth * NumBlks;
          RdR.Offset = 4 * (Width * NumBlks * Channel + TargetWidth * NumBlks * i);
          auto Rd = RdR.createRdRegion(V, "datasplit", CI, DL);
          if (NumChannels > 1) {
            Region WrR(DataTy);
            WrR.Width = WrR.NumElements = TargetWidth * NumBlks;
            WrR.Offset = ElmSz * TargetWidth * NumBlks * Channel;
            NewVec = WrR.createWrRegion(NewVec, Rd, "datasplit", CI, DL);
          } else
            NewVec = Rd;
        }
      }
      Args[DataIdx] = NewVec;
    }
    // atomic source operands
    if (AtomicSrcIdx != NONEED) {
      for (int SrcI = 0; SrcI < AtomicNumSrc; ++SrcI) {
        Value *V = CI->getArgOperand(AtomicSrcIdx + SrcI);
        Region R(V);
        R.Width = R.NumElements = TargetWidth;
        R.Offset = i * TargetWidth * V->getType()->getScalarSizeInBits()/8; // in bytes
        Args[AtomicSrcIdx + SrcI] = R.createRdRegion(V, "addrsplit", CI, DL);
        }
    }
    // now create the new narrower instruction
    if (NewResult) {
      Type *DstTy = nullptr;
      if (DataIdx != NONEED)
        DstTy = Args[DataIdx]->getType();
      else {
        DstTy =
            IGCLLVM::FixedVectorType::get(CI->getType()->getScalarType(),
                                          TargetWidth * NumBlks * NumChannels);
      }
      SmallVector<Type *, 4> Tys = {DstTy};

      // These two overloaded args can go in
      // arbitrary order in operand list
      unsigned IdxMin = std::min(AddrIdx, PredIdx);
      unsigned IdxMax = std::max(AddrIdx, PredIdx);
      if (IdxMin != NONEED)
        Tys.push_back(Args[IdxMin]->getType());
      if (IdxMax != NONEED)
        Tys.push_back(Args[IdxMax]->getType());

      auto Decl = GenXIntrinsic::getAnyDeclaration(CI->getModule(), IID, Tys);
      auto *Gather = CallInst::Create(Decl, Args, CI->getName() + ".split", CI);
      Gather->setDebugLoc(DL);
      if (IsNewLoad) {
        if (NumChannels == 1)
          NewResult = generate1ChannelWrrregion(NewResult, InitialOffset,
                                                Gather, LoadPred, i, CI);
        else
          NewResult =
              generateNChannelWrregion(NewResult, InitialOffset, Gather,
                                       LoadPred, i, NumSplits, NumChannels, CI);
        continue;
      }
      // Join the results together, starting with the old value.
      auto ElmSz = DstTy->getScalarSizeInBits() / 8;
      if (NumChannels > 1) {
        Region RdR(Gather);
        RdR.Width = RdR.NumElements = TargetWidth * NumBlks;
        Region WrR(NewResult);
        WrR.Width = WrR.NumElements = TargetWidth * NumBlks;
        WrR.Mask = Args[PredIdx];
        for (unsigned Channel = 0; Channel != NumChannels; ++Channel) {
          RdR.Offset = ElmSz * TargetWidth * NumBlks * Channel;
          auto Rd = RdR.createRdRegion(Gather, "joint", CI, DL);
          WrR.Offset = 4 * (Width * NumBlks * Channel + TargetWidth * NumBlks * i);
          NewResult = WrR.createWrRegion(NewResult, Rd, "join", CI, DL);
        }
      } else {
        Region WrR(NewResult);
        WrR.Width = WrR.NumElements = TargetWidth * NumBlks;
        WrR.Offset = ElmSz * TargetWidth * NumBlks * i;
        WrR.Mask = Args[PredIdx];
        NewResult = WrR.createWrRegion(NewResult, Gather, "join", CI, DL);
      }
    } else {
      IGC_ASSERT(CI->use_empty());
      IGC_ASSERT(DataIdx != NONEED);
      // Create the target-wide scatter instructions.
      Type *Tys[] = {Args[PredIdx]->getType(), Args[AddrIdx]->getType(),
                     Args[DataIdx]->getType()};
      auto Decl = GenXIntrinsic::getAnyDeclaration(CI->getModule(), IID, Tys);
      auto NewInst = CallInst::Create(Decl, Args, "", CI);
      NewInst->setDebugLoc(DL);
    }
  }

  if (NewResult)
    InstToReplace->replaceAllUsesWith(NewResult);

  if (InstToReplace != CI)
    ToErase.push_back(InstToReplace);
  ToErase.push_back(CI);
  return true;
}

static Constant* getConstVector(Type* ITy, unsigned int n, unsigned int step) {
    std::vector<Constant*> vConsts;
    unsigned v = 0;
    for (unsigned i = 0; i < n; i++) {
        vConsts.push_back(ConstantInt::get(ITy, v));
        v += step;
    }
    return ConstantVector::get(vConsts);
}


/***********************************************************************
 * translateSLMOWord : lower SLM OWord load/store to gathers/scatters
 * on legacy platform such as SKL.
 *
 * We only support the cases of 1,2,4,8 oword cases
 */
bool GenXLowering::translateSLMOWord(CallInst* CI, unsigned IID) {
  LLVMContext& CTX = CI->getContext();
  auto CIntTy = IntegerType::getInt32Ty(CTX);
  const DebugLoc& DL = CI->getDebugLoc();
  switch (IID) {
  case GenXIntrinsic::genx_oword_ld:
  case GenXIntrinsic::genx_oword_ld_unaligned: {
    constexpr unsigned BtiIdx = 1;
    constexpr unsigned AddrIdx = 2;
    Value* BtiV = CI->getArgOperand(BtiIdx);
    // only slm need this lowering
    if (!isa<ConstantInt>(BtiV))
      return false;
    if (cast<ConstantInt>(BtiV)->getZExtValue() !=
        visa::ReservedSurfaceIndex::RSI_Slm)
      return false;

    IRBuilder<> Builder(CI);
    Value* AddrV = CI->getArgOperand(AddrIdx);
    if (IID == GenXIntrinsic::genx_oword_ld) {
      AddrV = Builder.CreateShl(AddrV, llvm::ConstantInt::get(AddrV->getType(), 4));
    }
    auto OrigVT = cast<VectorType>(CI->getType());
    unsigned EltSize = OrigVT->getScalarSizeInBits();
    unsigned EltCount = OrigVT->getNumElements();
    // 1-oword is 16 bytes, using simd4 dword gather-scaled
    // 2-oword is 32 bytes, using simd8 dword gather-scaled
    // 4-oword is 64 bytes, using simd16 dword gather-scaled
    // 8-oword is 128 bytes, using 2*simd16 dword gather-scaled
    unsigned DWordCnt = (EltSize * EltCount) / 32;
    assert(DWordCnt == 4 || DWordCnt == 8 || DWordCnt == 16 || DWordCnt == 32);
    unsigned SimdWidth = (DWordCnt == 32) ? 16 : DWordCnt;
    auto NewVT = IGCLLVM::FixedVectorType::get(CIntTy, DWordCnt);
    auto GatherVT = IGCLLVM::FixedVectorType::get(CIntTy, SimdWidth);
    // generate gather-scaled
    auto VOffset = getConstVector(CIntTy, SimdWidth, 4);
    // create constant for predicate
    auto PredVTy = IGCLLVM::FixedVectorType::get(IntegerType::getInt1Ty(CTX), SimdWidth);
    auto OnePredV = Constant::getAllOnesValue(PredVTy);
    auto ScaleC = ConstantInt::get(Type::getInt16Ty(CTX), 0);
    std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "gather.scaled";
    auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
    // crease constant for num-blocks, 2 means 4-bytes
    auto NumBlksC = ConstantInt::get(CIntTy, 2);
    // create the intrinsic call
    Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), ID, { GatherVT, PredVTy, VOffset->getType() });
    Instruction* NewInst = nullptr;
    if (DWordCnt == SimdWidth) {
      NewInst = IntrinsicInst::Create(NewFDecl,
          { OnePredV, NumBlksC, ScaleC, BtiV, AddrV,
            VOffset, UndefValue::get(GatherVT) },
          CI->getName() + ".gather", CI);
      NewInst->setDebugLoc(DL);
      LLVM_DEBUG(dbgs() << "SLM OWord Load:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to gather:\n");
      LLVM_DEBUG(NewInst->dump());
    }
    else {  // need to two gathers for 8 owords
      // 1st gather
      auto New1st = IntrinsicInst::Create(NewFDecl,
           { OnePredV, NumBlksC, ScaleC, BtiV, AddrV,
             VOffset, UndefValue::get(GatherVT) },
           CI->getName() + ".gather1", CI);
      New1st->setDebugLoc(DL);
      // 2nd gather
      AddrV = Builder.CreateAdd(AddrV, llvm::ConstantInt::get(AddrV->getType(), 64));
      auto New2nd = IntrinsicInst::Create(NewFDecl,
           { OnePredV, NumBlksC, ScaleC, BtiV, AddrV,
             VOffset, UndefValue::get(GatherVT) },
           CI->getName() + ".gather2", CI);
      New2nd->setDebugLoc(DL);
      // write region, 1st half
      Region R(NewVT);
      R.Width = SimdWidth;
      R.NumElements = SimdWidth;
      R.Stride = 1;
      R.VStride = 0;
      R.Offset = 0;
      auto PartialV = R.createWrRegion(UndefValue::get(NewVT), New1st, "", CI, CI->getDebugLoc());
      // write region, 2nd half
      R.Offset = 64;
      NewInst = R.createWrRegion(PartialV, New2nd, "", CI, CI->getDebugLoc());
      LLVM_DEBUG(dbgs() << "SLM OWord Load:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to gather:\n");
      LLVM_DEBUG(New1st->dump());
      LLVM_DEBUG(New2nd->dump());
    }
    // cast back if required
    Value* Casted = NewInst;
    if (NewVT != OrigVT)
      Casted = CastInst::CreateBitOrPointerCast(
          Casted, OrigVT, Casted->getName() + VALUE_NAME(".cast"), CI);
    CI->replaceAllUsesWith(Casted);
    ToErase.push_back(CI);
    return true;
  }
  case GenXIntrinsic::genx_oword_st: {
    constexpr unsigned DataIdx = 2;
    constexpr unsigned AddrIdx = 1;
    constexpr unsigned BtiIdx = 0;
    Value* BtiV = CI->getArgOperand(BtiIdx);
    // Only slm need this lowering
    if (!isa<ConstantInt>(BtiV))
      return false;
    if (cast<ConstantInt>(BtiV)->getZExtValue() !=
        visa::ReservedSurfaceIndex::RSI_Slm)
      return false;

    IRBuilder<> Builder(CI);
    Value* AddrV = CI->getArgOperand(AddrIdx);
    AddrV = Builder.CreateShl(AddrV, llvm::ConstantInt::get(AddrV->getType(), 4));

    Value* Datum = CI->getArgOperand(DataIdx);
    auto OrigVT = cast<VectorType>(Datum->getType());
    unsigned EltSize = OrigVT->getScalarSizeInBits();
    unsigned EltCount = OrigVT->getNumElements();
    // 1-oword is 16 bytes, using simd4 dword scatter-scaled
    // 2-oword is 32 bytes, using simd8 dword scatter-scaled
    // 4-oword is 64 bytes, using simd16 dword scatter-scaled
    // 8-oword is 128 bytes, using 2*simd16 dword scatter-scaled
    unsigned DWordCnt = (EltSize * EltCount) / 32;
    assert(DWordCnt == 4 || DWordCnt == 8 || DWordCnt == 16 || DWordCnt == 32);
    auto NewVT = IGCLLVM::FixedVectorType::get(CIntTy, DWordCnt);
    IGC_ASSERT_MESSAGE(CastInst::isBitCastable(NewVT, OrigVT),
        "We expect resulting vectors to be bitcastable");
    if (NewVT != OrigVT)
      Datum = CastInst::CreateBitOrPointerCast(
          Datum, NewVT, Datum->getName() + VALUE_NAME(".cast"), CI);
    unsigned SimdWidth = (DWordCnt == 32) ? 16 : DWordCnt;

    // generate scatter-scaled
    auto VOffset = getConstVector(CIntTy, SimdWidth, 4);
    // create constant for predicate
    auto PredVTy = IGCLLVM::FixedVectorType::get(IntegerType::getInt1Ty(CTX), SimdWidth);
    auto OnePredV = Constant::getAllOnesValue(PredVTy);
    auto ScaleC = ConstantInt::get(Type::getInt16Ty(CTX), 0);
    // create constant for num-blocks
    auto NumBlksC = ConstantInt::get(CIntTy, 2);
    std::string IntrName =
        std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "scatter.scaled";
    auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
    // create the intrinsic call
    auto ScatterVT = IGCLLVM::FixedVectorType::get(CIntTy, SimdWidth);
    Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
              CI->getModule(), ID, { PredVTy, VOffset->getType(), ScatterVT });
    if (DWordCnt == SimdWidth) {
      // create one scatter
      auto NewInst = Builder.CreateCall(
               NewFDecl, { OnePredV, NumBlksC, ScaleC, BtiV, AddrV, VOffset, Datum });
      NewInst->setDebugLoc(DL);
      LLVM_DEBUG(dbgs() << "SLM OWord Store:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to scatter:\n");
      LLVM_DEBUG(NewInst->dump());
    }
    else {  // 8-oword (i.e 32 dword) case
      // scatter the 1st 16 dwords
      // read region then scatter
      Region R(ScatterVT);
      R.Width = SimdWidth;
      R.NumElements = SimdWidth;
      R.Stride = 1;
      R.VStride = 0;
      R.Offset = 0;
      auto Datum1st = R.createRdRegion(Datum, "", CI, CI->getDebugLoc());
      auto New1st = Builder.CreateCall(
          NewFDecl, { OnePredV, NumBlksC, ScaleC, BtiV, AddrV, VOffset, Datum1st });
      New1st->setDebugLoc(DL);
      // scatter the 2nd 16 dwords
      // read region then scatter
      AddrV = Builder.CreateAdd(AddrV, llvm::ConstantInt::get(AddrV->getType(), 64));
      R.Offset = 64;
      auto Datum2nd = R.createRdRegion(Datum, "", CI, CI->getDebugLoc());
      auto New2nd = Builder.CreateCall(
               NewFDecl, { OnePredV, NumBlksC, ScaleC, BtiV, AddrV, VOffset, Datum2nd });
      New2nd->setDebugLoc(DL);
      LLVM_DEBUG(dbgs() << "SLM OWord Store:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to scatter:\n");
      LLVM_DEBUG(New1st->dump());
      LLVM_DEBUG(New2nd->dump());
    }
    ToErase.push_back(CI);
    return true;
  }
  default:
    break;
  }
  return false;
}


/***********************************************************************
 * generatePrecicatedWrrForNewLoad : Generate predicated wrr if result
 *                                   of a load that needs no splits
 * Return: true if predicated wrr was generated
 */

bool GenXLowering::generatePredicatedWrrForNewLoad(CallInst *CI) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(CI), "New load expected");
  // Generate predicated wrr if result of a load is predicated with a select
  if (auto *SI = getLoadSelect(CI)) {
    Value *NewResult = SI->getFalseValue();
    Value *LoadPred = SI->getCondition();
    NewResult =
        generatePredicatedWrregion(NewResult, CI, LoadPred, 0 /* Offset */,
                                   SI->getNextNode(), "lowerpred");
    SI->replaceAllUsesWith(NewResult);
    ToErase.push_back(SI);
    return true;
  }
  return false;
}

/***********************************************************************
 * processInst : process one instruction in GenXLowering
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::processInst(Instruction *Inst) {
  if (isa<InsertElementInst>(Inst))
    return lowerInsertElement(Inst);
  if (isa<ExtractElementInst>(Inst))
    return lowerExtractElement(Inst);
  if (isa<TruncInst>(Inst))
    return lowerTrunc(Inst);
  if (isa<CastInst>(Inst))
    return lowerCast(Inst);
  if (auto SI = dyn_cast<SelectInst>(Inst)) {
    if (SI->getType()->getScalarType()->isIntegerTy(1)) {
      if (SI->getType() == SI->getCondition()->getType())
        return lowerBoolVectorSelect(SI);
      return lowerBoolScalarSelect(SI);
    }
    // Widen byte op if it necessary.
    return widenByteOp(SI);
  }
  if (auto SI = dyn_cast<ShuffleVectorInst>(Inst)) {
    if (SI->getType()->getScalarType()->isIntegerTy(1))
      return lowerBoolShuffle(SI);
    return lowerShuffle(SI);
  }
  if (isa<BinaryOperator>(Inst)) {
    if (widenByteOp(Inst))
      return true;
    if (Inst->getOpcode() == Instruction::AShr ||
        Inst->getOpcode() == Instruction::LShr)
      return lowerShr(Inst);
    if (Inst->getOpcode() == Instruction::Mul)
      return lowerMul64(Inst);
    return false;
  }
  if (Inst->getOpcode() == Instruction::ICmp)
    return widenByteOp(Inst);
  else if (auto CI = dyn_cast<FCmpInst>(Inst))
    return lowerFCmpInst(CI);
  if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
    if (CI->isInlineAsm())
      return false;
    processTwoAddressOpnd(CI);
    unsigned IntrinsicID = GenXIntrinsic::not_any_intrinsic;
    if (Function *Callee = CI->getCalledFunction()) {
      IntrinsicID = GenXIntrinsic::getAnyIntrinsicID(Callee);
      IGC_ASSERT(CI->getNumArgOperands() < GenXIntrinsicInfo::OPNDMASK);
    }
    if (ST) {
      // use gather/scatter to implement SLM oword load/store on
      // legacy platforms
      if (!ST->isCNLplus()) {
        if (translateSLMOWord(CI, IntrinsicID))
          return true;
      }
    }
       // split gather/scatter/atomic into the width legal to the target
    if (splitGatherScatter(CI, IntrinsicID))
      return true;
    else if (isNewLoadInst(CI))
      return generatePredicatedWrrForNewLoad(CI);

    switch (IntrinsicID) {
    case GenXIntrinsic::genx_rdregioni:
    case GenXIntrinsic::genx_rdregionf:
      return lowerRdRegion(Inst);
    case GenXIntrinsic::genx_wrregioni:
    case GenXIntrinsic::genx_wrregionf:
      return lowerWrRegion(Inst);
    case GenXIntrinsic::genx_rdpredregion:
      return lowerRdPredRegion(Inst);
    case GenXIntrinsic::genx_wrpredregion:
      return lowerWrPredRegion(Inst);
    case GenXIntrinsic::not_any_intrinsic:
      break;
    case Intrinsic::dbg_value:
    case GenXIntrinsic::genx_absf:
    case GenXIntrinsic::genx_absi:
      break;
    default:
    case GenXIntrinsic::genx_constantpred:
    case GenXIntrinsic::genx_constanti:
    case GenXIntrinsic::genx_constantf:
      break; // ignore
    case GenXIntrinsic::genx_vload: {
      if (!Inst->use_empty()) {
        Value *Ptr = Inst->getOperand(0);
        LoadInst *LI = new LoadInst(Ptr->getType()->getPointerElementType(),
                                    Ptr, "", /*volatile*/ true, Inst);
        LI->takeName(Inst);
        LI->setDebugLoc(Inst->getDebugLoc());
        Inst->replaceAllUsesWith(LI);
      }
      ToErase.push_back(Inst);
      return true;
    }
    case GenXIntrinsic::genx_vstore: {
      Value *Val = Inst->getOperand(0);
      Value *Ptr = Inst->getOperand(1);
      auto ST = new StoreInst(Val, Ptr, /*volatile*/ true, Inst);
      ST->setDebugLoc(Inst->getDebugLoc());
      ToErase.push_back(Inst);
      return true;
    }
    case GenXIntrinsic::genx_ssmul:
    case GenXIntrinsic::genx_sumul:
    case GenXIntrinsic::genx_usmul:
    case GenXIntrinsic::genx_uumul:
      return lowerGenXMul(CI, IntrinsicID);
    case GenXIntrinsic::genx_ssmul_sat:
    case GenXIntrinsic::genx_sumul_sat:
    case GenXIntrinsic::genx_usmul_sat:
    case GenXIntrinsic::genx_uumul_sat:
      return lowerGenXMulSat(CI, IntrinsicID);
    case GenXIntrinsic::genx_lzd:
      return lowerLzd(Inst);
    case Intrinsic::masked_load:
      return lowerLLVMMaskedLoad(CI);
    case Intrinsic::masked_store:
      return lowerLLVMMaskedStore(CI);
    case Intrinsic::masked_gather:
      return lowerLLVMMaskedGather(CI);
    case Intrinsic::masked_scatter:
      return lowerLLVMMaskedScatter(CI);
    case Intrinsic::trap:
      return lowerTrap(CI);
    case Intrinsic::ctpop:
      return lowerCtpop(CI);
    case Intrinsic::minnum:
    case Intrinsic::maxnum:
      return lowerMinMax(CI, IntrinsicID);
    case Intrinsic::sqrt:
      return lowerSqrt(CI);
    case Intrinsic::uadd_with_overflow:
      return lowerUAddWithOverflow(CI);
    case Intrinsic::sadd_with_overflow:
    case Intrinsic::ssub_with_overflow:
    case Intrinsic::usub_with_overflow:
    case Intrinsic::smul_with_overflow:
    case Intrinsic::umul_with_overflow:
      Inst->getContext().emitError(
          Inst, "GenX backend cannot handle overflowing intrinsics yet");
      break;
    case Intrinsic::assume:
      ToErase.push_back(Inst);
      return true;
    case Intrinsic::expect:
      llvm_unreachable("Expect intrinsic should be lowered before");
    }
    return false;
  }
  if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Inst))
    return lowerExtractValue(EV);
  if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Inst))
    return lowerInsertValue(IV);
  if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst))
    return lowerLoadStore(Inst);
  if (isa<AllocaInst>(Inst))
    Inst->getContext().emitError(Inst,
                                 "GenX backend cannot handle allocas yet");
  return false;
}

/***********************************************************************
 * lowerRdRegion : handle read region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * 1. If index is variable do add sinking on it. (This in itself does not
 *    cause this function to return true, because it does not cause the
 *    original instruction to be replaced.)
 */
bool GenXLowering::lowerRdRegion(Instruction *Inst) {
  // Sink add in address calculation.
  Use *U = &Inst->getOperandUse(GenXIntrinsic::GenXRegion::RdIndexOperandNum);
  *U = sinkAdd(*U);
  return false;
}

/***********************************************************************
 * lowerWrRegion : handle write region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * 1. If index is variable do add sinking on it. (This in itself does not
 *    cause this function to return true, because it does not cause the
 *    original instruction to be replaced.)
 *
 * 2. If it is a predicated byte wrregion, see if it can be widened.
 */
bool GenXLowering::lowerWrRegion(Instruction *Inst) {
  // Sink add in address calculation.
  Use *U = &Inst->getOperandUse(GenXIntrinsic::GenXRegion::WrIndexOperandNum);
  *U = sinkAdd(*U);
  // See if a predicated byte wrregion can be widened.
  return widenByteOp(Inst);
}

/***********************************************************************
 * lowerRdPredRegion : handle read predicate region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * rdpredregion is a GenX backend internal intrinsic, and was thus created
 * within this GenXLowering pass. However it is considered legal only if its
 * uses are all in select or wrregion or wrpredpredregion; if not we lower
 * it further here. If a use is in rdpredregion, we need to combine the two
 * rdpredregions into one.
 */
bool GenXLowering::lowerRdPredRegion(Instruction *Inst) {
  SmallVector<CallInst *, 4> RdPredRegionUsers;
  bool Ok = true;
  for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui) {
    auto User = cast<Instruction>(ui->getUser());
    if (isa<SelectInst>(User))
      continue;
    unsigned IID = GenXIntrinsic::getAnyIntrinsicID(User);
    if (GenXIntrinsic::isWrRegion(IID))
      continue;
    if (IID == GenXIntrinsic::genx_wrpredpredregion)
      continue;
    if (IID == GenXIntrinsic::genx_rdpredregion) {
      RdPredRegionUsers.push_back(cast<CallInst>(User));
      continue;
    }
    if (IID == GenXIntrinsic::not_any_intrinsic) {
      Ok = false;
      break;
    }
    if (cast<CallInst>(User)->doesNotAccessMemory()) {
      Ok = false;
      break;
    }
  }
  unsigned Start = cast<ConstantInt>(Inst->getOperand(1))->getZExtValue();
  unsigned Size = cast<VectorType>(Inst->getType())->getNumElements();
  if (Ok) {
    // All uses in select/wrregion/rdpredregion/non-ALU intrinsic, so we can
    // keep the rdpredregion.  Check for uses in another rdpredregion; we need
    // to combine those.
    for (auto ui = RdPredRegionUsers.begin(), ue = RdPredRegionUsers.end();
         ui != ue; ++ui) {
      auto User = *ui;
      unsigned UserStart =
          cast<ConstantInt>(User->getOperand(1))->getZExtValue();
      unsigned UserSize = cast<VectorType>(User->getType())->getNumElements();
      auto Combined =
          Region::createRdPredRegion(Inst->getOperand(0), Start + UserStart,
                                     UserSize, "", User, User->getDebugLoc());
      Combined->takeName(User);
      User->replaceAllUsesWith(Combined);
      ToErase.push_back(User);
    }
    return false;
  }
  // Need to lower it further.
  const DebugLoc &DL = Inst->getDebugLoc();
  // Convert input to vector of short.
  auto In = Inst->getOperand(0);
  Type *I16Ty = Type::getInt16Ty(Inst->getContext());
  Type *InI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<VectorType>(In->getType())->getNumElements());
  auto InI16 = CastInst::Create(Instruction::ZExt, In, InI16Ty,
                                Inst->getName() + ".lower1", Inst);
  InI16->setDebugLoc(DL);
  // Use rdregion to extract the region.
  Region R(InI16);
  R.getSubregion(Start, Size);
  auto Rd = R.createRdRegion(InI16, Inst->getName() + ".lower3", Inst, DL);
  // Convert back to predicate.
  auto Res = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, Rd,
                             Constant::getNullValue(Rd->getType()),
                             Inst->getName() + ".lower4", Inst);
  Res->setDebugLoc(DL);
  // Replace uses and erase.
  Inst->replaceAllUsesWith(Res);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerWrPredRegion : handle write predicate region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * wrpredregion is a GenX backend internal intrinsic, and was thus created
 * within this GenXLowering pass. However it is considered legal only if its
 * "new value" input is a compare; if not we lower it further here.
 */
bool GenXLowering::lowerWrPredRegion(Instruction *Inst) {
  auto NewVal = Inst->getOperand(1);
  if (isa<CmpInst>(NewVal))
    return false;
  // Need to lower it further.
  const DebugLoc &DL = Inst->getDebugLoc();
  // Convert "old value" input to vector of short.
  auto OldVal = Inst->getOperand(0);
  Type *I16Ty = Type::getInt16Ty(Inst->getContext());
  Type *OldValI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<VectorType>(OldVal->getType())->getNumElements());
  auto OldValI16 = CastInst::Create(Instruction::ZExt, OldVal, OldValI16Ty,
                                    Inst->getName() + ".lower1", Inst);
  OldValI16->setDebugLoc(DL);
  // Convert "new value" input to vector of short.
  Type *NewValI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<VectorType>(NewVal->getType())->getNumElements());
  auto NewValI16 = CastInst::Create(Instruction::ZExt, NewVal, NewValI16Ty,
                                    Inst->getName() + ".lower2", Inst);
  NewValI16->setDebugLoc(DL);
  // Use wrregion to write the new value into the old value.
  Region R(OldValI16);
  R.getSubregion(cast<ConstantInt>(Inst->getOperand(2))->getZExtValue(),
                 cast<VectorType>(NewValI16Ty)->getNumElements());
  auto Wr = R.createWrRegion(OldValI16, NewValI16, Inst->getName() + ".lower3",
                             Inst, DL);
  // Convert back to predicate.
  auto Res = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, Wr,
                             Constant::getNullValue(Wr->getType()),
                             Inst->getName() + ".lower4", Inst);
  Res->setDebugLoc(DL);
  // Replace uses and erase.
  Inst->replaceAllUsesWith(Res);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerInsertElement : lower InsertElement to wrregion, multiplying the
 *      index by the element size
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerInsertElement(Instruction *Inst) {
  Instruction *NewInst = NULL;
  // Special case - if the result has 1 element (usually turning scalar into 1
  // element vector) then simply transform the insert element into a bitcast We
  // don't need to worry about the index since if it is not zero the result is
  // undef anyway (and can be set to anything we like) We also don't need to
  // worry about what the original vector is (usually undef) since it will be
  // overwritten or undef
  VectorType *VT = dyn_cast<VectorType>(Inst->getType());
  IGC_ASSERT(VT);
  unsigned NumElements = VT->getNumElements();
  const DebugLoc &DL = Inst->getDebugLoc();
  if (NumElements == 1) {
    Value *ToInsert = Inst->getOperand(1);
    NewInst = CastInst::Create(Instruction::BitCast, ToInsert, VT,
                               Inst->getName(), Inst);
    NewInst->setDebugLoc(DL);
  } else if (!Inst->getType()->getScalarType()->isIntegerTy(1)) {
    // Cast and scale the index.
    Value *IdxVal = scaleInsertExtractElementIndex(
        Inst->getOperand(2), Inst->getOperand(1)->getType(), Inst);
    // Sink adds in the address calculation.
    IdxVal = sinkAdd(IdxVal);
    // Create the new wrregion
    Value *Src = Inst->getOperand(1);
    Region R(Src);
    R.Indirect = IdxVal;
    NewInst = R.createWrRegion(Inst->getOperand(0), Src, Inst->getName(),
                               Inst /*InsertBefore*/, DL);
  } else {
    // Boolean insertelement. We have to cast everything to i16, do the
    // insertelement, and cast it back again. All this gets further lowered
    // subsequently.
    auto I16Ty = Type::getIntNTy(Inst->getContext(), 16);
    auto VecTy = IGCLLVM::FixedVectorType::get(
        I16Ty, cast<VectorType>(Inst->getType())->getNumElements());
    auto CastVec =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(0), VecTy,
                         Inst->getOperand(0)->getName() + ".casti16", Inst);
    CastVec->setDebugLoc(DL);
    auto CastEl =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(1), I16Ty,
                         Inst->getOperand(1)->getName() + ".casti16", Inst);
    CastEl->setDebugLoc(DL);
    auto NewInsert = InsertElementInst::Create(CastVec, CastEl,
                                               Inst->getOperand(2), "", Inst);
    NewInsert->takeName(Inst);
    NewInsert->setDebugLoc(DL);
    NewInst = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, NewInsert,
                              Constant::getNullValue(VecTy),
                              NewInsert->getName() + ".casti1", Inst);
    NewInst->setDebugLoc(DL);
  }
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerExtractElement : lower ExtractElement to rdregion, multiplying the
 *      index by the element size
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerExtractElement(Instruction *Inst) {
  Instruction *NewInst = nullptr;
  if (!Inst->getType()->isIntegerTy(1)) {
    // Cast and scale the index.
    Type *ElTy = Inst->getType();
    Value *IdxVal =
        scaleInsertExtractElementIndex(Inst->getOperand(1), ElTy, Inst);
    // Sink adds in the address calculation.
    IdxVal = sinkAdd(IdxVal);
    // Create the new rdregion.
    Region R(Inst);
    R.Indirect = IdxVal;
    NewInst = R.createRdRegion(Inst->getOperand(0), Inst->getName(),
                               Inst /*InsertBefore*/, Inst->getDebugLoc(),
                               true /*AllowScalar*/);
  } else {
    // Boolean extractelement. We have to cast everything to i16, do the
    // extractelement, and cast it back again. All this gets further lowered
    // subsequently.
    auto I16Ty = Type::getIntNTy(Inst->getContext(), 16);
    auto VecTy = IGCLLVM::FixedVectorType::get(
        I16Ty,
        cast<VectorType>(Inst->getOperand(0)->getType())->getNumElements());
    auto CastVec =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(0), VecTy,
                         Inst->getOperand(0)->getName() + ".casti16", Inst);
    const DebugLoc &DL = Inst->getDebugLoc();
    CastVec->setDebugLoc(DL);
    auto NewExtract =
        ExtractElementInst::Create(CastVec, Inst->getOperand(1), "", Inst);
    NewExtract->takeName(Inst);
    NewExtract->setDebugLoc(DL);
    NewInst = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, NewExtract,
                              Constant::getNullValue(I16Ty),
                              NewExtract->getName() + ".casti1", Inst);
    NewInst->setDebugLoc(DL);
  }
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * scaleInsertExtractElementIndex : scale index by element byte size,
 *      and ensure it is an i16
 */
Value *GenXLowering::scaleInsertExtractElementIndex(Value *IdxVal, Type *ElTy,
                                                    Instruction *InsertBefore) {
  // Do the cast and multiply.
  static_assert(genx::ByteBits);
  unsigned ElementBytes = DL->getTypeSizeInBits(ElTy) / genx::ByteBits;
  IntegerType *I16Ty = Type::getInt16Ty(IdxVal->getContext());
  if (ConstantInt *CI = dyn_cast<ConstantInt>(IdxVal))
    return ConstantInt::get(I16Ty, CI->getSExtValue() * ElementBytes);
  // Ensure the variable offset is i16.
  Instruction *IdxInst = CastInst::CreateIntegerCast(
      IdxVal, I16Ty, false /*isSigned*/, "cast", InsertBefore);
  IdxInst->setDebugLoc(InsertBefore->getDebugLoc());
  // Multiply it by the element size in bytes.
  if (ElementBytes != 1) {
    IdxInst = BinaryOperator::Create(
        Instruction::Shl, IdxInst,
        ConstantInt::get(I16Ty, genx::log2(ElementBytes)), "scale",
        InsertBefore);
    IdxInst->setDebugLoc(InsertBefore->getDebugLoc());
  }
  return IdxInst;
}

/***********************************************************************
 * lowerTrunc : lower a TruncInst
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * A Trunc is lowered to a bitcast then a region/element read with a stride.
 * GenXCoalescing will coalesce the bitcast, so this will hopefully save
 * an instruction.
 */
bool GenXLowering::lowerTrunc(Instruction *Inst) {
  Value *InValue = Inst->getOperand(0);
  // Check for the trunc's input being a sext/zext where the original element
  // size is the same as the result of the trunc. We can just remove the
  // whole thing then. (This can arise from GenXReduceIntSize.)
  if (auto CI = dyn_cast<CastInst>(InValue)) {
    if ((isa<SExtInst>(CI) || isa<ZExtInst>(CI)) &&
        CI->getOperand(0)->getType() == Inst->getType()) {
      // Just replace uses with the original unextended value.
      Inst->replaceAllUsesWith(CI->getOperand(0));
      ToErase.push_back(Inst);
      return true;
    }
  }

  // Lower "trunc i8 %v to i1" into "cmp.ne (%v & 1), 0"
  if (Inst->getType()->isIntOrIntVectorTy(1)) {
    IRBuilder<> Builder(Inst);
    auto V =
        Builder.CreateAnd(InValue, ConstantInt::get(InValue->getType(), 1));
    V = Builder.CreateICmpNE(V, ConstantInt::get(V->getType(), 0));
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(V);
    ToErase.push_back(Inst);
    return true;
  }

  Type *InElementTy = InValue->getType();
  Type *OutElementTy = Inst->getType();
  unsigned NumElements = 1;
  if (VectorType *VT = dyn_cast<VectorType>(InElementTy)) {
    InElementTy = VT->getElementType();
    OutElementTy = cast<VectorType>(OutElementTy)->getElementType();
    NumElements = VT->getNumElements();
  }

  // Lower "trunc <32 x i16> %v to <32 x i1>" into "cmp.ne (%v & 1), 0"
  if (NumElements > 1 && OutElementTy->isIntegerTy(1)) {
    IRBuilder<> Builder(Inst);
    unsigned N = NumElements;
    Value *Os = ConstantVector::getSplat(IGCLLVM::getElementCount(N),
                                         ConstantInt::get(InElementTy, 1));
    Value *Zs = ConstantVector::getSplat(IGCLLVM::getElementCount(N),
                                         ConstantInt::get(InElementTy, 0));
    auto V = Builder.CreateAnd(InValue, Os);
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    V = Builder.CreateICmpNE(V, Zs);
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(V);
    ToErase.push_back(Inst);
    return true;
  }

  IGC_ASSERT(OutElementTy->getPrimitiveSizeInBits());
  unsigned Stride = InElementTy->getPrimitiveSizeInBits() /
                    OutElementTy->getPrimitiveSizeInBits();
  // Create the new bitcast.
  Instruction *BC = CastInst::Create(
      Instruction::BitCast, InValue,
      IGCLLVM::FixedVectorType::get(OutElementTy, Stride * NumElements),
      Inst->getName(), Inst /*InsertBefore*/);
  BC->setDebugLoc(Inst->getDebugLoc());
  // Create the new rdregion.
  Region R(BC);
  R.NumElements = NumElements;
  R.Stride = Stride;
  R.Width = NumElements;
  R.VStride = R.Stride * R.Width;
  Instruction *NewInst = R.createRdRegion(
      BC, Inst->getName(), Inst /*InsertBefore*/, Inst->getDebugLoc(),
      !isa<VectorType>(Inst->getType()) /*AllowScalar*/);
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerCast : lower a CastInst
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerCast(Instruction *Inst) {
  // if it is a bitcast for vector of i1 predicate to vector,
  // create bitcast to scalar and them bitcast to vector due to visa
  // requirements
  if (Inst->getOpcode() == Instruction::BitCast &&
      Inst->getOperand(0)->getType()->getScalarType()->isIntegerTy(1) &&
      Inst->getType()->isVectorTy() &&
      !Inst->getType()->getScalarType()->isIntegerTy(1)) {
    IRBuilder<> Builder(Inst);
    Value *operand = Inst->getOperand(0);
    Type *originTy = Inst->getType();
    Type *scalarIntTy =
        Type::getIntNTy(Inst->getContext(), DL->getTypeSizeInBits(originTy));
    const Twine Name = "pred";
    Value *Scalar = Builder.CreateBitCast(operand, scalarIntTy, Name);
    Value *Res = Builder.CreateBitCast(Scalar, originTy, Name);
    Inst->replaceAllUsesWith(Res);
    ToErase.push_back(Inst);
    return true;
  }

  // If it is zext/sext/UIToFP from (vector of) i1, turn into a select.
  if (Inst->getOperand(0)->getType()->getScalarType()->isIntegerTy(1) &&
      Inst->getOpcode() != Instruction::BitCast) {
    int OneVal = 0;
    switch (Inst->getOpcode()) {
    case Instruction::ZExt:
      OneVal = 1;
      break;
    case Instruction::SExt:
      OneVal = -1;
      break;
    case Instruction::UIToFP:
      OneVal = 1;
      break;
    default:
      IGC_ASSERT_MESSAGE(0, "unknown opcode in lowerCast");
    }

    Instruction *NewInst;
    if (Inst->getType()->isFPOrFPVectorTy())
      NewInst = SelectInst::Create(
          Inst->getOperand(0), ConstantFP::get(Inst->getType(), OneVal),
          ConstantFP::get(Inst->getType(), 0), Inst->getName(), Inst);
    else
      NewInst = SelectInst::Create(
          Inst->getOperand(0), ConstantInt::get(Inst->getType(), OneVal),
          ConstantInt::get(Inst->getType(), 0), Inst->getName(), Inst);
    NewInst->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(NewInst);
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

/***********************************************************************
 * lowerBoolScalarSelect : lower a SelectInst on vector of i1
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * This is a select on vector of i1 where the condition is scalar. This only
 * happens in simd control flow where an LLVM pass has optimized away the
 * conditional branch. We restore the conditional branch and create an
 * if..else..endif.
 */
bool GenXLowering::lowerBoolScalarSelect(SelectInst *SI) {
  //         BB1
  //        /  |
  // false /   | true
  //      /    |
  //   BB2     |
  //      \    |
  //       \   |
  //        \  |
  //         BB4
  //
  auto BB1 = SI->getParent();
  auto BB2 = SplitBlock(BB1, SI, DT);
  auto BB4 = SplitEdge(BB1, BB2, DT);
  BB2->setName("select.false");
  BB4->setName("select.true");

  auto OldTerm = BB1->getTerminator();
  BranchInst::Create(BB4, BB2, SI->getCondition(), OldTerm);
  OldTerm->eraseFromParent();
  // Since additional edge is added between BB1 and BB4 instead of through BB2
  // only. BB4 is not immediately dominated by BB2 anymore. Instead, BB4 is
  // dominated by BB1 immediately.
  if (DT)
    DT->changeImmediateDominator(BB4, BB1);
  // Replace 'select' with 'phi'
  auto Phi = PHINode::Create(SI->getType(), /*NumReservedValues=*/2, "",
                             &BB4->front());
  Phi->takeName(SI);
  Phi->addIncoming(SI->getTrueValue(), BB1);
  Phi->addIncoming(SI->getFalseValue(), BB2);
  SI->replaceAllUsesWith(Phi);
  ToErase.push_back(SI);
  // Split the (critical) edge from BB1 to BB4 to avoid having critical edge.
  auto BB3 = SplitEdge(BB1, BB4, DT);
  BB3->setName("select.crit");
  return true;
}

/***********************************************************************
 * lowerBoolVectorSelect : lower a SelectInst on (vector of) i1
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * A select on (vector of) i1 is lowered to the equivalent and/or/xor
 * instructions. No simplification is done even if an input is a constant.
 *
 * However, if the selector looks like an EM value, and the "true" operand is
 * a cmp, it is instead lowered to an llvm.genx.wrpredpredregion. Baling will
 * bale the cmp into it, resulting in a masked cmp instruction that sets bits
 * of the flag only if the corresponding EM bit is set.
 *
 * FIXME: I have seen a case where the two inputs are all false and all true.
 * Rather than try and simplify that here in the GenX backend, we should
 * try and work out how to stop LLVM generating it in the first place.
 */
bool GenXLowering::lowerBoolVectorSelect(SelectInst *Inst) {
  if (isa<CmpInst>(Inst->getTrueValue())) {
    // Check for the condition being an EM value. It might be a shufflevector
    // that slices the EM value at index 0.
    bool IsEM = GotoJoin::isEMValue(Inst->getCondition());
    if (!IsEM) {
      if (auto SV = dyn_cast<ShuffleVectorInst>(Inst->getCondition())) {
        ShuffleVectorAnalyzer SVA(SV);
        if (!SVA.getAsSlice()) {
          // Slice at index 0.
          IsEM = GotoJoin::isEMValue(SV->getOperand(0));
        }
      }
    }
    if (IsEM) {
      // Can be lowered to llvm.genx.wrpredpredregion. It always has an index of
      // 0 and the "new value" operand the same vector width as the whole vector
      // here. That might get changed if it is split up in legalization.
      auto NewInst = Region::createWrPredPredRegion(
          Inst->getFalseValue(), Inst->getTrueValue(), 0, Inst->getCondition(),
          "", Inst, Inst->getDebugLoc());
      NewInst->takeName(Inst);
      Inst->replaceAllUsesWith(NewInst);
      ToErase.push_back(Inst);
      return true;
    }
  }
  // Normal lowering to some bit twiddling.
  Instruction *NewInst1 =
      BinaryOperator::Create(BinaryOperator::And, Inst->getOperand(0),
                             Inst->getOperand(1), Inst->getName(), Inst);
  NewInst1->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst2 = BinaryOperator::Create(
      BinaryOperator::Xor, Inst->getOperand(0),
      Constant::getAllOnesValue(Inst->getType()), Inst->getName(), Inst);
  NewInst2->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst3 =
      BinaryOperator::Create(BinaryOperator::And, Inst->getOperand(2), NewInst2,
                             Inst->getName(), Inst);
  NewInst3->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst4 = BinaryOperator::Create(
      BinaryOperator::Or, NewInst1, NewInst3, Inst->getName(), Inst);
  NewInst4->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(NewInst4);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerBoolShuffle : lower a shufflevector (element type i1)
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * We handle four cases:
 *
 * 1. A slice of the vector, which can be turned into rdpredregion.
 *
 * 2. A splat. By default we need to lower that to a select to
 *    0 or -1 then a bitcast to the vector of i1. But if the input is the
 *    result of a cmp then we can splat the cmp as an optimization.
 *
 * 3. An unslice of the vector, which can be turned into wrpredregion.
 *
 * 4. General case. Like in the splat case we convert input via select and
 *    result is then bitcasted back to vector of i1. Converted vectors are
 *    then handled by lowerShuffleToMove
 */
bool GenXLowering::lowerBoolShuffle(ShuffleVectorInst *SI) {
  ShuffleVectorAnalyzer SVA(SI);
  // 1. Check for a slice.
  int SliceStart = SVA.getAsSlice();
  if (SliceStart >= 0) {
    unsigned Width = cast<VectorType>(SI->getType())->getNumElements();
    auto RPR = Region::createRdPredRegion(SI->getOperand(0), SliceStart, Width,
                                          "", SI, SI->getDebugLoc());
    RPR->takeName(SI);
    SI->replaceAllUsesWith(RPR);
    ToErase.push_back(SI);
    return true;
  }
  // 2. Check for a splat.
  auto Splat = SVA.getAsSplat();
  if (Splat.Input)
    return lowerBoolSplat(SI, Splat.Input, Splat.Index);
  // 3. Check for an unslice. The "old value" input is operand 0 of the
  // shufflevector; the "new value" input is operand 0 of the shufflevector
  // that is operand 1 of SI. We create a wrpredregion, but GenXLowering might
  // subsequently decide that it is illegal because its "new value" input is not
  // a compare, in which case it is further lowered.
  int UnsliceStart = SVA.getAsUnslice();
  if (UnsliceStart >= 0) {
    auto InnerSI = cast<ShuffleVectorInst>(SI->getOperand(1));
    auto WPR =
        Region::createWrPredRegion(SI->getOperand(0), InnerSI->getOperand(0),
                                   UnsliceStart, "", SI, SI->getDebugLoc());
    WPR->takeName(SI);
    SI->replaceAllUsesWith(WPR);
    // Undef out the operand for InnerSI in SI, so we can directly erase InnerSI
    // if SI was its only use.
    SI->setOperand(1, UndefValue::get(InnerSI->getType()));
    ToErase.push_back(SI);
    if (InnerSI->use_empty())
      InnerSI->eraseFromParent();
    return true;
  }

  // Do not lower replicated slices.
  if (SVA.isReplicatedSlice())
    return false;

  // 4. General case.

  // The idea is to convert input i1 vector to i16 vector via select,
  // then do a shufflevector lowering for non-bool case
  // and convert back to i1 vector via icmp instruction.

  IRBuilder<> B(SI);
  unsigned WidthInput =
      cast<VectorType>(SI->getOperand(0)->getType())->getNumElements();
  unsigned WidthResult = cast<VectorType>(SI->getType())->getNumElements();
  Constant *C1 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthInput),
                                          B.getInt16(1));
  Constant *C0 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthInput),
                                          B.getInt16(0));
  Value *V1 = B.CreateSelect(SI->getOperand(0), C1, C0);
  Value *V2 = B.CreateSelect(SI->getOperand(1), C1, C0);
  Value *SI1 = B.CreateShuffleVector(V1, V2, IGCLLVM::getShuffleMaskForBitcode(SI), SI->getName());
  Constant *C2 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthResult),
                                          B.getInt16(0));
  Value *Result = B.CreateICmpNE(SI1, C2);
  SI->replaceAllUsesWith(Result);
  ToErase.push_back(SI);

  return true;
}

/***********************************************************************
 * lowerBoolSplat : lower a shufflevector (element type i1) that is a splat
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerBoolSplat(ShuffleVectorInst *SI, Value *In,
                                  unsigned Idx) {
  auto IsFixedVectorOfWidth = [](const Type *Ty, unsigned Width) {
    IGC_ASSERT(Ty);
    const auto *VTy = dyn_cast<IGCLLVM::FixedVectorType>(Ty);
    if (!VTy)
      return false;
    return VTy->getNumElements() == Width;
  };

  unsigned Width = cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements();
  if (IsFixedVectorOfWidth(In->getType(), Width)) {
    IRBuilder<> B(SI);
    Constant *C1 = ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                            B.getInt16(1));
    Constant *C0 = ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                            B.getInt16(0));
    Value *V = B.CreateSelect(In, C1, C0);
    Region R(V);
    R.NumElements = Width;
    R.Stride = 0;
    R.VStride = 0;
    R.Offset = (int)Idx;
    V = R.createRdRegion(V, "splat", SI, SI->getDebugLoc());
    V = B.CreateICmpNE(V, C0);
    SI->replaceAllUsesWith(V);
    ToErase.push_back(SI);
    return true;
  }
  // This is a splat. See if the input is a cmp, possibly via a bitcast.
  if (auto BC = dyn_cast<BitCastInst>(In))
    In = BC->getOperand(0);
  if (auto Cmp = dyn_cast<CmpInst>(In)) {
    // Create a splatted version of the cmp.
    Value *CmpOpnds[2];
    Region R(Cmp->getOperand(0));
    R.NumElements = Width;
    R.Width = R.NumElements;
    R.Stride = 0;
    R.VStride = 0;
    for (unsigned i = 0; i != 2; ++i) {
      auto Opnd = Cmp->getOperand(i);
      if (auto C = dyn_cast<Constant>(Opnd)) {
        CmpOpnds[i] = ConstantVector::getSplat(
            IGCLLVM::getElementCount(R.NumElements), C);
        continue;
      }
      if (!isa<VectorType>(Opnd->getType())) {
        auto NewBC =
            CastInst::Create(Instruction::BitCast, Opnd,
                             IGCLLVM::FixedVectorType::get(Opnd->getType(), 1),
                             Opnd->getName() + ".bc", Cmp);
        NewBC->setDebugLoc(Cmp->getDebugLoc());
        Opnd = NewBC;
      }
      CmpOpnds[i] =
          R.createRdRegion(Opnd, Cmp->getOperand(i)->getName() + ".splat",
                           Cmp /*InsertBefore*/, Cmp->getDebugLoc());
    }
    auto NewCmp = CmpInst::Create(
        Cmp->getOpcode(), Cmp->getPredicate(), CmpOpnds[0], CmpOpnds[1],
        Cmp->getName() + ".splat", Cmp /*InsertBefore*/);
    NewCmp->setDebugLoc(Cmp->getDebugLoc());
    SI->replaceAllUsesWith(NewCmp);
    ToErase.push_back(SI);
    return true;
  }
  // Default code. Select int and bitcast to vector of i1.
  if (isa<VectorType>(In->getType())) {
    // First convert v1i1 to i1.
    auto NewBC = CastInst::Create(Instruction::BitCast, In,
                                  In->getType()->getScalarType(),
                                  In->getName() + ".scalar", SI);
    NewBC->setDebugLoc(SI->getDebugLoc());
    In = NewBC;
  }
  if (Width == 8 || Width == 16 || Width == 32) {
    auto IntTy = Type::getIntNTy(SI->getContext(), Width);
    auto Sel = SelectInst::Create(In, Constant::getAllOnesValue(IntTy),
                                  Constant::getNullValue(IntTy),
                                  SI->getName() + ".sel", SI);
    Sel->setDebugLoc(SI->getDebugLoc());
    auto NewBC =
        CastInst::Create(Instruction::BitCast, Sel, SI->getType(), "", SI);
    NewBC->takeName(SI);
    NewBC->setDebugLoc(SI->getDebugLoc());
    SI->replaceAllUsesWith(NewBC);
    ToErase.push_back(SI);
    return true;
  }

  IRBuilder<> Builder(SI);
  auto Val = Builder.CreateSelect(In, Builder.getInt16(1), Builder.getInt16(0),
                                  SI->getName() + ".sel");
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());
  Val = Builder.CreateBitCast(
      Val, IGCLLVM::FixedVectorType::get(Builder.getInt16Ty(), 1));
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());

  Region R(Val);
  R.Offset = 0;
  R.Width = 1;
  R.Stride = R.VStride = 0;
  R.NumElements = Width;
  Val = R.createRdRegion(Val, "", SI, SI->getDebugLoc());
  Val = Builder.CreateICmpNE(Val, ConstantVector::getNullValue(Val->getType()));
  Val->takeName(SI);
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(Val);
  ToErase.push_back(SI);
  return true;
}

/***********************************************************************
 * lowerShuffleSplat : lower a ShuffleInst (element type not i1) when it is
 *                     a splat (repetition of the same element)
 */
void GenXLowering::lowerShuffleSplat(ShuffleVectorInst *SI,
                                     ShuffleVectorAnalyzer::SplatInfo Splat) {
  // This is a splat. Turn it into a splatting rdregion.
  if (!isa<VectorType>(Splat.Input->getType())) {
    // The input is a scalar rather than a 1-vector. Bitcast it to a 1-vector.
    auto *BC = CastInst::Create(
        Instruction::BitCast, Splat.Input,
        IGCLLVM::FixedVectorType::get(Splat.Input->getType(), 1), SI->getName(),
        SI);
    BC->setDebugLoc(SI->getDebugLoc());
    Splat.Input = BC;
  }
  // Create a rdregion with a stride of 0 to represent this splat
  Region R(Splat.Input);
  R.NumElements = cast<VectorType>(SI->getType())->getNumElements();
  R.Width = R.NumElements;
  R.Stride = 0;
  R.VStride = 0;
  R.Offset = Splat.Index * R.ElementBytes;
  Instruction *NewInst =
      R.createRdRegion(Splat.Input, "", SI /*InsertBefore*/, SI->getDebugLoc());
  NewInst->takeName(SI);
  NewInst->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(NewInst);
  ToErase.push_back(SI);
}

/***********************************************************************
 * lowerShuffle : lower a ShuffleInst (element type not i1)
 *
 * Mostly these are splats. These are lowered to a rdregion
 * Any other shuffle is currently unsupported
 */
bool GenXLowering::lowerShuffle(ShuffleVectorInst *SI) {
  auto Splat = ShuffleVectorAnalyzer(SI).getAsSplat();
  if (Splat.Input) {
    lowerShuffleSplat(SI, Splat);
    return true;
  }
  if (lowerShuffleToSelect(SI))
    return true;
  lowerShuffleToMove(SI);
  return true;
}

// Lower those shufflevector that can be implemented efficiently as select.
bool GenXLowering::lowerShuffleToSelect(ShuffleVectorInst *SI) {
  int NumElements = cast<VectorType>(SI->getType())->getNumElements();
  int NumOpnd = SI->getNumOperands();
  for (int i = 0; i < NumOpnd; ++i) {
    if (cast<VectorType>(SI->getOperand(i)->getType())->getNumElements() !=
        NumElements)
      return false;
  }
  for (int i = 0; i < NumElements; ++i) {
    int idx = SI->getMaskValue(i);
    // undef index returns -1.
    if (idx < 0)
      continue;
    if (idx != i && idx != i + NumElements)
      return false;
  }
  IRBuilder<> Builder(SI);
  Type *Int1Ty = Builder.getInt1Ty();
  SmallVector<Constant *, 16> MaskVec;
  MaskVec.reserve(NumElements);
  for (int i = 0; i < NumElements; ++i) {
    int idx = SI->getMaskValue(i);
    // undef index returns -1.
    if (idx == i || idx < 0)
      MaskVec.push_back(ConstantInt::get(Int1Ty, 1));
    else
      MaskVec.push_back(ConstantInt::get(Int1Ty, 0));
  }
  Value *Mask = ConstantVector::get(MaskVec);
  auto NewSel =
      SelectInst::Create(Mask, SI->getOperand(0), SI->getOperand(1), "", SI);
  NewSel->takeName(SI);
  NewSel->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(NewSel);
  ToErase.push_back(SI);
  return true;
}

template <typename Iter> Iter skipUndefs(Iter First, Iter Last) {
  return std::find_if(First, Last, [](int MaskVal) { return MaskVal != -1; });
}

/***********************************************************************
 * lowerShuffleToMove : lower a ShuffleInst (element type is not i1) to a
 *                      sequence of rd/wrregion intrinsics
 */
void GenXLowering::lowerShuffleToMove(ShuffleVectorInst *SI) {
  ShuffleVectorAnalyzer Analyzer(SI);
  std::vector<ShuffleVectorAnalyzer::OperandRegionInfo> RdRegions;
  std::vector<Region> WrRegions;
  auto MaskVals = SI->getShuffleMask();

  // Filling read and write regions based on shuffle mask.
  for (auto It = skipUndefs(MaskVals.begin(), MaskVals.end());
       It != MaskVals.end();
       It = skipUndefs(std::next(It, RdRegions.back().R.NumElements),
                       MaskVals.end())) {
    int Idx = It - MaskVals.begin();
    auto OpRegion = Analyzer.getMaskRegionPrefix(Idx);
    IGC_ASSERT_MESSAGE(OpRegion.R.NumElements > 0,
      "should've match at least 1 element region");
    Region WrRegion(SI);
    WrRegion.Offset = Idx * WrRegion.ElementBytes;
    WrRegion.NumElements = WrRegion.Width = OpRegion.R.NumElements;
    RdRegions.push_back(std::move(OpRegion));
    WrRegions.push_back(std::move(WrRegion));
  }

  // Building rdregion intrinsics or promoting the operand if possible.
  std::vector<Value *> RdRegionInsts;
  std::transform(
      RdRegions.begin(), RdRegions.end(), std::back_inserter(RdRegionInsts),
      [SI](ShuffleVectorAnalyzer::OperandRegionInfo &OpRegion) -> Value * {
        if (OpRegion.R.isWhole(OpRegion.Op->getType()))
          return OpRegion.Op;
        return OpRegion.R.createRdRegion(
            OpRegion.Op, SI->getName() + ".shuffle.rd", SI, SI->getDebugLoc());
      });

  // Obtaining SI replacement (sequence of wrregion intrinsics in the
  // most common case).
  Value *Result;
  if (WrRegions.size() == 0)
    Result = UndefValue::get(SI->getType());
  else if (WrRegions.size() == 1 &&
           WrRegions.front().NumElements ==
               cast<VectorType>(SI->getType())->getNumElements())
    Result = RdRegionInsts.back();
  else {
    auto WrRegionArgs = zip(WrRegions, RdRegionInsts);
    Result = std::accumulate(
        WrRegionArgs.begin(), WrRegionArgs.end(),
        static_cast<Value *>(UndefValue::get(SI->getType())),
        [SI](Value *PrevWrRegionInst,
             const std::tuple<Region &, Value *> &Args) {
          return std::get<0>(Args).createWrRegion(
              PrevWrRegionInst, std::get<1>(Args),
              SI->getName() + ".shuffle.wr", SI, SI->getDebugLoc());
        });
  }

  SI->replaceAllUsesWith(Result);
  ToErase.push_back(SI);
}

/***********************************************************************
 * lowerShr : lower Shl followed by AShr/LShr by the same amount
 *    into trunc+sext/zext
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * See convertShlShr below.
 */
bool GenXLowering::lowerShr(Instruction *Inst) {
  Instruction *NewInst = convertShlShr(Inst);
  if (!NewInst)
    return false; // no conversion done
  ToErase.push_back(Inst);
  auto Shl = cast<Instruction>(Inst->getOperand(0));
  if (Shl->hasOneUse())
    ToErase.push_back(Shl);
  return true;
}

/***********************************************************************
 * convertShlShr : convert Shl followed by AShr/LShr by the same amount
 *    into trunc+sext/zext
 *
 * Enter:   Inst = the AShr or LShr instruction
 *
 * Return:  0 if no conversion done, else the new SExt/ZExt instruction.
 *          The original AShr/LShr is now unused, but neither original
 *          instruction is erased.
 *
 * This is the opposite to what instruction combining does! We want to change
 * it back to trunc then extend because the trunc can then be lowered into
 * a region, and the extend can sometimes be baled into whatever uses it.
 *
 * This is a separately callable global function so it can also be used
 * from GenXReduceIntSize, which for other reasons of convenience runs before
 * GenXLowering.
 */
Instruction *llvm::genx::convertShlShr(Instruction *Inst) {
  unsigned NumBits = Inst->getType()->getScalarType()->getPrimitiveSizeInBits();
  auto C = dyn_cast<Constant>(Inst->getOperand(1));
  if (!C)
    return nullptr;
  auto Shl = dyn_cast<Instruction>(Inst->getOperand(0));
  if (!Shl)
    return nullptr;
  if (Shl->getOpcode() != Instruction::Shl)
    return nullptr;
  if (Shl->getOperand(1) != C)
    return nullptr;
  if (isa<VectorType>(C->getType())) {
    C = C->getSplatValue();
    if (!C)
      return nullptr;
  }
  unsigned ShiftBits = cast<ConstantInt>(C)->getSExtValue();
  unsigned RemainingBits = NumBits - ShiftBits;
  if (RemainingBits != 8 && RemainingBits != 16)
    return nullptr;
  // We have Shl+AShr or Shl+LShr that can be turned into trunc+sext/zext.
  Type *ConvTy = Type::getIntNTy(Inst->getContext(), RemainingBits);
  if (auto VT = dyn_cast<VectorType>(Inst->getType()))
    ConvTy = IGCLLVM::FixedVectorType::get(ConvTy, VT->getNumElements());
  auto Trunc = CastInst::Create(Instruction::Trunc, Shl->getOperand(0), ConvTy,
                                "", Inst);
  Trunc->takeName(Shl);
  Trunc->setDebugLoc(Shl->getDebugLoc());
  auto Ext = CastInst::Create(Inst->getOpcode() == Instruction::AShr
                                  ? Instruction::SExt
                                  : Instruction::ZExt,
                              Trunc, Inst->getType(), "", Inst);
  Ext->takeName(Inst);
  Ext->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(Ext);
  return Ext;
}

/***********************************************************************
 * lowerExtractValue : remove extractvalue if possible
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * If we can trace the input of the extractvalue to the point where the
 * value was inserted, use that value instead.
 *
 * Because we have already split struct phi nodes, we should just be left
 * with insertvalue/extractvalue pairs that we can remove here. The
 * exception is when a struct is passed in to or returned from a call.
 * Then we leave the extractvalue for later handling in the register
 * allocator.
 */
bool GenXLowering::lowerExtractValue(ExtractValueInst *Inst) {
  ArrayRef<unsigned> EVIndices = Inst->getIndices();
  ArrayRef<unsigned> Indices = EVIndices;
  Value *V = Inst->getAggregateOperand();
  for (;;) {
    InsertValueInst *IV = dyn_cast<InsertValueInst>(V);
    if (!IV) {
      // If we used up any indices, create a new extractvalue for the
      // remaining ones.
      if (Indices.size() != EVIndices.size()) {
        Instruction *NewIV = ExtractValueInst::Create(
            Inst->getAggregateOperand(), Indices, Inst->getName(), Inst);
        NewIV->setDebugLoc(Inst->getDebugLoc());
        Inst->replaceAllUsesWith(NewIV);
        ToErase.push_back(Inst);
        return true;
      }
      return false;
    }
    // We have an insertvalue. See how many of the indices agree.
    ArrayRef<unsigned> IVIndices = IV->getIndices();
    unsigned Match = 0;
    while (Match < Indices.size() && Match < IVIndices.size() &&
           Indices[Match] == IVIndices[Match])
      ++Match;
    if (!Match) {
      // No match at all. Go back to the previous insertvalue.
      V = IV->getAggregateOperand();
      continue;
    }
    // Use the inserted value here.
    V = IV->getInsertedValueOperand();
    // Chop off the indices we have used up. If none left, we have finished.
    Indices = Indices.slice(Match);
    if (!Indices.size())
      break;
  }
  // We have found the struct element value V.
  Inst->replaceAllUsesWith(V);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerInsertValue : remove insertvalue if possible
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * In most cases, by the time we get to an insertvalue, it will be unused
 * because of extractvalue removal.
 *
 * In a case where it is still used (probably because this function has an
 * arg or return value that is a struct, or we call a function like that),
 * the struct value is dealt with in register allocation.
 */
bool GenXLowering::lowerInsertValue(InsertValueInst *Inst) {
  if (Inst->use_empty()) {
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

/***********************************************************************
 * lowerUAddWithOverflow : lower llvm.uadd.with.overflow
 *
 * This could potentially be implemented with the vISA addc instruction.
 * However an intrinsic for that would need extra GenX backend support for
 * returning a struct containing two vectors, and that support does not exist
 * now.
 *
 * So for now we use the old DEC Alpha trick of comparing the result with
 * one of the operands.
 */
bool GenXLowering::lowerUAddWithOverflow(CallInst *CI) {
  const DebugLoc &DL = CI->getDebugLoc();
  // Do the add.
  auto Add =
      BinaryOperator::Create(Instruction::Add, CI->getArgOperand(0),
                             CI->getArgOperand(1), CI->getName() + ".add", CI);
  Add->setDebugLoc(DL);
  // Do the comparison. (An unsigned add has overflowed if the result is
  // smaller than one of the operands, and, if it has overflowed, the result
  // is smaller than both of the operands. So it doesn't matter which operand
  // we use for the comparison.)
  auto Cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, Add,
                             CI->getArgOperand(1), CI->getName() + ".cmp", CI);
  Cmp->setDebugLoc(DL);
  // For any extractvalue use of the result of the original add with overflow,
  // replace it directly.
  SmallVector<ExtractValueInst *, 4> Extracts;
  for (auto ui = CI->use_begin(), ue = CI->use_end(); ui != ue; ++ui)
    if (auto EVI = dyn_cast<ExtractValueInst>(ui->getUser()))
      Extracts.push_back(EVI);
  for (auto ei = Extracts.begin(), ee = Extracts.end(); ei != ee; ++ei) {
    auto EVI = *ei;
    EVI->replaceAllUsesWith(EVI->getIndices()[0] ? (Value *)Cmp : (Value *)Add);
    EVI->setOperand(0, UndefValue::get(CI->getType()));
    ToErase.push_back(EVI);
  }
  // If any uses of the original intrinsic remain, recreate the struct value.
  if (!CI->use_empty()) {
    auto Insert = InsertValueInst::Create(UndefValue::get(CI->getType()), Add,
                                          0, CI->getName() + ".insertadd", CI);
    Insert->setDebugLoc(DL);
    Insert = InsertValueInst::Create(Insert, Cmp, 1,
                                     CI->getName() + ".insertcmp", CI);
    Insert->setDebugLoc(DL);
    // ... and use it to replace the original intrinsic.
    CI->replaceAllUsesWith(Insert);
  }
  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerTrap(CallInst *CI) {
  Module *M = CI->getModule();
  IRBuilder<> Builder(CI);
  auto &Ctx = CI->getContext();
  unsigned EMWidth = 32;
  Type *ArgTypes[] = {
      IGCLLVM::FixedVectorType::get(Type::getInt1Ty(Ctx), EMWidth),
      IGCLLVM::FixedVectorType::get(Type::getInt16Ty(Ctx), EMWidth)};
  auto Fn = GenXIntrinsic::getGenXDeclaration(M,
    GenXIntrinsic::genx_raw_send_noresult, ArgTypes);
  SmallVector<Value *, 8> Args;
  // send
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
  // predicate all lanes
  Args.push_back(ConstantVector::getSplat(IGCLLVM::getElementCount(EMWidth),
                                          ConstantInt::getTrue(Ctx)));
  // EOT
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0x27));
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0x02000010));
  Args.push_back(
      ConstantVector::getSplat(IGCLLVM::getElementCount(EMWidth),
                               Constant::getNullValue(Type::getInt16Ty(Ctx))));
  Builder.CreateCall(Fn, Args);
  ToErase.push_back(CI);

  return true;
}

bool GenXLowering::lowerCtpop(CallInst *CI) {
  Module *M = CI->getModule();
  IRBuilder<> Builder(CI);
  Builder.SetCurrentDebugLocation(CI->getDebugLoc());

  Type *Int32Ty = IntegerType::getInt32Ty(CI->getContext());
  Type *RetTy = nullptr;
  if (auto *VT = dyn_cast<VectorType>(CI->getType()))
    RetTy = IGCLLVM::FixedVectorType::get(Int32Ty, VT->getNumElements());
  else
    RetTy = Int32Ty;

  auto *CBitDecl = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_cbit, {RetTy, CI->getType()});
  Value *CBitInst =
      Builder.CreateCall(CBitDecl, CI->getOperand(0), CI->getName());

  CBitInst = Builder.CreateZExtOrTrunc(CBitInst, CI->getType());
  CI->replaceAllUsesWith(CBitInst);
  ToErase.push_back(CI);

  return true;
}

// Lower cmp instructions that GenX cannot deal with.
bool GenXLowering::lowerFCmpInst(FCmpInst *Inst) {
  IRBuilder<> Builder(Inst);
  Builder.SetCurrentDebugLocation(Inst->getDebugLoc());
  Value *Ops[] = {Inst->getOperand(0), Inst->getOperand(1)};

  switch (Inst->getPredicate()) {
  default:
    return lowerUnorderedFCmpInst(Inst);
  case CmpInst::FCMP_ORD: // True if ordered (no nans)
  {
    // %c = fcmp ord %a %b
    // =>
    // %1 = fcmp oeq %a %a
    // %2 = fcmp oeq %b %b
    // %c = and %1 %2
    Value *LHS = Builder.CreateFCmpOEQ(Ops[0], Ops[0]);
    Value *RHS = Builder.CreateFCmpOEQ(Ops[1], Ops[1]);
    Value *New = Builder.CreateAnd(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
  {
    // %c = fcmp uno %a %b
    // =>
    // %1 = fcmp une %a %a
    // %2 = fcmp une %b %b
    // %c = or %1 %2
    Value *LHS = Builder.CreateFCmpUNE(Ops[0], Ops[0]);
    Value *RHS = Builder.CreateFCmpUNE(Ops[1], Ops[1]);
    Value *New = Builder.CreateOr(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_UEQ: // UEQ cannot be replaced with NOT ONE because we do
                          // not have ONE
  {
    // %c = fcmp ueq %a %b
    // =>
    // %1 = fcmp olt %a %b
    // %2 = fcmp ogt %a %b
    // %3 = or %1 %2
    // %c = not %3
    Value *LHS = Builder.CreateFCmpOLT(Ops[0], Ops[1]);
    Value *RHS = Builder.CreateFCmpOGT(Ops[0], Ops[1]);
    Value *Or = Builder.CreateOr(LHS, RHS);
    Value *New = Builder.CreateNot(Or);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_ONE: // NE is unordered
  {
    // %c = fcmp one %a %b
    // =>
    // %1 = fcmp olt %a %b
    // %2 = fcmp ogt %a %b
    // %c = or %1 %2
    Value *LHS = Builder.CreateFCmpOLT(Ops[0], Ops[1]);
    Value *RHS = Builder.CreateFCmpOGT(Ops[0], Ops[1]);
    Value *New = Builder.CreateOr(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  }

  return false;
}

// FCmp with NE is the only one supported unordered cmp inst. All the rest must
// be lowered.
bool GenXLowering::lowerUnorderedFCmpInst(FCmpInst *Inst) {
  CmpInst::Predicate Pred = Inst->getPredicate();
  if (CmpInst::isOrdered(Pred))
    return false;

  // We support UNE.
  if (Pred == CmpInst::FCMP_UNE)
    return false;

  // For UNO and UEQ we have replacement in lowerFCmpInst.
  IGC_ASSERT(Pred != CmpInst::FCMP_UNO);
  IGC_ASSERT(Pred != CmpInst::FCMP_UEQ);

  CmpInst *InverseFCmp = CmpInst::Create(
      Inst->getOpcode(), CmpInst::getInversePredicate(Pred),
      Inst->getOperand(0), Inst->getOperand(1),
      Inst->getName() + ".ordered.inversed", Inst->getNextNode());
  Instruction *Result = BinaryOperator::CreateNot(
      InverseFCmp, InverseFCmp->getName() + ".not", InverseFCmp->getNextNode());
  Inst->replaceAllUsesWith(Result);
  ToErase.push_back(Inst);

  return true;
}

static GenXIntrinsic::ID convertMinMaxIntrinsic(unsigned ID) {
  switch (ID) {
  case Intrinsic::minnum:
    return GenXIntrinsic::genx_fmin;
  case Intrinsic::maxnum:
    return GenXIntrinsic::genx_fmax;
  };
  IGC_ASSERT_MESSAGE(0, "unknown min/max intrinsic");
  return GenXIntrinsic::not_any_intrinsic;
}

// Lower llvm minnum/maxnum to genx fmin/fmax.
bool GenXLowering::lowerMinMax(CallInst *CI, unsigned IntrinsicID) {
  auto *ResTy = CI->getType();
  auto *MinMaxDecl = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), convertMinMaxIntrinsic(IntrinsicID), {ResTy, ResTy});
  Value *Result = IRBuilder<>(CI).CreateCall(
      MinMaxDecl, {CI->getArgOperand(0), CI->getArgOperand(1)}, CI->getName());
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}

// Lower llvm.sqrt to genx.ieee.sqrt equivalent.
bool GenXLowering::lowerSqrt(CallInst *CI) {
  IGC_ASSERT_MESSAGE(GenXIntrinsic::getAnyIntrinsicID(CI) == Intrinsic::sqrt,
    "llvm.sqrt expected");
  auto *ResTy = CI->getType();

  GenXIntrinsic::ID SqrtID = (CI->getType()->getScalarType()->isDoubleTy() ||
                              !CI->getFastMathFlags().isFast())
                                 ? GenXIntrinsic::genx_ieee_sqrt
                                 : GenXIntrinsic::genx_sqrt;

  auto *SqrtDecl =
      GenXIntrinsic::getGenXDeclaration(CI->getModule(), SqrtID, {ResTy});
  Value *Result = IRBuilder<>(CI).CreateCall(SqrtDecl, {CI->getArgOperand(0)},
                                             CI->getName());
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}

// Some GenX-specific mul intrinsics require special lowering
bool GenXLowering::lowerGenXMul(CallInst *CI, unsigned IID) {
  // DxD -> Q mul should be handled specially if MulDDQ is NOT desired
  if (ST->useMulDDQ())
    return false;
  if (IID == GenXIntrinsic::genx_sumul || IID == GenXIntrinsic::genx_usmul)
    return false;
  IGC_ASSERT(IID == GenXIntrinsic::genx_uumul ||
             IID == GenXIntrinsic::genx_ssmul);

  auto ScalarType = [](Value *V) { return V->getType()->getScalarType(); };
  if (!ScalarType(CI)->isIntegerTy(64))
    return false;
  Value *LH = CI->getOperand(0);
  Value *RH = CI->getOperand(1);
  if (!ScalarType(LH)->isIntegerTy(32) || !ScalarType(RH)->isIntegerTy(32))
    return false;

  auto *M = CI->getModule();
  Type *tys[2] = {LH->getType(), RH->getType()};

  Function *MulHFunc =
      (IID == GenXIntrinsic::genx_uumul)
          ? GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_umulh, tys)
          : GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_smulh,
                                              tys);

  IRBuilder<> B(CI);
  Value *MulLo = B.CreateMul(LH, RH, CI->getName() + ".lo.");
  Value *MulHi = B.CreateCall(MulHFunc, {LH, RH}, CI->getName() + ".hi.");

  IGC_ASSERT(MulLo->getType() == MulHi->getType());
  if (!MulLo->getType()->isVectorTy()) {
    auto *VTy = IGCLLVM::FixedVectorType::get(B.getInt32Ty(), 1);
    MulLo = B.CreateBitCast(MulLo, VTy, MulLo->getName() + "v.");
    MulHi = B.CreateBitCast(MulHi, VTy, MulHi->getName() + "v.");
  }

  IVSplitter SplitBuilder(*CI);
  Value *Result = SplitBuilder.combineLoHiSplit(
      {MulLo, MulHi}, CI->getName() + ".", CI->getType()->isIntegerTy());
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}
// Lower integer mul with saturation since VISA support mul.sat only for float.
bool GenXLowering::lowerGenXMulSat(CallInst *CI, unsigned IntrinsicID) {
  auto IsSignedMulSat = [](unsigned ID) -> std::pair<bool, bool> {
    switch (ID) {
    case GenXIntrinsic::genx_uumul_sat:
      return {false, false};
    case GenXIntrinsic::genx_usmul_sat:
      return {false, true};
    case GenXIntrinsic::genx_sumul_sat:
      return {true, false};
    case GenXIntrinsic::genx_ssmul_sat:
      return {true, true};
    default:
      llvm_unreachable("Inst should be *mul.sat intrinsic");
    }
  };

  auto GetTruncSatIntrinsicId = [](bool ResSigned, bool OpSigned) {
    return ResSigned ? (OpSigned ? GenXIntrinsic::genx_sstrunc_sat
                                 : GenXIntrinsic::genx_sutrunc_sat)
                     : (OpSigned ? GenXIntrinsic::genx_ustrunc_sat
                                 : GenXIntrinsic::genx_uutrunc_sat);
  };

  auto [IsSignedRes, IsSignedOps] = IsSignedMulSat(IntrinsicID);

  Type *ResType = CI->getType();
  IGC_ASSERT(ResType->isIntOrIntVectorTy());

  IGC_ASSERT(CI->getOperand(0)->getType() == CI->getOperand(1)->getType());
  Type *OpType = CI->getOperand(0)->getType();
  IGC_ASSERT(OpType->isIntOrIntVectorTy());

  // Create type that doesn't overflow in multiplication.
  unsigned OpTypeWidth =
      cast<IntegerType>(OpType->getScalarType())->getBitWidth();
  IGC_ASSERT_MESSAGE(OpTypeWidth != 64, "i64 types are not supported");
  Type *MulType = IntegerType::get(OpType->getContext(), 2 * OpTypeWidth);
  if (OpType->isVectorTy())
    MulType =
        IGCLLVM::FixedVectorType::get(MulType, cast<VectorType>(OpType)->getNumElements());

  IRBuilder<> B(CI);
  auto ExtendMulOperand = [&, IsSignedOps=IsSignedOps](Value *Val) {
    return IsSignedOps ? B.CreateSExt(Val, MulType, Val->getName() + ".sext")
                       : B.CreateZExt(Val, MulType, Val->getName() + ".zext");
  };
  Value *Mul = B.CreateMul(ExtendMulOperand(CI->getOperand(0)),
                           ExtendMulOperand(CI->getOperand(1)), CI->getName());
  Function *TruncSatFunc = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), GetTruncSatIntrinsicId(IsSignedRes, IsSignedOps),
      {ResType, MulType});
  Value *Result = B.CreateCall(TruncSatFunc, {Mul}, CI->getName() + ".sat");
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}

// Generic lowering for mul64
// GenX backend does not support 64-bit multiplication, so we try
// To lower it to a sequence of 32-bit ops
bool GenXLowering::lowerMul64(Instruction *Inst) {

  IVSplitter SplitBuilder(*Inst);
  if (!SplitBuilder.IsI64Operation())
    return false;

  IRBuilder<> Builder(Inst);

  auto Src0 = SplitBuilder.splitOperandLoHi(0);
  auto Src1 = SplitBuilder.splitOperandLoHi(1);

  // create muls and adds
  auto *ResL = Builder.CreateMul(Src0.Lo, Src1.Lo);
  // create the mulh intrinsic to the get the carry-part
  Type *tys[2] = {ResL->getType(), Src0.Lo->getType()};
  // build argument list
  SmallVector<llvm::Value *, 2> args{Src0.Lo, Src1.Lo};
  auto *M = Inst->getModule();
  Function *IntrinFunc =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_umulh, tys);

  auto *Cari = Builder.CreateCall(IntrinFunc, args, ".cari");
  auto *Temp0 = Builder.CreateMul(Src0.Lo, Src1.Hi);
  auto *Temp1 = Builder.CreateAdd(Cari, Temp0);
  auto *Temp2 = Builder.CreateMul(Src0.Hi, Src1.Lo);
  auto *ResH = Builder.CreateAdd(Temp2, Temp1);

  // create the bitcast to the destination-type
  auto *Replace = SplitBuilder.combineLoHiSplit({ ResL, ResH }, "mul64",
                                                Inst->getType()->isIntegerTy());
  Inst->replaceAllUsesWith(Replace);
  ToErase.push_back(Inst);
  return true;
}
bool GenXLowering::lowerLzd(Instruction *Inst) {
  const unsigned OpIndex = 0;
  IVSplitter SplitBuilder(*Inst, &OpIndex);
  if (!SplitBuilder.IsI64Operation())
    return false;

  IRBuilder<> Builder(Inst);

  auto Src = SplitBuilder.splitOperandLoHi(0);

  auto *VTy32 = cast<VectorType>(Src.Lo->getType());
  IGC_ASSERT(VTy32->getScalarType() == Builder.getInt32Ty());
  auto *Zero = ConstantInt::getNullValue(VTy32);
  auto *K32 = ConstantInt::get(VTy32, 32);

  auto *LzdF = GenXIntrinsic::getAnyDeclaration(
      Inst->getModule(), GenXIntrinsic::genx_lzd, {Src.Lo->getType()});

  // Lzd64 is lowered as:
  // LoLzd  = lzd(Src.Lo)
  // HiLzd  = lzd(Src.Hi)
  // Result = (Src.Hi == 0) ? (LoLzd + 32) : HiLzd
  auto *VlzdLo = Builder.CreateCall(LzdF, Src.Lo, "lower.lzd64.lo.");
  auto *VlzdHi = Builder.CreateCall(LzdF, Src.Hi, "lower.lzd64.hi.");
  auto *FlagHiZero = Builder.CreateICmpEQ(Src.Hi, Zero, "lower.lzd64.hicmp.");
  auto *LoPathResult = Builder.CreateAdd(VlzdLo, K32, "lower.lzd64.lores.");
  auto *Result =
      Builder.CreateSelect(FlagHiZero, LoPathResult, VlzdHi, "lower.lzd32.");
  // TODO: allow lzd to have type of destination != operand type
  if (Inst->getType()->getScalarType()->isIntegerTy(64)) {
    Result = Builder.CreateZExt(Result,
                                VectorType::getExtendedElementVectorType(VTy32),
                                "lower.lzd64.");
  }
  auto *CastedResult =
      scalarizeOrVectorizeIfNeeded(cast<Instruction>(Result), Inst);
  if (CastedResult)
    Result = CastedResult;
  Inst->replaceAllUsesWith(Result);
  Result->takeName(Inst);
  ToErase.push_back(Inst);
  return true;
}

#define SYCL_SLM_AS  3
bool GenXLowering::lowerLLVMMaskedLoad(CallInst* CallOp) {
  auto PtrV = CallOp->getArgOperand(0);
  assert(PtrV->getType()->isPointerTy());
  auto AS = cast<PointerType>(PtrV->getType())->getAddressSpace();
  assert(AS != SYCL_SLM_AS && "do not expect masked load from SLM");
  auto DTy = CallOp->getType();
  // convert to unaligned-block-load then select
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) +
      "svm.block.ld.unaligned";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { DTy, PtrV->getType() });
  auto VecDT = CallInst::Create(NewFDecl, { PtrV },
      "svm.block.ld.unaligned", CallOp);
  IRBuilder<> Builder(CallOp);
  auto RepI = Builder.CreateSelect(CallOp->getArgOperand(2), VecDT,
      CallOp->getArgOperand(3), CallOp->getName());
  CallOp->replaceAllUsesWith(RepI);
  ToErase.push_back(CallOp);
  return true;
}

bool GenXLowering::lowerLLVMMaskedStore(CallInst* CallOp) {
  auto PtrV = CallOp->getArgOperand(1);
  assert(PtrV->getType()->isPointerTy());
  auto AS = cast<PointerType>(PtrV->getType())->getAddressSpace();
  assert(AS != SYCL_SLM_AS && "do not expected masked store to SLM");
  auto DTV = CallOp->getArgOperand(0);
  // convert to unaligned-block-load then select
  // then block-store
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) +
      "svm.block.ld.unaligned";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { DTV->getType(), PtrV->getType() });
  auto VecDT = CallInst::Create(NewFDecl, { PtrV },
      "svm.block.ld.unaligned", CallOp);
  IRBuilder<> Builder(CallOp);
  auto SelI = Builder.CreateSelect(CallOp->getArgOperand(3), DTV, VecDT);

  IntrName = std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) +
       "svm.block.st";
  ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { PtrV->getType(), DTV->getType() });
  CallInst::Create(
      NewFDecl, { PtrV, SelI },
      NewFDecl->getReturnType()->isVoidTy() ? "" : "svm.block.st",
      CallOp);
  ToErase.push_back(CallOp);
  return true;
}

bool GenXLowering::lowerLLVMMaskedGather(CallInst* CallOp) {
  auto PtrV = CallOp->getArgOperand(0);
  auto MaskV = CallOp->getArgOperand(2);
  auto OldV = CallOp->getArgOperand(3);
  auto DTy = CallOp->getType();
  assert(PtrV->getType()->isVectorTy() && DTy->isVectorTy());
  auto PtrETy = cast<VectorType>(PtrV->getType())->getElementType();
  assert(PtrETy->isPointerTy());
  auto AS = cast<PointerType>(PtrETy)->getAddressSpace();
  assert(AS != SYCL_SLM_AS && "do not expect masked gather from SLM");
  auto EltTy = cast<VectorType>(DTy)->getElementType();
  auto NumElts = cast<VectorType>(DTy)->getNumElements();
  auto EltBytes = EltTy->getPrimitiveSizeInBits() / 8;
  int EltsPerLane = 1;
  // for short or byte type, load-data is 4 bytes per lane
  if (EltBytes < 4) {
    EltTy = Type::getInt8Ty(EltTy->getContext());
    EltsPerLane = 4;
  }
  auto VDTy = IGCLLVM::FixedVectorType::get(EltTy, NumElts * EltsPerLane);
  auto CIntTy = Type::getInt32Ty(EltTy->getContext());
  int nb = (EltBytes == 2) ? 1 : 0; // 0/1/2/3 means 1/2/4/8 blks
  auto NumBlksC = ConstantInt::get(CIntTy, nb);
  Value* RepI = nullptr;
  IRBuilder<> Builder(CallOp);
  // need write-region for old-data in byte or short type
  if (EltBytes < 4) {
    if (isa<UndefValue>(OldV))
      OldV = UndefValue::get(VDTy);
    else {
      // zext to i32 type
      auto VInt32 = IGCLLVM::FixedVectorType::get(CIntTy, NumElts);
      auto ExtI = Builder.CreateZExt(OldV, VInt32, OldV->getName());
      // bitcast to 4xi8
      OldV = Builder.CreateBitCast(ExtI, VDTy, ExtI->getName());
    }
  }
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "svm.gather";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { VDTy, MaskV->getType(), PtrV->getType() });
  RepI = CallInst::Create(NewFDecl, { MaskV, NumBlksC, PtrV, OldV },
      CallOp->getName(), CallOp);
  if (EltBytes < 4) {
    // bitcast to i32 type
    auto VInt32 = IGCLLVM::FixedVectorType::get(CIntTy, NumElts);
    auto CastI = Builder.CreateBitCast(RepI, VInt32, RepI->getName());
    // trunc to the dst type
    RepI = Builder.CreateTrunc(CastI, DTy, CastI->getName());
  }
  CallOp->replaceAllUsesWith(RepI);
  ToErase.push_back(CallOp);
  return true;
}

bool GenXLowering::lowerLLVMMaskedScatter(CallInst* CallOp) {
  auto DataV = CallOp->getArgOperand(0);
  auto PtrV = CallOp->getArgOperand(1);
  auto MaskV = CallOp->getArgOperand(3);
  auto DTy = DataV->getType();
  assert(PtrV->getType()->isVectorTy() && DTy->isVectorTy());
  auto PtrETy = cast<VectorType>(PtrV->getType())->getElementType();
  assert(PtrETy->isPointerTy());
  auto AS = cast<PointerType>(PtrETy)->getAddressSpace();
  assert(AS != SYCL_SLM_AS && "do not expect masked scatter to SLM");
  auto EltTy = cast<VectorType>(DTy)->getElementType();
  auto NumElts = cast<VectorType>(DTy)->getNumElements();
  auto EltBytes = EltTy->getPrimitiveSizeInBits() / 8;
  int EltsPerLane = 1;
  // for short or byte type, store-data is 4 bytes per lane
  if (EltBytes < 4) {
    EltTy = Type::getInt8Ty(EltTy->getContext());
    EltsPerLane = 4;
  }
  auto VDTy = IGCLLVM::FixedVectorType::get(EltTy, NumElts * EltsPerLane);
  auto CIntTy = Type::getInt32Ty(EltTy->getContext());
  int nb = (EltBytes == 2) ? 1 : 0; // 0/1/2/3 means 1/2/4/8 blks
  auto NumBlksC = ConstantInt::get(CIntTy, nb);
  // need write-region for old-data in byte or short type
  IRBuilder<> Builder(CallOp);
  if (EltBytes < 4) {
    // zext to i32 type
    auto VInt32 = IGCLLVM::FixedVectorType::get(CIntTy, NumElts);
    auto ExtI = Builder.CreateZExt(DataV, VInt32, DataV->getName());
    // bitcast to 4xi8
    DataV = Builder.CreateBitCast(ExtI, VDTy, ExtI->getName());
  }
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "svm.scatter";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { MaskV->getType(), PtrV->getType(), VDTy });
  CallInst::Create(NewFDecl, { MaskV, NumBlksC, PtrV, DataV },
      CallOp->getName(), CallOp);
  ToErase.push_back(CallOp);
  return true;
}

/***********************************************************************
 * widenByteOp : widen a vector byte operation to short if that might
 *               improve code
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * Gen has restrictions on byte operands. The jitter copes with that, but
 * sometimes it needs to do even-odd splitting, which can lead to suboptimal
 * code if cmps and predicates are involved.
 * Here we attempt to pick up the common cases by converting a byte operation
 * to short.
 *
 * Note that we might end up with the extends being baled into the instruction
 * anyway, resulting in a byte operation in vISA.
 */
bool GenXLowering::widenByteOp(Instruction *Inst) {
  if (!EnableGenXByteWidening)
    return false;
  Type *Ty = Inst->getType();
  if (isa<CmpInst>(Inst))
    Ty = Inst->getOperand(0)->getType();
  if (!isa<VectorType>(Ty) || !Ty->getScalarType()->isIntegerTy(8))
    return false; // not byte operation
  if (Inst->use_empty())
    return false; // result unused
  // check use, if use is a phi, stop widenning
  if (!isa<CmpInst>(Inst)) {
    for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui) {
      auto User = cast<Instruction>(ui->getUser());
      if (isa<PHINode>(User))
        return false;
    }
  }
  // For a predicated wrregion, widen by separating the predication into a
  // rdregion and select, which can then be widened.
  if (GenXIntrinsic::isWrRegion(Inst)) {
    Region R(Inst, BaleInfo());
    if (R.NumElements == 1 || !R.Mask)
      return false;
    // Can only do this if the predicate is the right size. (We could handle
    // the wrong size case by adding an rdpredregion, but then we would need
    // to ensure that GenXLegalization can cope with an arbitrary size
    // rdpredregion.)
    if (cast<VectorType>(R.Mask->getType())->getNumElements() != R.NumElements)
      return false;
    // Create the rdregion and select.
    auto NewRd =
        R.createRdRegion(Inst->getOperand(0), Inst->getName() + ".byteselrdr",
                         Inst, Inst->getDebugLoc());
    auto NewSel =
        SelectInst::Create(R.Mask, Inst->getOperand(1), NewRd, "", Inst);
    NewSel->takeName(Inst);
    NewSel->setDebugLoc(Inst->getDebugLoc());
    // Modify the existing wrregion.
    Inst->setName(NewSel->getName() + ".byteselwrr");
    Inst->setOperand(1, NewSel);
    Inst->setOperand(GenXIntrinsic::GenXRegion::PredicateOperandNum,
                     Constant::getAllOnesValue(R.Mask->getType()));
    // Fall through for the select to get widened.
    Inst = NewSel;
  }
  // Do the widening for:
  // 1. a compare or select
  // 2. used in a zext that indicates that the user has probably already been
  //    widened by this code.
  bool Widen = false;
  if (isa<CmpInst>(Inst) || isa<SelectInst>(Inst))
    Widen = true;
  else {
    auto user = cast<Instruction>(Inst->use_begin()->getUser());
    if (isa<ZExtInst>(user))
      Widen = true;
  }
  if (!Widen)
    return false;
  // Widen to short.
  // Decide whether to zero or sign extend. Also decide whether the result is
  // guaranteed to have all 0 bits in the extended part.
  Instruction::CastOps ExtOpcode = Instruction::ZExt;
  bool ExtendedIsZero = false;
  switch (Inst->getOpcode()) {
  case Instruction::SDiv:
  case Instruction::AShr:
    ExtOpcode = Instruction::SExt;
    break;
  case Instruction::And:
  case Instruction::Or:
  case Instruction::Xor:
  case Instruction::LShr:
    ExtendedIsZero = true;
    break;
  case Instruction::ICmp:
    if (cast<CmpInst>(Inst)->isSigned())
      ExtOpcode = Instruction::SExt;
    break;
  default:
    break;
  }
  // Get the range of operands to process.
  unsigned StartIdx = 0, EndIdx = Inst->getNumOperands();
  if (auto CI = dyn_cast<CallInst>(Inst))
    EndIdx = CI->getNumArgOperands();
  else if (isa<SelectInst>(Inst))
    StartIdx = 1;
  // Extend the operands.
  auto ExtTy = IGCLLVM::FixedVectorType::get(
      Type::getInt16Ty(Inst->getContext()),
      cast<VectorType>(Inst->getOperand(StartIdx)->getType())
          ->getNumElements());
  SmallVector<Value *, 4> Opnds;
  for (unsigned Idx = 0; Idx != EndIdx; ++Idx) {
    Value *Opnd = Inst->getOperand(Idx);
    if (Idx >= StartIdx) {
      if (auto C = dyn_cast<Constant>(Opnd))
        Opnd = ConstantExpr::getCast(ExtOpcode, C, ExtTy);
      else {
        auto NewExt = CastInst::Create(ExtOpcode, Opnd, ExtTy,
                                       Inst->getName() + ".byteext", Inst);
        NewExt->setDebugLoc(Inst->getDebugLoc());
        Opnd = NewExt;
      }
    }
    Opnds.push_back(Opnd);
  }
  // Create the replacement instruction.
  Instruction *NewInst = nullptr;
  if (isa<BinaryOperator>(Inst))
    NewInst = BinaryOperator::Create((Instruction::BinaryOps)Inst->getOpcode(),
                                     Opnds[0], Opnds[1], "", Inst);
  else if (auto CI = dyn_cast<CmpInst>(Inst))
    NewInst = CmpInst::Create(CI->getOpcode(), CI->getPredicate(), Opnds[0],
                              Opnds[1], "", CI);
  else if (isa<SelectInst>(Inst))
    NewInst = SelectInst::Create(Opnds[0], Opnds[1], Opnds[2], "", Inst);
  else
    IGC_ASSERT_EXIT_MESSAGE(0, "unhandled instruction in widenByteOp");
  NewInst->takeName(Inst);
  NewInst->setDebugLoc(Inst->getDebugLoc());
  if (ExtendedIsZero) {
    // We know that the extended part of the result contains 0 bits. If we
    // find that any use is a zext (probably from also being byte widened
    // in this code), we can replace the use directly and save the
    // trunc/zext pair. First put the uses in a vector as the use list will
    // change under our feet.
    SmallVector<Use *, 4> Uses;
    for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui)
      Uses.push_back(&*ui);
    for (auto ui = Uses.begin(), ue = Uses.end(); ui != ue; ++ui) {
      if (auto user = dyn_cast<ZExtInst>((*ui)->getUser())) {
        if (user->getType() == NewInst->getType()) {
          user->replaceAllUsesWith(NewInst);
          ToErase.push_back(user);
          // Remove the use of Inst from the trunc so we can tell whether there
          // are any uses left below.
          *(*ui) = UndefValue::get(Inst->getType());
        }
      }
    }
  }
  if (!Inst->use_empty()) {
    // Truncate the result.
    if (!isa<CmpInst>(Inst)) {
      NewInst = CastInst::Create(Instruction::Trunc, NewInst, Inst->getType(),
                                 Inst->getName() + ".bytetrunc", Inst);
      NewInst->setDebugLoc(Inst->getDebugLoc());
    }
    // Replace uses.
    Inst->replaceAllUsesWith(NewInst);
  }
  ToErase.push_back(Inst);
  return true;
}

namespace {

// Helper class to translate load/store into proper GenX intrinsic calls.
class LoadStoreResolver {
  Instruction *Inst;
  const GenXSubtarget *ST;
  IRBuilder<> Builder;

public:
  LoadStoreResolver(Instruction *Inst, const GenXSubtarget *ST)
      : Inst(Inst), ST(ST), Builder(Inst) {}

  // Resolve this instruction and return true on success.
  bool resolve();

private:
  bool isLoad() const { return isa<LoadInst>(Inst); }
  bool isStore() const { return isa<StoreInst>(Inst); }

  const DataLayout &getDL() const {
    Function *F = Inst->getParent()->getParent();
    return F->getParent()->getDataLayout();
  }

  // Find a proper GenX intrinsic ID for this load/store instruction.
  GenXIntrinsic::ID getGenXIntrinsicID() const;

  unsigned getPointerSizeInBits() const {
    unsigned AddrSp = 0;
    if (auto LI = dyn_cast<LoadInst>(Inst))
      AddrSp = LI->getPointerAddressSpace();
    else if (auto SI = dyn_cast<StoreInst>(Inst))
      AddrSp = SI->getPointerAddressSpace();
    return getDL().getPointerSizeInBits(AddrSp);
  }

  unsigned getValueSizeInBits(Type *T) const {
    if (auto PT = dyn_cast<PointerType>(T)) {
      unsigned AddrSp = PT->getAddressSpace();
      return getDL().getPointerSizeInBits(AddrSp);
    }
    return T->getPrimitiveSizeInBits();
  }

  bool isVectorInst() const {
    if (auto *LI = dyn_cast<LoadInst>(Inst))
      return LI->getType()->isVectorTy();

    auto *SI = cast<StoreInst>(Inst);
    return SI->getValueOperand()->getType()->isVectorTy();
  }

  // Return true if this load/store can be translated.
  bool isSupported() const;

  // Emit actual intrinsic calls.
  bool emitGather();
  bool emitScatter();
  bool emitSVMGather();
  bool emitSVMScatter();
  bool emitSVMBlockLD();
  bool emitSVMBlockST();
  bool emitOwordLD();
  bool emitOwordST();

  // Helpers
  std::tuple<IGCLLVM::FixedVectorType *, Type *, Value *>
  castValueIfNeeded(IGCLLVM::FixedVectorType *ValTy, Value *Val = nullptr);
};

} // namespace

// Translate store instructions into genx builtins.
bool GenXLowering::lowerLoadStore(Instruction *Inst) {
  LoadStoreResolver Resolver(Inst, ST);
  if (Resolver.resolve()) {
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

bool LoadStoreResolver::resolve() {
  if (!isSupported())
    return false;

  GenXIntrinsic::ID ID = getGenXIntrinsicID();
  switch (ID) {
  case GenXIntrinsic::genx_gather_scaled:
    return emitGather();
  case GenXIntrinsic::genx_scatter_scaled:
    return emitScatter();
  case GenXIntrinsic::genx_svm_gather:
    return emitSVMGather();
  case GenXIntrinsic::genx_svm_scatter:
    return emitSVMScatter();
  case GenXIntrinsic::genx_svm_block_ld:
    return emitSVMBlockLD();
  case GenXIntrinsic::genx_svm_block_st:
    return emitSVMBlockST();
  case GenXIntrinsic::genx_oword_ld:
    return emitOwordLD();
  case GenXIntrinsic::genx_oword_st:
    return emitOwordST();
  default:
    break;
  }

  return false;
}

// Return true if this load/store can be translated.
bool LoadStoreResolver::isSupported() const {
  auto IsGlobalLoadStore = [=]() {
    Value *Ptr = nullptr;
    if (auto LI = dyn_cast<LoadInst>(Inst))
      Ptr = LI->getPointerOperand();
    if (auto SI = dyn_cast<StoreInst>(Inst))
      Ptr = SI->getPointerOperand();
    return getUnderlyingGlobalVariable(Ptr) != nullptr;
  };

  if (IsGlobalLoadStore())
    return false;

  Type *ValTy = Inst->getType();
  if (auto SI = dyn_cast<StoreInst>(Inst))
    ValTy = SI->getValueOperand()->getType();

  // Use scalar type for vector: it will be resolved separately
  unsigned VecWidth = 0;
  if (auto *VecTy = dyn_cast<IGCLLVM::FixedVectorType>(ValTy)) {
    ValTy = VecTy->getScalarType();
    VecWidth = VecTy->getNumElements();
  }

  // Only scalar data types.
  if (!ValTy->isFloatingPointTy() && !ValTy->isIntegerTy() &&
      !ValTy->isPointerTy()) {
    Inst->getContext().emitError(Inst, "unsupported type for load/store");
    return false;
  }

  // Only legal types: float, double, half, i8, i16, 132, i64, pointer types.
  unsigned NumBits = getValueSizeInBits(ValTy);
  if (NumBits < 8 || NumBits > 64 || !isPowerOf2_32(NumBits)) {
    Inst->getContext().emitError("unsupported integer type for load/store");
    return false;
  }

  // Only legal vec size: 16/32/64/128 bytes
  // TODO: it is possible to handle any vector by dividing it into
  // separate instructions. This is uncommon case so it is not
  // implemented now.
  if (VecWidth) {
    unsigned NumBytes = VecWidth * (NumBits / ByteBits);
    if (NumBytes < 16 || NumBytes > 128 || !isPowerOf2_32(NumBytes)) {
      Inst->getContext().emitError("unsupported vector type for load/store");
    }
  }

  // Translate this instruction.
  return true;
}

// Find a proper GenX intrinsic ID for this load/store instruction.
GenXIntrinsic::ID LoadStoreResolver::getGenXIntrinsicID() const {
  unsigned NBits = getPointerSizeInBits();
  if (isVectorInst()) {
    if (NBits == 32)
      return isLoad() ? GenXIntrinsic::genx_oword_ld
                      : GenXIntrinsic::genx_oword_st;
    return isLoad() ? GenXIntrinsic::genx_svm_block_ld
                    : GenXIntrinsic::genx_svm_block_st;
  }

  // A32 byte scattered stateless messages only work on CNL+.
  if (NBits == 32 && ST && !ST->WaNoA32ByteScatteredStatelessMessages())
    return isLoad() ? GenXIntrinsic::genx_gather_scaled
                    : GenXIntrinsic::genx_scatter_scaled;

  return isLoad() ? GenXIntrinsic::genx_svm_gather
                  : GenXIntrinsic::genx_svm_scatter;
}

bool LoadStoreResolver::emitGather() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto LI = cast<LoadInst>(Inst);

  // Global offset.
  Value *Addr = LI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);

  unsigned NBlocks = getValueSizeInBits(LI->getType()) / 8;
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  // If this is more than 4 bytes, use a larger SIMD size.
  unsigned SIMD = 1;
  if (NBlocks > 4) {
    SIMD = NBlocks / 4;
    NBlocksLog2 = 2;
  }

  // The old value is undef.
  Type *ValTy = LI->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Type *DataTy = IGCLLVM::FixedVectorType::get(ValTy, 1);
  if (SIMD > 1)
    DataTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *OldVal = UndefValue::get(DataTy);

  // Offset.
  Type *EltOffsetTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *EltOffset = Constant::getNullValue(EltOffsetTy);
  if (SIMD > 1) {
    SmallVector<uint32_t, 2> Offsets(SIMD);
    for (unsigned i = 0; i < SIMD; ++i)
      // Increase offset by 4 bytes for each lane.
      Offsets[i] = i * 4;
    EltOffset = ConstantDataVector::get(Inst->getContext(), Offsets);
  }

  // Arguments.
  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), SIMD)),
      Builder.getInt32(NBlocksLog2), // log[2](NBlocks)
      Builder.getInt16(0),           // scale
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // global offset
      EltOffset,                                         // element offset
      OldVal                                             // old value
  };

  // Overload with return type, predicate type and element offset type
  Type *Tys[] = {OldVal->getType(), Args[0]->getType(), EltOffsetTy};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_gather_scaled, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  NewVal = Builder.CreateBitCast(NewVal, ValTy);
  LI->replaceAllUsesWith(NewVal);
  return true;
}

bool LoadStoreResolver::emitScatter() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto SI = cast<StoreInst>(Inst);

  // Global offset.
  Value *Addr = SI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);

  Value *Val = SI->getValueOperand();
  unsigned NBlocks = getValueSizeInBits(Val->getType()) / 8;
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  // If this is more than 4 bytes, use a larger SIMD size.
  unsigned SIMD = 1;
  if (NBlocks > 4) {
    SIMD = NBlocks / 4;
    NBlocksLog2 = 2;
  }

  // Value to write.
  Type *ValTy = (SIMD > 1) ? Builder.getInt32Ty() : Val->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Val = Builder.CreateBitCast(Val, IGCLLVM::FixedVectorType::get(ValTy, SIMD));

  // Offset.
  Type *EltOffsetTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *EltOffset = Constant::getNullValue(EltOffsetTy);
  if (SIMD > 1) {
    SmallVector<uint32_t, 2> Offsets(SIMD);
    // Increase offset by 4 bytes for each lane.
    for (unsigned i = 0; i < SIMD; ++i)
      Offsets[i] = i * 4;
    EltOffset = ConstantDataVector::get(Inst->getContext(), Offsets);
  }

  // Arguments.
  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), SIMD)),
      Builder.getInt32(NBlocksLog2), // log[2](NBlocks)
      Builder.getInt16(0),           // scale
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // global offset
      EltOffset,                                         // element offset
      Val                                                // value to write
  };

  // Overload with predicate type, element offset type, value to write type.
  Type *Tys[] = {Args[0]->getType(), EltOffsetTy, Val->getType()};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_scatter_scaled, Tys);
  Builder.CreateCall(Fn, Args);
  return true;
}

// Compute the block size and the number of blocks for svm gather/scatter.
//
// Block_Size, 1, 4, 8
// Num_Blocks, 1, 2, 4,
//             8 only valid for 4 byte blocks and execution size 8.
//
static unsigned getBlockCount(Type *Ty) {
  unsigned NumBytes = Ty->getPrimitiveSizeInBits() / 8;
  IGC_ASSERT_MESSAGE(NumBytes <= 8, "out of sync");

  // If this is N = 2 byte data, use 2 blocks;
  // otherwise, use 1 block of N bytes.
  return (NumBytes == 2) ? NumBytes : 1U;
}

// Translate store to svm scatter.
bool LoadStoreResolver::emitSVMGather() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto LI = cast<LoadInst>(Inst);

  // Address.
  Value *Addr = LI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);
  if (NBits == 32)
    Addr = Builder.CreateZExt(Addr, Builder.getInt64Ty());
  Addr = Builder.CreateBitCast(
      Addr, IGCLLVM::FixedVectorType::get(Addr->getType(), 1));

  // The old value is undef.
  Type *ValTy = LI->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Type *DataTy = IGCLLVM::FixedVectorType::get(ValTy, 1);
  Value *OldVal = UndefValue::get(DataTy);

  // Num of blocks.
  unsigned NBlocks = getBlockCount(OldVal->getType());
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), 1)),
      Builder.getInt32(NBlocksLog2), // log2(num_of_blocks)
      Addr,                          // addresses
      OldVal                         // old value
  };

  // Overload with return type, predicate type and address vector type
  Type *Tys[] = {OldVal->getType(), Args[0]->getType(), Addr->getType()};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_svm_gather, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  NewVal = Builder.CreateBitCast(NewVal, ValTy);
  if (LI->getType()->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

bool LoadStoreResolver::emitSVMScatter() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto SI = cast<StoreInst>(Inst);

  // Address
  Value *Addr = SI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);
  if (NBits == 32)
    Addr = Builder.CreateZExt(Addr, Builder.getInt64Ty());
  Addr = Builder.CreateBitCast(
      Addr, IGCLLVM::FixedVectorType::get(Addr->getType(), 1));

  // data to write.
  Value *Val = SI->getValueOperand();
  Type *ValTy = Val->getType();
  if (ValTy->isPointerTy()) {
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
    Val = Builder.CreatePtrToInt(Val, ValTy);
  }
  Val = Builder.CreateBitCast(Val, IGCLLVM::FixedVectorType::get(ValTy, 1));

  // Num of blocks.
  unsigned NBlocks = getBlockCount(Val->getType());
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), 1)),
      Builder.getInt32(NBlocksLog2), // log2(num_of_blocks)
      Addr,                          // addresses
      Val                            // value to write
  };

  // Overload with predicate type, address vector type, and data type
  Type *Tys[] = {Args[0]->getType(), Addr->getType(), Val->getType()};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_svm_scatter, Tys);

  Builder.CreateCall(Fn, Args);
  return true;
}

// Helper for pointer value cast.
// Returns [Casted ValTy, Old ScalarTy, Casted Value]
std::tuple<IGCLLVM::FixedVectorType *, Type *, Value *>
LoadStoreResolver::castValueIfNeeded(IGCLLVM::FixedVectorType *ValTy,
                                     Value *Val) {
  Type *ScalarTy = ValTy->getScalarType();
  if (ScalarTy->isPointerTy()) {
    unsigned VecWidth = ValTy->getNumElements();
    ValTy = IGCLLVM::FixedVectorType::get(
        Builder.getIntNTy(getValueSizeInBits(ScalarTy)), VecWidth);
    if (Val)
      Val = Builder.CreatePtrToInt(Val, ValTy);
  }

  return {ValTy, ScalarTy, Val};
}

// Translate load to svm block ld.
bool LoadStoreResolver::emitSVMBlockLD() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 64,
                     "Incorrect ptr size for svm block ld");
  auto *LI = cast<LoadInst>(Inst);

  // Address argument should be int
  Value *Addr = LI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Analyze types
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(LI->getType());
  Type *BasicScalarTy = nullptr;
  std::tie(ValTy, BasicScalarTy, std::ignore) = castValueIfNeeded(ValTy);

  // Overload with return type and address type
  Type *Tys[] = {ValTy, Addr->getType()};
  Module *M = Inst->getModule();
  auto *Fn = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_svm_block_ld, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Addr);
  if (BasicScalarTy->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

// Translate store to svm block st.
bool LoadStoreResolver::emitSVMBlockST() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 64,
                     "Incorrect ptr size for svm block st");
  auto *SI = cast<StoreInst>(Inst);

  // Address argument should be int
  Value *Addr = SI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Data to write
  Value *Val = SI->getValueOperand();
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(Val->getType());
  std::tie(ValTy, std::ignore, Val) = castValueIfNeeded(ValTy, Val);

  // Overload with predicate type, address vector type, and data type
  Type *Tys[] = {Addr->getType(), ValTy};
  Module *M = Inst->getModule();
  auto *Fn = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_svm_block_st, Tys);

  Builder.CreateCall(Fn, {Addr, Val});
  return true;
}

// Translate load to oword ld.
bool LoadStoreResolver::emitOwordLD() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 32,
                     "Incorrect ptr size for oword ld");
  auto *LI = cast<LoadInst>(Inst);

  // Offset argument should be int
  Value *Addr = LI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Analyze types
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(LI->getType());
  Type *BasicScalarTy = nullptr;
  std::tie(ValTy, BasicScalarTy, std::ignore) = castValueIfNeeded(ValTy);

  // Arguments
  Value *Args[] = {
      Builder.getInt32(1), // is modified (ignored according to vISA spec)
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr                                               // offset
  };

  // Overload with return type
  Module *M = Inst->getModule();
  auto *Fn =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_oword_ld, ValTy);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  if (BasicScalarTy->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

// Translate store to oword st.
bool LoadStoreResolver::emitOwordST() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 32,
                     "Incorrect ptr size for oword st");
  auto *SI = cast<StoreInst>(Inst);

  // Offset argument should be int
  Value *Addr = SI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Value to write
  Value *Val = SI->getValueOperand();
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(Val->getType());
  std::tie(ValTy, std::ignore, Val) = castValueIfNeeded(ValTy, Val);

  // Arguments.
  Value *Args[] = {
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // offset
      Val                                                // value to write
  };

  // Overload with value to write type.
  Module *M = Inst->getModule();
  auto *Fn =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_oword_st, ValTy);
  Builder.CreateCall(Fn, Args);
  return true;
}
