mirror of
https://github.com/intel/llvm.git
synced 2026-01-21 04:14:03 +08:00
48ae32a33b
Summary: Refactor architecture-specific code out of llvm into llvm-bolt. Introduce MCPlusBuilder, a class that is taking over MCInstrAnalysis responsibilities, i.e. creating, analyzing, and modifying instructions. To access the builder use BC->MIB, i.e. substitute MIA with MIB. MIB is an acronym for MCInstBuilder, that's what MCPlusBuilder used to be. The name stuck, and I find it better than MPB. Instructions are still MCInst, and a bunch of BOLT-specific code still lives in LLVM, but the staff under Target/* is significantly reduced. (cherry picked from FBD7300101)
1584 lines
51 KiB
C++
1584 lines
51 KiB
C++
//===--- BinaryPasses.cpp - Binary-level analysis/optimization passes -----===//
|
|
//
|
|
// The LLVM Compiler Infrastructure
|
|
//
|
|
// This file is distributed under the University of Illinois Open Source
|
|
// License. See LICENSE.TXT for details.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "BinaryPasses.h"
|
|
#include "Passes/ReorderAlgorithm.h"
|
|
#include "llvm/Support/Options.h"
|
|
#include <numeric>
|
|
|
|
#define DEBUG_TYPE "bolt"
|
|
|
|
using namespace llvm;
|
|
|
|
namespace {
|
|
|
|
const char* dynoStatsOptName(const bolt::DynoStats::Category C) {
|
|
if (C == bolt::DynoStats::FIRST_DYNO_STAT)
|
|
return "none";
|
|
else if (C == bolt::DynoStats::LAST_DYNO_STAT)
|
|
return "all";
|
|
|
|
static std::string OptNames[bolt::DynoStats::LAST_DYNO_STAT+1];
|
|
|
|
OptNames[C] = bolt::DynoStats::Description(C);
|
|
|
|
std::replace(OptNames[C].begin(), OptNames[C].end(), ' ', '-');
|
|
|
|
return OptNames[C].c_str();
|
|
}
|
|
|
|
const char* dynoStatsOptDesc(const bolt::DynoStats::Category C) {
|
|
if (C == bolt::DynoStats::FIRST_DYNO_STAT)
|
|
return "unsorted";
|
|
else if (C == bolt::DynoStats::LAST_DYNO_STAT)
|
|
return "sorted by all stats";
|
|
|
|
return bolt::DynoStats::Description(C);
|
|
}
|
|
|
|
}
|
|
|
|
namespace opts {
|
|
|
|
extern cl::OptionCategory BoltCategory;
|
|
extern cl::OptionCategory BoltOptCategory;
|
|
|
|
extern cl::opt<unsigned> Verbosity;
|
|
extern cl::opt<bolt::BinaryFunction::SplittingType> SplitFunctions;
|
|
extern bool shouldProcess(const bolt::BinaryFunction &Function);
|
|
|
|
enum DynoStatsSortOrder : char {
|
|
Ascending,
|
|
Descending
|
|
};
|
|
|
|
static cl::opt<bool>
|
|
AggressiveSplitting("split-all-cold",
|
|
cl::desc("outline as many cold basic blocks as possible"),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<DynoStatsSortOrder>
|
|
DynoStatsSortOrderOpt("print-sorted-by-order",
|
|
cl::desc("use ascending or descending order when printing functions "
|
|
"ordered by dyno stats"),
|
|
cl::ZeroOrMore,
|
|
cl::init(DynoStatsSortOrder::Descending),
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<bool>
|
|
ICFUseDFS("icf-dfs",
|
|
cl::desc("use DFS ordering when using -icf option"),
|
|
cl::ReallyHidden,
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<bool>
|
|
MinBranchClusters("min-branch-clusters",
|
|
cl::desc("use a modified clustering algorithm geared towards minimizing "
|
|
"branches"),
|
|
cl::ZeroOrMore,
|
|
cl::Hidden,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
enum PeepholeOpts : char {
|
|
PEEP_NONE = 0x0,
|
|
PEEP_SHORTEN = 0x1,
|
|
PEEP_DOUBLE_JUMPS = 0x2,
|
|
PEEP_TAILCALL_TRAPS = 0x4,
|
|
PEEP_USELESS_BRANCHES = 0x8,
|
|
PEEP_ALL = 0xf
|
|
};
|
|
|
|
static cl::list<PeepholeOpts>
|
|
Peepholes("peepholes",
|
|
cl::CommaSeparated,
|
|
cl::desc("enable peephole optimizations"),
|
|
cl::value_desc("opt1,opt2,opt3,..."),
|
|
cl::values(
|
|
clEnumValN(PEEP_NONE, "none", "disable peepholes"),
|
|
clEnumValN(PEEP_SHORTEN, "shorten", "perform instruction shortening"),
|
|
clEnumValN(PEEP_DOUBLE_JUMPS, "double-jumps",
|
|
"remove double jumps when able"),
|
|
clEnumValN(PEEP_TAILCALL_TRAPS, "tailcall-traps", "insert tail call traps"),
|
|
clEnumValN(PEEP_USELESS_BRANCHES, "useless-branches",
|
|
"remove useless conditional branches"),
|
|
clEnumValN(PEEP_ALL, "all", "enable all peephole optimizations")),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<unsigned>
|
|
PrintFuncStat("print-function-statistics",
|
|
cl::desc("print statistics about basic block ordering"),
|
|
cl::init(0),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::list<bolt::DynoStats::Category>
|
|
PrintSortedBy("print-sorted-by",
|
|
cl::CommaSeparated,
|
|
cl::desc("print functions sorted by order of dyno stats"),
|
|
cl::value_desc("key1,key2,key3,..."),
|
|
cl::values(
|
|
#define D(name, ...) \
|
|
clEnumValN(bolt::DynoStats::name, \
|
|
dynoStatsOptName(bolt::DynoStats::name), \
|
|
dynoStatsOptDesc(bolt::DynoStats::name)),
|
|
DYNO_STATS
|
|
#undef D
|
|
clEnumValN(0xffff, ".", ".")
|
|
),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<bolt::ReorderBasicBlocks::LayoutType>
|
|
ReorderBlocks("reorder-blocks",
|
|
cl::desc("change layout of basic blocks in a function"),
|
|
cl::init(bolt::ReorderBasicBlocks::LT_NONE),
|
|
cl::values(
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_NONE,
|
|
"none",
|
|
"do not reorder basic blocks"),
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_REVERSE,
|
|
"reverse",
|
|
"layout blocks in reverse order"),
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE,
|
|
"normal",
|
|
"perform optimal layout based on profile"),
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_BRANCH,
|
|
"branch-predictor",
|
|
"perform optimal layout prioritizing branch "
|
|
"predictions"),
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_CACHE,
|
|
"cache",
|
|
"perform optimal layout prioritizing I-cache "
|
|
"behavior"),
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_CACHE_PLUS,
|
|
"cache+",
|
|
"perform layout optimizing I-cache behavior"),
|
|
clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_SHUFFLE,
|
|
"cluster-shuffle",
|
|
"perform random layout of clusters")),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<bool>
|
|
ReportStaleFuncs("report-stale",
|
|
cl::desc("print the list of functions with stale profile"),
|
|
cl::init(false),
|
|
cl::ZeroOrMore,
|
|
cl::Hidden,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
enum SctcModes : char {
|
|
SctcAlways,
|
|
SctcPreserveDirection,
|
|
SctcHeuristic
|
|
};
|
|
|
|
static cl::opt<SctcModes>
|
|
SctcMode("sctc-mode",
|
|
cl::desc("mode for simplify conditional tail calls"),
|
|
cl::init(SctcAlways),
|
|
cl::values(clEnumValN(SctcAlways, "always", "always perform sctc"),
|
|
clEnumValN(SctcPreserveDirection,
|
|
"preserve",
|
|
"only perform sctc when branch direction is "
|
|
"preserved"),
|
|
clEnumValN(SctcHeuristic,
|
|
"heuristic",
|
|
"use branch prediction data to control sctc")),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<bool>
|
|
SplitEH("split-eh",
|
|
cl::desc("split C++ exception handling code (experimental)"),
|
|
cl::ZeroOrMore,
|
|
cl::Hidden,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<unsigned>
|
|
TSPThreshold("tsp-threshold",
|
|
cl::desc("maximum number of hot basic blocks in a function for which to use "
|
|
"a precise TSP solution while re-ordering basic blocks"),
|
|
cl::init(10),
|
|
cl::ZeroOrMore,
|
|
cl::Hidden,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<unsigned>
|
|
TopCalledLimit("top-called-limit",
|
|
cl::desc("maximum number of functions to print in top called "
|
|
"functions section"),
|
|
cl::init(100),
|
|
cl::ZeroOrMore,
|
|
cl::Hidden,
|
|
cl::cat(BoltCategory));
|
|
|
|
} // namespace opts
|
|
|
|
namespace llvm {
|
|
namespace bolt {
|
|
|
|
bool BinaryFunctionPass::shouldOptimize(const BinaryFunction &BF) const {
|
|
return BF.isSimple() &&
|
|
BF.getState() == BinaryFunction::State::CFG &&
|
|
opts::shouldProcess(BF) &&
|
|
(BF.getSize() > 0);
|
|
}
|
|
|
|
bool BinaryFunctionPass::shouldPrint(const BinaryFunction &BF) const {
|
|
return BF.isSimple() && opts::shouldProcess(BF);
|
|
}
|
|
|
|
void OptimizeBodylessFunctions::analyze(
|
|
BinaryFunction &BF,
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs) {
|
|
if (BF.size() != 1 || BF.front().getNumNonPseudos() != 1)
|
|
return;
|
|
|
|
const auto *FirstInstr = BF.front().getFirstNonPseudoInstr();
|
|
if (!FirstInstr)
|
|
return;
|
|
if (!BC.MIB->isTailCall(*FirstInstr))
|
|
return;
|
|
const auto *TargetSymbol = BC.MIB->getTargetSymbol(*FirstInstr);
|
|
if (!TargetSymbol)
|
|
return;
|
|
const auto *Function = BC.getFunctionForSymbol(TargetSymbol);
|
|
if (!Function)
|
|
return;
|
|
|
|
EquivalentCallTarget[BF.getSymbol()] = Function;
|
|
}
|
|
|
|
void OptimizeBodylessFunctions::optimizeCalls(BinaryFunction &BF,
|
|
BinaryContext &BC) {
|
|
for (auto *BB : BF.layout()) {
|
|
for (auto &Inst : *BB) {
|
|
if (!BC.MIB->isCall(Inst))
|
|
continue;
|
|
const auto *OriginalTarget = BC.MIB->getTargetSymbol(Inst);
|
|
if (!OriginalTarget)
|
|
continue;
|
|
const auto *Target = OriginalTarget;
|
|
// Iteratively update target since we could have f1() calling f2()
|
|
// calling f3() calling f4() and we want to output f1() directly
|
|
// calling f4().
|
|
unsigned CallSites = 0;
|
|
while (EquivalentCallTarget.count(Target)) {
|
|
Target = EquivalentCallTarget.find(Target)->second->getSymbol();
|
|
++CallSites;
|
|
}
|
|
if (Target == OriginalTarget)
|
|
continue;
|
|
DEBUG(dbgs() << "BOLT-DEBUG: Optimizing " << BB->getName()
|
|
<< " (executed " << BB->getKnownExecutionCount()
|
|
<< " times) in " << BF
|
|
<< ": replacing call to " << OriginalTarget->getName()
|
|
<< " by call to " << Target->getName()
|
|
<< " while folding " << CallSites << " call sites\n");
|
|
BC.MIB->replaceBranchTarget(Inst, Target, BC.Ctx.get());
|
|
|
|
NumOptimizedCallSites += CallSites;
|
|
if (BB->hasProfile()) {
|
|
NumEliminatedCalls += CallSites * BB->getExecutionCount();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void OptimizeBodylessFunctions::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
if (shouldOptimize(Function)) {
|
|
analyze(Function, BC, BFs);
|
|
}
|
|
}
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
if (shouldOptimize(Function)) {
|
|
optimizeCalls(Function, BC);
|
|
}
|
|
}
|
|
|
|
if (NumEliminatedCalls || NumOptimizedCallSites) {
|
|
outs() << "BOLT-INFO: optimized " << NumOptimizedCallSites
|
|
<< " redirect call sites to eliminate " << NumEliminatedCalls
|
|
<< " dynamic calls.\n";
|
|
}
|
|
}
|
|
|
|
void EliminateUnreachableBlocks::runOnFunction(BinaryFunction& Function) {
|
|
if (Function.layout_size() > 0) {
|
|
unsigned Count;
|
|
uint64_t Bytes;
|
|
Function.markUnreachable();
|
|
DEBUG({
|
|
for (auto *BB : Function.layout()) {
|
|
if (!BB->isValid()) {
|
|
dbgs() << "BOLT-INFO: UCE found unreachable block " << BB->getName()
|
|
<< " in function " << Function << "\n";
|
|
BB->dump();
|
|
}
|
|
}
|
|
});
|
|
std::tie(Count, Bytes) = Function.eraseInvalidBBs();
|
|
DeletedBlocks += Count;
|
|
DeletedBytes += Bytes;
|
|
if (Count && opts::Verbosity > 0) {
|
|
Modified.insert(&Function);
|
|
outs() << "BOLT-INFO: Removed " << Count
|
|
<< " dead basic block(s) accounting for " << Bytes
|
|
<< " bytes in function " << Function << '\n';
|
|
}
|
|
}
|
|
}
|
|
|
|
void EliminateUnreachableBlocks::runOnFunctions(
|
|
BinaryContext&,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &
|
|
) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
if (shouldOptimize(Function)) {
|
|
runOnFunction(Function);
|
|
}
|
|
}
|
|
outs() << "BOLT-INFO: UCE removed " << DeletedBlocks << " blocks and "
|
|
<< DeletedBytes << " bytes of code.\n";
|
|
}
|
|
|
|
bool ReorderBasicBlocks::shouldPrint(const BinaryFunction &BF) const {
|
|
return (BinaryFunctionPass::shouldPrint(BF) &&
|
|
opts::ReorderBlocks != ReorderBasicBlocks::LT_NONE);
|
|
}
|
|
|
|
void ReorderBasicBlocks::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &LargeFunctions) {
|
|
if (opts::ReorderBlocks == ReorderBasicBlocks::LT_NONE)
|
|
return;
|
|
|
|
uint64_t ModifiedFuncCount = 0;
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
|
|
if (!shouldOptimize(Function))
|
|
continue;
|
|
|
|
const bool ShouldSplit =
|
|
(opts::SplitFunctions == BinaryFunction::ST_ALL) ||
|
|
(opts::SplitFunctions == BinaryFunction::ST_EH &&
|
|
Function.hasEHRanges()) ||
|
|
(LargeFunctions.find(It.first) != LargeFunctions.end());
|
|
modifyFunctionLayout(Function, opts::ReorderBlocks, opts::MinBranchClusters,
|
|
ShouldSplit);
|
|
|
|
if (opts::PrintFuncStat > 0 && Function.hasLayoutChanged()) {
|
|
++ModifiedFuncCount;
|
|
}
|
|
}
|
|
|
|
if (opts::PrintFuncStat > 0) {
|
|
raw_ostream &OS = outs();
|
|
// Copy all the values into vector in order to sort them
|
|
std::map<uint64_t, BinaryFunction &> ScoreMap;
|
|
for (auto It = BFs.begin(); It != BFs.end(); ++It) {
|
|
ScoreMap.insert(std::pair<uint64_t, BinaryFunction &>(
|
|
It->second.getFunctionScore(), It->second));
|
|
}
|
|
|
|
OS << "\nBOLT-INFO: Printing Function Statistics:\n\n";
|
|
OS << " There are " << BFs.size() << " functions in total. \n";
|
|
OS << " Number of functions being modified: " << ModifiedFuncCount
|
|
<< "\n";
|
|
OS << " User asks for detailed information on top "
|
|
<< opts::PrintFuncStat << " functions. (Ranked by function score)"
|
|
<< "\n\n";
|
|
uint64_t I = 0;
|
|
for (std::map<uint64_t, BinaryFunction &>::reverse_iterator
|
|
Rit = ScoreMap.rbegin();
|
|
Rit != ScoreMap.rend() && I < opts::PrintFuncStat; ++Rit, ++I) {
|
|
auto &Function = Rit->second;
|
|
|
|
OS << " Information for function of top: " << (I + 1) << ": \n";
|
|
OS << " Function Score is: " << Function.getFunctionScore()
|
|
<< "\n";
|
|
OS << " There are " << Function.size()
|
|
<< " number of blocks in this function.\n";
|
|
OS << " There are " << Function.getInstructionCount()
|
|
<< " number of instructions in this function.\n";
|
|
OS << " The edit distance for this function is: "
|
|
<< Function.getEditDistance() << "\n\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
void ReorderBasicBlocks::modifyFunctionLayout(BinaryFunction &BF,
|
|
LayoutType Type, bool MinBranchClusters, bool Split) const {
|
|
if (BF.size() == 0 || Type == LT_NONE)
|
|
return;
|
|
|
|
BinaryFunction::BasicBlockOrderType NewLayout;
|
|
std::unique_ptr<ReorderAlgorithm> Algo;
|
|
|
|
// Cannot do optimal layout without profile.
|
|
if (Type != LT_REVERSE && !BF.hasValidProfile())
|
|
return;
|
|
|
|
if (Type == LT_REVERSE) {
|
|
Algo.reset(new ReverseReorderAlgorithm());
|
|
} else if (BF.size() <= opts::TSPThreshold && Type != LT_OPTIMIZE_SHUFFLE) {
|
|
// Work on optimal solution if problem is small enough
|
|
DEBUG(dbgs() << "finding optimal block layout for " << BF << "\n");
|
|
Algo.reset(new OptimalReorderAlgorithm());
|
|
} else {
|
|
DEBUG(dbgs() << "running block layout heuristics on " << BF << "\n");
|
|
|
|
std::unique_ptr<ClusterAlgorithm> CAlgo;
|
|
if (MinBranchClusters)
|
|
CAlgo.reset(new MinBranchGreedyClusterAlgorithm());
|
|
else
|
|
CAlgo.reset(new PHGreedyClusterAlgorithm());
|
|
|
|
switch(Type) {
|
|
case LT_OPTIMIZE:
|
|
Algo.reset(new OptimizeReorderAlgorithm(std::move(CAlgo)));
|
|
break;
|
|
|
|
case LT_OPTIMIZE_BRANCH:
|
|
Algo.reset(new OptimizeBranchReorderAlgorithm(std::move(CAlgo)));
|
|
break;
|
|
|
|
case LT_OPTIMIZE_CACHE:
|
|
Algo.reset(new OptimizeCacheReorderAlgorithm(std::move(CAlgo)));
|
|
break;
|
|
|
|
case LT_OPTIMIZE_CACHE_PLUS:
|
|
Algo.reset(new CachePlusReorderAlgorithm(std::move(CAlgo)));
|
|
break;
|
|
|
|
case LT_OPTIMIZE_SHUFFLE:
|
|
Algo.reset(new RandomClusterReorderAlgorithm(std::move(CAlgo)));
|
|
break;
|
|
|
|
default:
|
|
llvm_unreachable("unexpected layout type");
|
|
}
|
|
}
|
|
|
|
Algo->reorderBasicBlocks(BF, NewLayout);
|
|
|
|
BF.updateBasicBlockLayout(NewLayout, /*SavePrevLayout=*/opts::PrintFuncStat);
|
|
|
|
if (Split)
|
|
splitFunction(BF);
|
|
}
|
|
|
|
void ReorderBasicBlocks::splitFunction(BinaryFunction &BF) const {
|
|
if (!BF.size())
|
|
return;
|
|
|
|
bool AllCold = true;
|
|
for (auto *BB : BF.layout()) {
|
|
auto ExecCount = BB->getExecutionCount();
|
|
if (ExecCount == BinaryBasicBlock::COUNT_NO_PROFILE)
|
|
return;
|
|
if (ExecCount != 0)
|
|
AllCold = false;
|
|
}
|
|
|
|
if (AllCold)
|
|
return;
|
|
|
|
// Never outline the first basic block.
|
|
BF.layout_front()->setCanOutline(false);
|
|
for (auto *BB : BF.layout()) {
|
|
if (!BB->canOutline())
|
|
continue;
|
|
if (BB->getExecutionCount() != 0) {
|
|
BB->setCanOutline(false);
|
|
continue;
|
|
}
|
|
if (BF.hasEHRanges() && !opts::SplitEH) {
|
|
// We cannot move landing pads (or rather entry points for landing
|
|
// pads).
|
|
if (BB->isLandingPad()) {
|
|
BB->setCanOutline(false);
|
|
continue;
|
|
}
|
|
// We cannot move a block that can throw since exception-handling
|
|
// runtime cannot deal with split functions. However, if we can guarantee
|
|
// that the block never throws, it is safe to move the block to
|
|
// decrease the size of the function.
|
|
for (auto &Instr : *BB) {
|
|
if (BF.getBinaryContext().MIB->isInvoke(Instr)) {
|
|
BB->setCanOutline(false);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (opts::AggressiveSplitting) {
|
|
// All blocks with 0 count that we can move go to the end of the function.
|
|
// Even if they were natural to cluster formation and were seen in-between
|
|
// hot basic blocks.
|
|
std::stable_sort(BF.layout_begin(), BF.layout_end(),
|
|
[&] (BinaryBasicBlock *A, BinaryBasicBlock *B) {
|
|
return A->canOutline() < B->canOutline();
|
|
});
|
|
} else if (BF.hasEHRanges() && !opts::SplitEH) {
|
|
// Typically functions with exception handling have landing pads at the end.
|
|
// We cannot move beginning of landing pads, but we can move 0-count blocks
|
|
// comprising landing pads to the end and thus facilitate splitting.
|
|
auto FirstLP = BF.layout_begin();
|
|
while ((*FirstLP)->isLandingPad())
|
|
++FirstLP;
|
|
|
|
std::stable_sort(FirstLP, BF.layout_end(),
|
|
[&] (BinaryBasicBlock *A, BinaryBasicBlock *B) {
|
|
return A->canOutline() < B->canOutline();
|
|
});
|
|
}
|
|
|
|
// Separate hot from cold starting from the bottom.
|
|
for (auto I = BF.layout_rbegin(), E = BF.layout_rend();
|
|
I != E; ++I) {
|
|
BinaryBasicBlock *BB = *I;
|
|
if (!BB->canOutline())
|
|
break;
|
|
BB->setIsCold(true);
|
|
}
|
|
}
|
|
|
|
void FixupBranches::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
if (BC.HasRelocations || shouldOptimize(Function)) {
|
|
if (BC.TheTriple->getArch() == llvm::Triple::aarch64 &&
|
|
!Function.isSimple())
|
|
continue;
|
|
Function.fixBranches();
|
|
}
|
|
}
|
|
}
|
|
|
|
void FinalizeFunctions::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &
|
|
) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
const auto ShouldOptimize = shouldOptimize(Function);
|
|
|
|
// Always fix functions in relocation mode.
|
|
if (!BC.HasRelocations && !ShouldOptimize)
|
|
continue;
|
|
|
|
// Fix the CFI state.
|
|
if (ShouldOptimize && !Function.fixCFIState()) {
|
|
if (BC.HasRelocations) {
|
|
errs() << "BOLT-ERROR: unable to fix CFI state for function "
|
|
<< Function << ". Exiting.\n";
|
|
exit(1);
|
|
}
|
|
Function.setSimple(false);
|
|
continue;
|
|
}
|
|
|
|
Function.setFinalized();
|
|
|
|
// Update exception handling information.
|
|
Function.updateEHRanges();
|
|
}
|
|
}
|
|
|
|
void StripAnnotations::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &
|
|
) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
|
|
for (auto &BB : Function) {
|
|
for (auto &Inst : BB) {
|
|
BC.MIB->removeAllAnnotations(Inst);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
|
|
// This peephole fixes jump instructions that jump to another basic
|
|
// block with a single jump instruction, e.g.
|
|
//
|
|
// B0: ...
|
|
// jmp B1 (or jcc B1)
|
|
//
|
|
// B1: jmp B2
|
|
//
|
|
// ->
|
|
//
|
|
// B0: ...
|
|
// jmp B2 (or jcc B2)
|
|
//
|
|
uint64_t fixDoubleJumps(BinaryContext &BC,
|
|
BinaryFunction &Function,
|
|
bool MarkInvalid) {
|
|
uint64_t NumDoubleJumps = 0;
|
|
|
|
for (auto &BB : Function) {
|
|
auto checkAndPatch = [&](BinaryBasicBlock *Pred,
|
|
BinaryBasicBlock *Succ,
|
|
const MCSymbol *SuccSym) {
|
|
// Ignore infinite loop jumps or fallthrough tail jumps.
|
|
if (Pred == Succ || Succ == &BB)
|
|
return false;
|
|
|
|
if (Succ) {
|
|
const MCSymbol *TBB = nullptr;
|
|
const MCSymbol *FBB = nullptr;
|
|
MCInst *CondBranch = nullptr;
|
|
MCInst *UncondBranch = nullptr;
|
|
auto Res = Pred->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
|
|
if(!Res) {
|
|
DEBUG(dbgs() << "analyzeBranch failed in peepholes in block:\n";
|
|
Pred->dump());
|
|
return false;
|
|
}
|
|
Pred->replaceSuccessor(&BB, Succ);
|
|
|
|
// We must patch up any existing branch instructions to match up
|
|
// with the new successor.
|
|
auto *Ctx = BC.Ctx.get();
|
|
assert((CondBranch || (!CondBranch && Pred->succ_size() == 1)) &&
|
|
"Predecessor block has inconsistent number of successors");
|
|
if (CondBranch &&
|
|
BC.MIB->getTargetSymbol(*CondBranch) == BB.getLabel()) {
|
|
BC.MIB->replaceBranchTarget(*CondBranch, Succ->getLabel(), Ctx);
|
|
} else if (UncondBranch &&
|
|
BC.MIB->getTargetSymbol(*UncondBranch) == BB.getLabel()) {
|
|
BC.MIB->replaceBranchTarget(*UncondBranch, Succ->getLabel(), Ctx);
|
|
} else if (!UncondBranch) {
|
|
assert(Function.getBasicBlockAfter(Pred, false) != Succ &&
|
|
"Don't add an explicit jump to a fallthrough block.");
|
|
Pred->addBranchInstruction(Succ);
|
|
}
|
|
} else {
|
|
// Succ will be null in the tail call case. In this case we
|
|
// need to explicitly add a tail call instruction.
|
|
auto *Branch = Pred->getLastNonPseudoInstr();
|
|
if (Branch && BC.MIB->isUnconditionalBranch(*Branch)) {
|
|
assert(BC.MIB->getTargetSymbol(*Branch) == BB.getLabel());
|
|
Pred->removeSuccessor(&BB);
|
|
Pred->eraseInstruction(Branch);
|
|
Pred->addTailCallInstruction(SuccSym);
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
++NumDoubleJumps;
|
|
DEBUG(dbgs() << "Removed double jump in " << Function << " from "
|
|
<< Pred->getName() << " -> " << BB.getName() << " to "
|
|
<< Pred->getName() << " -> " << SuccSym->getName()
|
|
<< (!Succ ? " (tail)\n" : "\n"));
|
|
|
|
return true;
|
|
};
|
|
|
|
if (BB.getNumNonPseudos() != 1 || BB.isLandingPad())
|
|
continue;
|
|
|
|
auto *Inst = BB.getFirstNonPseudoInstr();
|
|
const bool IsTailCall = BC.MIB->isTailCall(*Inst);
|
|
|
|
if (!BC.MIB->isUnconditionalBranch(*Inst) && !IsTailCall)
|
|
continue;
|
|
|
|
// If we operate after SCTC make sure it's not a conditional tail call.
|
|
if (IsTailCall && BC.MIB->isConditionalBranch(*Inst))
|
|
continue;
|
|
|
|
const auto *SuccSym = BC.MIB->getTargetSymbol(*Inst);
|
|
auto *Succ = BB.getSuccessor();
|
|
|
|
if (((!Succ || &BB == Succ) && !IsTailCall) || (IsTailCall && !SuccSym))
|
|
continue;
|
|
|
|
std::vector<BinaryBasicBlock *> Preds{BB.pred_begin(), BB.pred_end()};
|
|
|
|
for (auto *Pred : Preds) {
|
|
if (Pred->isLandingPad())
|
|
continue;
|
|
|
|
if (Pred->getSuccessor() == &BB ||
|
|
(Pred->getConditionalSuccessor(true) == &BB && !IsTailCall) ||
|
|
Pred->getConditionalSuccessor(false) == &BB) {
|
|
if (checkAndPatch(Pred, Succ, SuccSym) && MarkInvalid) {
|
|
BB.markValid(BB.pred_size() != 0 ||
|
|
BB.isLandingPad() ||
|
|
BB.isEntryPoint());
|
|
}
|
|
assert(Function.validateCFG());
|
|
}
|
|
}
|
|
}
|
|
|
|
return NumDoubleJumps;
|
|
}
|
|
|
|
}
|
|
|
|
bool SimplifyConditionalTailCalls::shouldRewriteBranch(
|
|
const BinaryBasicBlock *PredBB,
|
|
const MCInst &CondBranch,
|
|
const BinaryBasicBlock *BB,
|
|
const bool DirectionFlag
|
|
) {
|
|
if (BeenOptimized.count(PredBB))
|
|
return false;
|
|
|
|
const bool IsForward = BinaryFunction::isForwardBranch(PredBB, BB);
|
|
|
|
if (IsForward)
|
|
++NumOrigForwardBranches;
|
|
else
|
|
++NumOrigBackwardBranches;
|
|
|
|
if (opts::SctcMode == opts::SctcAlways)
|
|
return true;
|
|
|
|
if (opts::SctcMode == opts::SctcPreserveDirection)
|
|
return IsForward == DirectionFlag;
|
|
|
|
const auto Frequency = PredBB->getBranchStats(BB);
|
|
|
|
// It's ok to rewrite the conditional branch if the new target will be
|
|
// a backward branch.
|
|
|
|
// If no data available for these branches, then it should be ok to
|
|
// do the optimization since it will reduce code size.
|
|
if (Frequency.getError())
|
|
return true;
|
|
|
|
// TODO: should this use misprediction frequency instead?
|
|
const bool Result =
|
|
(IsForward && Frequency.get().first >= 0.5) ||
|
|
(!IsForward && Frequency.get().first <= 0.5);
|
|
|
|
return Result == DirectionFlag;
|
|
}
|
|
|
|
uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
|
|
BinaryFunction &BF) {
|
|
// Need updated indices to correctly detect branch' direction.
|
|
BF.updateLayoutIndices();
|
|
BF.markUnreachable();
|
|
|
|
auto &MIB = BC.MIB;
|
|
uint64_t NumLocalCTCCandidates = 0;
|
|
uint64_t NumLocalCTCs = 0;
|
|
uint64_t LocalCTCTakenCount = 0;
|
|
uint64_t LocalCTCExecCount = 0;
|
|
std::vector<std::pair<BinaryBasicBlock *,
|
|
const BinaryBasicBlock *>> NeedsUncondBranch;
|
|
|
|
// Will block be deleted by UCE?
|
|
auto isValid = [](const BinaryBasicBlock *BB) {
|
|
return (BB->pred_size() != 0 ||
|
|
BB->isLandingPad() ||
|
|
BB->isEntryPoint());
|
|
};
|
|
|
|
for (auto *BB : BF.layout()) {
|
|
// Locate BB with a single direct tail-call instruction.
|
|
if (BB->getNumNonPseudos() != 1)
|
|
continue;
|
|
|
|
auto *Instr = BB->getFirstNonPseudoInstr();
|
|
if (!MIB->isTailCall(*Instr) || BC.MIB->isConditionalBranch(*Instr))
|
|
continue;
|
|
|
|
auto *CalleeSymbol = MIB->getTargetSymbol(*Instr);
|
|
if (!CalleeSymbol)
|
|
continue;
|
|
|
|
// Detect direction of the possible conditional tail call.
|
|
const bool IsForwardCTC = BF.isForwardCall(CalleeSymbol);
|
|
|
|
// Iterate through all predecessors.
|
|
for (auto *PredBB : BB->predecessors()) {
|
|
auto *CondSucc = PredBB->getConditionalSuccessor(true);
|
|
if (!CondSucc)
|
|
continue;
|
|
|
|
++NumLocalCTCCandidates;
|
|
|
|
const MCSymbol *TBB = nullptr;
|
|
const MCSymbol *FBB = nullptr;
|
|
MCInst *CondBranch = nullptr;
|
|
MCInst *UncondBranch = nullptr;
|
|
auto Result = PredBB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
|
|
|
|
// analyzeBranch() can fail due to unusual branch instructions, e.g. jrcxz
|
|
if (!Result) {
|
|
DEBUG(dbgs() << "analyzeBranch failed in SCTC in block:\n";
|
|
PredBB->dump());
|
|
continue;
|
|
}
|
|
|
|
assert(Result && "internal error analyzing conditional branch");
|
|
assert(CondBranch && "conditional branch expected");
|
|
|
|
// It's possible that PredBB is also a successor to BB that may have
|
|
// been processed by a previous iteration of the SCTC loop, in which
|
|
// case it may have been marked invalid. We should skip rewriting in
|
|
// this case.
|
|
if (!PredBB->isValid()) {
|
|
assert(PredBB->isSuccessor(BB) &&
|
|
"PredBB should be valid if it is not a successor to BB");
|
|
continue;
|
|
}
|
|
|
|
// We don't want to reverse direction of the branch in new order
|
|
// without further profile analysis.
|
|
const bool DirectionFlag = CondSucc == BB ? IsForwardCTC : !IsForwardCTC;
|
|
if (!shouldRewriteBranch(PredBB, *CondBranch, BB, DirectionFlag))
|
|
continue;
|
|
|
|
// Record this block so that we don't try to optimize it twice.
|
|
BeenOptimized.insert(PredBB);
|
|
|
|
bool BranchForStats;
|
|
if (CondSucc != BB) {
|
|
// Patch the new target address into the conditional branch.
|
|
MIB->reverseBranchCondition(*CondBranch, CalleeSymbol, BC.Ctx.get());
|
|
// Since we reversed the condition on the branch we need to change
|
|
// the target for the unconditional branch or add a unconditional
|
|
// branch to the old target. This has to be done manually since
|
|
// fixupBranches is not called after SCTC.
|
|
NeedsUncondBranch.emplace_back(std::make_pair(PredBB, CondSucc));
|
|
BranchForStats = false;
|
|
} else {
|
|
// Change destination of the conditional branch.
|
|
MIB->replaceBranchTarget(*CondBranch, CalleeSymbol, BC.Ctx.get());
|
|
BranchForStats = true;
|
|
}
|
|
const auto Count = PredBB->getBranchInfo(BranchForStats).Count;
|
|
const uint64_t CTCTakenFreq =
|
|
Count == BinaryBasicBlock::COUNT_NO_PROFILE ? 0 : Count;
|
|
|
|
// Annotate it, so "isCall" returns true for this jcc
|
|
MIB->setConditionalTailCall(*CondBranch);
|
|
// Add info abount the conditional tail call frequency, otherwise this
|
|
// info will be lost when we delete the associated BranchInfo entry
|
|
BC.MIB->removeAnnotation(*CondBranch, "CTCTakenCount");
|
|
BC.MIB->addAnnotation(BC.Ctx.get(), *CondBranch, "CTCTakenCount",
|
|
CTCTakenFreq);
|
|
|
|
// Remove the unused successor which may be eliminated later
|
|
// if there are no other users.
|
|
PredBB->removeSuccessor(BB);
|
|
// Update BB execution count
|
|
if (CTCTakenFreq && CTCTakenFreq <= BB->getKnownExecutionCount()) {
|
|
BB->setExecutionCount(BB->getExecutionCount() - CTCTakenFreq);
|
|
} else if (CTCTakenFreq > BB->getKnownExecutionCount()) {
|
|
BB->setExecutionCount(0);
|
|
}
|
|
|
|
++NumLocalCTCs;
|
|
LocalCTCTakenCount += CTCTakenFreq;
|
|
LocalCTCExecCount += PredBB->getKnownExecutionCount();
|
|
}
|
|
|
|
// Remove the block from CFG if all predecessors were removed.
|
|
BB->markValid(isValid(BB));
|
|
}
|
|
|
|
// Add unconditional branches at the end of BBs to new successors
|
|
// as long as the successor is not a fallthrough.
|
|
for (auto &Entry : NeedsUncondBranch) {
|
|
auto *PredBB = Entry.first;
|
|
auto *CondSucc = Entry.second;
|
|
|
|
const MCSymbol *TBB = nullptr;
|
|
const MCSymbol *FBB = nullptr;
|
|
MCInst *CondBranch = nullptr;
|
|
MCInst *UncondBranch = nullptr;
|
|
PredBB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
|
|
|
|
// Find the next valid block. Invalid blocks will be deleted
|
|
// so they shouldn't be considered fallthrough targets.
|
|
const auto *NextBlock = BF.getBasicBlockAfter(PredBB, false);
|
|
while (NextBlock && !isValid(NextBlock)) {
|
|
NextBlock = BF.getBasicBlockAfter(NextBlock, false);
|
|
}
|
|
|
|
// Get the unconditional successor to this block.
|
|
const auto *PredSucc = PredBB->getSuccessor();
|
|
assert(PredSucc && "The other branch should be a tail call");
|
|
|
|
const bool HasFallthrough = (NextBlock && PredSucc == NextBlock);
|
|
|
|
if (UncondBranch) {
|
|
if (HasFallthrough)
|
|
PredBB->eraseInstruction(UncondBranch);
|
|
else
|
|
MIB->replaceBranchTarget(*UncondBranch,
|
|
CondSucc->getLabel(),
|
|
BC.Ctx.get());
|
|
} else if (!HasFallthrough) {
|
|
MCInst Branch;
|
|
MIB->createUncondBranch(Branch, CondSucc->getLabel(), BC.Ctx.get());
|
|
PredBB->addInstruction(Branch);
|
|
}
|
|
}
|
|
|
|
if (NumLocalCTCs > 0) {
|
|
NumDoubleJumps += fixDoubleJumps(BC, BF, true);
|
|
// Clean-up unreachable tail-call blocks.
|
|
const auto Stats = BF.eraseInvalidBBs();
|
|
DeletedBlocks += Stats.first;
|
|
DeletedBytes += Stats.second;
|
|
}
|
|
|
|
DEBUG(dbgs() << "BOLT: created " << NumLocalCTCs
|
|
<< " conditional tail calls from a total of "
|
|
<< NumLocalCTCCandidates << " candidates in function " << BF
|
|
<< ". CTCs execution count for this function is "
|
|
<< LocalCTCExecCount << " and CTC taken count is "
|
|
<< LocalCTCTakenCount << "\n";);
|
|
|
|
NumTailCallsPatched += NumLocalCTCs;
|
|
NumCandidateTailCalls += NumLocalCTCCandidates;
|
|
CTCExecCount += LocalCTCExecCount;
|
|
CTCTakenCount += LocalCTCTakenCount;
|
|
|
|
return NumLocalCTCs > 0;
|
|
}
|
|
|
|
void SimplifyConditionalTailCalls::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &
|
|
) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
|
|
if (!shouldOptimize(Function))
|
|
continue;
|
|
|
|
if (fixTailCalls(BC, Function)) {
|
|
Modified.insert(&Function);
|
|
}
|
|
}
|
|
|
|
outs() << "BOLT-INFO: SCTC: patched " << NumTailCallsPatched
|
|
<< " tail calls (" << NumOrigForwardBranches << " forward)"
|
|
<< " tail calls (" << NumOrigBackwardBranches << " backward)"
|
|
<< " from a total of " << NumCandidateTailCalls << " while removing "
|
|
<< NumDoubleJumps << " double jumps"
|
|
<< " and removing " << DeletedBlocks << " basic blocks"
|
|
<< " totalling " << DeletedBytes
|
|
<< " bytes of code. CTCs total execution count is " << CTCExecCount
|
|
<< " and the number of times CTCs are taken is " << CTCTakenCount
|
|
<< ".\n";
|
|
}
|
|
|
|
uint64_t Peepholes::shortenInstructions(BinaryContext &BC,
|
|
BinaryFunction &Function) {
|
|
MCInst DebugInst;
|
|
uint64_t Count = 0;
|
|
for (auto &BB : Function) {
|
|
for (auto &Inst : BB) {
|
|
if (opts::Verbosity > 1) {
|
|
DebugInst = Inst;
|
|
}
|
|
if (BC.MIB->shortenInstruction(Inst)) {
|
|
if (opts::Verbosity > 1) {
|
|
outs() << "BOLT-INFO: peephole, shortening:\n"
|
|
<< "BOLT-INFO: ";
|
|
BC.printInstruction(outs(), DebugInst, 0, &Function);
|
|
outs() << "BOLT-INFO: to:";
|
|
BC.printInstruction(outs(), Inst, 0, &Function);
|
|
}
|
|
++Count;
|
|
}
|
|
}
|
|
}
|
|
return Count;
|
|
}
|
|
|
|
void Peepholes::addTailcallTraps(BinaryContext &BC,
|
|
BinaryFunction &Function) {
|
|
for (auto &BB : Function) {
|
|
auto *Inst = BB.getLastNonPseudoInstr();
|
|
if (Inst && BC.MIB->isTailCall(*Inst) && BC.MIB->isIndirectBranch(*Inst)) {
|
|
MCInst Trap;
|
|
if (BC.MIB->createTrap(Trap)) {
|
|
BB.addInstruction(Trap);
|
|
++TailCallTraps;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void Peepholes::removeUselessCondBranches(BinaryContext &BC,
|
|
BinaryFunction &Function) {
|
|
for (auto &BB : Function) {
|
|
if (BB.succ_size() != 2)
|
|
continue;
|
|
|
|
auto *CondBB = BB.getConditionalSuccessor(true);
|
|
auto *UncondBB = BB.getConditionalSuccessor(false);
|
|
if (CondBB != UncondBB)
|
|
continue;
|
|
|
|
const MCSymbol *TBB = nullptr;
|
|
const MCSymbol *FBB = nullptr;
|
|
MCInst *CondBranch = nullptr;
|
|
MCInst *UncondBranch = nullptr;
|
|
auto Result = BB.analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
|
|
|
|
// analyzeBranch() can fail due to unusual branch instructions,
|
|
// e.g. jrcxz, or jump tables (indirect jump).
|
|
if (!Result || !CondBranch)
|
|
continue;
|
|
|
|
BB.removeDuplicateConditionalSuccessor(CondBranch);
|
|
++NumUselessCondBranches;
|
|
}
|
|
}
|
|
|
|
void Peepholes::runOnFunctions(BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &LargeFunctions) {
|
|
const char Opts =
|
|
std::accumulate(opts::Peepholes.begin(),
|
|
opts::Peepholes.end(),
|
|
0,
|
|
[](const char A, const opts::PeepholeOpts B) {
|
|
return A | B;
|
|
});
|
|
if (Opts == opts::PEEP_NONE)
|
|
return;
|
|
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
if (shouldOptimize(Function)) {
|
|
if (Opts & opts::PEEP_SHORTEN)
|
|
NumShortened += shortenInstructions(BC, Function);
|
|
if (Opts & opts::PEEP_DOUBLE_JUMPS)
|
|
NumDoubleJumps += fixDoubleJumps(BC, Function, false);
|
|
if (Opts & opts::PEEP_TAILCALL_TRAPS)
|
|
addTailcallTraps(BC, Function);
|
|
if (Opts & opts::PEEP_USELESS_BRANCHES)
|
|
removeUselessCondBranches(BC, Function);
|
|
}
|
|
}
|
|
outs() << "BOLT-INFO: Peephole: " << NumShortened
|
|
<< " instructions shortened.\n"
|
|
<< "BOLT-INFO: Peephole: " << NumDoubleJumps
|
|
<< " double jumps patched.\n"
|
|
<< "BOLT-INFO: Peephole: " << TailCallTraps
|
|
<< " tail call traps inserted.\n"
|
|
<< "BOLT-INFO: Peephole: " << NumUselessCondBranches
|
|
<< " useless conditional branches removed.\n";
|
|
}
|
|
|
|
bool SimplifyRODataLoads::simplifyRODataLoads(
|
|
BinaryContext &BC, BinaryFunction &BF) {
|
|
auto &MIB = BC.MIB;
|
|
|
|
uint64_t NumLocalLoadsSimplified = 0;
|
|
uint64_t NumDynamicLocalLoadsSimplified = 0;
|
|
uint64_t NumLocalLoadsFound = 0;
|
|
uint64_t NumDynamicLocalLoadsFound = 0;
|
|
|
|
for (auto *BB : BF.layout()) {
|
|
for (auto &Inst : *BB) {
|
|
unsigned Opcode = Inst.getOpcode();
|
|
const MCInstrDesc &Desc = BC.MII->get(Opcode);
|
|
|
|
// Skip instructions that do not load from memory.
|
|
if (!Desc.mayLoad())
|
|
continue;
|
|
|
|
// Try to statically evaluate the target memory address;
|
|
uint64_t TargetAddress;
|
|
|
|
if (MIB->hasPCRelOperand(Inst)) {
|
|
// Try to find the symbol that corresponds to the PC-relative operand.
|
|
auto DispOpI = MIB->getMemOperandDisp(Inst);
|
|
assert(DispOpI != Inst.end() && "expected PC-relative displacement");
|
|
assert(DispOpI->isExpr() &&
|
|
"found PC-relative with non-symbolic displacement");
|
|
|
|
// Get displacement symbol.
|
|
const MCSymbol *DisplSymbol;
|
|
uint64_t DisplOffset;
|
|
|
|
std::tie(DisplSymbol, DisplOffset) =
|
|
BC.MIB->getTargetSymbolInfo(DispOpI->getExpr());
|
|
|
|
if (!DisplSymbol)
|
|
continue;
|
|
|
|
// Look up the symbol address in the global symbols map of the binary
|
|
// context object.
|
|
auto *BD = BC.getBinaryDataByName(DisplSymbol->getName());
|
|
if (!BD)
|
|
continue;
|
|
TargetAddress = BD->getAddress() + DisplOffset;
|
|
} else if (!MIB->evaluateMemOperandTarget(Inst, TargetAddress)) {
|
|
continue;
|
|
}
|
|
|
|
// Get the contents of the section containing the target address of the
|
|
// memory operand. We are only interested in read-only sections.
|
|
auto DataSection = BC.getSectionForAddress(TargetAddress);
|
|
if (!DataSection || !DataSection->isReadOnly())
|
|
continue;
|
|
|
|
if (BC.getRelocationAt(TargetAddress))
|
|
continue;
|
|
|
|
uint32_t Offset = TargetAddress - DataSection->getAddress();
|
|
StringRef ConstantData = DataSection->getContents();
|
|
|
|
++NumLocalLoadsFound;
|
|
if (BB->hasProfile())
|
|
NumDynamicLocalLoadsFound += BB->getExecutionCount();
|
|
|
|
if (MIB->replaceMemOperandWithImm(Inst, ConstantData, Offset)) {
|
|
++NumLocalLoadsSimplified;
|
|
if (BB->hasProfile())
|
|
NumDynamicLocalLoadsSimplified += BB->getExecutionCount();
|
|
}
|
|
}
|
|
}
|
|
|
|
NumLoadsFound += NumLocalLoadsFound;
|
|
NumDynamicLoadsFound += NumDynamicLocalLoadsFound;
|
|
NumLoadsSimplified += NumLocalLoadsSimplified;
|
|
NumDynamicLoadsSimplified += NumDynamicLocalLoadsSimplified;
|
|
|
|
return NumLocalLoadsSimplified > 0;
|
|
}
|
|
|
|
void SimplifyRODataLoads::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &
|
|
) {
|
|
for (auto &It : BFs) {
|
|
auto &Function = It.second;
|
|
if (shouldOptimize(Function) && simplifyRODataLoads(BC, Function)) {
|
|
Modified.insert(&Function);
|
|
}
|
|
}
|
|
|
|
outs() << "BOLT-INFO: simplified " << NumLoadsSimplified << " out of "
|
|
<< NumLoadsFound << " loads from a statically computed address.\n"
|
|
<< "BOLT-INFO: dynamic loads simplified: " << NumDynamicLoadsSimplified
|
|
<< "\n"
|
|
<< "BOLT-INFO: dynamic loads found: " << NumDynamicLoadsFound << "\n";
|
|
}
|
|
|
|
void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &) {
|
|
const auto OriginalFunctionCount = BFs.size();
|
|
uint64_t NumFunctionsFolded = 0;
|
|
uint64_t NumJTFunctionsFolded = 0;
|
|
uint64_t BytesSavedEstimate = 0;
|
|
uint64_t CallsSavedEstimate = 0;
|
|
static bool UseDFS = opts::ICFUseDFS;
|
|
|
|
// This hash table is used to identify identical functions. It maps
|
|
// a function to a bucket of functions identical to it.
|
|
struct KeyHash {
|
|
std::size_t operator()(const BinaryFunction *F) const {
|
|
return F->hash(/*Recompute=*/false);
|
|
}
|
|
};
|
|
struct KeyCongruent {
|
|
bool operator()(const BinaryFunction *A, const BinaryFunction *B) const {
|
|
return A->isIdenticalWith(*B, /*IgnoreSymbols=*/true, /*UseDFS=*/UseDFS);
|
|
}
|
|
};
|
|
struct KeyEqual {
|
|
bool operator()(const BinaryFunction *A, const BinaryFunction *B) const {
|
|
return A->isIdenticalWith(*B, /*IgnoreSymbols=*/false, /*UseDFS=*/UseDFS);
|
|
}
|
|
};
|
|
|
|
// Create buckets with congruent functions - functions that potentially could
|
|
// be folded.
|
|
std::unordered_map<BinaryFunction *, std::set<BinaryFunction *>,
|
|
KeyHash, KeyCongruent> CongruentBuckets;
|
|
for (auto &BFI : BFs) {
|
|
auto &BF = BFI.second;
|
|
if (!shouldOptimize(BF) || BF.isFolded())
|
|
continue;
|
|
|
|
// Make sure indices are in-order.
|
|
BF.updateLayoutIndices();
|
|
|
|
// Pre-compute hash before pushing into hashtable.
|
|
BF.hash(/*Recompute=*/true, /*UseDFS=*/UseDFS);
|
|
|
|
CongruentBuckets[&BF].emplace(&BF);
|
|
}
|
|
|
|
// We repeat the pass until no new modifications happen.
|
|
unsigned Iteration = 1;
|
|
uint64_t NumFoldedLastIteration;
|
|
do {
|
|
NumFoldedLastIteration = 0;
|
|
|
|
DEBUG(dbgs() << "BOLT-DEBUG: ICF iteration " << Iteration << "...\n");
|
|
|
|
for (auto &CBI : CongruentBuckets) {
|
|
auto &Candidates = CBI.second;
|
|
if (Candidates.size() < 2)
|
|
continue;
|
|
|
|
// Identical functions go into the same bucket.
|
|
std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>,
|
|
KeyHash, KeyEqual> IdenticalBuckets;
|
|
for (auto *BF : Candidates) {
|
|
IdenticalBuckets[BF].emplace_back(BF);
|
|
}
|
|
|
|
for (auto &IBI : IdenticalBuckets) {
|
|
// Functions identified as identical.
|
|
auto &Twins = IBI.second;
|
|
if (Twins.size() < 2)
|
|
continue;
|
|
|
|
// Fold functions. Keep the order consistent across invocations with
|
|
// different options.
|
|
std::stable_sort(Twins.begin(), Twins.end(),
|
|
[](const BinaryFunction *A, const BinaryFunction *B) {
|
|
return A->getFunctionNumber() < B->getFunctionNumber();
|
|
});
|
|
|
|
BinaryFunction *ParentBF = Twins[0];
|
|
for (unsigned i = 1; i < Twins.size(); ++i) {
|
|
auto *ChildBF = Twins[i];
|
|
DEBUG(dbgs() << "BOLT-DEBUG: folding " << *ChildBF << " into "
|
|
<< *ParentBF << '\n');
|
|
|
|
// Remove child function from the list of candidates.
|
|
auto FI = Candidates.find(ChildBF);
|
|
assert(FI != Candidates.end() &&
|
|
"function expected to be in the set");
|
|
Candidates.erase(FI);
|
|
|
|
// Fold the function and remove from the list of processed functions.
|
|
BytesSavedEstimate += ChildBF->getSize();
|
|
CallsSavedEstimate += std::min(ChildBF->getKnownExecutionCount(),
|
|
ParentBF->getKnownExecutionCount());
|
|
BC.foldFunction(*ChildBF, *ParentBF, BFs);
|
|
|
|
++NumFoldedLastIteration;
|
|
|
|
if (ParentBF->hasJumpTables())
|
|
++NumJTFunctionsFolded;
|
|
}
|
|
}
|
|
|
|
}
|
|
NumFunctionsFolded += NumFoldedLastIteration;
|
|
++Iteration;
|
|
|
|
} while (NumFoldedLastIteration > 0);
|
|
|
|
DEBUG(
|
|
// Print functions that are congruent but not identical.
|
|
for (auto &CBI : CongruentBuckets) {
|
|
auto &Candidates = CBI.second;
|
|
if (Candidates.size() < 2)
|
|
continue;
|
|
dbgs() << "BOLT-DEBUG: the following " << Candidates.size()
|
|
<< " functions (each of size " << (*Candidates.begin())->getSize()
|
|
<< " bytes) are congruent but not identical:\n";
|
|
for (auto *BF : Candidates) {
|
|
dbgs() << " " << *BF;
|
|
if (BF->getKnownExecutionCount()) {
|
|
dbgs() << " (executed " << BF->getKnownExecutionCount() << " times)";
|
|
}
|
|
dbgs() << '\n';
|
|
}
|
|
}
|
|
);
|
|
|
|
if (NumFunctionsFolded) {
|
|
outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded
|
|
<< " out of " << OriginalFunctionCount << " functions in "
|
|
<< Iteration << " passes. "
|
|
<< NumJTFunctionsFolded << " functions had jump tables.\n"
|
|
<< "BOLT-INFO: Removing all identical functions will save "
|
|
<< format("%.2lf", (double) BytesSavedEstimate / 1024)
|
|
<< " KB of code space. Folded functions were called "
|
|
<< CallsSavedEstimate << " times based on profile.\n";
|
|
}
|
|
}
|
|
|
|
void
|
|
PrintProgramStats::runOnFunctions(BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &) {
|
|
uint64_t NumSimpleFunctions{0};
|
|
uint64_t NumStaleProfileFunctions{0};
|
|
uint64_t NumNonSimpleProfiledFunctions{0};
|
|
std::vector<BinaryFunction *> ProfiledFunctions;
|
|
const char *StaleFuncsHeader = "BOLT-INFO: Functions with stale profile:\n";
|
|
for (auto &BFI : BFs) {
|
|
auto &Function = BFI.second;
|
|
if (!Function.isSimple()) {
|
|
if (Function.hasProfile()) {
|
|
++NumNonSimpleProfiledFunctions;
|
|
}
|
|
continue;
|
|
}
|
|
++NumSimpleFunctions;
|
|
if (!Function.hasProfile())
|
|
continue;
|
|
if (Function.hasValidProfile()) {
|
|
ProfiledFunctions.push_back(&Function);
|
|
} else {
|
|
if (opts::ReportStaleFuncs) {
|
|
outs() << StaleFuncsHeader;
|
|
StaleFuncsHeader = "";
|
|
outs() << " " << Function << '\n';
|
|
}
|
|
++NumStaleProfileFunctions;
|
|
}
|
|
}
|
|
BC.NumProfiledFuncs = ProfiledFunctions.size();
|
|
|
|
const auto NumAllProfiledFunctions =
|
|
ProfiledFunctions.size() + NumStaleProfileFunctions;
|
|
outs() << "BOLT-INFO: "
|
|
<< NumAllProfiledFunctions
|
|
<< " functions out of " << NumSimpleFunctions << " simple functions ("
|
|
<< format("%.1f", NumAllProfiledFunctions /
|
|
(float) NumSimpleFunctions * 100.0f)
|
|
<< "%) have non-empty execution profile.\n";
|
|
if (NumNonSimpleProfiledFunctions) {
|
|
outs() << "BOLT-INFO: " << NumNonSimpleProfiledFunctions
|
|
<< " non-simple function(s) have profile.\n";
|
|
}
|
|
if (NumStaleProfileFunctions) {
|
|
outs() << "BOLT-INFO: " << NumStaleProfileFunctions
|
|
<< format(" (%.1f%% of all profiled)",
|
|
NumStaleProfileFunctions /
|
|
(float) NumAllProfiledFunctions * 100.0f)
|
|
<< " function" << (NumStaleProfileFunctions == 1 ? "" : "s")
|
|
<< " have invalid (possibly stale) profile."
|
|
" Use -report-stale to see the list.\n";
|
|
}
|
|
|
|
// Profile is marked as 'Used' if it either matches a function name
|
|
// exactly or if it 100% matches any of functions with matching common
|
|
// LTO names.
|
|
auto getUnusedObjects = [&]() -> Optional<std::vector<StringRef>> {
|
|
std::vector<StringRef> UnusedObjects;
|
|
for (const auto &Func : BC.DR.getAllFuncsData()) {
|
|
if (!Func.getValue().Used) {
|
|
UnusedObjects.emplace_back(Func.getKey());
|
|
}
|
|
}
|
|
if (UnusedObjects.empty())
|
|
return NoneType();
|
|
return UnusedObjects;
|
|
};
|
|
|
|
if (const auto UnusedObjects = getUnusedObjects()) {
|
|
outs() << "BOLT-INFO: profile for " << UnusedObjects->size()
|
|
<< " objects was ignored\n";
|
|
if (opts::Verbosity >= 1) {
|
|
for (auto Name : *UnusedObjects) {
|
|
outs() << " " << Name << '\n';
|
|
}
|
|
}
|
|
}
|
|
|
|
if (ProfiledFunctions.size() > 10) {
|
|
if (opts::Verbosity >= 1) {
|
|
outs() << "BOLT-INFO: top called functions are:\n";
|
|
std::sort(ProfiledFunctions.begin(), ProfiledFunctions.end(),
|
|
[](BinaryFunction *A, BinaryFunction *B) {
|
|
return B->getExecutionCount() < A->getExecutionCount();
|
|
}
|
|
);
|
|
auto SFI = ProfiledFunctions.begin();
|
|
auto SFIend = ProfiledFunctions.end();
|
|
for (auto i = 0u; i < opts::TopCalledLimit && SFI != SFIend; ++SFI, ++i) {
|
|
outs() << " " << **SFI << " : "
|
|
<< (*SFI)->getExecutionCount() << '\n';
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!opts::PrintSortedBy.empty() &&
|
|
std::find(opts::PrintSortedBy.begin(),
|
|
opts::PrintSortedBy.end(),
|
|
DynoStats::FIRST_DYNO_STAT) == opts::PrintSortedBy.end()) {
|
|
|
|
std::vector<const BinaryFunction *> Functions;
|
|
std::map<const BinaryFunction *, DynoStats> Stats;
|
|
|
|
for (const auto &BFI : BFs) {
|
|
const auto &BF = BFI.second;
|
|
if (shouldOptimize(BF) && BF.hasValidProfile()) {
|
|
Functions.push_back(&BF);
|
|
Stats.emplace(&BF, BF.getDynoStats());
|
|
}
|
|
}
|
|
|
|
const bool SortAll =
|
|
std::find(opts::PrintSortedBy.begin(),
|
|
opts::PrintSortedBy.end(),
|
|
DynoStats::LAST_DYNO_STAT) != opts::PrintSortedBy.end();
|
|
|
|
const bool Ascending =
|
|
opts::DynoStatsSortOrderOpt == opts::DynoStatsSortOrder::Ascending;
|
|
|
|
if (SortAll) {
|
|
std::stable_sort(
|
|
Functions.begin(),
|
|
Functions.end(),
|
|
[Ascending,&Stats](const BinaryFunction *A, const BinaryFunction *B) {
|
|
return Ascending ?
|
|
Stats.at(A) < Stats.at(B) : Stats.at(B) < Stats.at(A);
|
|
}
|
|
);
|
|
} else {
|
|
std::stable_sort(
|
|
Functions.begin(),
|
|
Functions.end(),
|
|
[Ascending,&Stats](const BinaryFunction *A, const BinaryFunction *B) {
|
|
const auto &StatsA = Stats.at(A);
|
|
const auto &StatsB = Stats.at(B);
|
|
return Ascending
|
|
? StatsA.lessThan(StatsB, opts::PrintSortedBy)
|
|
: StatsB.lessThan(StatsA, opts::PrintSortedBy);
|
|
}
|
|
);
|
|
}
|
|
|
|
outs() << "BOLT-INFO: top functions sorted by ";
|
|
if (SortAll) {
|
|
outs() << "dyno stats";
|
|
} else {
|
|
outs() << "(";
|
|
bool PrintComma = false;
|
|
for (const auto Category : opts::PrintSortedBy) {
|
|
if (PrintComma) outs() << ", ";
|
|
outs() << DynoStats::Description(Category);
|
|
PrintComma = true;
|
|
}
|
|
outs() << ")";
|
|
}
|
|
|
|
outs() << " are:\n";
|
|
auto SFI = Functions.begin();
|
|
for (unsigned i = 0; i < 100 && SFI != Functions.end(); ++SFI, ++i) {
|
|
const auto Stats = (*SFI)->getDynoStats();
|
|
outs() << " " << **SFI;
|
|
if (!SortAll) {
|
|
outs() << " (";
|
|
bool PrintComma = false;
|
|
for (const auto Category : opts::PrintSortedBy) {
|
|
if (PrintComma) outs() << ", ";
|
|
outs() << dynoStatsOptName(Category) << "=" << Stats[Category];
|
|
PrintComma = true;
|
|
}
|
|
outs() << ")";
|
|
}
|
|
outs() << "\n";
|
|
}
|
|
}
|
|
|
|
if (!BC.TrappedFunctions.empty()) {
|
|
errs() << "BOLT-WARNING: " << BC.TrappedFunctions.size()
|
|
<< " functions will trap on entry";
|
|
if (opts::Verbosity >= 1) {
|
|
errs() << ".\n";
|
|
for (const auto *Function : BC.TrappedFunctions)
|
|
errs() << " " << *Function << '\n';
|
|
} else {
|
|
errs() << " (use -v=1 to see the list).\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
void InstructionLowering::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &LargeFunctions) {
|
|
for (auto &BFI : BFs) {
|
|
for (auto &BB : BFI.second) {
|
|
for (auto &Instruction : BB) {
|
|
BC.MIB->lowerTailCall(Instruction);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void StripRepRet::runOnFunctions(
|
|
BinaryContext &BC,
|
|
std::map<uint64_t, BinaryFunction> &BFs,
|
|
std::set<uint64_t> &LargeFunctions) {
|
|
uint64_t NumPrefixesRemoved = 0;
|
|
uint64_t NumBytesSaved = 0;
|
|
for (auto &BFI : BFs) {
|
|
for (auto &BB : BFI.second) {
|
|
auto LastInstRIter = BB.getLastNonPseudo();
|
|
if (LastInstRIter == BB.rend() ||
|
|
!BC.MIB->isReturn(*LastInstRIter) ||
|
|
!BC.MIB->deleteREPPrefix(*LastInstRIter))
|
|
continue;
|
|
|
|
NumPrefixesRemoved += BB.getKnownExecutionCount();
|
|
++NumBytesSaved;
|
|
}
|
|
}
|
|
|
|
if (NumBytesSaved) {
|
|
outs() << "BOLT-INFO: removed " << NumBytesSaved << " 'repz' prefixes"
|
|
" with estimated execution count of " << NumPrefixesRemoved
|
|
<< " times.\n";
|
|
}
|
|
}
|
|
|
|
} // namespace bolt
|
|
} // namespace llvm
|