-- Adding Veneer elimination pass and Veneer count to dyno stats.

Summary: Create a pass that performs veneers elimination .

(cherry picked from FBD8359299)

bolt/src/BinaryFunction.cpp

@@ -1361,6 +1361,14 @@ void BinaryFunction::disassemble(ArrayRef<uint8_t> FunctionData) {
             MIB->matchLinkerVeneer(Instructions.begin(), Instructions.end(),
                                    AbsoluteInstrAddr, Instruction, TargetHiBits,
                                    TargetLowBits, TargetAddress)) {
+          MIB->addAnnotation(Instruction, "AArch64Veneer", true);
+
+          uint8_t Counter = 0;
+          for (auto It = std::prev(Instructions.end()); Counter != 2;
+               --It, ++Counter) {
+            MIB->addAnnotation(It->second, "AArch64Veneer", true);
+          }
+
           fixStubTarget(*TargetLowBits, *TargetHiBits, TargetAddress);
         }
       }
@@ -3753,7 +3761,7 @@ void BinaryFunction::printLoopInfo(raw_ostream &OS) const {
 }
 
 DynoStats BinaryFunction::getDynoStats() const {
-  DynoStats Stats;
+  DynoStats Stats(/*PrintAArch64Stats*/ BC.isAArch64());
 
   // Return empty-stats about the function we don't completely understand.
   if (!isSimple() || !hasValidProfile())
@@ -3779,6 +3787,10 @@ DynoStats BinaryFunction::getDynoStats() const {
     if (BB->getNumNonPseudos() == 0 || BBExecutionCount == 0)
       continue;
 
+    // Count AArch64 linker-inserted veneers
+    if(isAArch64Veneer())
+        Stats[DynoStats::VENEER_CALLS_AARCH64] += getKnownExecutionCount();
+
     // Count the number of calls by iterating through all instructions.
     for (const auto &Instr : *BB) {
       if (BC.MIB->isStore(Instr)) {
@@ -3887,6 +3899,22 @@ DynoStats BinaryFunction::getDynoStats() const {
   return Stats;
 }
 
+bool BinaryFunction::isAArch64Veneer() const {
+  if (BasicBlocks.size() != 1)
+    return false;
+
+  auto &BB = **BasicBlocks.begin();
+  if (BB.size() != 3)
+    return false;
+
+  for (auto &Inst : BB) {
+    if (!BC.MIB->hasAnnotation(Inst, "AArch64Veneer"))
+      return false;
+  }
+
+  return true;
+}
+
 void DynoStats::print(raw_ostream &OS, const DynoStats *Other) const {
   auto printStatWithDelta = [&](const std::string &Name, uint64_t Stat,
                                 uint64_t OtherStat) {
@@ -3907,6 +3935,10 @@ void DynoStats::print(raw_ostream &OS, const DynoStats *Other) const {
   for (auto Stat = DynoStats::FIRST_DYNO_STAT + 1;
        Stat < DynoStats::LAST_DYNO_STAT;
        ++Stat) {
+
+    if (!PrintAArch64Stats && Stat == DynoStats::VENEER_CALLS_AARCH64)
+      continue;
+
     printStatWithDelta(Desc[Stat], Stats[Stat], Other ? (*Other)[Stat] : 0);
   }
 }

bolt/src/BinaryFunction.h

@@ -80,6 +80,7 @@ class DynoStats {
       Fadd(FORWARD_COND_BRANCHES_TAKEN, BACKWARD_COND_BRANCHES_TAKEN))\
   D(ALL_CONDITIONAL,              "all conditional branches",\
       Fadd(FORWARD_COND_BRANCHES, BACKWARD_COND_BRANCHES))\
+  D(VENEER_CALLS_AARCH64,         "linker-inserted veneer calls", Fn)\
   D(LAST_DYNO_STAT,               "<reserved>", 0)
 
 public:
@@ -90,13 +91,15 @@ public:
 
 private:
   uint64_t Stats[LAST_DYNO_STAT+1];
+  bool PrintAArch64Stats;
 
 #define D(name, desc, ...) desc,
   static constexpr const char *Desc[] = { DYNO_STATS };
 #undef D
 
 public:
-  DynoStats() {
+  DynoStats(bool PrintAArch64Stats ) {
+    this->PrintAArch64Stats = PrintAArch64Stats;
     for (auto Stat = FIRST_DYNO_STAT + 0; Stat < LAST_DYNO_STAT; ++Stat)
       Stats[Stat] = 0;
   }
@@ -2198,6 +2201,9 @@ public:
       const DWARFDebugLoc::LocationList &InputLL,
       BaseAddress BaseAddr) const;
 
+  /// Return true if the function is an AArch64 linker inserted veneer
+  bool isAArch64Veneer() const;
+
   virtual ~BinaryFunction();
 
   /// Info for fragmented functions.
@@ -2230,7 +2236,8 @@ public:
 /// Return program-wide dynostats.
 template <typename FuncsType>
 inline DynoStats getDynoStats(const FuncsType &Funcs) {
-  DynoStats dynoStats;
+  bool IsAArch64 = Funcs.begin()->second.getBinaryContext().isAArch64();
+  DynoStats dynoStats(IsAArch64);
   for (auto &BFI : Funcs) {
     auto &BF = BFI.second;
     if (BF.isSimple()) {
@@ -2247,7 +2254,8 @@ callWithDynoStats(FnType &&Func,
                   const FuncsType &Funcs,
                   StringRef Phase,
                   const bool Flag) {
-  DynoStats DynoStatsBefore;
+  bool IsAArch64 = Funcs.begin()->second.getBinaryContext().isAArch64();
+  DynoStats DynoStatsBefore(IsAArch64);
   if (Flag) {
     DynoStatsBefore = getDynoStats(Funcs);
   }

bolt/src/BinaryPassManager.cpp

@@ -24,6 +24,7 @@
 #include "Passes/ReorderData.h"
 #include "Passes/StokeInfo.h"
 #include "Passes/ValidateInternalCalls.h"
+#include "Passes/VeneerElimination.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include <numeric>
@@ -274,6 +275,13 @@ PrintStoke("print-stoke",
   cl::ZeroOrMore,
   cl::cat(BoltOptCategory));
 
+static llvm::cl::opt<bool>
+  PrintVeneerElimination("print-veneer-elimination",
+    cl::desc("print functions after veneer elimination pass"),
+    cl::init(false),
+    cl::ZeroOrMore,
+    cl::cat(BoltOptCategory));
+
 } // namespace opts
 
 namespace llvm {
@@ -368,6 +376,9 @@ void BinaryFunctionPassManager::runAllPasses(
   Manager.registerPass(llvm::make_unique<IdenticalCodeFolding>(PrintICF),
                        opts::ICF);
 
+  if (BC.isAArch64())
+      Manager.registerPass(llvm::make_unique<VeneerElimination>(PrintVeneerElimination));
+
   Manager.registerPass(llvm::make_unique<InlineMemcpy>(NeverPrint),
                        opts::StringOps);
 

bolt/src/Passes/CMakeLists.txt

@@ -33,6 +33,7 @@ add_llvm_library(LLVMBOLTPasses
   StackReachingUses.cpp
   StokeInfo.cpp
   ValidateInternalCalls.cpp
+  VeneerElimination.cpp
 
   DEPENDS
   intrinsics_gen

bolt/src/Passes/VeneerElimination.cpp

@@ -0,0 +1,106 @@
+//===--- Passes/VeneerElimination.cpp--------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This class implements a pass that removes linker-inserted veneers from the
+// code and redirects veneer callers to call to veneers destinations
+//
+//===----------------------------------------------------------------------===//
+#include "VeneerElimination.h"
+#define DEBUG_TYPE "veneer-elim"
+
+using namespace llvm;
+
+namespace opts {
+
+extern cl::OptionCategory BoltOptCategory;
+
+static llvm::cl::opt<bool>
+EliminateVeneers("elim-link-veneers",
+  cl::desc("run veneer elimination pass"),
+  cl::init(false),
+  cl::ZeroOrMore,
+  cl::Hidden,
+  cl::cat(BoltOptCategory));
+} // namespace opts
+
+namespace llvm {
+namespace bolt {
+
+void VeneerElimination::runOnFunctions(BinaryContext &BC,
+                                       std::map<uint64_t, BinaryFunction> &BFs,
+                                       std::set<uint64_t> &LargeFunctions) {
+  if (!opts::EliminateVeneers || !BC.isAArch64())
+    return;
+
+  std::unordered_map<const MCSymbol *, const MCSymbol *> VeneerDestinations;
+  uint64_t VeneersCount = 0;
+  for (auto It = BFs.begin(); It != BFs.end();) {
+    auto CurrentIt = It;
+    ++It;
+
+    if (CurrentIt->second.isAArch64Veneer()) {
+      VeneersCount++;
+      BinaryFunction &VeneerFunction = CurrentIt->second;
+
+      auto &FirstInstruction = *(VeneerFunction.begin()->begin());
+      const MCSymbol *VeneerTargetSymbol =
+          BC.MIB->getTargetSymbol(FirstInstruction, 1);
+
+      // Functions can have multiple symbols
+      for (auto &Name : VeneerFunction.getNames()) {
+        auto *Symbol = BC.Ctx->lookupSymbol(Name);
+        VeneerDestinations[Symbol] = VeneerTargetSymbol;
+        BC.SymbolToFunctionMap.erase(Symbol);
+      }
+
+      BC.BinaryDataMap.erase(VeneerFunction.getAddress());
+      BFs.erase(CurrentIt);
+    }
+  }
+
+  DEBUG(dbgs() << "BOLT-INFO: number of removed linker-inserted veneers :" << VeneersCount
+               << "\n");
+
+  // Handle veneers to veneers in case they occur
+  for (auto entry : VeneerDestinations) {
+    const MCSymbol *src = entry.first;
+    const MCSymbol *dest = entry.second;
+    while (VeneerDestinations.find(dest) != VeneerDestinations.end()) {
+      dest = VeneerDestinations[dest];
+    }
+    VeneerDestinations[src] = dest;
+  }
+
+  uint64_t VeneerCallers = 0;
+  for (auto &It : BFs) {
+    auto &Function = It.second;
+    for (auto &BB : Function) {
+      for (auto &Instr : BB) {
+        if (!BC.MIB->isCall(Instr) || BC.MIB->isIndirectCall(Instr))
+          continue;
+
+        auto *TargetSymbol = BC.MIB->getTargetSymbol(Instr, 0);
+        if (VeneerDestinations.find(TargetSymbol) == VeneerDestinations.end())
+          continue;
+
+        VeneerCallers++;
+        if (!BC.MIB->replaceBranchTarget(
+                Instr, VeneerDestinations[TargetSymbol], BC.Ctx.get())) {
+          assert(false && "updating veneer call destination failed");
+        }
+      }
+    }
+  }
+
+  DEBUG(dbgs() << "BOLT-INFO: number of linker-inserted veneers call sites :" << VeneerCallers
+               << "\n");
+}
+
+} // namespace bolt
+} // namespace llvm

bolt/src/Passes/VeneerElimination.h

@@ -0,0 +1,40 @@
+//===--- Passes/VeneerElimination.h ---------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_BOLT_VENEER_ELIMINATION_H
+#define LLVM_TOOLS_LLVM_BOLT_VENEER_ELIMINATION_H
+
+#include "BinaryFunctionCallGraph.h"
+#include "BinaryPasses.h"
+#include "MCPlus.h"
+#include "MCPlusBuilder.h"
+
+namespace llvm {
+namespace bolt {
+
+class VeneerElimination : public BinaryFunctionPass {
+public:
+  /// BinaryPass public interface
+  explicit VeneerElimination(const cl::opt<bool> &PrintPass)
+      : BinaryFunctionPass(PrintPass) {
+    ;
+  }
+
+  const char *getName() const override { return "veneer-elimination"; }
+
+  void runOnFunctions(BinaryContext &BC,
+                      std::map<uint64_t, BinaryFunction> &BFs,
+                      std::set<uint64_t> &LargeFunctions) override;
+};
+} // namespace bolt
+} // namespace llvm
+
+#endif