diff --git a/bolt/BinaryFunction.cpp b/bolt/BinaryFunction.cpp index 0f77c756e36c..ca359385987b 100644 --- a/bolt/BinaryFunction.cpp +++ b/bolt/BinaryFunction.cpp @@ -213,6 +213,9 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation, else OS << '0'; OS << "; action: " << Action; + auto GnuArgsSize = BC.MIA->getGnuArgsSize(Instruction); + if (GnuArgsSize >= 0) + OS << "; GNU_args_size = " << GnuArgsSize; } } if (opts::PrintDebugInfo && LineTable) { @@ -825,6 +828,9 @@ bool BinaryFunction::buildCFG() { // Update the state. CurrentState = State::CFG; + // Annotate invoke instructions with GNU_args_size data. + propagateGnuArgsSizeInfo(); + return true; } @@ -939,15 +945,13 @@ void BinaryFunction::annotateCFIState() { ++HighestState; if (CFI->getOperation() == MCCFIInstruction::OpRememberState) { StateStack.push(State); - continue; - } - if (CFI->getOperation() == MCCFIInstruction::OpRestoreState) { + } else if (CFI->getOperation() == MCCFIInstruction::OpRestoreState) { assert(!StateStack.empty() && "Corrupt CFI stack"); State = StateStack.top(); StateStack.pop(); - continue; + } else if (CFI->getOperation() != MCCFIInstruction::OpGnuArgsSize) { + State = HighestState; } - State = HighestState; } } @@ -995,8 +999,12 @@ bool BinaryFunction::fixCFIState() { } for (auto CFI : NewCFIs) { - InsertIt = addCFIPseudo(InBB, InsertIt, CFI); - ++InsertIt; + // Ignore GNU_args_size instructions. + if (FrameInstructions[CFI].getOperation() != + MCCFIInstruction::OpGnuArgsSize) { + InsertIt = addCFIPseudo(InBB, InsertIt, CFI); + ++InsertIt; + } } return true; @@ -1658,5 +1666,45 @@ void BinaryFunction::splitFunction() { } } +void BinaryFunction::propagateGnuArgsSizeInfo() { + assert(CurrentState == State::CFG && "unexpected function state"); + + if (!hasEHRanges() || !usesGnuArgsSize()) + return; + + // The current value of DW_CFA_GNU_args_size affects all following + // invoke instructions untill the next CFI overrides it. + // It is important to iterate basic blocks in the original order when + // assigning the value. + uint64_t CurrentGnuArgsSize = 0; + for (auto &BB : BasicBlocks) { + for (auto II = BB.begin(); II != BB.end(); ) { + auto &Instr = *II; + if (BC.MIA->isCFI(Instr)) { + auto CFI = getCFIFor(Instr); + if (CFI->getOperation() == MCCFIInstruction::OpGnuArgsSize) { + CurrentGnuArgsSize = CFI->getOffset(); + // Delete DW_CFA_GNU_args_size instructions and only regenerate + // during the final code emission. The information is embedded + // inside call instructions. + II = BB.Instructions.erase(II); + } else { + ++II; + } + continue; + } + + if (BC.MIA->isInvoke(Instr)) { + // Add the value of GNU_args_size as an extra operand if landing pad + // is non-emptry. + if (BC.MIA->getEHInfo(Instr).first) { + Instr.addOperand(MCOperand::createImm(CurrentGnuArgsSize)); + } + } + ++II; + } + } +} + } // namespace bolt } // namespace llvm diff --git a/bolt/BinaryFunction.h b/bolt/BinaryFunction.h index de1c0fbabe08..3df5247adb06 100644 --- a/bolt/BinaryFunction.h +++ b/bolt/BinaryFunction.h @@ -122,13 +122,6 @@ private: /// Alignment requirements for the function. uint64_t Alignment{1}; - /// True if this function needs to be emitted in two separate parts, one for - /// the hot basic blocks and another for the cold basic blocks. - bool IsSplit{false}; - - /// Indicate if this function has associated exception handling metadata. - bool HasEHRanges{false}; - MCSymbol *PersonalityFunction{nullptr}; uint8_t PersonalityEncoding{dwarf::DW_EH_PE_sdata4 | dwarf::DW_EH_PE_pcrel}; @@ -138,6 +131,16 @@ private: /// flow graph and re-assemble. bool IsSimple{true}; + /// True if this function needs to be emitted in two separate parts, one for + /// the hot basic blocks and another for the cold basic blocks. + bool IsSplit{false}; + + /// Indicate if this function has associated exception handling metadata. + bool HasEHRanges{false}; + + /// True if the function uses DW_CFA_GNU_args_size CFIs. + bool UsesGnuArgsSize{false}; + /// The address for the code for this function in codegen memory. uint64_t ImageAddress{0}; @@ -440,10 +443,21 @@ public: return IsSimple; } + /// Return true if the function body is non-contiguous. bool isSplit() const { return IsSplit; } + /// Return true if the function has exception handling tables. + bool hasEHRanges() const { + return HasEHRanges; + } + + /// Return true if the function uses DW_CFA_GNU_args_size CFIs. + bool usesGnuArgsSize() const { + return UsesGnuArgsSize; + } + MCSymbol *getPersonalityFunction() const { return PersonalityFunction; } @@ -531,7 +545,8 @@ public: // with NOPs and then reorder it away. // We fix this by moving the CFI instruction just before any NOPs. auto I = Instructions.lower_bound(Offset); - if (I == Instructions.end() && Offset == getSize()) { + if (Offset == getSize()) { + assert(I == Instructions.end() && "unexpected iterator value"); // Sometimes compiler issues restore_state after all instructions // in the function (even after nop). --I; @@ -593,6 +608,11 @@ public: return *this; } + BinaryFunction &setUsesGnuArgsSize(bool Uses = true) { + UsesGnuArgsSize = Uses; + return *this; + } + BinaryFunction &setPersonalityFunction(uint64_t Addr) { PersonalityFunction = BC.getOrCreateGlobalSymbol(Addr, "FUNCat"); return *this; @@ -737,6 +757,10 @@ public: /// is corrupted. If it is unable to fix it, it returns false. bool fixCFIState(); + /// Associate DW_CFA_GNU_args_size info with invoke instructions + /// (call instructions with non-empty landing pad). + void propagateGnuArgsSizeInfo(); + /// Traverse the CFG checking branches, inverting their condition, removing or /// adding jumps based on a new layout order. void fixBranches(); @@ -751,9 +775,6 @@ public: /// Update exception handling ranges for the function. void updateEHRanges(); - /// Return true if the function has exception handling tables. - bool hasEHRanges() const { return HasEHRanges; } - /// Emit exception handling ranges for the function. void emitLSDA(MCStreamer *Streamer); diff --git a/bolt/Exceptions.cpp b/bolt/Exceptions.cpp index c746cf6dca95..2734d4e2c389 100644 --- a/bolt/Exceptions.cpp +++ b/bolt/Exceptions.cpp @@ -611,6 +611,7 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const { Function.addCFIInstruction( Offset, MCCFIInstruction::createGnuArgsSize(nullptr, Instr.Ops[0])); + Function.setUsesGnuArgsSize(); break; case DW_CFA_val_offset_sf: case DW_CFA_val_offset: diff --git a/bolt/RewriteInstance.cpp b/bolt/RewriteInstance.cpp index 5d8e7cc71970..1650d220aa27 100644 --- a/bolt/RewriteInstance.cpp +++ b/bolt/RewriteInstance.cpp @@ -1168,6 +1168,7 @@ void emitFunction(MCStreamer &Streamer, BinaryFunction &Function, "first basic block should never be cold"); // Emit code. + int64_t CurrentGnuArgsSize = 0; for (auto BB : Function.layout()) { if (EmitColdPart != BB->isCold()) continue; @@ -1189,41 +1190,50 @@ void emitFunction(MCStreamer &Streamer, BinaryFunction &Function, Streamer.EmitLabel(const_cast(Label)); continue; } - if (!BC.MIA->isCFI(Instr)) { - if (opts::UpdateDebugSections) { - auto RowReference = DebugLineTableRowRef::fromSMLoc(Instr.getLoc()); - if (RowReference != DebugLineTableRowRef::NULL_ROW && - Instr.getLoc().getPointer() != LastLocSeen.getPointer()) { - auto CompileUnit = - BC.OffsetToDwarfCU[RowReference.DwCompileUnitIndex]; - assert(CompileUnit && - "Invalid CU offset set in instruction debug info."); - - auto OriginalLineTable = - BC.DwCtx->getLineTableForUnit( - CompileUnit); - const auto &OriginalRow = - OriginalLineTable->Rows[RowReference.RowIndex - 1]; - - BC.Ctx->setCurrentDwarfLoc( - OriginalRow.File, - OriginalRow.Line, - OriginalRow.Column, - (DWARF2_FLAG_IS_STMT * OriginalRow.IsStmt) | - (DWARF2_FLAG_BASIC_BLOCK * OriginalRow.BasicBlock) | - (DWARF2_FLAG_PROLOGUE_END * OriginalRow.PrologueEnd) | - (DWARF2_FLAG_EPILOGUE_BEGIN * OriginalRow.EpilogueBegin), - OriginalRow.Isa, - OriginalRow.Discriminator); - BC.Ctx->setDwarfCompileUnitID(CompileUnit->getOffset()); - LastLocSeen = Instr.getLoc(); - } - } - - Streamer.EmitInstruction(Instr, *BC.STI); + if (BC.MIA->isCFI(Instr)) { + emitCFIInstr(*Function.getCFIFor(Instr)); continue; } - emitCFIInstr(*Function.getCFIFor(Instr)); + if (opts::UpdateDebugSections) { + auto RowReference = DebugLineTableRowRef::fromSMLoc(Instr.getLoc()); + if (RowReference != DebugLineTableRowRef::NULL_ROW && + Instr.getLoc().getPointer() != LastLocSeen.getPointer()) { + auto CompileUnit = + BC.OffsetToDwarfCU[RowReference.DwCompileUnitIndex]; + assert(CompileUnit && + "Invalid CU offset set in instruction debug info."); + + auto OriginalLineTable = + BC.DwCtx->getLineTableForUnit( + CompileUnit); + const auto &OriginalRow = + OriginalLineTable->Rows[RowReference.RowIndex - 1]; + + BC.Ctx->setCurrentDwarfLoc( + OriginalRow.File, + OriginalRow.Line, + OriginalRow.Column, + (DWARF2_FLAG_IS_STMT * OriginalRow.IsStmt) | + (DWARF2_FLAG_BASIC_BLOCK * OriginalRow.BasicBlock) | + (DWARF2_FLAG_PROLOGUE_END * OriginalRow.PrologueEnd) | + (DWARF2_FLAG_EPILOGUE_BEGIN * OriginalRow.EpilogueBegin), + OriginalRow.Isa, + OriginalRow.Discriminator); + BC.Ctx->setDwarfCompileUnitID(CompileUnit->getOffset()); + LastLocSeen = Instr.getLoc(); + } + } + + // Emit GNU_args_size CFIs as necessary. + if (Function.usesGnuArgsSize() && BC.MIA->isInvoke(Instr)) { + auto NewGnuArgsSize = BC.MIA->getGnuArgsSize(Instr); + if (NewGnuArgsSize >= 0 && NewGnuArgsSize != CurrentGnuArgsSize) { + CurrentGnuArgsSize = NewGnuArgsSize; + Streamer.EmitCFIGnuArgsSize(CurrentGnuArgsSize); + } + } + + Streamer.EmitInstruction(Instr, *BC.STI); } MCSymbol *BBEndLabel = BC.Ctx->createTempSymbol();