diff --git a/bolt/BinaryFunction.cpp b/bolt/BinaryFunction.cpp index ca359385987b..843f3ced3c73 100644 --- a/bolt/BinaryFunction.cpp +++ b/bolt/BinaryFunction.cpp @@ -36,6 +36,11 @@ namespace bolt { namespace opts { +static cl::opt +AgressiveSplitting("split-all-cold", + cl::desc("outline as many cold basic blocks as possible"), + cl::Optional); + static cl::opt PrintClusters("print-clusters", cl::desc("print clusters"), cl::Optional); @@ -1611,34 +1616,26 @@ void BinaryFunction::splitFunction() { assert(BasicBlocksLayout.size() > 0); - // Separate hot from cold - if (!hasEHRanges()) { - for (auto I = BasicBlocksLayout.rbegin(), E = BasicBlocksLayout.rend(); - I != E; ++I) { - BinaryBasicBlock *BB = *I; - if (BB->getExecutionCount() != 0) - break; - BB->IsCold = true; - IsSplit = true; + // Never outline the first basic block. + BasicBlocks.front().CanOutline = false; + for (auto &BB : BasicBlocks) { + if (!BB.CanOutline) + continue; + if (BB.getExecutionCount() != 0) { + BB.CanOutline = false; + continue; } - } else { - // We cannot move a block that can throw since exception-handling - // runtime cannot deal with split functions. However, if we can guarantee - // that the block never throws, it is safe to move the block to - // decrease the size of the function. - // - // We also cannot move landing pads (or rather entry points for landing - // pads) for the same reason. - // - // Never move the first basic block. - BasicBlocks.front().CanOutline = false; - for (auto &BB : BasicBlocks) { - if (!BB.CanOutline) - continue; + if (hasEHRanges()) { + // We cannot move landing pads (or rather entry points for landing + // pads). if (LandingPads.find(BB.getLabel()) != LandingPads.end()) { BB.CanOutline = false; continue; } + // We cannot move a block that can throw since exception-handling + // runtime cannot deal with split functions. However, if we can guarantee + // that the block never throws, it is safe to move the block to + // decrease the size of the function. for (auto &Instr : BB) { if (BC.MIA->isInvoke(Instr)) { BB.CanOutline = false; @@ -1646,23 +1643,36 @@ void BinaryFunction::splitFunction() { } } } + } + + if (opts::AgressiveSplitting) { + // All blocks with 0 count that we can move go to the end of the function. std::stable_sort(BasicBlocksLayout.begin(), BasicBlocksLayout.end(), [&] (BinaryBasicBlock *A, BinaryBasicBlock *B) { - if (A->getExecutionCount() != 0 || B->getExecutionCount() != 0) - return false; return A->canOutline() < B->canOutline(); }); + } else if (hasEHRanges()) { + // Typically functions with exception handling have landing pads at the end. + // We cannot move beginning of landing pads, but we can move 0-count blocks + // comprising landing pads to the end and thus facilitating splitting. + auto FirstLP = BasicBlocksLayout.begin(); + while (LandingPads.find((*FirstLP)->getLabel()) != LandingPads.end()) + ++FirstLP; - for (auto I = BasicBlocksLayout.rbegin(), E = BasicBlocksLayout.rend(); - I != E; ++I) { - BinaryBasicBlock *BB = *I; - if (BB->getExecutionCount() != 0) - break; - if (!BB->canOutline()) - break; - BB->IsCold = true; - IsSplit = true; - } + std::stable_sort(FirstLP, BasicBlocksLayout.end(), + [&] (BinaryBasicBlock *A, BinaryBasicBlock *B) { + return A->canOutline() < B->canOutline(); + }); + } + + // Separate hot from cold + for (auto I = BasicBlocksLayout.rbegin(), E = BasicBlocksLayout.rend(); + I != E; ++I) { + BinaryBasicBlock *BB = *I; + if (!BB->canOutline()) + break; + BB->IsCold = true; + IsSplit = true; } } @@ -1673,7 +1683,7 @@ void BinaryFunction::propagateGnuArgsSizeInfo() { return; // The current value of DW_CFA_GNU_args_size affects all following - // invoke instructions untill the next CFI overrides it. + // invoke instructions until the next CFI overrides it. // It is important to iterate basic blocks in the original order when // assigning the value. uint64_t CurrentGnuArgsSize = 0; diff --git a/bolt/BinaryFunction.h b/bolt/BinaryFunction.h index 3df5247adb06..721a5598ee59 100644 --- a/bolt/BinaryFunction.h +++ b/bolt/BinaryFunction.h @@ -62,8 +62,10 @@ public: /// Settings for splitting function bodies into hot/cold partitions. enum SplittingType : char { ST_NONE = 0, /// Do not split functions - ST_LARGE = 1, /// Only split functions that exceed maximum size - ST_ALL =2, /// Split all functions + ST_EH, /// Split blocks comprising landing pads + ST_LARGE, /// Split functions that exceed maximum size in addition + /// to landing pads. + ST_ALL, /// Split all functions }; /// Choose which strategy should the block layout heuristic prioritize when diff --git a/bolt/RewriteInstance.cpp b/bolt/RewriteInstance.cpp index 1650d220aa27..02390eec4457 100644 --- a/bolt/RewriteInstance.cpp +++ b/bolt/RewriteInstance.cpp @@ -104,9 +104,11 @@ SplitFunctions("split-functions", cl::init(BinaryFunction::ST_NONE), cl::values(clEnumValN(BinaryFunction::ST_NONE, "0", "do not split any function"), - clEnumValN(BinaryFunction::ST_LARGE, "1", - "split if function is too large to fit"), - clEnumValN(BinaryFunction::ST_ALL, "2", + clEnumValN(BinaryFunction::ST_EH, "1", + "split all landing pads"), + clEnumValN(BinaryFunction::ST_LARGE, "2", + "also split if function too large to fit"), + clEnumValN(BinaryFunction::ST_ALL, "3", "split all functions"), clEnumValEnd), cl::Optional); @@ -1017,7 +1019,9 @@ void RewriteInstance::runOptimizationPasses() { if (opts::ReorderBlocks != BinaryFunction::LT_NONE) { bool ShouldSplit = (opts::SplitFunctions == BinaryFunction::ST_ALL) || - LargeFunctions.find(BFI.first) != LargeFunctions.end(); + (opts::SplitFunctions == BinaryFunction::ST_EH && + Function.hasEHRanges()) || + (LargeFunctions.find(BFI.first) != LargeFunctions.end()); BFI.second.modifyLayout(opts::ReorderBlocks, ShouldSplit); if (opts::PrintAll || opts::PrintReordered) Function.print(errs(), "after reordering blocks", true);