[ELF] - Implemented --compress-debug-sections option.

Patch implements --compress-debug-sections=zlib.

In compare with D20211 (a year old patch, abandoned), it implementation
uses streaming and fully reimplemented, does not support zlib-gnu for
simplification.

This is PR32308.

Differential revision: https://reviews.llvm.org/D31941

llvm-svn: 300444

lld/ELF/Config.h

@@ -104,6 +104,7 @@ struct Configuration {
   bool Bsymbolic;
   bool BsymbolicFunctions;
   bool ColorDiagnostics = false;
+  bool CompressDebugSections;
   bool DefineCommon;
   bool Demangle = true;
   bool DisableVerify;

lld/ELF/Driver.cpp

@@ -45,6 +45,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Object/Decompressor.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
@@ -564,12 +565,25 @@ static std::vector<StringRef> getLines(MemoryBufferRef MB) {
   return Ret;
 }
 
+static bool getCompressDebugSections(opt::InputArgList &Args) {
+  if (auto *Arg = Args.getLastArg(OPT_compress_debug_sections)) {
+    StringRef S = Arg->getValue();
+    if (S == "none")
+      return false;
+    if (S == "zlib")
+      return zlib::isAvailable();
+    error("unknown --compress-debug-sections value: " + S);
+  }
+  return false;
+}
+
 // Initializes Config members by the command line options.
 void LinkerDriver::readConfigs(opt::InputArgList &Args) {
   Config->AllowMultipleDefinition = Args.hasArg(OPT_allow_multiple_definition);
   Config->AuxiliaryList = getArgs(Args, OPT_auxiliary);
   Config->Bsymbolic = Args.hasArg(OPT_Bsymbolic);
   Config->BsymbolicFunctions = Args.hasArg(OPT_Bsymbolic_functions);
+  Config->CompressDebugSections = getCompressDebugSections(Args);
   Config->DefineCommon = getArg(Args, OPT_define_common, OPT_no_define_common,
                                 !Args.hasArg(OPT_relocatable));
   Config->Demangle = getArg(Args, OPT_demangle, OPT_no_demangle, true);

lld/ELF/Options.td

@@ -22,6 +22,9 @@ def build_id: F<"build-id">, HelpText<"Generate build ID note">;
 
 def build_id_eq: J<"build-id=">, HelpText<"Generate build ID note">;
 
+def compress_debug_sections : J<"compress-debug-sections=">,
+  HelpText<"Compress DWARF debug sections">;
+
 def L: JoinedOrSeparate<["-"], "L">, MetaVarName<"<dir>">,
   HelpText<"Add a directory to the library search path">;
 

lld/ELF/OutputSections.cpp

@@ -16,6 +16,7 @@
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "Threads.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
@@ -83,6 +84,55 @@ static bool compareByFilePosition(InputSection *A, InputSection *B) {
   return LA->OutSecOff < LB->OutSecOff;
 }
 
+// Compressed sections has header which we create in this function.
+// Format is explaned here:
+// https://docs.oracle.com/cd/E53394_01/html/E54813/section_compression.html
+template <class ELFT>
+static std::vector<uint8_t> createHeader(size_t Size, uint32_t Alignment) {
+  const endianness E = ELFT::TargetEndianness;
+
+  std::vector<uint8_t> Ret(sizeof(typename ELFT::Chdr));
+  uint8_t *Buf = &Ret[0];
+  write32<E>(Buf, ELFCOMPRESS_ZLIB);
+  Buf += 4;
+
+  if (Config->Is64) {
+    Buf += sizeof(Elf64_Word); // Skip ch_reserved field.
+    write64<E>(Buf, Size);
+    Buf += sizeof(ELFT::Chdr::ch_size);
+    write64<E>(Buf, Alignment);
+    Buf += sizeof(ELFT::Chdr::ch_addralign);
+  } else {
+    write32<E>(Buf, Size);
+    Buf += sizeof(ELFT::Chdr::ch_size);
+    write32<E>(Buf, Alignment);
+    Buf += sizeof(ELFT::Chdr::ch_addralign);
+  }
+
+  return Ret;
+}
+
+template <class ELFT> void OutputSection::maybeCompress() {
+  // If -compress-debug-sections is specified, we compress output debug
+  // sections.
+  if (!Config->CompressDebugSections || !Name.startswith(".debug_") ||
+      (Flags & SHF_ALLOC))
+    return;
+
+  this->Flags |= SHF_COMPRESSED;
+  CompressedHeader = createHeader<ELFT>(this->Size, this->Alignment);
+
+  // Here we write relocated content of sections and compress it.
+  std::vector<uint8_t> Data(this->Size);
+  this->writeTo<ELFT>(&Data[0]);
+
+  if (Error E = zlib::compress(StringRef((char *)Data.data(), Data.size()),
+                               CompressedData))
+    fatal("compress failed: " + llvm::toString(std::move(E)));
+
+  this->Size = this->CompressedHeader.size() + this->CompressedData.size();
+}
+
 template <class ELFT> void OutputSection::finalize() {
   if ((this->Flags & SHF_LINK_ORDER) && !this->Sections.empty()) {
     std::sort(Sections.begin(), Sections.end(), compareByFilePosition);
@@ -245,6 +295,15 @@ uint32_t OutputSection::getFiller() {
 template <class ELFT> void OutputSection::writeTo(uint8_t *Buf) {
   Loc = Buf;
 
+  // We may have already rendered compressed content when using
+  // -compress-debug-sections option. Write it together with header.
+  if (!CompressedData.empty()) {
+    memcpy(Buf, CompressedHeader.data(), CompressedHeader.size());
+    Buf += CompressedHeader.size();
+    memcpy(Buf, CompressedData.data(), CompressedData.size());
+    return;
+  }
+
   // Write leading padding.
   uint32_t Filler = getFiller();
   if (Filler)
@@ -422,6 +481,11 @@ template void OutputSection::finalize<ELF32BE>();
 template void OutputSection::finalize<ELF64LE>();
 template void OutputSection::finalize<ELF64BE>();
 
+template void OutputSection::maybeCompress<ELF32LE>();
+template void OutputSection::maybeCompress<ELF32BE>();
+template void OutputSection::maybeCompress<ELF64LE>();
+template void OutputSection::maybeCompress<ELF64BE>();
+
 template void OutputSection::writeTo<ELF32LE>(uint8_t *Buf);
 template void OutputSection::writeTo<ELF32BE>(uint8_t *Buf);
 template void OutputSection::writeTo<ELF64LE>(uint8_t *Buf);

lld/ELF/OutputSections.h

@@ -84,9 +84,14 @@ public:
   uint32_t getFiller();
   template <class ELFT> void writeTo(uint8_t *Buf);
   template <class ELFT> void finalize();
+  template <class ELFT> void maybeCompress();
   void assignOffsets();
   std::vector<InputSection *> Sections;
 
+  // Used for implementation of --compress-debug-sections option.
+  llvm::SmallVector<char, 1> CompressedData;
+  std::vector<uint8_t> CompressedHeader;
+
   // Location in the output buffer.
   uint8_t *Loc = nullptr;
 };

lld/ELF/Writer.cpp

@@ -19,6 +19,7 @@
 #include "SymbolTable.h"
 #include "SyntheticSections.h"
 #include "Target.h"
+#include "Threads.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/FileOutputBuffer.h"
@@ -1216,6 +1217,11 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
   for (OutputSection *Sec : OutputSections)
     Sec->finalize<ELFT>();
 
+  // Some sections may require compression. That happens for
+  // debug sections when --compress-debug-sections option used.
+  parallelForEach(OutputSections.begin(), OutputSections.end(),
+                  [](OutputSection *S) { S->maybeCompress<ELFT>(); });
+
   // createThunks may have added local symbols to the static symbol table
   applySynthetic({In<ELFT>::SymTab, In<ELFT>::ShStrTab, In<ELFT>::StrTab},
                  [](SyntheticSection *SS) { SS->postThunkContents(); });

lld/test/ELF/compress-debug-output.s

@@ -0,0 +1,33 @@
+# REQUIRES: x86
+# REQUIRES: zlib
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t1 --compress-debug-sections=zlib
+
+# RUN: llvm-objdump -s %t1 | FileCheck %s --check-prefix=ZLIBCONTENT
+# ZLIBCONTENT:     Contents of section .debug_str:
+# ZLIBCONTENT-NOT: AAAAAAAAA
+
+# RUN: llvm-readobj -s %t1 | FileCheck %s --check-prefix=ZLIBFLAGS
+# ZLIBFLAGS:       Section {
+# ZLIBFLAGS:         Index:
+# ZLIBFLAGS:         Name: .debug_str
+# ZLIBFLAGS-NEXT:    Type: SHT_PROGBITS
+# ZLIBFLAGS-NEXT:    Flags [
+# ZLIBFLAGS-NEXT:      SHF_COMPRESSED
+
+# RUN: llvm-dwarfdump %t1 -debug-dump=str | \
+# RUN:   FileCheck %s --check-prefix=DEBUGSTR
+# DEBUGSTR:     .debug_str contents:
+# DEBUGSTR-NEXT:  AAAAAAAAAAAAAAAAAAAAAAAAAAA
+# DEBUGSTR-NEXT:  BBBBBBBBBBBBBBBBBBBBBBBBBBB
+
+# RUN: not ld.lld %t.o -o %t1 --compress-debug-sections=zlib-gabi 2>&1 | \
+# RUN:   FileCheck -check-prefix=ERR %s
+# ERR: unknown --compress-debug-sections value: zlib-gabi
+
+.section .debug_str,"MS",@progbits,1
+.Linfo_string0:
+  .asciz "AAAAAAAAAAAAAAAAAAAAAAAAAAA"
+.Linfo_string1:
+  .asciz "BBBBBBBBBBBBBBBBBBBBBBBBBBB"