feature: Adding elf rewriter utility

It will aid in various elf patching tasks both in ocloc
and UMD

Related-To: NEO-10190

Signed-off-by: Chodor, Jaroslaw <jaroslaw.chodor@intel.com>

shared/source/device_binary_format/elf/elf_decoder.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -8,6 +8,7 @@
 #include "shared/source/device_binary_format/elf/elf_decoder.h"
 
 #include "shared/source/device_binary_format/elf/elf.h"
+#include "shared/source/helpers/aligned_memory.h"
 #include "shared/source/helpers/ptr_math.h"
 
 #include <string.h>
@@ -16,6 +17,30 @@ namespace NEO {
 
 namespace Elf {
 
+template <ElfIdentifierClass numBits>
+bool decodeNoteSection(ArrayRef<const uint8_t> sectionData, std::vector<DecodedNote> &out, std::string &outErrReason, std::string &outWarning) {
+    uint64_t pos = 0;
+    auto sectionSize = sectionData.size();
+    auto base = sectionData.begin();
+    while (pos < sectionSize) {
+        auto note = reinterpret_cast<const ElfNoteSection *>(base + pos);
+        auto alignedEntrySize = alignUp(sizeof(ElfNoteSection) + note->nameSize + note->descSize, 4);
+        if (pos + alignedEntrySize > sectionSize) {
+            outErrReason.append("Invalid elf note section - not enough data\n");
+            return false;
+        }
+        ConstStringRef name{reinterpret_cast<const char *>(note + 1), note->nameSize};
+        ConstStringRef desc{reinterpret_cast<const char *>(note + 1) + note->nameSize, note->descSize};
+        pos += alignedEntrySize;
+
+        out.push_back(DecodedNote{name, desc, note->type});
+    }
+    return true;
+}
+
+template bool decodeNoteSection<EI_CLASS_32>(ArrayRef<const uint8_t> sectionData, std::vector<DecodedNote> &out, std::string &outErrReason, std::string &outWarning);
+template bool decodeNoteSection<EI_CLASS_64>(ArrayRef<const uint8_t> sectionData, std::vector<DecodedNote> &out, std::string &outErrReason, std::string &outWarning);
+
 template <ElfIdentifierClass numBits>
 const ElfFileHeader<numBits> *decodeElfFileHeader(const ArrayRef<const uint8_t> binary) {
     if (binary.size() < sizeof(ElfFileHeader<numBits>)) {
@@ -87,7 +112,7 @@ Elf<numBits> decodeElf(const ArrayRef<const uint8_t> binary, std::string &outErr
 }
 
 template <ElfIdentifierClass numBits>
-bool Elf<numBits>::decodeSymTab(SectionHeaderAndData &sectionHeaderData, std::string &outError) {
+bool Elf<numBits>::decodeSymTab(SectionHeaderAndData<numBits> &sectionHeaderData, std::string &outError) {
     if (sectionHeaderData.header->type == SectionHeaderType::SHT_SYMTAB) {
         auto symSize = sizeof(ElfSymbolEntry<numBits>);
         if (symSize != sectionHeaderData.header->entsize) {
@@ -107,7 +132,7 @@ bool Elf<numBits>::decodeSymTab(SectionHeaderAndData &sectionHeaderData, std::st
 }
 
 template <ElfIdentifierClass numBits>
-bool Elf<numBits>::decodeRelocations(SectionHeaderAndData &sectionHeaderData, std::string &outError) {
+bool Elf<numBits>::decodeRelocations(SectionHeaderAndData<numBits> &sectionHeaderData, std::string &outError) {
     if (sectionHeaderData.header->type == SectionHeaderType::SHT_RELA) {
         auto relaSize = sizeof(ElfRela<numBits>);
         if (relaSize != sectionHeaderData.header->entsize) {

shared/source/device_binary_format/elf/elf_decoder.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -22,18 +22,20 @@ enum class RelocationX8664Type : uint32_t {
     relocation32 = 0xa
 };
 
+template <ElfIdentifierClass numBits = EI_CLASS_64>
+struct ProgramHeaderAndData {
+    const ElfProgramHeader<numBits> *header = nullptr;
+    ArrayRef<const uint8_t> data;
+};
+
+template <ElfIdentifierClass numBits = EI_CLASS_64>
+struct SectionHeaderAndData {
+    const ElfSectionHeader<numBits> *header;
+    ArrayRef<const uint8_t> data;
+};
+
 template <ElfIdentifierClass numBits = EI_CLASS_64>
 struct Elf {
-    struct ProgramHeaderAndData {
-        const ElfProgramHeader<numBits> *header = nullptr;
-        ArrayRef<const uint8_t> data;
-    };
-
-    struct SectionHeaderAndData {
-        const ElfSectionHeader<numBits> *header;
-        ArrayRef<const uint8_t> data;
-    };
-
     struct RelocationInfo {
         int symbolSectionIndex;
         int symbolTableIndex;
@@ -72,11 +74,15 @@ struct Elf {
         }
     }
 
-    MOCKABLE_VIRTUAL std::string getSymbolName(uint32_t nameOffset) const {
+    std::string getName(uint32_t nameOffset) const {
         auto sectionHeaderNamesData = sectionHeaders[elfFileHeader->shStrNdx].data;
         return std::string(reinterpret_cast<const char *>(sectionHeaderNamesData.begin()) + nameOffset);
     }
 
+    MOCKABLE_VIRTUAL std::string getSymbolName(uint32_t nameOffset) const {
+        return getName(nameOffset);
+    }
+
     decltype(ElfSymbolEntry<numBits>::value) getSymbolValue(uint32_t idx) const {
         return symbolTable[idx].value;
     }
@@ -98,12 +104,12 @@ struct Elf {
     }
 
     const ElfFileHeader<numBits> *elfFileHeader = nullptr;
-    StackVec<ProgramHeaderAndData, 32> programHeaders;
-    StackVec<SectionHeaderAndData, 32> sectionHeaders;
+    StackVec<ProgramHeaderAndData<numBits>, 32> programHeaders;
+    StackVec<SectionHeaderAndData<numBits>, 32> sectionHeaders;
 
   protected:
-    bool decodeSymTab(SectionHeaderAndData &sectionHeaderData, std::string &outError);
-    bool decodeRelocations(SectionHeaderAndData &sectionHeaderData, std::string &outError);
+    bool decodeSymTab(SectionHeaderAndData<numBits> &sectionHeaderData, std::string &outError);
+    bool decodeRelocations(SectionHeaderAndData<numBits> &sectionHeaderData, std::string &outError);
     bool isDebugDataRelocation(ConstStringRef sectionName);
 
     SymbolsTable symbolTable;
@@ -111,6 +117,17 @@ struct Elf {
     Relocations debugInfoRelocations;
 };
 
+struct DecodedNote {
+    ConstStringRef name;
+    ConstStringRef desc;
+    uint32_t type;
+};
+
+template <ElfIdentifierClass numBits = EI_CLASS_64>
+bool decodeNoteSection(ArrayRef<const uint8_t> sectionData, std::vector<DecodedNote> &out, std::string &outErrReason, std::string &outWarning);
+extern template bool decodeNoteSection<EI_CLASS_32>(ArrayRef<const uint8_t> sectionData, std::vector<DecodedNote> &out, std::string &outErrReason, std::string &outWarning);
+extern template bool decodeNoteSection<EI_CLASS_64>(ArrayRef<const uint8_t> sectionData, std::vector<DecodedNote> &out, std::string &outErrReason, std::string &outWarning);
+
 template <ElfIdentifierClass numBits = EI_CLASS_64>
 const ElfFileHeader<numBits> *decodeElfFileHeader(const ArrayRef<const uint8_t> binary);
 extern template const ElfFileHeader<EI_CLASS_32> *decodeElfFileHeader<EI_CLASS_32>(const ArrayRef<const uint8_t>);

shared/source/device_binary_format/elf/elf_encoder.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -29,7 +29,7 @@ ElfEncoder<numBits>::ElfEncoder(bool addUndefSectionHeader, bool addHeaderSectio
 }
 
 template <ElfIdentifierClass numBits>
-void ElfEncoder<numBits>::appendSection(const ElfSectionHeader<numBits> &sectionHeader, const ArrayRef<const uint8_t> sectionData) {
+ElfSectionHeader<numBits> &ElfEncoder<numBits>::appendSection(const ElfSectionHeader<numBits> &sectionHeader, const ArrayRef<const uint8_t> sectionData) {
     sectionHeaders.push_back(sectionHeader);
     if ((SHT_NOBITS != sectionHeader.type) && (false == sectionData.empty())) {
         auto sectionDataAlignment = std::min<uint64_t>(defaultDataAlignment, 8U);
@@ -42,10 +42,11 @@ void ElfEncoder<numBits>::appendSection(const ElfSectionHeader<numBits> &section
         sectionHeaders.rbegin()->offset = static_cast<decltype(sectionHeaders.rbegin()->offset)>(alignedOffset);
         sectionHeaders.rbegin()->size = static_cast<decltype(sectionHeaders.rbegin()->size)>(sectionData.size());
     }
+    return *sectionHeaders.rbegin();
 }
 
 template <ElfIdentifierClass numBits>
-void ElfEncoder<numBits>::appendSegment(const ElfProgramHeader<numBits> &programHeader, const ArrayRef<const uint8_t> segmentData) {
+ElfProgramHeader<numBits> &ElfEncoder<numBits>::appendSegment(const ElfProgramHeader<numBits> &programHeader, const ArrayRef<const uint8_t> segmentData) {
     maxDataAlignmentNeeded = std::max<uint64_t>(maxDataAlignmentNeeded, static_cast<uint64_t>(programHeader.align));
     programHeaders.push_back(programHeader);
     if (false == segmentData.empty()) {
@@ -59,6 +60,7 @@ void ElfEncoder<numBits>::appendSegment(const ElfProgramHeader<numBits> &program
         programHeaders.rbegin()->offset = static_cast<decltype(programHeaders.rbegin()->offset)>(alignedOffset);
         programHeaders.rbegin()->fileSz = static_cast<decltype(programHeaders.rbegin()->fileSz)>(segmentData.size());
     }
+    return *programHeaders.rbegin();
 }
 
 template <ElfIdentifierClass numBits>
@@ -89,8 +91,7 @@ ElfSectionHeader<numBits> &ElfEncoder<numBits>::appendSection(SectionHeaderType
     default:
         break;
     }
-    appendSection(section, sectionData);
-    return *sectionHeaders.rbegin();
+    return appendSection(section, sectionData);
 }
 
 template <ElfIdentifierClass numBits>
@@ -100,8 +101,7 @@ ElfProgramHeader<numBits> &ElfEncoder<numBits>::appendSegment(ProgramHeaderType
     segment.flags = static_cast<decltype(segment.flags)>(PF_NONE);
     segment.offset = 0U;
     segment.align = static_cast<decltype(segment.align)>(defaultDataAlignment);
-    appendSegment(segment, segmentData);
-    return *programHeaders.rbegin();
+    return appendSegment(segment, segmentData);
 }
 
 template <ElfIdentifierClass numBits>
@@ -200,6 +200,37 @@ std::vector<uint8_t> ElfEncoder<numBits>::encode() const {
     return ret;
 }
 
+template <ElfIdentifierClass numBits>
+std::vector<uint8_t> encodeNoteSectionData(ArrayRef<const NoteToEncode> notes) {
+    std::vector<uint8_t> ret;
+    {
+        size_t dataSize = 0U;
+        for (auto &note : notes) {
+            dataSize += sizeof(ElfNoteSection);
+            dataSize += note.name.size();
+            dataSize += note.desc.size();
+            dataSize = alignUp(dataSize, 4);
+        }
+        ret.reserve(dataSize);
+    }
+
+    for (auto &note : notes) {
+        ElfNoteSection entry;
+        entry.nameSize = static_cast<uint32_t>(note.name.size());
+        entry.descSize = static_cast<uint32_t>(note.desc.size());
+        entry.type = note.type;
+        ret.insert(ret.end(), reinterpret_cast<uint8_t *>(&entry), reinterpret_cast<uint8_t *>((&entry) + 1));
+        ret.insert(ret.end(), reinterpret_cast<const uint8_t *>(note.name.c_str()), reinterpret_cast<const uint8_t *>(note.name.c_str() + note.name.size()));
+        ret.insert(ret.end(), reinterpret_cast<const uint8_t *>(note.desc.c_str()), reinterpret_cast<const uint8_t *>(note.desc.c_str() + note.desc.size()));
+        ret.resize(alignUp(ret.size(), 4), 0U);
+    }
+
+    return ret;
+}
+
+template std::vector<uint8_t> encodeNoteSectionData<EI_CLASS_64>(ArrayRef<const NoteToEncode> notes);
+template std::vector<uint8_t> encodeNoteSectionData<EI_CLASS_32>(ArrayRef<const NoteToEncode> notes);
+
 template struct ElfEncoder<EI_CLASS_32>;
 template struct ElfEncoder<EI_CLASS_64>;
 

shared/source/device_binary_format/elf/elf_encoder.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2023 Intel Corporation
+ * Copyright (C) 2020-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -13,6 +13,7 @@
 
 #include <queue>
 #include <string>
+#include <vector>
 
 namespace NEO {
 
@@ -54,8 +55,8 @@ struct ElfEncoder {
     ElfEncoder(bool addUndefSectionHeader = true, bool addHeaderSectionNamesSection = true,
                typename ElfSectionHeaderTypes<numBits>::AddrAlign defaultDataAlignment = 8U);
 
-    void appendSection(const ElfSectionHeader<numBits> &sectionHeader, const ArrayRef<const uint8_t> sectionData);
-    void appendSegment(const ElfProgramHeader<numBits> &programHeader, const ArrayRef<const uint8_t> segmentData);
+    ElfSectionHeader<numBits> &appendSection(const ElfSectionHeader<numBits> &sectionHeader, const ArrayRef<const uint8_t> sectionData);
+    ElfProgramHeader<numBits> &appendSegment(const ElfProgramHeader<numBits> &programHeader, const ArrayRef<const uint8_t> segmentData);
 
     ElfSectionHeader<numBits> &appendSection(SectionHeaderType sectionType, ConstStringRef sectionLabel, const ArrayRef<const uint8_t> sectionData);
     ElfProgramHeader<numBits> &appendSegment(ProgramHeaderType segmentType, const ArrayRef<const uint8_t> segmentData);
@@ -99,6 +100,17 @@ struct ElfEncoder {
     uint32_t shStrTabNameOffset = 0;
 };
 
+struct NoteToEncode {
+    std::string name;
+    std::string desc;
+    uint32_t type;
+};
+
+template <ElfIdentifierClass numBits = EI_CLASS_64>
+std::vector<uint8_t> encodeNoteSectionData(ArrayRef<const NoteToEncode> notes);
+extern template std::vector<uint8_t> encodeNoteSectionData<EI_CLASS_64>(ArrayRef<const NoteToEncode> notes);
+extern template std::vector<uint8_t> encodeNoteSectionData<EI_CLASS_32>(ArrayRef<const NoteToEncode> notes);
+
 extern template struct ElfEncoder<EI_CLASS_32>;
 extern template struct ElfEncoder<EI_CLASS_64>;
 

shared/source/device_binary_format/elf/elf_rewriter.h

@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2020-2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/device_binary_format/elf/elf.h"
+#include "shared/source/device_binary_format/elf/elf_decoder.h"
+#include "shared/source/device_binary_format/elf/elf_encoder.h"
+#include "shared/source/utilities/arrayref.h"
+#include "shared/source/utilities/stackvec.h"
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace NEO {
+
+namespace Elf {
+
+template <ElfIdentifierClass NumBits = NEO::Elf::EI_CLASS_64>
+struct MutableSectionHeader {
+    MutableSectionHeader(const std::string &name, const NEO::Elf::ElfSectionHeader<NumBits> &header, const std::vector<uint8_t> &data)
+        : name(name), header(header), data(data) {
+    }
+    std::string name;
+    NEO::Elf::ElfSectionHeader<NumBits> header{};
+    std::vector<uint8_t> data;
+};
+
+template <ElfIdentifierClass NumBits = NEO::Elf::EI_CLASS_64>
+struct MutableProgramHeader {
+    MutableProgramHeader(const NEO::Elf::ElfProgramHeader<NumBits> &header, const std::vector<uint8_t> &data)
+        : header(header), data(data) {
+    }
+    NEO::Elf::ElfProgramHeader<NumBits> header = {};
+    std::vector<uint8_t> data;
+    MutableSectionHeader<NumBits> *referencedSectionData = nullptr;
+};
+
+template <ElfIdentifierClass NumBits = NEO::Elf::EI_CLASS_64>
+struct ElfRewriter {
+    using SectionId = uint32_t;
+
+    ElfRewriter(NEO::Elf::Elf<NumBits> &src) {
+        elfFileHeader = *src.elfFileHeader;
+        for (const auto &sh : src.sectionHeaders) {
+            this->sectionHeaders.push_back(std::make_unique<MutableSectionHeader<NumBits>>(src.getName(sh.header->name), *sh.header, std::vector<uint8_t>{sh.data.begin(), sh.data.end()}));
+        }
+        for (const auto &ph : src.programHeaders) {
+            this->programHeaders.push_back(std::make_unique<MutableProgramHeader<NumBits>>(*ph.header, std::vector<uint8_t>{ph.data.begin(), ph.data.end()}));
+            for (const auto &sh : this->sectionHeaders) {
+                if ((sh->header.offset == ph.header->offset) && (sh->header.size == ph.header->fileSz)) {
+                    (*this->programHeaders.rbegin())->referencedSectionData = sh.get();
+                }
+            }
+        }
+    }
+
+    std::vector<uint8_t> encode() const {
+        NEO::Elf::ElfEncoder<NumBits> encoder;
+        encoder.getElfFileHeader() = elfFileHeader;
+        std::unordered_map<MutableSectionHeader<NumBits> *, decltype(NEO::Elf::ElfSectionHeader<NumBits>::offset)> encodedSectionsOffsets;
+        for (const auto &sh : this->sectionHeaders) {
+            if ((sh->header.type == SHT_NULL) || (sh->header.type == SHT_STRTAB)) {
+                continue;
+            }
+            auto nameIdx = encoder.appendSectionName(sh->name);
+            NEO::Elf::ElfSectionHeader<NumBits> header = sh->header;
+            header.name = nameIdx;
+            encodedSectionsOffsets[sh.get()] = encoder.appendSection(header, sh->data).offset;
+        }
+        for (const auto &ph : this->programHeaders) {
+            if (ph->referencedSectionData) {
+                auto &header = ph->header;
+                header.offset = encodedSectionsOffsets[ph->referencedSectionData];
+                encoder.appendSegment(ph->header, {});
+            } else {
+                encoder.appendSegment(ph->header, ph->data);
+            }
+        }
+        return encoder.encode();
+    }
+
+    StackVec<SectionId, 16> findSections(typename ElfSectionHeaderTypes<NumBits>::Type type, ConstStringRef name) {
+        StackVec<SectionId, 16> ret;
+        for (size_t i = 0; i < this->sectionHeaders.size(); i++) {
+            auto &section = this->sectionHeaders[i];
+            if ((type == section->header.type) && (name == section->name)) {
+                ret.push_back(static_cast<SectionId>(i));
+            }
+        }
+        return ret;
+    }
+
+    MutableSectionHeader<NumBits> &getSection(SectionId idx) {
+        return *sectionHeaders[idx];
+    }
+
+    void removeSection(SectionId idx) {
+        auto sectionHeaderToRemove = std::move(sectionHeaders[idx]);
+        for (auto it = idx + 1; it < sectionHeaders.size(); ++it) { // preserve order
+            sectionHeaders[it - 1] = std::move(sectionHeaders[it]);
+        }
+        sectionHeaders.pop_back();
+        for (auto &programHeader : programHeaders) {
+            if (sectionHeaderToRemove.get() == programHeader->referencedSectionData) {
+                programHeader->referencedSectionData = nullptr;
+            }
+        }
+    }
+
+    ElfFileHeader<NumBits> elfFileHeader = {};
+
+  protected:
+    StackVec<std::unique_ptr<MutableSectionHeader<NumBits>>, 32> sectionHeaders;
+    StackVec<std::unique_ptr<MutableProgramHeader<NumBits>>, 32> programHeaders;
+};
+
+} // namespace Elf
+
+} // namespace NEO

shared/source/device_binary_format/zebin/zebin_decoder.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "shared/source/device_binary_format/device_binary_formats.h"
+#include "shared/source/device_binary_format/elf/elf_decoder.h"
 #include "shared/source/device_binary_format/zebin/zebin_elf.h"
 #include "shared/source/utilities/stackvec.h"
 
@@ -29,7 +30,7 @@ namespace Zebin {
 
 template <Elf::ElfIdentifierClass numBits = Elf::EI_CLASS_64>
 struct ZebinSections {
-    using SectionHeaderData = typename NEO::Elf::Elf<numBits>::SectionHeaderAndData;
+    using SectionHeaderData = NEO::Elf::SectionHeaderAndData<numBits>;
     StackVec<SectionHeaderData *, 32> textKernelSections;
     StackVec<SectionHeaderData *, 32> gtpinInfoSections;
     StackVec<SectionHeaderData *, 1> zeInfoSections;