[lldb] Make ObjectFileJSON loadable as a module

This patch adds support for creating modules from JSON object files.
This is necessary for the crashlog use case where we don't have either a
module or a symbol file. In that case the ObjectFileJSON serves as both.

The patch adds support for an object file type (i.e. executable, shared
library, etc). It also adds the ability to specify sections, which is
necessary in order specify symbols by address. Finally, this patch
improves error handling and fixes a bug where we wouldn't read more than
the initial 512 bytes in GetModuleSpecifications.

Differential revision: https://reviews.llvm.org/D148062

lldb/include/lldb/Core/Section.h

@@ -17,6 +17,7 @@
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
+#include "llvm/Support/JSON.h"
 
 #include <memory>
 #include <vector>
@@ -99,6 +100,13 @@ protected:
   collection m_sections;
 };
 
+struct JSONSection {
+  std::string name;
+  std::optional<lldb::SectionType> type;
+  std::optional<uint64_t> address;
+  std::optional<uint64_t> size;
+};
+
 class Section : public std::enable_shared_from_this<Section>,
                 public ModuleChild,
                 public UserID,
@@ -287,4 +295,16 @@ private:
 
 } // namespace lldb_private
 
+namespace llvm {
+namespace json {
+
+bool fromJSON(const llvm::json::Value &value,
+              lldb_private::JSONSection &section, llvm::json::Path path);
+
+bool fromJSON(const llvm::json::Value &value, lldb::SectionType &type,
+              llvm::json::Path path);
+
+} // namespace json
+} // namespace llvm
+
 #endif // LLDB_CORE_SECTION_H

lldb/include/lldb/Symbol/ObjectFile.h

@@ -768,6 +768,11 @@ template <> struct format_provider<lldb_private::ObjectFile::Strata> {
   static void format(const lldb_private::ObjectFile::Strata &strata,
                      raw_ostream &OS, StringRef Style);
 };
+
+namespace json {
+bool fromJSON(const llvm::json::Value &value, lldb_private::ObjectFile::Type &,
+              llvm::json::Path path);
+} // namespace json
 } // namespace llvm
 
 #endif // LLDB_SYMBOL_OBJECTFILE_H

lldb/source/Core/Section.cpp

@@ -673,3 +673,36 @@ uint64_t SectionList::GetDebugInfoSize() const {
   }
   return debug_info_size;
 }
+
+namespace llvm {
+namespace json {
+
+bool fromJSON(const llvm::json::Value &value,
+              lldb_private::JSONSection &section, llvm::json::Path path) {
+  llvm::json::ObjectMapper o(value, path);
+  return o && o.map("name", section.name) && o.map("type", section.type) &&
+         o.map("size", section.address) && o.map("size", section.size);
+}
+
+bool fromJSON(const llvm::json::Value &value, lldb::SectionType &type,
+              llvm::json::Path path) {
+  if (auto str = value.getAsString()) {
+    type = llvm::StringSwitch<lldb::SectionType>(*str)
+               .Case("code", eSectionTypeCode)
+               .Case("container", eSectionTypeContainer)
+               .Case("data", eSectionTypeData)
+               .Case("debug", eSectionTypeDebug)
+               .Default(eSectionTypeInvalid);
+
+    if (type == eSectionTypeInvalid) {
+      path.report("invalid section type");
+      return false;
+    }
+
+    return true;
+  }
+  path.report("expected string");
+  return false;
+}
+} // namespace json
+} // namespace llvm

lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.cpp

@@ -49,6 +49,7 @@ ObjectFileJSON::CreateInstance(const ModuleSP &module_sp, DataBufferSP data_sp,
   if (!MagicBytesMatch(data_sp, 0, data_sp->GetByteSize()))
     return nullptr;
 
+  // Update the data to contain the entire file if it doesn't already.
   if (data_sp->GetByteSize() < length) {
     data_sp = MapFileData(*file, length, file_offset);
     if (!data_sp)
@@ -56,30 +57,41 @@ ObjectFileJSON::CreateInstance(const ModuleSP &module_sp, DataBufferSP data_sp,
     data_offset = 0;
   }
 
+  Log *log = GetLog(LLDBLog::Symbols);
+
   auto text =
       llvm::StringRef(reinterpret_cast<const char *>(data_sp->GetBytes()));
 
   Expected<json::Value> json = json::parse(text);
   if (!json) {
-    llvm::consumeError(json.takeError());
+    LLDB_LOG_ERROR(log, json.takeError(),
+                   "failed to parse JSON object file: {0}");
     return nullptr;
   }
 
   json::Path::Root root;
   Header header;
-  if (!fromJSON(*json, header, root))
+  if (!fromJSON(*json, header, root)) {
+    LLDB_LOG_ERROR(log, root.getError(),
+                   "failed to parse JSON object file header: {0}");
     return nullptr;
+  }
 
   ArchSpec arch(header.triple);
   UUID uuid;
   uuid.SetFromStringRef(header.uuid);
+  Type type = header.type.value_or(eTypeDebugInfo);
 
   Body body;
-  fromJSON(*json, body, root);
+  if (!fromJSON(*json, body, root)) {
+    LLDB_LOG_ERROR(log, root.getError(),
+                   "failed to parse JSON object file body: {0}");
+    return nullptr;
+  }
 
   return new ObjectFileJSON(module_sp, data_sp, data_offset, file, file_offset,
-                            length, std::move(arch), std::move(uuid),
-                            std::move(body.symbols));
+                            length, std::move(arch), std::move(uuid), type,
+                            std::move(body.symbols), std::move(body.sections));
 }
 
 ObjectFile *ObjectFileJSON::CreateMemoryInstance(const ModuleSP &module_sp,
@@ -92,23 +104,36 @@ ObjectFile *ObjectFileJSON::CreateMemoryInstance(const ModuleSP &module_sp,
 size_t ObjectFileJSON::GetModuleSpecifications(
     const FileSpec &file, DataBufferSP &data_sp, offset_t data_offset,
     offset_t file_offset, offset_t length, ModuleSpecList &specs) {
-
   if (!MagicBytesMatch(data_sp, data_offset, data_sp->GetByteSize()))
     return 0;
 
+  // Update the data to contain the entire file if it doesn't already.
+  if (data_sp->GetByteSize() < length) {
+    data_sp = MapFileData(file, length, file_offset);
+    if (!data_sp)
+      return 0;
+    data_offset = 0;
+  }
+
+  Log *log = GetLog(LLDBLog::Symbols);
+
   auto text =
       llvm::StringRef(reinterpret_cast<const char *>(data_sp->GetBytes()));
 
   Expected<json::Value> json = json::parse(text);
   if (!json) {
-    llvm::consumeError(json.takeError());
+    LLDB_LOG_ERROR(log, json.takeError(),
+                   "failed to parse JSON object file: {0}");
     return 0;
   }
 
   json::Path::Root root;
   Header header;
-  if (!fromJSON(*json, header, root))
+  if (!fromJSON(*json, header, root)) {
+    LLDB_LOG_ERROR(log, root.getError(),
+                   "failed to parse JSON object file header: {0}");
     return 0;
+  }
 
   ArchSpec arch(header.triple);
   UUID uuid;
@@ -123,10 +148,12 @@ size_t ObjectFileJSON::GetModuleSpecifications(
 ObjectFileJSON::ObjectFileJSON(const ModuleSP &module_sp, DataBufferSP &data_sp,
                                offset_t data_offset, const FileSpec *file,
                                offset_t offset, offset_t length, ArchSpec arch,
-                               UUID uuid, std::vector<JSONSymbol> symbols)
+                               UUID uuid, Type type,
+                               std::vector<JSONSymbol> symbols,
+                               std::vector<JSONSection> sections)
     : ObjectFile(module_sp, file, offset, length, data_sp, data_offset),
-      m_arch(std::move(arch)), m_uuid(std::move(uuid)),
-      m_symbols(std::move(symbols)) {}
+      m_arch(std::move(arch)), m_uuid(std::move(uuid)), m_type(type),
+      m_symbols(std::move(symbols)), m_sections(std::move(sections)) {}
 
 bool ObjectFileJSON::ParseHeader() {
   // We already parsed the header during initialization.
@@ -139,7 +166,7 @@ void ObjectFileJSON::ParseSymtab(Symtab &symtab) {
   for (JSONSymbol json_symbol : m_symbols) {
     llvm::Expected<Symbol> symbol = Symbol::FromJSON(json_symbol, section_list);
     if (!symbol) {
-      LLDB_LOG_ERROR(log, symbol.takeError(), "invalid symbol");
+      LLDB_LOG_ERROR(log, symbol.takeError(), "invalid symbol: {0}");
       continue;
     }
     symtab.AddSymbol(*symbol);
@@ -147,7 +174,21 @@ void ObjectFileJSON::ParseSymtab(Symtab &symtab) {
   symtab.Finalize();
 }
 
-void ObjectFileJSON::CreateSections(SectionList &unified_section_list) {}
+void ObjectFileJSON::CreateSections(SectionList &unified_section_list) {
+  if (m_sections_up)
+    return;
+  m_sections_up = std::make_unique<SectionList>();
+
+  lldb::user_id_t id = 1;
+  for (const auto &section : m_sections) {
+    auto section_sp = std::make_shared<Section>(
+        GetModule(), this, id++, ConstString(section.name),
+        section.type.value_or(eSectionTypeCode), 0, section.size.value_or(0), 0,
+        section.size.value_or(0), /*log2align*/ 0, /*flags*/ 0);
+    m_sections_up->AddSection(section_sp);
+    unified_section_list.AddSection(section_sp);
+  }
+}
 
 bool ObjectFileJSON::MagicBytesMatch(DataBufferSP data_sp,
                                      lldb::addr_t data_offset,
@@ -164,13 +205,15 @@ namespace lldb_private {
 bool fromJSON(const json::Value &value, ObjectFileJSON::Header &header,
               json::Path path) {
   json::ObjectMapper o(value, path);
-  return o && o.map("triple", header.triple) && o.map("uuid", header.uuid);
+  return o && o.map("triple", header.triple) && o.map("uuid", header.uuid) &&
+         o.map("type", header.type);
 }
 
 bool fromJSON(const json::Value &value, ObjectFileJSON::Body &body,
               json::Path path) {
   json::ObjectMapper o(value, path);
-  return o && o.map("symbols", body.symbols);
+  return o && o.mapOptional("symbols", body.symbols) &&
+         o.mapOptional("sections", body.sections);
 }
 
 } // namespace lldb_private

lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.h

@@ -82,7 +82,7 @@ public:
 
   uint32_t GetDependentModules(FileSpecList &files) override { return 0; }
 
-  Type CalculateType() override { return eTypeDebugInfo; }
+  Type CalculateType() override { return m_type; }
 
   Strata CalculateStrata() override { return eStrataUser; }
 
@@ -92,21 +92,27 @@ public:
   struct Header {
     std::string triple;
     std::string uuid;
+    std::optional<ObjectFile::Type> type;
   };
 
   struct Body {
+    std::vector<JSONSection> sections;
     std::vector<JSONSymbol> symbols;
   };
 
 private:
   ArchSpec m_arch;
   UUID m_uuid;
+  ObjectFile::Type m_type;
+  std::optional<uint64_t> m_size;
   std::vector<JSONSymbol> m_symbols;
+  std::vector<JSONSection> m_sections;
 
   ObjectFileJSON(const lldb::ModuleSP &module_sp, lldb::DataBufferSP &data_sp,
                  lldb::offset_t data_offset, const FileSpec *file,
                  lldb::offset_t offset, lldb::offset_t length, ArchSpec arch,
-                 UUID uuid, std::vector<JSONSymbol> symbols);
+                 UUID uuid, Type type, std::vector<JSONSymbol> symbols,
+                 std::vector<JSONSection> sections);
 };
 
 bool fromJSON(const llvm::json::Value &value, ObjectFileJSON::Header &header,

lldb/source/Symbol/ObjectFile.cpp

@@ -761,3 +761,34 @@ uint32_t ObjectFile::GetCacheHash() {
   m_cache_hash = llvm::djbHash(strm.GetString());
   return *m_cache_hash;
 }
+
+namespace llvm {
+namespace json {
+
+bool fromJSON(const llvm::json::Value &value,
+              lldb_private::ObjectFile::Type &type, llvm::json::Path path) {
+  if (auto str = value.getAsString()) {
+    type = llvm::StringSwitch<ObjectFile::Type>(*str)
+               .Case("corefile", ObjectFile::eTypeCoreFile)
+               .Case("executable", ObjectFile::eTypeExecutable)
+               .Case("debuginfo", ObjectFile::eTypeDebugInfo)
+               .Case("dynamiclinker", ObjectFile::eTypeDynamicLinker)
+               .Case("objectfile", ObjectFile::eTypeObjectFile)
+               .Case("sharedlibrary", ObjectFile::eTypeSharedLibrary)
+               .Case("stublibrary", ObjectFile::eTypeStubLibrary)
+               .Case("jit", ObjectFile::eTypeJIT)
+               .Case("unknown", ObjectFile::eTypeUnknown)
+               .Default(ObjectFile::eTypeInvalid);
+
+    if (type == ObjectFile::eTypeInvalid) {
+      path.report("invalid object type");
+      return false;
+    }
+
+    return true;
+  }
+  path.report("expected string");
+  return false;
+}
+} // namespace json
+} // namespace llvm

lldb/test/API/macosx/symbols/TestObjectFileJSON.py

@@ -0,0 +1,100 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+import json
+import uuid
+import os
+import shutil
+
+
+class TestObjectFileJSON(TestBase):
+    TRIPLE = "arm64-apple-macosx13.0.0"
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self.source = "main.c"
+
+    def emitJSON(self, data, path):
+        json_object = json.dumps(data, indent=4)
+        json_object_file = self.getBuildArtifact("a.json")
+        with open(json_object_file, "w") as outfile:
+            outfile.write(json_object)
+
+    def toModuleSpec(self, path):
+        module_spec = lldb.SBModuleSpec()
+        module_spec.SetFileSpec(lldb.SBFileSpec(path))
+        return module_spec
+
+    @no_debug_info_test
+    def test_target(self):
+        data = {
+            "triple": self.TRIPLE,
+            "uuid": str(uuid.uuid4()),
+            "type": "executable",
+        }
+
+        json_object_file = self.getBuildArtifact("a.json")
+        self.emitJSON(data, json_object_file)
+
+        target = self.dbg.CreateTarget(json_object_file)
+        self.assertTrue(target.IsValid())
+        self.assertEqual(target.GetTriple(), self.TRIPLE)
+
+    @no_debug_info_test
+    def test_module(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        target = self.dbg.CreateTarget(exe)
+
+        data = {
+            "triple": self.TRIPLE,
+            "uuid": str(uuid.uuid4()),
+        }
+
+        json_object_file = self.getBuildArtifact("a.json")
+        self.emitJSON(data, json_object_file)
+
+        module = target.AddModule(self.toModuleSpec(json_object_file))
+        self.assertFalse(module.IsValid())
+
+        data = {
+            "triple": self.TRIPLE,
+            "uuid": str(uuid.uuid4()),
+            "type": "sharedlibrary",
+            "sections": [
+                {
+                    "name": "__TEXT",
+                    "type": "code",
+                    "address": 0,
+                    "size": 0x222,
+                }
+            ],
+            "symbols": [
+                {
+                    "name": "foo",
+                    "address": 0x100,
+                    "size": 0x11,
+                }
+            ],
+        }
+        self.emitJSON(data, json_object_file)
+
+        module = target.AddModule(self.toModuleSpec(json_object_file))
+        self.assertTrue(module.IsValid())
+
+        section = module.GetSectionAtIndex(0)
+        self.assertTrue(section.IsValid())
+        self.assertEqual(section.GetName(), "__TEXT")
+        self.assertEqual(section.file_addr, 0x0)
+        self.assertEqual(section.size, 0x222)
+
+        symbol = module.FindSymbol("foo")
+        self.assertTrue(symbol.IsValid())
+        self.assertEqual(symbol.addr.GetFileAddress(), 0x100)
+        self.assertEqual(symbol.GetSize(), 0x11)
+
+        error = target.SetSectionLoadAddress(section, 0x1000)
+        self.assertSuccess(error)
+        self.assertEqual(symbol.addr.GetLoadAddress(target), 0x1100)