[libclang] Add index option to store preambles in memory

This commit allows libclang API users to opt into storing PCH in memory
instead of temporary files. The option can be set only during CXIndex
construction to avoid multithreading issues and confusion or bugs if
some preambles are stored in temporary files and others - in memory.

The added API works as expected in KDevelop:
https://invent.kde.org/kdevelop/kdevelop/-/merge_requests/283

Differential Revision: https://reviews.llvm.org/D145974

clang/docs/ReleaseNotes.rst

@@ -330,8 +330,8 @@ libclang
   was marked with the explicit identifier.
 
 - Introduced the new ``CXIndex`` constructor function
-  ``clang_createIndexWithOptions``, which allows overriding precompiled preamble
-  storage path.
+  ``clang_createIndexWithOptions``, which allows storing precompiled preambles
+  in memory or overriding the precompiled preamble storage path.
 
 - Deprecated two functions ``clang_CXIndex_setGlobalOptions`` and
   ``clang_CXIndex_setInvocationEmissionPathOption`` in favor of the new

clang/include/clang-c/Index.h

@@ -372,13 +372,19 @@ typedef struct CXIndexOptions {
    * \see clang_createIndex()
    */
   unsigned DisplayDiagnostics : 1;
-  unsigned /*Reserved*/ : 14;
+  /**
+   * Store PCH in memory. If zero, PCH are stored in temporary files.
+   */
+  unsigned StorePreamblesInMemory : 1;
+  unsigned /*Reserved*/ : 13;
 
   /**
    * The path to a directory, in which to store temporary PCH files. If null or
    * empty, the default system temporary directory is used. These PCH files are
    * deleted on clean exit but stay on disk if the program crashes or is killed.
    *
+   * This option is ignored if \a StorePreamblesInMemory is non-zero.
+   *
    * Libclang does not create the directory at the specified path in the file
    * system. Therefore it must exist, or storing PCH files will fail.
    */

clang/include/clang/Frontend/ASTUnit.h

@@ -119,6 +119,7 @@ private:
   std::shared_ptr<PreprocessorOptions>    PPOpts;
   IntrusiveRefCntPtr<ASTReader> Reader;
   bool HadModuleLoaderFatalFailure = false;
+  bool StorePreamblesInMemory = false;
 
   struct ASTWriterData;
   std::unique_ptr<ASTWriterData> WriterData;
@@ -803,9 +804,12 @@ public:
   ///
   /// \param ResourceFilesPath - The path to the compiler resource files.
   ///
+  /// \param StorePreamblesInMemory - Whether to store PCH in memory. If false,
+  /// PCH are stored in temporary files.
+  ///
   /// \param PreambleStoragePath - The path to a directory, in which to create
   /// temporary PCH files. If empty, the default system temporary directory is
-  /// used.
+  /// used. This parameter is ignored if \p StorePreamblesInMemory is true.
   ///
   /// \param ModuleFormat - If provided, uses the specific module format.
   ///
@@ -825,6 +829,7 @@ public:
       const char **ArgBegin, const char **ArgEnd,
       std::shared_ptr<PCHContainerOperations> PCHContainerOps,
       IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
+      bool StorePreamblesInMemory = false,
       StringRef PreambleStoragePath = StringRef(), bool OnlyLocalDecls = false,
       CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
       ArrayRef<RemappedFile> RemappedFiles = std::nullopt,

clang/lib/Frontend/ASTUnit.cpp

@@ -1397,7 +1397,7 @@ ASTUnit::getMainBufferWithPrecompiledPreamble(
 
     llvm::ErrorOr<PrecompiledPreamble> NewPreamble = PrecompiledPreamble::Build(
         PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS,
-        PCHContainerOps, /*StoreInMemory=*/false, PreambleStoragePath,
+        PCHContainerOps, StorePreamblesInMemory, PreambleStoragePath,
         Callbacks);
 
     PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies =
@@ -1742,13 +1742,13 @@ ASTUnit *ASTUnit::LoadFromCommandLine(
     const char **ArgBegin, const char **ArgEnd,
     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
     IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
-    StringRef PreambleStoragePath, bool OnlyLocalDecls,
-    CaptureDiagsKind CaptureDiagnostics, ArrayRef<RemappedFile> RemappedFiles,
-    bool RemappedFilesKeepOriginalName, unsigned PrecompilePreambleAfterNParses,
-    TranslationUnitKind TUKind, bool CacheCodeCompletionResults,
-    bool IncludeBriefCommentsInCodeCompletion, bool AllowPCHWithCompilerErrors,
-    SkipFunctionBodiesScope SkipFunctionBodies, bool SingleFileParse,
-    bool UserFilesAreVolatile, bool ForSerialization,
+    bool StorePreamblesInMemory, StringRef PreambleStoragePath,
+    bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
+    ArrayRef<RemappedFile> RemappedFiles, bool RemappedFilesKeepOriginalName,
+    unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind,
+    bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion,
+    bool AllowPCHWithCompilerErrors, SkipFunctionBodiesScope SkipFunctionBodies,
+    bool SingleFileParse, bool UserFilesAreVolatile, bool ForSerialization,
     bool RetainExcludedConditionalBlocks, std::optional<StringRef> ModuleFormat,
     std::unique_ptr<ASTUnit> *ErrAST,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
@@ -1803,6 +1803,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(
     VFS = llvm::vfs::getRealFileSystem();
   VFS = createVFSFromCompilerInvocation(*CI, *Diags, VFS);
   AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
+  AST->StorePreamblesInMemory = StorePreamblesInMemory;
   AST->PreambleStoragePath = PreambleStoragePath;
   AST->ModuleCache = new InMemoryModuleCache;
   AST->OnlyLocalDecls = OnlyLocalDecls;

clang/tools/libclang/CIndex.cpp

@@ -3742,6 +3742,7 @@ CXIndex clang_createIndexWithOptions(const CXIndexOptions *options) {
       options->ExcludeDeclarationsFromPCH, options->DisplayDiagnostics,
       options->ThreadBackgroundPriorityForIndexing,
       options->ThreadBackgroundPriorityForEditing);
+  CIdxr->setStorePreamblesInMemory(options->StorePreamblesInMemory);
   CIdxr->setPreambleStoragePath(options->PreambleStoragePath);
   CIdxr->setInvocationEmissionPath(options->InvocationEmissionPath);
   return CIdxr;
@@ -3956,8 +3957,9 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
   std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCommandLine(
       Args->data(), Args->data() + Args->size(),
       CXXIdx->getPCHContainerOperations(), Diags,
-      CXXIdx->getClangResourcesPath(), CXXIdx->getPreambleStoragePath(),
-      CXXIdx->getOnlyLocalDecls(), CaptureDiagnostics, *RemappedFiles.get(),
+      CXXIdx->getClangResourcesPath(), CXXIdx->getStorePreamblesInMemory(),
+      CXXIdx->getPreambleStoragePath(), CXXIdx->getOnlyLocalDecls(),
+      CaptureDiagnostics, *RemappedFiles.get(),
       /*RemappedFilesKeepOriginalName=*/true, PrecompilePreambleAfterNParses,
       TUKind, CacheCodeCompletionResults, IncludeBriefCommentsInCodeCompletion,
       /*AllowPCHWithCompilerErrors=*/true, SkipFunctionBodies, SingleFileParse,

clang/tools/libclang/CIndexer.h

@@ -34,6 +34,7 @@ class IdentifierInfo;
 class CIndexer {
   bool OnlyLocalDecls;
   bool DisplayDiagnostics;
+  bool StorePreamblesInMemory = false;
   unsigned Options; // CXGlobalOptFlags.
 
   std::string ResourcesPath;
@@ -78,6 +79,11 @@ public:
 
   StringRef getClangToolchainPath();
 
+  void setStorePreamblesInMemory(bool StoreInMemory) {
+    StorePreamblesInMemory = StoreInMemory;
+  }
+  bool getStorePreamblesInMemory() const { return StorePreamblesInMemory; }
+
   void setPreambleStoragePath(StringRef Str) {
     PreambleStoragePath = Str.str();
   }

clang/unittests/Frontend/ASTUnitTest.cpp

@@ -167,7 +167,7 @@ TEST_F(ASTUnitTest, LoadFromCommandLineEarlyError) {
   std::unique_ptr<clang::ASTUnit> ErrUnit;
 
   ASTUnit *AST = ASTUnit::LoadFromCommandLine(
-      &Args[0], &Args[4], PCHContainerOps, Diags, "", "", false,
+      &Args[0], &Args[4], PCHContainerOps, Diags, "", false, "", false,
       CaptureDiagsKind::All, std::nullopt, true, 0, TU_Complete, false, false,
       false, SkipFunctionBodiesScope::None, false, true, false, false,
       std::nullopt, &ErrUnit, nullptr);

clang/unittests/libclang/LibclangTest.cpp

@@ -479,6 +479,7 @@ protected:
 };
 
 class LibclangSetPreambleStoragePathTest : public LibclangPreambleStorageTest {
+  virtual bool StorePreamblesInMemory() { return false; }
   virtual const char *PreambleStoragePath() = 0;
 
 protected:
@@ -487,6 +488,7 @@ protected:
 
     CXIndexOptions Opts{};
     Opts.Size = sizeof(CXIndexOptions);
+    Opts.StorePreamblesInMemory = StorePreamblesInMemory();
     Opts.PreambleStoragePath = PreambleStoragePath();
     Index = clang_createIndexWithOptions(&Opts);
     ASSERT_TRUE(Index);
@@ -506,6 +508,19 @@ class LibclangPreambleDirPreambleStoragePathTest
   const char *PreambleStoragePath() override { return PreambleDir.c_str(); }
 };
 
+class LibclangStoreInMemoryNullPreambleStoragePathTest
+    : public LibclangNullPreambleStoragePathTest {
+  bool StorePreamblesInMemory() override { return true; }
+};
+class LibclangStoreInMemoryEmptyPreambleStoragePathTest
+    : public LibclangEmptyPreambleStoragePathTest {
+  bool StorePreamblesInMemory() override { return true; }
+};
+class LibclangStoreInMemoryPreambleDirPreambleStoragePathTest
+    : public LibclangPreambleDirPreambleStoragePathTest {
+  bool StorePreamblesInMemory() override { return true; }
+};
+
 TEST_F(LibclangNotOverriddenPreambleStoragePathTest, CountPreambles) {
   CountPreamblesInPreambleDir(0);
 }
@@ -518,6 +533,16 @@ TEST_F(LibclangEmptyPreambleStoragePathTest, CountPreambles) {
 TEST_F(LibclangPreambleDirPreambleStoragePathTest, CountPreambles) {
   CountPreamblesInPreambleDir(1);
 }
+TEST_F(LibclangStoreInMemoryNullPreambleStoragePathTest, CountPreambles) {
+  CountPreamblesInPreambleDir(0);
+}
+TEST_F(LibclangStoreInMemoryEmptyPreambleStoragePathTest, CountPreambles) {
+  CountPreamblesInPreambleDir(0);
+}
+TEST_F(LibclangStoreInMemoryPreambleDirPreambleStoragePathTest,
+       CountPreambles) {
+  CountPreamblesInPreambleDir(0);
+}
 
 TEST_F(LibclangParseTest, AllSkippedRanges) {
   std::string Header = "header.h", Main = "main.cpp";