From Vassil Vassilev:

Enable incremental parsing by the Preprocessor,
where more code can be provided after an EOF.
It mainly prevents the tearing down of the topmost lexer.
To be used like this:
PP.enableIncrementalProcessing();
while (getMoreSource()) {
  while (Parser.ParseTopLevelDecl(ADecl)) {...}
}
PP.enableIncrementalProcessing(false);

llvm-svn: 152914

clang/include/clang/Lex/Preprocessor.h

@@ -148,6 +148,10 @@ class Preprocessor : public RefCountedBase<Preprocessor> {
   /// with this preprocessor.
   std::vector<CommentHandler *> CommentHandlers;
 
+  /// \brief True if we want to ignore EOF token and continue later on (thus 
+  /// avoid tearing the Lexer and etc. down).
+  bool IncrementalProcessing;
+
   /// \brief The code-completion handler.
   CodeCompletionHandler *CodeComplete;
 
@@ -344,7 +348,8 @@ public:
                ModuleLoader &TheModuleLoader,
                IdentifierInfoLookup *IILookup = 0,
                bool OwnsHeaderSearch = false,
-               bool DelayInitialization = false);
+               bool DelayInitialization = false,
+               bool IncrProcessing = false);
 
   ~Preprocessor();
 
@@ -691,6 +696,14 @@ public:
   /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
   /// CurTokenLexer pointers.
   void recomputeCurLexerKind();
+
+  /// \brief Returns true if incremental processing is enabled
+  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
+
+  /// \brief Enables the incremental processing
+  void enableIncrementalProcessing(bool value = true) {
+    IncrementalProcessing = value;
+  }
   
   /// \brief Specify the point at which code-completion will be performed.
   ///

clang/lib/Lex/PPLexerChange.cpp

@@ -328,15 +328,17 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
     CurLexer->BufferPtr = EndPos;
     CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
 
-    // We're done with the #included file.
-    CurLexer.reset();
+    if (!isIncrementalProcessingEnabled())
+      // We're done with lexing.
+      CurLexer.reset();
   } else {
     assert(CurPTHLexer && "Got EOF but no current lexer set!");
     CurPTHLexer->getEOF(Result);
     CurPTHLexer.reset();
   }
-
-  CurPPLexer = 0;
+  
+  if (!isIncrementalProcessingEnabled())
+    CurPPLexer = 0;
 
   // This is the end of the top-level file. 'WarnUnusedMacroLocs' has collected
   // all macro locations that we need to warn because they are not used.

clang/lib/Lex/Preprocessor.cpp

@@ -54,18 +54,19 @@ Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
                            HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
                            IdentifierInfoLookup* IILookup,
                            bool OwnsHeaders,
-                           bool DelayInitialization)
+                           bool DelayInitialization,
+                           bool IncrProcessing)
   : Diags(&diags), LangOpts(opts), Target(target),FileMgr(Headers.getFileMgr()),
     SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
-    ExternalSource(0), 
-    Identifiers(opts, IILookup), CodeComplete(0),
+    ExternalSource(0), Identifiers(opts, IILookup), 
+    IncrementalProcessing(IncrProcessing), CodeComplete(0), 
     CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
     SkipMainFilePreamble(0, true), CurPPLexer(0), 
     CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 
     Record(0), MIChainHead(0), MICache(0) 
 {
   OwnsHeaderSearch = OwnsHeaders;
-  
+
   if (!DelayInitialization) {
     assert(Target && "Must provide target information for PP initialization");
     Initialize(*Target);

clang/lib/Parse/Parser.cpp

@@ -475,6 +475,11 @@ void Parser::Initialize() {
 bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result) {
   DelayedCleanupPoint CleanupRAII(TopLevelDeclCleanupPool);
 
+  // Skip over the EOF token, flagging end of previous input for incremental 
+  // processing
+  if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof))
+    ConsumeToken();
+
   while (Tok.is(tok::annot_pragma_unused))
     HandlePragmaUnused();
 
@@ -483,15 +488,17 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result) {
     // Late template parsing can begin.
     if (getLangOpts().DelayedTemplateParsing)
       Actions.SetLateTemplateParser(LateTemplateParserCallback, this);
+    if (!PP.isIncrementalProcessingEnabled())
+      Actions.ActOnEndOfTranslationUnit();
+    //else don't tell Sema that we ended parsing: more input might come.
 
-    Actions.ActOnEndOfTranslationUnit();
     return true;
   }
 
   ParsedAttributesWithRange attrs(AttrFactory);
   MaybeParseCXX0XAttributes(attrs);
   MaybeParseMicrosoftAttributes(attrs);
-  
+
   Result = ParseExternalDeclaration(attrs);
   return false;
 }