make the -rewrite-test a bit more interesting: it now

wraps comments in <i> tags.  Extend rewrite tokens to support
this minimal functionality.

llvm-svn: 57409

clang/Driver/RewriteTest.cpp

@@ -11,9 +11,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/TokenRewriter.h"
 #include "clang.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Rewrite/TokenRewriter.h"
 #include <iostream>
 
 void clang::DoRewriteTest(Preprocessor &PP, const std::string &InFileName,
@@ -22,27 +22,18 @@ void clang::DoRewriteTest(Preprocessor &PP, const std::string &InFileName,
   const LangOptions &LangOpts = PP.getLangOptions();
 
   TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts);
-  
-  
-  
-  
-  
-  std::pair<const char*,const char*> File =SM.getBufferData(SM.getMainFileID());
-  
-  // Create a lexer to lex all the tokens of the main file in raw mode.  Even
-  // though it is in raw mode, it will not return comments.
-  Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
-               LangOpts, File.first, File.second);
-  
-  RawLex.SetKeepWhitespaceMode(true);
-  
-  Token RawTok;
-  RawLex.LexFromRawLexer(RawTok);
-  while (RawTok.isNot(tok::eof)) {
-    std::cout << PP.getSpelling(RawTok);
-    RawLex.LexFromRawLexer(RawTok);
+
+  // Throw <i> </i> tags around comments.
+  for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
+       E = Rewriter.token_end(); I != E; ++I) {
+    if (I->isNot(tok::comment)) continue;
+
+    Rewriter.AddTokenBefore(I, "<i>");
+    Rewriter.AddTokenAfter(I, "</i>");
   }
   
+  
+  // Print out the output.
   for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
        E = Rewriter.token_end(); I != E; ++I)
     std::cout << PP.getSpelling(*I);

clang/include/clang/Rewrite/TokenRewriter.h

@@ -16,12 +16,14 @@
 #define LLVM_CLANG_TOKENREWRITER_H
 
 #include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/OwningPtr.h"
 #include <list>
 #include <map>
 
 namespace clang {
   class Token;
   class LangOptions;
+  class ScratchBuffer;
   
   class TokenRewriter {
     /// TokenList - This is the list of raw tokens that make up this file.  Each
@@ -37,20 +39,37 @@ namespace clang {
     /// backwards.
     std::map<SourceLocation, TokenRefTy> TokenAtLoc;
     
+    /// ScratchBuf - This is the buffer that we create scratch tokens from.
+    ///
+    llvm::OwningPtr<ScratchBuffer> ScratchBuf;
+    
+    TokenRewriter(const TokenRewriter&);  // DO NOT IMPLEMENT
+    void operator=(const TokenRewriter&); // DO NOT IMPLEMENT.
   public:
     /// TokenRewriter - This creates a TokenRewriter for the file with the
     /// specified FileID.
     TokenRewriter(unsigned FileID, SourceManager &SM, const LangOptions &LO);
-    
+    ~TokenRewriter();
     
     typedef std::list<Token>::const_iterator token_iterator;
     token_iterator token_begin() const { return TokenList.begin(); }
     token_iterator token_end() const { return TokenList.end(); }
     
+    
+    token_iterator AddTokenBefore(token_iterator I, const char *Val);
+    token_iterator AddTokenAfter(token_iterator I, const char *Val) {
+      assert(I != token_end() && "Cannot insert after token_end()!");
+      return AddTokenBefore(++I, Val);
+    }
+    
   private:
+    /// RemapIterator - Convert from token_iterator (a const iterator) to
+    /// TokenRefTy (a non-const iterator).
+    TokenRefTy RemapIterator(token_iterator I);
+    
     /// AddToken - Add the specified token into the Rewriter before the other
     /// position.
-    void AddToken(const Token &T, TokenRefTy Where);
+    TokenRefTy AddToken(const Token &T, TokenRefTy Where);
   };
   
   

clang/lib/Rewrite/TokenRewriter.cpp

@@ -14,11 +14,13 @@
 
 #include "clang/Rewrite/TokenRewriter.h"
 #include "clang/Lex/Lexer.h"
+#include "clang/Lex/ScratchBuffer.h"
 #include "clang/Basic/SourceManager.h"
 using namespace clang;
 
 TokenRewriter::TokenRewriter(unsigned FileID, SourceManager &SM,
                              const LangOptions &LangOpts) {
+  ScratchBuf.reset(new ScratchBuffer(SM));
   
   std::pair<const char*,const char*> File = SM.getBufferData(FileID);
   
@@ -33,21 +35,66 @@ TokenRewriter::TokenRewriter(unsigned FileID, SourceManager &SM,
   Token RawTok;
   RawLex.LexFromRawLexer(RawTok);
   while (RawTok.isNot(tok::eof)) {
+#if 0
+    if (Tok.is(tok::identifier)) {
+      // Look up the identifier info for the token.  This should use
+      // IdentifierTable directly instead of PP.
+      Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
+    }
+#endif
+      
     AddToken(RawTok, TokenList.end());
     RawLex.LexFromRawLexer(RawTok);
   }
-  
-  
 }
 
+TokenRewriter::~TokenRewriter() {
+}
+
+
+/// RemapIterator - Convert from token_iterator (a const iterator) to
+/// TokenRefTy (a non-const iterator).
+TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
+  if (I == token_end()) return TokenList.end();
+  
+  // FIXME: This is horrible, we should use our own list or something to avoid
+  // this.
+  std::map<SourceLocation, TokenRefTy>::iterator MapIt = 
+    TokenAtLoc.find(I->getLocation());
+  assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
+  return MapIt->second;
+}
+
+
 /// AddToken - Add the specified token into the Rewriter before the other
 /// position.
-void TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
+TokenRewriter::TokenRefTy 
+TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
   Where = TokenList.insert(Where, T);
   
   bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
                                                         Where)).second;
   assert(InsertSuccess && "Token location already in rewriter!");
   InsertSuccess = InsertSuccess;
+  return Where;
 }
-    
+  
+
+TokenRewriter::token_iterator
+TokenRewriter::AddTokenBefore(token_iterator I, const char *Val){
+  unsigned Len = strlen(Val);
+  
+  // Plop the string into the scratch buffer, then create a token for this
+  // string.
+  Token Tok;
+  Tok.startToken();
+  Tok.setLocation(ScratchBuf->getToken(Val, Len));
+  Tok.setLength(Len);
+  
+  // TODO: Form a whole lexer around this and relex the token!  For now, just
+  // set kind to tok::unknown.
+  Tok.setKind(tok::unknown);
+
+  return AddToken(Tok, RemapIterator(I));
+}
+