Code completion: implement delayed parsing and code completion for

TopLevelCodeDecls


Swift SVN r6638

Author: gribozavr

Diff for: include/swift/Parse/Parser.h

@@ -392,12 +392,17 @@ class Parser {
   bool parseList(tok RightK, SourceLoc LeftLoc, SourceLoc &RightLoc,
                  tok SeparatorK, bool OptionalSep, Diag<> ErrorDiag,
                  std::function<bool()> callback);
-  
+
+  void consumeTopLevelDecl(TopLevelCodeDecl *TLCD,
+                           ParserPosition BeginParserPosition);
+
   void parseBraceItems(SmallVectorImpl<ExprStmtOrDecl> &Decls,
                        bool IsTopLevel,
                        BraceItemListKind Kind = BraceItemListKind::Brace);
   NullablePtr<BraceStmt> parseBraceItemList(Diag<> ID);
 
+  void parseTopLevelCodeDeclDelayed();
+
   //===--------------------------------------------------------------------===//
   // Decl Parsing
   static bool isStartOfDecl(const Token &Tok, const Token &Tok2);

Diff for: include/swift/Parse/PersistentParserState.h

@@ -22,7 +22,9 @@
 #include "llvm/ADT/DenseMap.h"
 
 namespace swift {
+  class Decl;
   class FuncExpr;
+  class TopLevelCodeDecl;
 
 /// \brief Parser state persistent across multiple parses.
 class PersistentParserState {
@@ -50,21 +52,56 @@ class PersistentParserState {
     {}
   };
 
+  class DelayedDeclState {
+    friend class PersistentParserState;
+    friend class Parser;
+    Decl *D;
+    ParserPos BodyPos;
+    SourceLoc BodyEnd;
+    SavedScope Scope;
+
+    SavedScope takeScope() {
+      return std::move(Scope);
+    }
+
+  public:
+    DelayedDeclState(Decl *D, SourceRange BodyRange, SourceLoc PreviousLoc,
+                     SavedScope &&Scope)
+      : D(D), BodyPos{BodyRange.Start, PreviousLoc},
+        BodyEnd(BodyRange.End), Scope(std::move(Scope))
+    {}
+  };
+
 private:
   ScopeInfo ScopeInfo;
   typedef llvm::DenseMap<FuncExpr *, std::unique_ptr<FunctionBodyState>>
       DelayedBodiesTy;
   DelayedBodiesTy DelayedBodies;
+
   /// \brief Parser sets this if it stopped parsing before the buffer ended.
   ParserPos MarkedPos;
 
+  std::unique_ptr<DelayedDeclState> CodeCompletionDelayedDeclState;
+
 public:
   swift::ScopeInfo &getScopeInfo() { return ScopeInfo; }
 
   void delayFunctionBodyParsing(FuncExpr *FE, SourceRange BodyRange,
                                 SourceLoc PreviousLoc);
   std::unique_ptr<FunctionBodyState> takeBodyState(FuncExpr *FE);
 
+  void delayTopLevelCodeDecl(TopLevelCodeDecl *TLCD, SourceRange BodyRange,
+                             SourceLoc PreviousLoc);
+  bool hasDelayedDecl() {
+    return CodeCompletionDelayedDeclState.get() != nullptr;
+  }
+  SourceLoc getDelayedDeclLoc() {
+    return CodeCompletionDelayedDeclState->BodyPos.Loc;
+  }
+  std::unique_ptr<DelayedDeclState> takeDelayedDeclState() {
+    return std::move(CodeCompletionDelayedDeclState);
+  }
+
   void markParserPosition(SourceLoc Loc, SourceLoc PrevLoc) {
     MarkedPos = {Loc, PrevLoc};
   }

Diff for: lib/Parse/ParseStmt.cpp

@@ -132,6 +132,16 @@ static bool isTerminatorForBraceItemListKind(const Token &Tok,
   }
 }
 
+void Parser::consumeTopLevelDecl(TopLevelCodeDecl *TLCD,
+                                 ParserPosition BeginParserPosition) {
+  backtrackToPosition(BeginParserPosition);
+  SourceLoc BeginLoc = Tok.getLoc();
+  skipUntilDeclStmtRBrace();
+  SourceLoc EndLoc = Tok.getLoc();
+  State->delayTopLevelCodeDecl(TLCD, { BeginLoc, EndLoc },
+                               BeginParserPosition.PreviousLoc);
+}
+
 ///   brace-item:
 ///     decl
 ///     expr
@@ -194,8 +204,19 @@ void Parser::parseBraceItems(SmallVectorImpl<ExprStmtOrDecl> &Entries,
       auto *TLCD = new (Context) TopLevelCodeDecl(CurDeclContext);
       ContextChange CC(*this, TLCD);
       SourceLoc StartLoc = Tok.getLoc();
-      
-      if (parseExprOrStmt(Result)) {
+
+      ParserPosition BeginParserPosition;
+      if (isCodeCompletionFirstPass())
+        BeginParserPosition = getParserPosition();
+
+      bool FailedToParse = parseExprOrStmt(Result);
+
+      if (Tok.is(tok::code_complete)) {
+        consumeTopLevelDecl(TLCD, BeginParserPosition);
+        return;
+      }
+
+      if (FailedToParse) {
         NeedParseErrorRecovery = true;
       } else {
         auto Brace = BraceStmt::create(Context, StartLoc, Result, Tok.getLoc());
@@ -244,6 +265,41 @@ void Parser::parseBraceItems(SmallVectorImpl<ExprStmtOrDecl> &Entries,
   }
 }
 
+void Parser::parseTopLevelCodeDeclDelayed() {
+  auto DelayedState = State->takeDelayedDeclState();
+  assert(DelayedState.get() && "should have delayed state");
+
+  auto *TLCD = dyn_cast<TopLevelCodeDecl>(DelayedState->D);
+  assert(!TLCD->getBody() && "should not have a parsed body");
+
+  auto BeginParserPosition = getParserPosition(DelayedState->BodyPos);
+  auto EndLexerState = L->getStateForEndOfTokenLoc(DelayedState->BodyEnd);
+
+  // ParserPositionRAII needs a primed parser to restore to.
+  if (Tok.is(tok::NUM_TOKENS))
+    consumeToken();
+
+  // Ensure that we restore the parser state at exit.
+  ParserPositionRAII PPR(*this);
+
+  // Create a lexer that can not go past the end state.
+  Lexer LocalLex(*L, BeginParserPosition.LS, EndLexerState, SourceMgr, &Diags,
+                 nullptr /*not SIL*/);
+
+  // Temporarily swap out the parser's current lexer with our new one.
+  llvm::SaveAndRestore<Lexer *> T(L, &LocalLex);
+
+  // Rewind to '{' of the function body.
+  restoreParserPosition(BeginParserPosition);
+
+  // Re-enter the lexical scope.
+  Scope S(this, DelayedState->takeScope());
+  ContextChange CC(*this, TLCD);
+
+  ExprStmtOrDecl Result;
+  parseExprOrStmt(Result);
+}
+
 /// Recover from a 'case' or 'default' outside of a 'switch' by consuming up to
 /// the next ':'.
 static NullablePtr<Stmt> recoverFromInvalidCase(Parser &P) {

Diff for: lib/Parse/Parser.cpp

@@ -99,6 +99,30 @@ class ParseDelayedFunctionBodies : public ASTWalker {
     TheParser.parseDeclFuncBodyDelayed(FD);
   }
 };
+
+void
+parseDelayedTopLevelDecl(TranslationUnit *TU,
+                         PersistentParserState &ParserState,
+                         CodeCompletionCallbacksFactory *CodeCompletionFactory,
+                         unsigned CodeCompletionOffset) {
+  assert(CodeCompletionFactory &&
+         "delayed parsing of decls only makes sense for code completion");
+
+  if (!ParserState.hasDelayedDecl())
+    return;
+
+  int BufferID = TU->Ctx.SourceMgr
+      .FindBufferContainingLoc(ParserState.getDelayedDeclLoc().Value);
+  Parser TheParser(BufferID, TU,
+                   TU->Kind == TranslationUnit::Main ||
+                   TU->Kind == TranslationUnit::REPL, nullptr, &ParserState);
+
+  std::unique_ptr<CodeCompletionCallbacks> CodeCompletion(
+      CodeCompletionFactory->createCodeCompletionCallbacks(TheParser));
+  TheParser.setCodeCompletion(CodeCompletionOffset);
+  TheParser.setCodeCompletionCallbacks(CodeCompletion.get());
+  TheParser.parseTopLevelCodeDeclDelayed();
+}
 } // unnamed namespace
 
 /// parseIntoTranslationUnit - Entrypoint for the parser.
@@ -138,6 +162,10 @@ void swift::performDelayedParsing(
   for (Decl *D : TU->Decls) {
     D->walk(Walker);
   }
+
+  if (CodeCompletionFactory)
+    parseDelayedTopLevelDecl(TU, PersistentState, CodeCompletionFactory,
+                             CodeCompletionOffset);
 }
 
 std::vector<Token> swift::tokenize(llvm::SourceMgr &SM, unsigned BufferID,

Diff for: lib/Parse/PersistentParserState.cpp

@@ -14,6 +14,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "swift/AST/Decl.h"
 #include "swift/AST/Expr.h"
 #include "swift/Parse/PersistentParserState.h"
 
@@ -39,3 +40,13 @@ PersistentParserState::takeBodyState(FuncExpr *FE) {
   DelayedBodies.erase(I);
   return State;
 }
+
+void PersistentParserState::delayTopLevelCodeDecl(TopLevelCodeDecl *TLCD,
+                                                  SourceRange BodyRange,
+                                                  SourceLoc PreviousLoc) {
+  assert(!CodeCompletionDelayedDeclState.get() &&
+         "only one decl can be delayed for code completion");
+  CodeCompletionDelayedDeclState.reset(new DelayedDeclState(
+      TLCD, BodyRange, PreviousLoc, ScopeInfo.saveCurrentScope()));
+}
+