libSyntax: create a basic infrastructure for generating libSyntax entities by using Parser.

Author: nkcsgexi

Diff for: include/swift/AST/Module.h

@@ -28,6 +28,8 @@
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Parse/Token.h"
+#include "swift/Syntax/SyntaxNodes.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
@@ -1082,9 +1084,23 @@ class SourceFile final : public FileUnit {
     return (bool)AllCorrectedTokens;
   }
 
+  syntax::SourceFileSyntax getSyntaxRoot() const {
+    assert(SyntaxRoot && "no syntax root is set.");
+    return *SyntaxRoot;
+  }
+
 private:
+  friend class syntax::SyntaxParsingContext;
+  friend class syntax::SyntaxParsingContextRoot;
+
   /// If not None, the underlying vector should contain tokens of this source file.
   Optional<std::vector<Token>> AllCorrectedTokens;
+
+  /// All of the raw token syntax nodes in the underlying source.
+  std::vector<syntax::RawTokenInfo> AllRawTokenSyntax;
+
+  /// The root of the syntax tree representing the source file.
+  Optional<syntax::SourceFileSyntax> SyntaxRoot;
 };
 
 

Diff for: include/swift/Basic/OwnedString.h

@@ -49,10 +49,6 @@ class OwnedString {
     assert(Length >= 0 && "expected length to be non-negative");
 
     if (Ownership == StringOwnership::Copied && Data) {
-      assert(
-        Length <= strlen(Data) &&
-        "expected length to be a valid index, within the length of the string");
-
       char *substring = static_cast<char *>(malloc(Length + 1));
       assert(substring && "expected successful malloc of copy");
 

Diff for: include/swift/Parse/Lexer.h

@@ -23,6 +23,7 @@
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/References.h"
 #include "swift/Syntax/Trivia.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -245,7 +246,7 @@ class Lexer {
   }
 
   /// Lex a full token including leading and trailing trivia.
-  RC<syntax::RawTokenSyntax> fullLex();
+  syntax::RawTokenInfo fullLex();
 
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;

Diff for: include/swift/Parse/Parser.h

@@ -25,6 +25,8 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
+#include "swift/Syntax/RawTokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/LocalContext.h"
@@ -325,6 +327,9 @@ class Parser {
   /// This vector is managed by \c StructureMarkerRAII objects.
   llvm::SmallVector<StructureMarker, 16> StructureMarkers;
 
+  /// Current syntax parsing context where call backs should be directed to.
+  syntax::SyntaxParsingContext *SyntaxContext;
+
 public:
   Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
          PersistentParserState *PersistentState = nullptr);
@@ -1417,6 +1422,11 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
                    unsigned Offset = 0,
                    unsigned EndOffset = 0);
 
+
+void populateTokenSyntaxMap(const LangOptions &LangOpts,
+                            const SourceManager &SM,
+                            unsigned BufferID,
+                            std::vector<syntax::RawTokenInfo> &Result);
 } // end namespace swift
 
 #endif

Diff for: include/swift/Syntax/Syntax.h

@@ -127,6 +127,9 @@ class Syntax {
   /// if it has one, otherwise 0.
   CursorIndex getIndexInParent() const;
 
+  /// Returns true if this syntax node represents a token.
+  bool isToken() const;
+
   /// Returns true if this syntax node represents a statement.
   bool isStmt() const;
 
@@ -152,9 +155,6 @@ class Syntax {
   /// Returns true if the node is "present" in the source.
   bool isPresent() const;
 
-
-  bool isToken() const;
-
   /// Print the syntax node with full fidelity to the given output stream.
   void print(llvm::raw_ostream &OS) const;
 

Diff for: include/swift/Syntax/SyntaxFactory.h.gyb

@@ -54,6 +54,8 @@ struct SyntaxFactory {
   static Optional<Syntax>
   createSyntax(SyntaxKind Kind, llvm::ArrayRef<Syntax> Elements);
 
+  static SyntaxKind getUnknownKind(SyntaxKind Kind);
+
 % for node in SYNTAX_NODES:
 %   if node.children:
 %     child_params = []
@@ -105,7 +107,7 @@ struct SyntaxFactory {
     llvm::Optional<TokenSyntax> TrailingComma = llvm::None);
 
   /// Creates a TypeIdentifierSyntax with the provided name and leading/trailing
-  /// trivia. 
+  /// trivia.
   static TypeIdentifierSyntax makeTypeIdentifier(OwnedString TypeName,
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});
 
@@ -117,7 +119,7 @@ struct SyntaxFactory {
   /// Creates a TypeIdentifierSyntax for the `Any` type.
   static TypeIdentifierSyntax makeAnyTypeIdentifier(
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});
-  
+
   /// Creates a TypeIdentifierSyntax for the `Self` type.
   static TypeIdentifierSyntax makeSelfTypeIdentifier(
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});

Diff for: include/swift/Syntax/SyntaxParsingContext.h

@@ -0,0 +1,110 @@
+//===----------- SyntaxParsingContext.h -==============----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SYNTAX_PARSING_CONTEXT_H
+#define SWIFT_SYNTAX_PARSING_CONTEXT_H
+
+#include "swift/Syntax/RawTokenSyntax.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/References.h"
+#include "swift/Syntax/RawSyntax.h"
+#include "swift/Syntax/Syntax.h"
+#include "swift/Syntax/TokenKinds.h"
+#include "swift/Syntax/Trivia.h"
+
+namespace swift {
+  class SourceFile;
+
+namespace syntax {
+
+struct RawTokenInfo {
+  SourceLoc Loc;
+  RC<RawTokenSyntax> Token;
+};
+
+enum class SyntaxParsingContextKind: uint8_t {
+  Root,
+  Child,
+};
+
+/// The base class of different kinds of Syntax context that Parser should use to
+/// create syntax nodes.
+class SyntaxParsingContext {
+protected:
+  SyntaxParsingContext(bool Enabled);
+  SyntaxParsingContext(SyntaxParsingContext &Another);
+public:
+  struct ContextInfo;
+  ContextInfo &ContextData;
+
+  // Add a token syntax at the given source location to the context; this
+  // token node can be used to build more complex syntax nodes in later call
+  // back.
+  virtual void addTokenSyntax(SourceLoc Loc) = 0;
+
+  // Get the context kind.
+  virtual SyntaxParsingContextKind getKind() = 0;
+
+  // Create a syntax node of the given kind.
+  virtual void makeNode(SyntaxKind Kind) = 0;
+  virtual ~SyntaxParsingContext();
+
+  // Disable the building of syntax tree in the current context.
+  void disable();
+};
+
+// The start point of syntax tree parsing. This context is the root
+// of all other entity-specific contexts. This is the context Parser
+// has when the parser instance is firstly created.
+class SyntaxParsingContextRoot: public SyntaxParsingContext {
+public:
+  struct GlobalInfo;
+
+  // Contains global information of the source file under parsing.
+  GlobalInfo &GlobalData;
+  SyntaxParsingContextRoot(SourceFile &SF, unsigned BufferID);
+  ~SyntaxParsingContextRoot();
+  void addTokenSyntax(SourceLoc Loc) override {};
+  void makeNode(SyntaxKind Kind) override {};
+  SyntaxParsingContextKind getKind() override {
+    return SyntaxParsingContextKind::Root;
+  };
+};
+
+// The base class for contexts that are created from a parent context.
+// The stack instance will set the context holder when the context
+// is firstly created and reset the context holder to the parent when
+// it's destructed.
+class SyntaxParsingContextChild: public SyntaxParsingContext {
+  SyntaxParsingContext *Parent;
+  SyntaxParsingContext *&ContextHolder;
+  const SyntaxKind FinalKind;
+public:
+  SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
+                            SyntaxKind FinalKind):
+    SyntaxParsingContext(*ContextHolder), Parent(ContextHolder),
+    ContextHolder(ContextHolder), FinalKind(FinalKind) {
+      ContextHolder = this;
+  }
+  ~SyntaxParsingContextChild();
+  void makeNode(SyntaxKind Kind) override;
+  void addTokenSyntax(SourceLoc Loc) override;
+  SyntaxParsingContext* getParent() { return Parent; }
+  SyntaxParsingContextRoot &getRoot();
+  SyntaxParsingContextKind getKind() override {
+    return SyntaxParsingContextKind::Child;
+  };
+};
+}
+}
+#endif // SWIFT_SYNTAX_PARSING_CONTEXT_H
+

Diff for: lib/Parse/Lexer.cpp

@@ -20,7 +20,7 @@
 #include "swift/AST/Identifier.h"
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/SourceManager.h"
-#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -737,11 +737,12 @@ static bool rangeContainsPlaceholderEnd(const char *CurPtr,
   return false;
 }
 
-RC<syntax::RawTokenSyntax> Lexer::fullLex() {
+syntax::RawTokenInfo Lexer::fullLex() {
   if (NextToken.isEscapedIdentifier()) {
     LeadingTrivia.push_back(syntax::TriviaPiece::backtick());
     TrailingTrivia.push_front(syntax::TriviaPiece::backtick());
   }
+  auto Loc = NextToken.getLoc();
   auto Result = syntax::RawTokenSyntax::make(NextToken.getKind(),
                                         OwnedString(NextToken.getText()).copy(),
                                         syntax::SourcePresence::Present,
@@ -751,7 +752,7 @@ RC<syntax::RawTokenSyntax> Lexer::fullLex() {
   if (NextToken.isNot(tok::eof)) {
     lexImpl();
   }
-  return Result;
+  return {Loc, Result};
 }
 
 /// lexOperatorIdentifier - Match identifiers formed out of punctuation.

Diff for: lib/Parse/ParseExpr.cpp

@@ -18,6 +18,9 @@
 #include "swift/AST/DiagnosticsParse.h"
 #include "swift/Basic/EditorPlaceholder.h"
 #include "swift/Parse/CodeCompletionCallbacks.h"
+#include "swift/Syntax/SyntaxFactory.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
@@ -27,6 +30,7 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace swift;
+using namespace swift::syntax;
 
 /// parseExpr
 ///
@@ -35,6 +39,9 @@ using namespace swift;
 ///
 /// \param isExprBasic Whether we're only parsing an expr-basic.
 ParserResult<Expr> Parser::parseExprImpl(Diag<> Message, bool isExprBasic) {
+  // Start a context for creating expression syntax.
+  SyntaxParsingContextChild ExprParsingContext(SyntaxContext, SyntaxKind::Expr);
+
   // If we are parsing a refutable pattern, check to see if this is the start
   // of a let/var/is pattern.  If so, parse it to an UnresolvedPatternExpr and
   // name binding will perform final validation.
@@ -465,6 +472,7 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
     Tok.setKind(tok::oper_prefix);
     LLVM_FALLTHROUGH;
   case tok::oper_prefix:
+    SyntaxContext->addTokenSyntax(Tok.getLoc());
     Operator = parseExprOperator();
     break;
   case tok::oper_binary_spaced:
@@ -757,7 +765,6 @@ UnresolvedDeclRefExpr *Parser::parseExprOperator() {
   SourceLoc loc = Tok.getLoc();
   Identifier name = Context.getIdentifier(Tok.getText());
   consumeToken();
-
   // Bypass local lookup.
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
@@ -1381,6 +1388,8 @@ ParserResult<Expr> Parser::parseExprPostfix(Diag<> ID, bool isExprBasic) {
   case tok::integer_literal: {
     StringRef Text = copyAndStripUnderscores(Context, Tok.getText());
     SourceLoc Loc = consumeToken(tok::integer_literal);
+    SyntaxContext->addTokenSyntax(Loc);
+    SyntaxContext->makeNode(SyntaxKind::IntegerLiteralExpr);
     Result = makeParserResult(new (Context) IntegerLiteralExpr(Text, Loc,
                                                            /*Implicit=*/false));
     break;
@@ -1786,11 +1795,20 @@ createStringLiteralExprFromSegment(ASTContext &Ctx,
 ///   expr-literal:
 ///     string_literal
 ParserResult<Expr> Parser::parseExprStringLiteral() {
+
   SmallVector<Lexer::StringSegment, 1> Segments;
   L->getStringLiteralSegments(Tok, Segments);
 
   Token EntireTok = Tok;
 
+  // Create a syntax node for string literal.
+  SyntaxContext->addTokenSyntax(Tok.getLoc());
+  SyntaxContext->makeNode(SyntaxKind::StringLiteralExpr);
+  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxKind::Expr);
+
+  // FIXME: Avoid creating syntax nodes for string interpolation.
+  LocalContext.disable();
+
   // The start location of the entire string literal.
   SourceLoc Loc = Tok.getLoc();