Re-apply "libSyntax: create a basic infrastructure for generating libSyntax entities by using Parser." (#12538)

Author: nkcsgexi

Diff for: include/swift/AST/Module.h

@@ -79,6 +79,13 @@ namespace swift {
   class VarDecl;
   class VisibleDeclConsumer;
 
+namespace syntax {
+  class SourceFileSyntax;
+  class SyntaxParsingContext;
+  class SyntaxParsingContextRoot;
+  struct RawTokenInfo;
+}
+
 /// Discriminator for file-units.
 enum class FileUnitKind {
   /// For a .swift source file.
@@ -765,6 +772,7 @@ class SourceFile final : public FileUnit {
 public:
   class LookupCache;
   class Impl;
+  struct SourceFileSyntaxInfo;
 
   /// The implicit module import that the SourceFile should get.
   enum class ImplicitModuleImportKind {
@@ -1082,9 +1090,22 @@ class SourceFile final : public FileUnit {
     return (bool)AllCorrectedTokens;
   }
 
+  syntax::SourceFileSyntax getSyntaxRoot() const;
+
 private:
+  friend class syntax::SyntaxParsingContext;
+  friend class syntax::SyntaxParsingContextRoot;
+
   /// If not None, the underlying vector should contain tokens of this source file.
   Optional<std::vector<Token>> AllCorrectedTokens;
+
+  /// All of the raw token syntax nodes in the underlying source.
+  std::vector<syntax::RawTokenInfo> AllRawTokenSyntax;
+
+  SourceFileSyntaxInfo &SyntaxInfo;
+
+  void setSyntaxRoot(syntax::SourceFileSyntax &&Root);
+  bool hasSyntaxRoot() const;
 };
 
 

Diff for: include/swift/Basic/OwnedString.h

@@ -49,10 +49,6 @@ class OwnedString {
     assert(Length >= 0 && "expected length to be non-negative");
 
     if (Ownership == StringOwnership::Copied && Data) {
-      assert(
-        Length <= strlen(Data) &&
-        "expected length to be a valid index, within the length of the string");
-
       char *substring = static_cast<char *>(malloc(Length + 1));
       assert(substring && "expected successful malloc of copy");
 

Diff for: include/swift/Parse/Lexer.h

@@ -23,6 +23,7 @@
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/References.h"
 #include "swift/Syntax/Trivia.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -245,7 +246,7 @@ class Lexer {
   }
 
   /// Lex a full token including leading and trailing trivia.
-  RC<syntax::RawTokenSyntax> fullLex();
+  syntax::RawTokenInfo fullLex();
 
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;

Diff for: include/swift/Parse/Parser.h

@@ -25,6 +25,8 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
+#include "swift/Syntax/RawTokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/LocalContext.h"
@@ -325,6 +327,9 @@ class Parser {
   /// This vector is managed by \c StructureMarkerRAII objects.
   llvm::SmallVector<StructureMarker, 16> StructureMarkers;
 
+  /// Current syntax parsing context where call backs should be directed to.
+  syntax::SyntaxParsingContext *SyntaxContext;
+
 public:
   Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
          PersistentParserState *PersistentState = nullptr);
@@ -1417,6 +1422,11 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
                    unsigned Offset = 0,
                    unsigned EndOffset = 0);
 
+
+void populateTokenSyntaxMap(const LangOptions &LangOpts,
+                            const SourceManager &SM,
+                            unsigned BufferID,
+                            std::vector<syntax::RawTokenInfo> &Result);
 } // end namespace swift
 
 #endif

Diff for: include/swift/Syntax/Syntax.h

@@ -127,6 +127,9 @@ class Syntax {
   /// if it has one, otherwise 0.
   CursorIndex getIndexInParent() const;
 
+  /// Returns true if this syntax node represents a token.
+  bool isToken() const;
+
   /// Returns true if this syntax node represents a statement.
   bool isStmt() const;
 
@@ -152,9 +155,6 @@ class Syntax {
   /// Returns true if the node is "present" in the source.
   bool isPresent() const;
 
-
-  bool isToken() const;
-
   /// Print the syntax node with full fidelity to the given output stream.
   void print(llvm::raw_ostream &OS) const;
 

Diff for: include/swift/Syntax/SyntaxFactory.h.gyb

@@ -54,6 +54,8 @@ struct SyntaxFactory {
   static Optional<Syntax>
   createSyntax(SyntaxKind Kind, llvm::ArrayRef<Syntax> Elements);
 
+  static SyntaxKind getUnknownKind(SyntaxKind Kind);
+
 % for node in SYNTAX_NODES:
 %   if node.children:
 %     child_params = []
@@ -105,7 +107,7 @@ struct SyntaxFactory {
     llvm::Optional<TokenSyntax> TrailingComma = llvm::None);
 
   /// Creates a TypeIdentifierSyntax with the provided name and leading/trailing
-  /// trivia. 
+  /// trivia.
   static TypeIdentifierSyntax makeTypeIdentifier(OwnedString TypeName,
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});
 
@@ -117,7 +119,7 @@ struct SyntaxFactory {
   /// Creates a TypeIdentifierSyntax for the `Any` type.
   static TypeIdentifierSyntax makeAnyTypeIdentifier(
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});
-  
+
   /// Creates a TypeIdentifierSyntax for the `Self` type.
   static TypeIdentifierSyntax makeSelfTypeIdentifier(
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});

Diff for: include/swift/Syntax/SyntaxParsingContext.h

@@ -0,0 +1,110 @@
+//===----------- SyntaxParsingContext.h -==============----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SYNTAX_PARSING_CONTEXT_H
+#define SWIFT_SYNTAX_PARSING_CONTEXT_H
+
+#include "swift/Syntax/RawTokenSyntax.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/References.h"
+#include "swift/Syntax/RawSyntax.h"
+#include "swift/Syntax/Syntax.h"
+#include "swift/Syntax/TokenKinds.h"
+#include "swift/Syntax/Trivia.h"
+
+namespace swift {
+  class SourceFile;
+
+namespace syntax {
+
+struct RawTokenInfo {
+  SourceLoc Loc;
+  RC<RawTokenSyntax> Token;
+};
+
+enum class SyntaxParsingContextKind: uint8_t {
+  Root,
+  Child,
+};
+
+/// The base class of different kinds of Syntax context that Parser should use to
+/// create syntax nodes.
+class SyntaxParsingContext {
+protected:
+  SyntaxParsingContext(bool Enabled);
+  SyntaxParsingContext(SyntaxParsingContext &Another);
+public:
+  struct ContextInfo;
+  ContextInfo &ContextData;
+
+  // Add a token syntax at the given source location to the context; this
+  // token node can be used to build more complex syntax nodes in later call
+  // back.
+  virtual void addTokenSyntax(SourceLoc Loc) = 0;
+
+  // Get the context kind.
+  virtual SyntaxParsingContextKind getKind() = 0;
+
+  // Create a syntax node of the given kind.
+  virtual void makeNode(SyntaxKind Kind) = 0;
+  virtual ~SyntaxParsingContext();
+
+  // Disable the building of syntax tree in the current context.
+  void disable();
+};
+
+// The start point of syntax tree parsing. This context is the root
+// of all other entity-specific contexts. This is the context Parser
+// has when the parser instance is firstly created.
+class SyntaxParsingContextRoot: public SyntaxParsingContext {
+public:
+  struct GlobalInfo;
+
+  // Contains global information of the source file under parsing.
+  GlobalInfo &GlobalData;
+  SyntaxParsingContextRoot(SourceFile &SF, unsigned BufferID);
+  ~SyntaxParsingContextRoot();
+  void addTokenSyntax(SourceLoc Loc) override {};
+  void makeNode(SyntaxKind Kind) override {};
+  SyntaxParsingContextKind getKind() override {
+    return SyntaxParsingContextKind::Root;
+  };
+};
+
+// The base class for contexts that are created from a parent context.
+// The stack instance will set the context holder when the context
+// is firstly created and reset the context holder to the parent when
+// it's destructed.
+class SyntaxParsingContextChild: public SyntaxParsingContext {
+  SyntaxParsingContext *Parent;
+  SyntaxParsingContext *&ContextHolder;
+  const SyntaxKind FinalKind;
+public:
+  SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
+                            SyntaxKind FinalKind):
+    SyntaxParsingContext(*ContextHolder), Parent(ContextHolder),
+    ContextHolder(ContextHolder), FinalKind(FinalKind) {
+      ContextHolder = this;
+  }
+  ~SyntaxParsingContextChild();
+  void makeNode(SyntaxKind Kind) override;
+  void addTokenSyntax(SourceLoc Loc) override;
+  SyntaxParsingContext* getParent() { return Parent; }
+  SyntaxParsingContextRoot &getRoot();
+  SyntaxParsingContextKind getKind() override {
+    return SyntaxParsingContextKind::Child;
+  };
+};
+}
+}
+#endif // SWIFT_SYNTAX_PARSING_CONTEXT_H
+

Diff for: lib/AST/Module.cpp

@@ -34,6 +34,8 @@
 #include "swift/AST/ProtocolConformance.h"
 #include "swift/Basic/Compiler.h"
 #include "swift/Basic/SourceManager.h"
+#include "swift/Syntax/SyntaxNodes.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "clang/Basic/Module.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
@@ -761,6 +763,23 @@ class SourceFile::Impl {
   }
 };
 
+struct SourceFile::SourceFileSyntaxInfo {
+  /// The root of the syntax tree representing the source file.
+  Optional<syntax::SourceFileSyntax> SyntaxRoot;
+};
+
+bool SourceFile::hasSyntaxRoot() const {
+  return SyntaxInfo.SyntaxRoot.hasValue();
+}
+
+syntax::SourceFileSyntax SourceFile::getSyntaxRoot() const {
+  assert(hasSyntaxRoot() && "no syntax root is set.");
+  return *SyntaxInfo.SyntaxRoot;
+}
+
+void SourceFile::setSyntaxRoot(syntax::SourceFileSyntax &&Root) {
+  SyntaxInfo.SyntaxRoot.emplace(Root);
+}
 
 template<typename OP_DECL>
 static Optional<OP_DECL *>
@@ -1325,7 +1344,7 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
                        bool KeepTokens)
   : FileUnit(FileUnitKind::Source, M),
     BufferID(bufferID ? *bufferID : -1),
-    Kind(K) {
+    Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo()) {
   M.getASTContext().addDestructorCleanup(*this);
   performAutoImport(*this, ModImpKind);
 
@@ -1339,7 +1358,7 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
   }
 }
 
-SourceFile::~SourceFile() {}
+SourceFile::~SourceFile() { delete &SyntaxInfo; }
 
 bool FileUnit::walk(ASTWalker &walker) {
   SmallVector<Decl *, 64> Decls;

Diff for: lib/Parse/Lexer.cpp

@@ -20,7 +20,7 @@
 #include "swift/AST/Identifier.h"
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/SourceManager.h"
-#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -737,11 +737,12 @@ static bool rangeContainsPlaceholderEnd(const char *CurPtr,
   return false;
 }
 
-RC<syntax::RawTokenSyntax> Lexer::fullLex() {
+syntax::RawTokenInfo Lexer::fullLex() {
   if (NextToken.isEscapedIdentifier()) {
     LeadingTrivia.push_back(syntax::TriviaPiece::backtick());
     TrailingTrivia.push_front(syntax::TriviaPiece::backtick());
   }
+  auto Loc = NextToken.getLoc();
   auto Result = syntax::RawTokenSyntax::make(NextToken.getKind(),
                                         OwnedString(NextToken.getText()).copy(),
                                         syntax::SourcePresence::Present,
@@ -751,7 +752,7 @@ RC<syntax::RawTokenSyntax> Lexer::fullLex() {
   if (NextToken.isNot(tok::eof)) {
     lexImpl();
   }
-  return Result;
+  return {Loc, Result};
 }
 
 /// lexOperatorIdentifier - Match identifiers formed out of punctuation.