[Lexer] Push trivia piece lexing down to SyntaxParsingContext

This is again a transitional state before SyntaxParsingContext hands
the responsibility over to SyntaxTreeCreator and from there to
SyntaxParseActions.

Author: ahoppen

Diff for: include/swift/Parse/SyntaxParsingContext.h

@@ -264,8 +264,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   void addRawSyntax(ParsedRawSyntaxNode Raw);
 
   /// Add Token with Trivia to the parts.
-  void addToken(Token &Tok, const ParsedTrivia &LeadingTrivia,
-                const ParsedTrivia &TrailingTrivia);
+  void addToken(Token &Tok, StringRef LeadingTrivia, StringRef TrailingTrivia);
 
   /// Add Syntax to the parts.
   void addSyntax(ParsedSyntax Node);

Diff for: lib/Parse/ParseDecl.cpp

@@ -210,9 +210,7 @@ void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
   }
 
   // Finalize the syntax context.
-  auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
-  auto TrailingTriviaPieces = TriviaLexer::lexTrivia(TrailingTrivia);
-  SyntaxContext->addToken(Tok, LeadingTriviaPieces, TrailingTriviaPieces);
+  SyntaxContext->addToken(Tok, LeadingTrivia, TrailingTrivia);
 }
 
 bool Parser::parseTopLevelSIL() {

Diff for: lib/Parse/ParseExpr.cpp

@@ -1730,7 +1730,6 @@ parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,
                     unsigned &InterpolationCount) {
   SourceLoc Loc = EntireTok.getLoc();
   ParserStatus Status;
-  ParsedTrivia EmptyTrivia;
   bool First = true;
 
   DeclNameRef appendLiteral(
@@ -1785,7 +1784,7 @@ parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,
       // such token to the context.
       Token content(tok::string_segment,
                     CharSourceRange(Segment.Loc, Segment.Length).str());
-      SyntaxContext->addToken(content, EmptyTrivia, EmptyTrivia);
+      SyntaxContext->addToken(content, StringRef(), StringRef());
       break;
     }
 
@@ -1799,14 +1798,14 @@ parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,
       // Backslash is part of an expression segment.
       SourceLoc BackSlashLoc = Segment.Loc.getAdvancedLoc(-1 - DelimiterLen);
       Token BackSlash(tok::backslash, CharSourceRange(BackSlashLoc, 1).str());
-      ExprContext.addToken(BackSlash, EmptyTrivia, EmptyTrivia);
+      ExprContext.addToken(BackSlash, StringRef(), StringRef());
 
       // Custom delimiter may be a part of an expression segment.
       if (HasCustomDelimiter) {
         SourceLoc DelimiterLoc = Segment.Loc.getAdvancedLoc(-DelimiterLen);
         Token Delimiter(tok::raw_string_delimiter,
                         CharSourceRange(DelimiterLoc, DelimiterLen).str());
-        ExprContext.addToken(Delimiter, EmptyTrivia, EmptyTrivia);
+        ExprContext.addToken(Delimiter, StringRef(), StringRef());
       }
 
       // Create a temporary lexer that lexes from the body of the string.
@@ -1909,20 +1908,17 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
   // Make unknown tokens to represent the open and close quote.
   Token OpenQuote(QuoteKind, OpenQuoteStr);
   Token CloseQuote(QuoteKind, CloseQuoteStr);
-  ParsedTrivia EmptyTrivia;
   StringRef EntireTrailingTrivia = TrailingTrivia;
 
   if (HasCustomDelimiter) {
-    auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
     Token OpenDelimiter(tok::raw_string_delimiter, OpenDelimiterStr);
     // When a custom delimiter is present, it owns the leading trivia.
-    SyntaxContext->addToken(OpenDelimiter, LeadingTriviaPieces, EmptyTrivia);
+    SyntaxContext->addToken(OpenDelimiter, LeadingTrivia, StringRef());
 
-    SyntaxContext->addToken(OpenQuote, EmptyTrivia, EmptyTrivia);
+    SyntaxContext->addToken(OpenQuote, StringRef(), StringRef());
   } else {
-    auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
     // Without custom delimiter the quote owns trailing trivia.
-    SyntaxContext->addToken(OpenQuote, LeadingTriviaPieces, EmptyTrivia);
+    SyntaxContext->addToken(OpenQuote, LeadingTrivia, StringRef());
   }
 
   // The simple case: just a single literal segment.
@@ -1943,24 +1939,19 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
       auto Segment = Segments.front();
       Token content(tok::string_segment,
                     CharSourceRange(Segment.Loc, Segment.Length).str());
-      SyntaxContext->addToken(content, EmptyTrivia, EmptyTrivia);
+      SyntaxContext->addToken(content, StringRef(), StringRef());
     }
 
     if (HasCustomDelimiter) {
-      SyntaxContext->addToken(CloseQuote, EmptyTrivia, EmptyTrivia);
+      SyntaxContext->addToken(CloseQuote, StringRef(), StringRef());
 
       Token CloseDelimiter(tok::raw_string_delimiter, CloseDelimiterStr);
       // When a custom delimiter is present it owns the trailing trivia.
-      auto EntireTrailingTriviaPieces =
-          TriviaLexer::lexTrivia(EntireTrailingTrivia);
-      SyntaxContext->addToken(CloseDelimiter, EmptyTrivia,
-                              EntireTrailingTriviaPieces);
+      SyntaxContext->addToken(CloseDelimiter, StringRef(),
+                              EntireTrailingTrivia);
     } else {
       // Without custom delimiter the quote owns trailing trivia.
-      auto EntireTrailingTriviaPieces =
-          TriviaLexer::lexTrivia(EntireTrailingTrivia);
-      SyntaxContext->addToken(CloseQuote, EmptyTrivia,
-                              EntireTrailingTriviaPieces);
+      SyntaxContext->addToken(CloseQuote, StringRef(), EntireTrailingTrivia);
     }
 
     return makeParserResult(
@@ -2019,20 +2010,16 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
   }
 
   if (HasCustomDelimiter) {
-    SyntaxContext->addToken(CloseQuote, EmptyTrivia, EmptyTrivia);
+    SyntaxContext->addToken(CloseQuote, StringRef(), StringRef());
 
     Token CloseDelimiter(tok::raw_string_delimiter, CloseDelimiterStr);
     // When a custom delimiter is present it owns the trailing trivia.
-    auto EntireTrailingTriviaPieces =
-        TriviaLexer::lexTrivia(EntireTrailingTrivia);
-    SyntaxContext->addToken(CloseDelimiter, EmptyTrivia,
-                            EntireTrailingTriviaPieces);
+    SyntaxContext->addToken(CloseDelimiter, StringRef(), EntireTrailingTrivia);
   } else {
     // Without custom delimiter the quote owns trailing trivia.
     auto EntireTrailingTriviaPieces =
         TriviaLexer::lexTrivia(EntireTrailingTrivia);
-    SyntaxContext->addToken(CloseQuote, EmptyTrivia,
-                            EntireTrailingTriviaPieces);
+    SyntaxContext->addToken(CloseQuote, StringRef(), EntireTrailingTrivia);
   }
 
   if (AppendingExpr->getBody()->getNumElements() == 1) {

Diff for: lib/Parse/Parser.cpp

@@ -606,9 +606,7 @@ void Parser::consumeExtraToken(Token Extra) {
 
 SourceLoc Parser::consumeToken() {
   TokReceiver->receive(Tok);
-  auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
-  auto TrailingTriviaPieces = TriviaLexer::lexTrivia(TrailingTrivia);
-  SyntaxContext->addToken(Tok, LeadingTriviaPieces, TrailingTriviaPieces);
+  SyntaxContext->addToken(Tok, LeadingTrivia, TrailingTrivia);
   return consumeTokenWithoutFeedingReceiver();
 }
 
@@ -643,9 +641,7 @@ SourceLoc Parser::consumeStartingCharacterOfCurrentToken(tok Kind, size_t Len) {
 void Parser::markSplitToken(tok Kind, StringRef Txt) {
   SplitTokens.emplace_back();
   SplitTokens.back().setToken(Kind, Txt);
-  ParsedTrivia EmptyTrivia;
-  auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
-  SyntaxContext->addToken(SplitTokens.back(), LeadingTriviaPieces, EmptyTrivia);
+  SyntaxContext->addToken(SplitTokens.back(), LeadingTrivia, StringRef());
   TokReceiver->receive(SplitTokens.back());
 }
 

Diff for: lib/Parse/SyntaxParsingContext.cpp

@@ -18,8 +18,9 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/SourceFile.h"
 #include "swift/Basic/Defer.h"
-#include "swift/Parse/ParsedSyntax.h"
+#include "swift/Parse/Lexer.h"
 #include "swift/Parse/ParsedRawSyntaxRecorder.h"
+#include "swift/Parse/ParsedSyntax.h"
 #include "swift/Parse/ParsedSyntaxRecorder.h"
 #include "swift/Parse/SyntaxParseActions.h"
 #include "swift/Parse/SyntaxParsingCache.h"
@@ -203,18 +204,22 @@ ParsedTokenSyntax SyntaxParsingContext::popToken() {
 }
 
 /// Add Token with Trivia to the parts.
-void SyntaxParsingContext::addToken(Token &Tok,
-                                    const ParsedTrivia &LeadingTrivia,
-                                    const ParsedTrivia &TrailingTrivia) {
+void SyntaxParsingContext::addToken(Token &Tok, StringRef LeadingTrivia,
+                                    StringRef TrailingTrivia) {
   if (!Enabled)
     return;
 
+  auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
+  auto TrailingTriviaPieces = TriviaLexer::lexTrivia(TrailingTrivia);
+
   ParsedRawSyntaxNode raw;
-  if (shouldDefer())
-    raw = ParsedRawSyntaxNode::makeDeferred(Tok, LeadingTrivia, TrailingTrivia,
-                                            *this);
-  else
-    raw = getRecorder().recordToken(Tok, LeadingTrivia, TrailingTrivia);
+  if (shouldDefer()) {
+    raw = ParsedRawSyntaxNode::makeDeferred(Tok, LeadingTriviaPieces,
+                                            TrailingTriviaPieces, *this);
+  } else {
+    raw = getRecorder().recordToken(Tok, LeadingTriviaPieces,
+                                    TrailingTriviaPieces);
+  }
   addRawSyntax(std::move(raw));
 }