[Lexer] Handle SwiftInterface files as well as SIL

Previously, the Lexer kept a single flag whether we’re lexing Swift or SIL. Instead, keep track if we’re parsing Swift, SIL, or a Swiftinterface file. .swiftinterface files allow $-prefixed identifiers anywhere.

Author: Harlan Haskins

include/swift/Parse/Lexer.h

@@ -55,6 +55,12 @@ enum class HashbangMode : bool {
   Allowed,
 };
 
+enum class LexerMode {
+  Swift,
+  SwiftInterface,
+  SIL
+};
+
 /// Kinds of conflict marker which the lexer might encounter.
 enum class ConflictMarkerKind {
   /// A normal or diff3 conflict marker, initiated by at least 7 "<"s,
@@ -98,9 +104,10 @@ class Lexer {
 
   Token NextToken;
 
-  /// This is true if we're lexing a .sil file instead of a .swift
-  /// file.  This enables the 'sil' keyword.
-  const bool InSILMode;
+  /// The kind of source we're lexing. This either enables special behavior for
+  /// parseable interfaces, or enables things like the 'sil' keyword if lexing
+  /// a .sil file.
+  const LexerMode LexMode;
 
   /// True if we should skip past a `#!` line at the start of the file.
   const bool IsHashbangAllowed;
@@ -135,8 +142,8 @@ class Lexer {
   /// everything.
   Lexer(const PrincipalTag &, const LangOptions &LangOpts,
         const SourceManager &SourceMgr, unsigned BufferID,
-        DiagnosticEngine *Diags, bool InSILMode, HashbangMode HashbangAllowed,
-        CommentRetentionMode RetainComments,
+        DiagnosticEngine *Diags, LexerMode LexMode,
+        HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
         TriviaRetentionMode TriviaRetention);
 
   void initialize(unsigned Offset, unsigned EndOffset);
@@ -150,21 +157,21 @@ class Lexer {
   ///   identifier), but not things like how many characters are
   ///   consumed.  If that changes, APIs like getLocForEndOfToken will
   ///   need to take a LangOptions explicitly.
-  /// \param InSILMode - whether we're parsing a SIL source file.
+  /// \param LexMode - the kind of source file we're lexing.
   ///   Unlike language options, this does affect primitive lexing, which
   ///   means that APIs like getLocForEndOfToken really ought to take
   ///   this flag; it's just that we don't care that much about fidelity
   ///   when parsing SIL files.
   Lexer(
       const LangOptions &Options, const SourceManager &SourceMgr,
-      unsigned BufferID, DiagnosticEngine *Diags, bool InSILMode,
+      unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
       HashbangMode HashbangAllowed = HashbangMode::Disallowed,
       CommentRetentionMode RetainComments = CommentRetentionMode::None,
       TriviaRetentionMode TriviaRetention = TriviaRetentionMode::WithoutTrivia);
 
   /// Create a lexer that scans a subrange of the source buffer.
   Lexer(const LangOptions &Options, const SourceManager &SourceMgr,
-        unsigned BufferID, DiagnosticEngine *Diags, bool InSILMode,
+        unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
         HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
         TriviaRetentionMode TriviaRetention, unsigned Offset,
         unsigned EndOffset);

lib/AST/RawComment.cpp

@@ -108,7 +108,7 @@ static RawComment toRawComment(ASTContext &Context, CharSourceRange Range) {
   unsigned Offset = SourceMgr.getLocOffsetInBuffer(Range.getStart(), BufferID);
   unsigned EndOffset = SourceMgr.getLocOffsetInBuffer(Range.getEnd(), BufferID);
   LangOptions FakeLangOpts;
-  Lexer L(FakeLangOpts, SourceMgr, BufferID, nullptr, /*InSILMode=*/false,
+  Lexer L(FakeLangOpts, SourceMgr, BufferID, nullptr, LexerMode::Swift,
           HashbangMode::Disallowed,
           CommentRetentionMode::ReturnAsTokens,
           TriviaRetentionMode::WithoutTrivia,

lib/Immediate/REPL.cpp

@@ -1053,7 +1053,7 @@ class REPLEnvironment {
         unsigned BufferID =
             CI.getSourceMgr().addMemBufferCopy(Line, "<REPL Input>");
         Lexer L(CI.getASTContext().LangOpts,
-                CI.getSourceMgr(), BufferID, nullptr, false /*not SIL*/);
+                CI.getSourceMgr(), BufferID, nullptr, LexerMode::Swift);
         Token Tok;
         L.lex(Tok);
         assert(Tok.is(tok::colon));

lib/Parse/Lexer.cpp

@@ -171,11 +171,11 @@ uint32_t swift::validateUTF8CharacterAndAdvance(const char *&Ptr,
 
 Lexer::Lexer(const PrincipalTag &, const LangOptions &LangOpts,
              const SourceManager &SourceMgr, unsigned BufferID,
-             DiagnosticEngine *Diags, bool InSILMode,
+             DiagnosticEngine *Diags, LexerMode LexMode,
              HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
              TriviaRetentionMode TriviaRetention)
     : LangOpts(LangOpts), SourceMgr(SourceMgr), BufferID(BufferID),
-      Diags(Diags), InSILMode(InSILMode),
+      Diags(Diags), LexMode(LexMode),
       IsHashbangAllowed(HashbangAllowed == HashbangMode::Allowed),
       RetainComments(RetainComments), TriviaRetention(TriviaRetention) {}
 
@@ -216,28 +216,28 @@ void Lexer::initialize(unsigned Offset, unsigned EndOffset) {
 }
 
 Lexer::Lexer(const LangOptions &Options, const SourceManager &SourceMgr,
-             unsigned BufferID, DiagnosticEngine *Diags, bool InSILMode,
+             unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
              HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
              TriviaRetentionMode TriviaRetention)
-    : Lexer(PrincipalTag(), Options, SourceMgr, BufferID, Diags, InSILMode,
+    : Lexer(PrincipalTag(), Options, SourceMgr, BufferID, Diags, LexMode,
             HashbangAllowed, RetainComments, TriviaRetention) {
   unsigned EndOffset = SourceMgr.getRangeForBuffer(BufferID).getByteLength();
   initialize(/*Offset=*/0, EndOffset);
 }
 
 Lexer::Lexer(const LangOptions &Options, const SourceManager &SourceMgr,
-             unsigned BufferID, DiagnosticEngine *Diags, bool InSILMode,
+             unsigned BufferID, DiagnosticEngine *Diags, LexerMode LexMode,
              HashbangMode HashbangAllowed, CommentRetentionMode RetainComments,
              TriviaRetentionMode TriviaRetention, unsigned Offset,
              unsigned EndOffset)
-    : Lexer(PrincipalTag(), Options, SourceMgr, BufferID, Diags, InSILMode,
+    : Lexer(PrincipalTag(), Options, SourceMgr, BufferID, Diags, LexMode,
             HashbangAllowed, RetainComments, TriviaRetention) {
   initialize(Offset, EndOffset);
 }
 
 Lexer::Lexer(Lexer &Parent, State BeginState, State EndState)
     : Lexer(PrincipalTag(), Parent.LangOpts, Parent.SourceMgr, Parent.BufferID,
-            Parent.Diags, Parent.InSILMode,
+            Parent.Diags, Parent.LexMode,
             Parent.IsHashbangAllowed
                 ? HashbangMode::Allowed
                 : HashbangMode::Disallowed,
@@ -264,7 +264,7 @@ Token Lexer::getTokenAt(SourceLoc Loc) {
                          SourceMgr.findBufferContainingLoc(Loc)) &&
          "location from the wrong buffer");
 
-  Lexer L(LangOpts, SourceMgr, BufferID, Diags, InSILMode,
+  Lexer L(LangOpts, SourceMgr, BufferID, Diags, LexMode,
           HashbangMode::Allowed, CommentRetentionMode::None,
           TriviaRetentionMode::WithoutTrivia);
   L.restoreState(State(Loc));
@@ -672,7 +672,8 @@ void Lexer::lexIdentifier() {
   // Lex [a-zA-Z_$0-9[[:XID_Continue:]]]*
   while (advanceIfValidContinuationOfIdentifier(CurPtr, BufferEnd));
 
-  tok Kind = kindOfIdentifier(StringRef(TokStart, CurPtr-TokStart), InSILMode);
+  tok Kind = kindOfIdentifier(StringRef(TokStart, CurPtr-TokStart),
+                              LexMode == LexerMode::SIL);
   return formToken(Kind, TokStart);
 }
 
@@ -2515,7 +2516,7 @@ Token Lexer::getTokenAtLocation(const SourceManager &SM, SourceLoc Loc) {
   // comments and normally we won't be at the beginning of a comment token
   // (making this option irrelevant), or the caller lexed comments and
   // we need to lex just the comment token.
-  Lexer L(FakeLangOpts, SM, BufferID, nullptr, /*InSILMode=*/ false,
+  Lexer L(FakeLangOpts, SM, BufferID, nullptr, LexerMode::Swift,
           HashbangMode::Allowed, CommentRetentionMode::ReturnAsTokens);
   L.restoreState(State(Loc));
   return L.peekNextToken();
@@ -2671,7 +2672,7 @@ static SourceLoc getLocForStartOfTokenInBuf(SourceManager &SM,
   // and the exact token produced.
   LangOptions FakeLangOptions;
 
-  Lexer L(FakeLangOptions, SM, BufferID, nullptr, /*InSILMode=*/false,
+  Lexer L(FakeLangOptions, SM, BufferID, nullptr, LexerMode::Swift,
           HashbangMode::Allowed, CommentRetentionMode::None,
           TriviaRetentionMode::WithoutTrivia, BufferStart, BufferEnd);
 
@@ -2799,7 +2800,7 @@ SourceLoc Lexer::getLocForEndOfLine(SourceManager &SM, SourceLoc Loc) {
   // comments and normally we won't be at the beginning of a comment token
   // (making this option irrelevant), or the caller lexed comments and
   // we need to lex just the comment token.
-  Lexer L(FakeLangOpts, SM, BufferID, nullptr, /*InSILMode=*/ false,
+  Lexer L(FakeLangOpts, SM, BufferID, nullptr, LexerMode::Swift,
           HashbangMode::Allowed, CommentRetentionMode::ReturnAsTokens);
   L.restoreState(State(Loc));
   L.skipToEndOfLine(/*EatNewline=*/true);

lib/Parse/Parser.cpp

@@ -59,7 +59,7 @@ void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
   if (Offset == 0 && EndOffset == 0)
     EndOffset = SM.getRangeForBuffer(BufferID).getByteLength();
 
-  Lexer L(LangOpts, SM, BufferID, Diags, /*InSILMode=*/false,
+  Lexer L(LangOpts, SM, BufferID, Diags, LexerMode::Swift,
           HashbangMode::Allowed, RetainComments, TriviaRetention, Offset,
           EndOffset);
 
@@ -345,6 +345,19 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
 //===----------------------------------------------------------------------===//
 
 
+static LexerMode sourceFileKindToLexerMode(SourceFileKind kind) {
+  switch (kind) {
+    case swift::SourceFileKind::Interface:
+      return LexerMode::SwiftInterface;
+    case swift::SourceFileKind::SIL:
+      return LexerMode::SIL;
+    case swift::SourceFileKind::Library:
+    case swift::SourceFileKind::Main:
+    case swift::SourceFileKind::REPL:
+      return LexerMode::Swift;
+  }
+}
+
 Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
                PersistentParserState *PersistentState,
                std::shared_ptr<SyntaxParseActions> SPActions,
@@ -361,7 +374,7 @@ Parser::Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
           std::unique_ptr<Lexer>(new Lexer(
               SF.getASTContext().LangOpts, SF.getASTContext().SourceMgr,
               BufferID, LexerDiags,
-              /*InSILMode=*/SIL != nullptr,
+              sourceFileKindToLexerMode(SF.Kind),
               SF.Kind == SourceFileKind::Main
                   ? HashbangMode::Allowed
                   : HashbangMode::Disallowed,
@@ -399,7 +412,7 @@ class TokenRecorder: public ConsumeTokenReceiver {
 
   void relexComment(CharSourceRange CommentRange,
                     llvm::SmallVectorImpl<Token> &Scratch) {
-    Lexer L(Ctx.LangOpts, Ctx.SourceMgr, BufferID, nullptr, /*InSILMode=*/false,
+    Lexer L(Ctx.LangOpts, Ctx.SourceMgr, BufferID, nullptr, LexerMode::Swift,
             HashbangMode::Disallowed,
             CommentRetentionMode::ReturnAsTokens,
             TriviaRetentionMode::WithoutTrivia,
@@ -1119,7 +1132,7 @@ ParserUnit::ParserUnit(SourceManager &SM, SourceFileKind SFKind, unsigned Buffer
   std::unique_ptr<Lexer> Lex;
   Lex.reset(new Lexer(Impl.LangOpts, SM,
                       BufferID, &Impl.Diags,
-                      /*InSILMode=*/false,
+                      LexerMode::Swift,
                       HashbangMode::Allowed,
                       CommentRetentionMode::None,
                       TriviaRetentionMode::WithoutTrivia,