Propagate an "Error" Bit Into the Lexemes

Author: CodaFi

Sources/SwiftParser/Expressions.swift

@@ -1382,6 +1382,7 @@ extension Parser {
       wholeText: wholeText,
       textRange: textRange,
       presence: .present,
+      hasError: false,
       arena: self.arena
     )
   }

Sources/SwiftParser/Lexer.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 @_spi(RawSyntax) import SwiftSyntax
+import SwiftDiagnostics
 
 /// A lexical analyzer for the Swift programming language.
 ///
@@ -32,6 +33,7 @@ public struct Lexer {
 
       public static let isAtStartOfLine = Flags(rawValue: 1 << 0)
       public static let isMultilineStringLiteral = Flags(rawValue: 1 << 1)
+      public static let isErroneous = Flags(rawValue: 1 << 2)
     }
 
     @_spi(RawSyntax)
@@ -219,6 +221,10 @@ extension Lexer {
       return self.pointer.distance(to: other.pointer)
     }
 
+    func absoluteDistance(to other: Self) -> AbsolutePosition {
+      return AbsolutePosition(utf8Offset: self.distance(to: other))
+    }
+
     func peek(at offset: Int = 0) -> UInt8 {
       assert(!self.isAtEndOfFile)
       assert(offset >= 0)
@@ -750,14 +756,19 @@ extension Lexer.Cursor {
 }
 
 extension Lexer.Cursor {
-  mutating func nextToken(_ ContentStart: Lexer.Cursor) -> Lexer.Lexeme {
+  mutating func nextToken(
+    _ ContentStart: Lexer.Cursor,
+    diagnosticHandler: ((Int, DiagnosticMessage) -> Void)? = nil
+  ) -> Lexer.Lexeme {
     // Leading trivia.
     let leadingTriviaStart = self
     let newlineInLeadingTrivia = self.lexTrivia(.leading)
 
     // Token text.
     let textStart = self
-    var (kind, flags) = self.lexImpl(ContentStart: ContentStart)
+    var (kind, flags) = self.lexImpl(
+      ContentStart: ContentStart,
+      diagnosticHandler: diagnosticHandler)
 
     // Trailing trivia.
     let trailingTriviaStart = self
@@ -777,7 +788,10 @@ extension Lexer.Cursor {
       trailingTriviaLength: trailingTriviaStart.distance(to: self))
   }
 
-  private mutating func lexImpl(ContentStart: Lexer.Cursor) -> (RawTokenKind, Lexer.Lexeme.Flags) {
+  private mutating func lexImpl(
+    ContentStart: Lexer.Cursor,
+    diagnosticHandler: ((Int, DiagnosticMessage) -> Void)?
+  ) -> (RawTokenKind, Lexer.Lexeme.Flags) {
     let start = self
     switch self.advance() {
     case UInt8(ascii: "@"): return (.atSign, [])
@@ -866,7 +880,7 @@ extension Lexer.Cursor {
          UInt8(ascii: "3"), UInt8(ascii: "4"), UInt8(ascii: "5"),
          UInt8(ascii: "6"), UInt8(ascii: "7"), UInt8(ascii: "8"),
          UInt8(ascii: "9"):
-      return self.lexNumber(start, ContentStart)
+      return self.lexNumber(start, ContentStart, diagnosticHandler)
     case UInt8(ascii: #"'"#), UInt8(ascii: #"""#):
       return self.lexStringLiteral(start)
 
@@ -1109,11 +1123,15 @@ extension Lexer.Cursor {
 //      diagnoseSingleQuoteStringLiteral(TokStart, CurPtr)
 //    }
 
+    var flags: Lexer.Lexeme.Flags = []
+    if IsMultilineString {
+      flags.insert(.isMultilineStringLiteral)
+    }
     if wasErroneous {
-      return (.unknown, [])
+      flags.insert(.isErroneous)
     }
 
-    return (.stringLiteral, IsMultilineString ? .isMultilineStringLiteral : [])
+    return (.stringLiteral, flags)
   }
 }
 
@@ -1298,7 +1316,11 @@ extension Lexer.Cursor {
   ///   floating_literal ::= [0-9][0-9_]*[eE][+-]?[0-9][0-9_]*
   ///   floating_literal ::= 0x[0-9A-Fa-f][0-9A-Fa-f_]*
   ///                          (\.[0-9A-Fa-f][0-9A-Fa-f_]*)?[pP][+-]?[0-9][0-9_]*
-  mutating func lexNumber(_ TokStart: Lexer.Cursor, _ ContentStart: Lexer.Cursor) -> (RawTokenKind, Lexer.Lexeme.Flags) {
+  mutating func lexNumber(
+    _ TokStart: Lexer.Cursor,
+    _ ContentStart: Lexer.Cursor,
+    _ diagnosticHandler: ((Int, DiagnosticMessage) -> Void)? = nil
+  ) -> (RawTokenKind, Lexer.Lexeme.Flags) {
     assert((Unicode.Scalar(self.previous).isDigit || self.previous == UInt8(ascii: ".")),
            "Unexpected start")
 
@@ -1312,7 +1334,7 @@ extension Lexer.Cursor {
 //    }
 
     if !self.isAtEndOfFile && self.previous == UInt8(ascii: "0") && self.peek() == UInt8(ascii: "x") {
-      return self.lexHexNumber(TokStart)
+      return self.lexHexNumber(TokStart, diagnosticHandler)
     }
 
     if !self.isAtEndOfFile && self.previous == UInt8(ascii: "0") && self.peek() == UInt8(ascii: "o") {
@@ -1427,7 +1449,10 @@ extension Lexer.Cursor {
     return (.floatingLiteral, [])
   }
 
-  mutating func lexHexNumber(_ TokStart: Lexer.Cursor) -> (RawTokenKind, Lexer.Lexeme.Flags) {
+  mutating func lexHexNumber(
+    _ TokStart: Lexer.Cursor,
+    _ diagnosticHandler: ((Int, DiagnosticMessage) -> Void)?
+  ) -> (RawTokenKind, Lexer.Lexeme.Flags) {
     // We assume we're starting from the 'x' in a '0x...' floating-point literal.
     assert(self.peek() == UInt8(ascii: "x"), "not a hex literal")
     assert(self.previous == UInt8(ascii: "0"), "not a hex literal")
@@ -1438,9 +1463,10 @@ extension Lexer.Cursor {
       return (.unknown, [])
     }
 
-                                                   let expected_hex_digit = { (loc: Lexer.Cursor) -> (RawTokenKind, Lexer.Lexeme.Flags) in
+    let expected_hex_digit = { (loc: Lexer.Cursor) -> (RawTokenKind, Lexer.Lexeme.Flags) in
 //      diagnose(loc, diag::lex_invalid_digit_in_hex_literal, StringRef(loc, 1),
 //               (unsigned)kind)
+
       return expected_digit(loc)
     }
 
@@ -1477,14 +1503,14 @@ extension Lexer.Cursor {
 
       self.advance(while: { $0.isHexDigit || $0 == Unicode.Scalar("_") })
 
-      if !self.isAtEndOfFile, self.peek() != UInt8(ascii: "p") && self.peek() != UInt8(ascii: "P") {
+      if self.isAtEndOfFile || (self.peek() != UInt8(ascii: "p") && self.peek() != UInt8(ascii: "P")) {
         if !Unicode.Scalar(PtrOnDot!.peek(at: 1)).isDigit {
           // e.g: 0xff.description
           self = PtrOnDot!
           return (.integerLiteral, [])
         }
-//        diagnose(CurPtr, diag::lex_expected_binary_exponent_in_hex_float_literal)
-        return (.unknown, [])
+        diagnosticHandler?(TokStart.distance(to: self), StaticLexerError.lex_expected_binary_exponent_in_hex_float_literal)
+        return (.integerLiteral, [ .isErroneous ])
       }
     } else {
       PtrOnDot = nil

Sources/SwiftParser/Parser.swift

@@ -196,6 +196,7 @@ public struct Parser: TokenConsumer {
       wholeText: tok.wholeText,
       textRange: tok.textRange,
       presence: .present,
+      hasError: tok.flags.contains(.isErroneous),
       arena: arena
     )
   }
@@ -551,6 +552,7 @@ extension Parser {
       wholeText: SyntaxText(rebasing: current.wholeText[..<endIndex]),
       textRange: current.textRange.lowerBound ..< endIndex,
       presence: .present,
+      hasError: current.flags.contains(.isErroneous),
       arena: self.arena
     )
 

Sources/SwiftParserDiagnostics/CMakeLists.txt

@@ -8,6 +8,7 @@
 
 add_library(SwiftParserDiagnostics STATIC
   DiagnosticExtensions.swift
+  LexerDiagnosticMessages.swift
   MissingNodesError.swift
   ParserDiagnosticMessages.swift
   ParseDiagnosticsGenerator.swift

Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift

@@ -0,0 +1,79 @@
+//===--- LexerDiagnosticMessages.swift ------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftDiagnostics
+@_spi(RawSyntax) import SwiftSyntax
+
+fileprivate let diagnosticDomain: String = "SwiftLexer"
+
+/// A error diagnostic whose ID is determined by the diagnostic's type.
+public protocol LexerError: DiagnosticMessage {
+  var diagnosticID: MessageID { get }
+}
+
+public extension LexerError {
+  static var diagnosticID: MessageID {
+    return MessageID(domain: diagnosticDomain, id: "\(self)")
+  }
+
+  var diagnosticID: MessageID {
+    return Self.diagnosticID
+  }
+
+  var severity: DiagnosticSeverity {
+    return .error
+  }
+}
+
+public protocol LexerNote: NoteMessage {
+  var fixItID: MessageID { get }
+}
+
+public extension LexerNote {
+  static var fixItID: MessageID {
+    return MessageID(domain: diagnosticDomain, id: "\(self)")
+  }
+
+  var fixItID: MessageID {
+    return Self.fixItID
+  }
+}
+
+public protocol LexerFixIt: FixItMessage {
+  var fixItID: MessageID { get }
+}
+
+public extension LexerFixIt {
+  static var fixItID: MessageID {
+    return MessageID(domain: diagnosticDomain, id: "\(self)")
+  }
+
+  var fixItID: MessageID {
+    return Self.fixItID
+  }
+}
+
+// MARK: - Errors (please sort alphabetically)
+
+/// Please order the cases in this enum alphabetically by case name.
+public enum StaticLexerError: String, DiagnosticMessage {
+  case lex_expected_binary_exponent_in_hex_float_literal = "hexadecimal floating point literal must end with an exponent"
+  
+  public var message: String { self.rawValue }
+
+  public var diagnosticID: MessageID {
+    MessageID(domain: diagnosticDomain, id: "\(type(of: self)).\(self)")
+  }
+
+  public var severity: DiagnosticSeverity { .error }
+}
+

Sources/SwiftParserDiagnostics/ParseDiagnosticsGenerator.swift

@@ -242,6 +242,13 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       } else {
         return handleMissingSyntax(node)
       }
+    } else if node.hasError {
+      node.syntaxTextBytes.withUnsafeBufferPointer { buf in
+        var cursor = Lexer.Cursor(input: buf, previous: 0)
+        _ = cursor.nextToken(cursor) { [self] offset, diagnostic in
+          self.addDiagnostic(node, position: node.position.advanced(by: offset), diagnostic)
+        }
+      }
     }
     return .skipChildren
   }

Sources/SwiftParserDiagnostics/ParserDiagnosticMessages.swift

@@ -13,7 +13,7 @@
 import SwiftDiagnostics
 @_spi(RawSyntax) import SwiftSyntax
 
-let diagnosticDomain: String = "SwiftParser"
+fileprivate let diagnosticDomain: String = "SwiftParser"
 
 /// A error diagnostic whose ID is determined by the diagnostic's type.
 public protocol ParserError: DiagnosticMessage {

Sources/SwiftSyntax/AbsolutePosition.swift

@@ -21,6 +21,10 @@ public struct AbsolutePosition: Comparable {
     self.utf8Offset = utf8Offset
   }
 
+  public func advanced(by offset: Int) -> AbsolutePosition {
+    return AbsolutePosition(utf8Offset: self.utf8Offset + offset)
+  }
+
   public static func <(lhs: AbsolutePosition, rhs: AbsolutePosition) -> Bool {
     return lhs.utf8Offset < rhs.utf8Offset
   }

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -54,6 +54,8 @@ internal struct RawSyntaxData {
     var textRange: Range<SyntaxText.Index>
 
     var presence: SourcePresence
+
+    var hasError: Bool
   }
 
   /// Token typically created with `TokenSyntax.<someToken>`.
@@ -168,7 +170,7 @@ extension RawSyntax {
     switch view {
     case .token(let tokenView):
       var recursiveFlags: RecursiveRawSyntaxFlags = []
-      if tokenView.presence == .missing {
+      if tokenView.hasError || tokenView.presence == .missing {
         recursiveFlags.insert(.hasError)
       }
       return recursiveFlags
@@ -450,15 +452,17 @@ extension RawSyntax {
     wholeText: SyntaxText,
     textRange: Range<SyntaxText.Index>,
     presence: SourcePresence,
-    arena: SyntaxArena
+    arena: SyntaxArena,
+    hasError: Bool
   ) -> RawSyntax {
     assert(arena.contains(text: wholeText),
            "token text must be managed by the arena")
     let payload = RawSyntaxData.ParsedToken(
       tokenKind: kind,
       wholeText: wholeText,
       textRange: textRange,
-      presence: presence
+      presence: presence,
+      hasError: hasError
     )
     return RawSyntax(arena: arena, payload: .parsedToken(payload))
   }

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -132,14 +132,16 @@ public struct RawTokenSyntax: RawSyntaxToSyntax, RawSyntaxNodeProtocol {
     wholeText: SyntaxText,
     textRange: Range<SyntaxText.Index>,
     presence: SourcePresence,
+    hasError: Bool,
     arena: __shared SyntaxArena
   ) {
     let raw = RawSyntax.parsedToken(
       kind: kind,
       wholeText: wholeText,
       textRange: textRange,
       presence: presence,
-      arena: arena
+      arena: arena,
+      hasError: hasError
     )
     self = RawTokenSyntax(raw: raw)
   }
@@ -161,6 +163,7 @@ public struct RawTokenSyntax: RawSyntaxToSyntax, RawSyntaxNodeProtocol {
         wholeText: text,
         textRange: 0 ..< text.count,
         presence: presence,
+        hasError: false,
         arena: arena
       )
     } else {

Sources/SwiftSyntax/Raw/RawSyntaxTokenView.swift

@@ -216,4 +216,15 @@ public struct RawSyntaxTokenView {
       preconditionFailure("'presence' is a token-only property")
     }
   }
+
+  var hasError: Bool {
+    switch raw.rawData.payload {
+    case .parsedToken(let dat):
+      return dat.hasError
+    case .materializedToken(_):
+      return false
+    case .layout(_):
+      preconditionFailure("'hasError' is a token-only property")
+    }
+  }
 }