[libSyntax] Make the ByteTree protocol version consist of a major and minor component

Author: ahoppen

Diff for: docs/ByteTree.md

@@ -21,7 +21,7 @@ Arrays are modelled as objects whose fields are all of the same type and whose l
 
 ## Versioning
 
-The ByteTree format is prepended by a 4-byte protocol version number that describes the version of the object tree that was serialized. Its exact semantics are up to each specific application, but it is encouraged to interpret it as a two-comentent number where the first component, consisting of the first three bytes, is incremented for breaking changes and the last byte is incremented for backwards-compatible changes.
+The ByteTree format is prepended by a 4-byte protocol version number that describes the version of the object tree that was serialized. Its exact semantics are up to each specific application, but it is encouraged to interpret it as a two-comentent number where the first component, consisting of the three most significant bytes, is incremented for breaking changes and the last byte is incremented for backwards-compatible changes.
 
 ## Forward compatilibity
 

Diff for: include/swift/Syntax/Serialization/SyntaxSerialization.h

@@ -203,6 +203,20 @@ struct NullableTraits<RC<syntax::RawSyntax>> {
 
 namespace byteTree {
 
+/// Increase the major version for every change that is not just adding a new
+/// field at the end of an object. Older swiftSyntax clients will no longer be
+/// able to deserialize the format.
+const uint16_t SYNTAX_TREE_VERSION_MAJOR = 1; // Last change: initial version
+/// Increase the minor version if only new field has been added at the end of
+/// an object. Older swiftSyntax clients will still be able to deserialize the
+/// format.
+const uint8_t SYNTAX_TREE_VERSION_MINOR = 0; // Last change: initial version
+
+// Combine the major and minor version into one. The first three bytes
+// represent the major version, the last byte the minor version.
+const uint32_t SYNTAX_TREE_VERSION =
+    SYNTAX_TREE_VERSION_MAJOR << 8 | SYNTAX_TREE_VERSION_MINOR;
+
 /// The key for a ByteTree serializion user info of type
 /// `std::unordered_set<unsigned> *`. Specifies the IDs of syntax nodes that
 /// shall be omitted when the syntax tree gets serialized.

Diff for: tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -2448,7 +2448,8 @@ void serializeSyntaxTreeAsByteTree(
   Stream.reserve(32 * 1024);
   std::map<void *, void *> UserInfo;
   UserInfo[swift::byteTree::UserInfoKeyReusedNodeIds] = &ReusedNodeIds;
-  swift::byteTree::ByteTreeWriter::write(Stream, /*ProtocolVersion=*/1,
+  swift::byteTree::ByteTreeWriter::write(Stream,
+                                         swift::byteTree::SYNTAX_TREE_VERSION,
                                          *SyntaxTree.getRaw(), UserInfo);
 
   std::unique_ptr<llvm::WritableMemoryBuffer> Buf =

Diff for: tools/SwiftSyntax/ByteTreeDeserialization.swift

@@ -129,10 +129,15 @@ struct ByteTreeObjectReader {
   }
 }
 
+struct ByteTreeProtocolVersion {
+  let major: Int
+  let minor: Int
+}
+
 /// Reader for reading the ByteTree format into Swift objects
 struct ByteTreeReader {
   enum DeserializationError: Error, CustomStringConvertible {
-    case versionValidationFailed(ByteTreeReader.ProtocolVersion)
+    case versionValidationFailed(ByteTreeProtocolVersion)
 
     public var description: String {
       switch self {
@@ -143,9 +148,6 @@ struct ByteTreeReader {
     }
   }
 
-  /// The type as which the protocol version is encoded in ByteTree
-  typealias ProtocolVersion = UInt32
-
   /// A pointer pointing to the next byte of serialized data to be read
   private var pointer: UnsafeRawPointer
 
@@ -173,7 +175,7 @@ struct ByteTreeReader {
   static func read<T: ByteTreeObjectDecodable>(
     _ rootObjectType: T.Type, from pointer: UnsafeRawPointer,
     userInfo: UnsafePointer<[ByteTreeUserInfoKey: Any]>,
-    protocolVersionValidation: (ProtocolVersion) -> Bool
+    protocolVersionValidation: (ByteTreeProtocolVersion) -> Bool
   ) throws -> T {
     var reader = ByteTreeReader(pointer: pointer, userInfo: userInfo)
     try reader.readAndValidateProtocolVersion(protocolVersionValidation)
@@ -193,7 +195,7 @@ struct ByteTreeReader {
   static func read<T: ByteTreeObjectDecodable>(
     _ rootObjectType: T.Type, from data: Data,
     userInfo: UnsafePointer<[ByteTreeUserInfoKey: Any]>,
-    protocolVersionValidation versionValidate: (ProtocolVersion) -> Bool
+    protocolVersionValidation versionValidate: (ByteTreeProtocolVersion) -> Bool
   ) throws -> T {
     return try data.withUnsafeBytes { (pointer: UnsafePointer<UInt8>) in
       let rawPointer = UnsafeRawPointer(pointer)
@@ -249,10 +251,17 @@ struct ByteTreeReader {
   /// - Parameter validationCallback: A callback that determines if the given
   ///             protocol version can be read
   private mutating func readAndValidateProtocolVersion(
-    _ validationCallback: (ProtocolVersion) -> Bool
+    _ validationCallback: (ByteTreeProtocolVersion) -> Bool
   ) throws {
-    let protocolVersion = ProtocolVersion(littleEndian: 
-      readRaw(ProtocolVersion.self))
+    // The first three bytes of the four byte version number make up the major 
+    // version
+    let version = UInt32(littleEndian: readRaw(UInt32.self))
+    // The most significant three bytes make up the major version
+    let majorVersion = Int(version >> 8)
+    // The least significant byte constitutes the minor version
+    let minorVersion = Int(version & 0xff)
+    let protocolVersion = ByteTreeProtocolVersion(major: majorVersion, 
+                                                  minor: minorVersion)
     let result = validationCallback(protocolVersion)
     if !result {
       throw DeserializationError.versionValidationFailed(protocolVersion)

Diff for: tools/SwiftSyntax/SwiftSyntax.swift

@@ -80,8 +80,8 @@ public final class SyntaxTreeDeserializer {
     userInfo[.omittedNodeLookupFunction] = self.lookupNode
     return try ByteTreeReader.read(RawSyntax.self, from: data,
                                             userInfo: &userInfo) {
-      (version: ByteTreeReader.ProtocolVersion) in
-      return version == 1
+      (version: ByteTreeProtocolVersion) in
+      return version.major == 1
     }
   }
 

Diff for: tools/swift-syntax-test/swift-syntax-test.cpp

@@ -737,7 +737,8 @@ int doSerializeRawTree(const char *MainExecutablePath,
       if (options::AddByteTreeFields) {
         UserInfo[swift::byteTree::UserInfoKeyAddInvalidFields] = (void *)true;
       }
-      swift::byteTree::ByteTreeWriter::write(Stream, /*ProtocolVersion=*/1,
+      swift::byteTree::ByteTreeWriter::write(Stream,
+                                             byteTree::SYNTAX_TREE_VERSION,
                                              *Root, UserInfo);
       auto OutputBufferOrError = llvm::FileOutputBuffer::create(
           options::OutputFilename, Stream.data().size());