crasher in postgres-kit/Sources/PostgresKit/PostgresDataDecoder.swift, line 143

[lsh-silpion]

Hi,

I've got a crash in postgres-kit/Sources/PostgresKit/PostgresDataDecoder.swift, line 143:

Fatal error: Unexpectedly found nil while unwrapping an Optional value

looking at the code there:

    struct _ValueDecoder: SingleValueDecodingContainer {
        let data: PostgresData
        let json: JSONDecoder
        var codingPath: [CodingKey] {
            []
        }

        func decodeNil() -> Bool {
            return self.data.value == nil
        }

        func decode<T>(_ type: T.Type) throws -> T where T : Decodable {
            if let convertible = T.self as? PostgresDataConvertible.Type {
                return convertible.init(postgresData: self.data)! as! T
            } else {
                return try T.init(from: _Decoder(data: self.data, json: self.json))
            }
        }
    }

it says in line 143:

return convertible.init(postgresData: self.data)! as! T

looking at convertible.init(postgresData: self.data) I see in PostgresNIO/Data/PostgresData+Bool.swift lines 57-62:

    public init?(postgresData: PostgresData) {
        guard let bool = postgresData.bool else {
            return nil
        }
        self = bool
    }

that a call to convertible.init(postgresData: self.data) can indeed return nil, so I guess the forced unwrapping in line 143 of PostgresDataDecoder.swift is wrong.

See also on this: https://forums.swift.org/t/vapor-4-and-fluent-bug-in-postgres-kit-sources-postgreskit-postgresdatadecoder-swift-or-my-sources/34585

If you need more information please let me know!

[tanner0101]

Thanks for reporting @lsh-silpion. That should throw a value not found error instead of crashing.

[lsh-silpion]

This seems to have something to do with me using a JSONBcolumn in the database.

I refactored my code so it now looks like this (no more joins, everything in a single table):

GeoJSONController:

import Fluent
import Vapor

struct GeoJSONController {
    func index(req: Request) throws -> EventLoopFuture<[GeoJSONDTO]> {
        return GeoJSON.query(on: req.db).all()
            .flatMapThrowing { geoJSONArray in
                try geoJSONArray.map { geoJSON in try GeoJSONDTO(model: geoJSON) }
            }.flatMapErrorThrowing { error in
                throw Abort(.internalServerError)
            }
    }

    func create(req: Request) throws -> EventLoopFuture<GeoJSON> {
        let geoJSON = try req.content.decode(GeoJSON.self)
        return geoJSON.save(on: req.db).map { geoJSON }
    }

    func get(req: Request) throws -> EventLoopFuture<String> {
        return GeoJSON.find(req.parameters.get("GeoJSONID"), on: req.db)
            .unwrap(or: Abort(.notFound))
            .flatMapThrowing { found in
                try JSONEncoder().encode(found.json)
            }.map { data in
                String(decoding: data, as: UTF8.self)
            }.flatMapErrorThrowing { error in
                throw Abort(.internalServerError)
            }
    }
    
    func put(req: Request) throws -> EventLoopFuture<HTTPStatus> {
        guard let payload = try? req.content.decode(AnyCodable.self) else {
            throw Abort(.badRequest, reason: "Not a valid payload")
        }
        
        return GeoJSON.find(req.parameters.get("GeoJSONID"), on: req.db)
            .unwrap(or: Abort(.notFound))
            .map { found in
                found.json = payload
                
                found.save(on: req.db)
                
                return .noContent
            }
    }
    
    func delete(req: Request) throws -> EventLoopFuture<HTTPStatus> {
        return GeoJSON.find(req.parameters.get("GeoJSONID"), on: req.db)
            .unwrap(or: Abort(.notFound))
            .flatMap { found in found.delete(on: req.db) }
            .transform(to: .noContent)
    }
}

GeoJSON:

import Fluent
import Vapor

final class GeoJSON: Model, Content {
    static let schema = "geo_jsons"
    
    @ID(key: .id)
    var id: UUID?

    @Field(key: "description")
    var description: String

    @Field(key: "json")
    var json: AnyCodable?

    init() { }

    init(id: UUID? = nil, description: String) {
        self.id = id
        self.description = description
    }
}

GeoJSONDTO:

import Foundation
import Vapor

struct GeoJSONDTO: Content {
    var id: GeoJSON.IDValue
    var description: String?

    init(model: GeoJSON) throws {
        self.id = try model.requireID()
        self.description = model.description
    }
}

CreateGeoJSON:

import Fluent

struct CreateGeoJSON: Migration {
    func prepare(on database: Database) -> EventLoopFuture<Void> {
        return database.schema("geo_jsons")
            .id()
            .field("description", .string, .required)
            .field("json", .json)
            .create()
    }

    func revert(on database: Database) -> EventLoopFuture<Void> {
        return database.schema("geo_jsons").delete()
    }
}

configure:

import Fluent
import FluentPostgresDriver
import Vapor

// configures your application
public func configure(_ app: Application) throws {
    // uncomment to serve files from /Public folder
    // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory))
    
    print(app.directory.workingDirectory)

    app.databases.use(.postgres(
        hostname: Environment.get("DATABASE_HOST") ?? "localhost",
        username: Environment.get("DATABASE_USERNAME") ?? "MapEditor",
        password: Environment.get("DATABASE_PASSWORD") ?? "mapeditor",
        database: Environment.get("DATABASE_NAME") ?? "MapEditor"
    ), as: .psql)

    app.migrations.add(CreateGeoJSON())

    // register routes
    try routes(app)
}

routes:

import Fluent
import Vapor

func routes(_ app: Application) throws {
    app.get { req in
        return "It works!"
    }

    app.group("MapEditorBackend") { mapEditorBackend in
        let geoJSONController = GeoJSONController()
    
        mapEditorBackend.get(use: geoJSONController.index)
        mapEditorBackend.post(use: geoJSONController.create)
        
        mapEditorBackend.group(":GeoJSONID") { geoJSON in
            geoJSON.get(use: geoJSONController.get)
            geoJSON.put(use: geoJSONController.put)
            geoJSON.delete(use: geoJSONController.delete)
        }
    }
}

and AnyCodable, AnyDecodable and AnyEncodable from https://github.com/Flight-School/AnyCodable

Steps to reproduce:

• run the app

• fire some POST requests like

  {
      "description": "some description"
  }
  

  to http://127.0.0.1:8080/MapEditorBackend

• fire a GET request to http://127.0.0.1:8080/MapEditorBackend -> you get a list of GeoJSONs back containing UUIDs

• fire a DELETE request to http://127.0.0.1:8080/MapEditorBackend/(some UUID copied from above list) -> it works

• fire a PUT request to http://127.0.0.1:8080/MapEditorBackend/(some other UUID copied from above list) -> it works (have a look into the database)

• fire a GET request to http://127.0.0.1:8080/MapEditorBackend -> it no longer works, the app crashes

• restart the app

• fire a DELETE request to http://127.0.0.1:8080/MapEditorBackend/(same UUID to which you fired the PUT request) -> it no longer works, the app crashes

[lsh-silpion]

It looks that as soon as a value is in the JSONB column of the database the described crash occurs.

[grosch]

Potential fix in #176

[lsh-silpion]

I created a somewhat reduced test case:

Package.swift:

// swift-tools-version:5.2
import PackageDescription

let package = Package(
    name: "postgres-kit_issues_175_test",
    platforms: [
       .macOS(.v10_15)
    ],
    dependencies: [
        // 💧 A server-side Swift web framework.
        .package(url: "https://github.com/vapor/vapor.git", from: "4.0.0-rc"),
        .package(url: "https://github.com/vapor/fluent.git", from: "4.0.0-rc"),
        .package(url: "https://github.com/vapor/fluent-postgres-driver.git", from: "2.0.0-rc"),
        .package(url: "https://github.com/Flight-School/AnyCodable.git", from: "0.2.3")
    ],
    targets: [
        .target(name: "App", dependencies: [
            .product(name: "Fluent", package: "fluent"),
            .product(name: "FluentPostgresDriver", package: "fluent-postgres-driver"),
            .product(name: "Vapor", package: "vapor"),
            .product(name: "AnyCodable", package: "AnyCodable")
        ]),
        .target(name: "Run", dependencies: ["App"]),
        .testTarget(name: "AppTests", dependencies: [
            .target(name: "App"),
            .product(name: "XCTVapor", package: "vapor"),
        ])
    ]
)

Sources/App/Controllers/Issue175Controller.swift:

import Fluent
import Vapor

struct Issue175Controller {
    func index(req: Request) throws -> EventLoopFuture<[Issue175]> {
        return Issue175.query(on: req.db).all()
    }

    func create(req: Request) throws -> EventLoopFuture<Issue175> {
        let issue175 = try req.content.decode(Issue175.self)
        return issue175.save(on: req.db).map { issue175 }
    }

    func delete(req: Request) throws -> EventLoopFuture<HTTPStatus> {
        return Issue175.find(req.parameters.get("issue175ID"), on: req.db)
            .unwrap(or: Abort(.notFound))
            .flatMap { $0.delete(on: req.db) }
            .transform(to: .ok)
    }
}

Sources/App/Migrations/CreateIssue175.swift:

import Fluent

struct CreateIssue175: Migration {
    func prepare(on database: Database) -> EventLoopFuture<Void> {
        return database.schema("issue_175")
            .id()
            .field("json", .json)
            .create()
    }

    func revert(on database: Database) -> EventLoopFuture<Void> {
        return database.schema("issue_175").delete()
    }
}

Sources/App/Models/Issue175.swift:

import Fluent
import Vapor
import AnyCodable

final class Issue175: Model, Content {
    static let schema = "issue_175"
    
    @ID(key: .id)
    var id: UUID?

    @Field(key: "json")
    var json: AnyCodable?

    init() { }

    init(id: UUID? = nil, json: AnyCodable? = nil) {
        self.id = id
        self.json = json
    }
}

Sources/App/configure.swift:

import Fluent
import FluentPostgresDriver
import Vapor

// configures your application
public func configure(_ app: Application) throws {
    // uncomment to serve files from /Public folder
    // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory))

    app.databases.use(.postgres(
        hostname: Environment.get("DATABASE_HOST") ?? "localhost",
        username: Environment.get("DATABASE_USERNAME") ?? "issue_175",
        password: Environment.get("DATABASE_PASSWORD") ?? "issue_175",
        database: Environment.get("DATABASE_NAME") ?? "issue_175"
    ), as: .psql)

    app.migrations.add(CreateIssue175())

    // register routes
    try routes(app)
}

Sources/App/routes.swift:

import Fluent
import Vapor

func routes(_ app: Application) throws {
    app.get { req in
        return "It works!"
    }

    let issue175Controller = Issue175Controller()
    app.get("postgres-kit_issues_175_test", use: issue175Controller.index)
    app.post("postgres-kit_issues_175_test", use: issue175Controller.create)
    app.delete("postgres-kit_issues_175_test", ":issue175ID", use: issue175Controller.delete)
}

Sources/Run/main.swift is unchanged from the template

Steps to reproduce the issue:

• fire a GET onto http://127.0.0.1:8080/postgres-kit_issues_175_test
  ->

  []
  

• fire a POST onto http://127.0.0.1:8080/postgres-kit_issues_175_test with a body like this

  {}
  

  ->
  something like:

  {
      "id": "FC8B217B-DA17-4CCA-A984-BF1D37D65B36",
      "json": null
  }
  

• fire a GET onto http://127.0.0.1:8080/postgres-kit_issues_175_test
  ->
  something like:

  [
      {
          "id": "FC8B217B-DA17-4CCA-A984-BF1D37D65B36",
          "json": null
      }
  ]
  

• fire a POST onto http://127.0.0.1:8080/postgres-kit_issues_175_test with a body like this

  {
      "json": { "postgres-kit_issues_175_test": "postgres-kit_issues_175_test" }
  }
  

  ->
  something like:

  {
      "id": "95C6C312-0636-4B61-9227-8A69DCB9A547",
      "json": {
          "postgres-kit_issues_175_test": "postgres-kit_issues_175_test"
      }
  }
  

• fire a GET onto http://127.0.0.1:8080/postgres-kit_issues_175_test
  -> crash like described above
• add:

  .package(url: "https://github.com/vapor/postgres-kit", .branch("issue175"))

  to the dependencies of Package.swift
• restart the Vapor-App
• fire a GET onto http://127.0.0.1:8080/postgres-kit_issues_175_test
  ->
  something like:

  [
      {
          "id": "FC8B217B-DA17-4CCA-A984-BF1D37D65B36",
          "json": null
      },
      {
          "id": "95C6C312-0636-4B61-9227-8A69DCB9A547",
          "json": {
              "postgres-kit_issues_175_test": "postgres-kit_issues_175_test"
          }
      }
  ]
  

[tanner0101]

Fixed in #182