AtomicStorage.swift.gyb

//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift Atomics open source project
//
// Copyright (c) 2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

import Builtin

% from SwiftAtomics import *

% for (type, size, alignment, builtin, swift) in atomicTypes:

//===----------------------------------------------------------------------===//
// ${size} bit Atomic Storage
//===----------------------------------------------------------------------===//

@available(SwiftStdlib 6.0, *)
@frozen
@_alignment(${alignment})
public struct ${type} {
  public var _storage: ${builtin}

  @available(SwiftStdlib 6.0, *)
  @_alwaysEmitIntoClient
  @_transparent
  public init(_ _builtin: ${builtin}) {
    self._storage = _builtin
  }
}

//===----------------------------------------------------------------------===//
// ${size} bit atomic load
//===----------------------------------------------------------------------===//

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Atomically loads and returns the current value, applying the specified
  /// memory ordering.
  ///
  /// - Parameter ordering: The memory ordering to apply on this operation.
  /// - Returns: The current value.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func load(ordering: AtomicLoadOrdering) -> Value {
    let result = switch ordering {
%   for (name, api, doc, llvm) in loadOrderings:
    case .${name}:
      Builtin.atomicload_${llvm}_Int${size}(rawAddress)
%   end

    default:
      Builtin.unreachable()
    }

    return Value.decodeAtomicRepresentation(Value.AtomicRepresentation(result))
  }
}

//===----------------------------------------------------------------------===//
// ${size} bit atomic store
//===----------------------------------------------------------------------===//

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Atomically sets the current value to `desired`, applying the specified
  /// memory ordering.
  ///
  /// - Parameter desired: The desired new value.
  /// - Parameter ordering: The memory ordering to apply on this operation.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func store(
    _ desired: consuming Value,
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
%   for (name, api, doc, llvm) in storeOrderings:
    case .${name}:
      Builtin.atomicstore_${llvm}_Int${size}(
        rawAddress,
        Value.encodeAtomicRepresentation(desired)._storage
      )
%   end

    default:
      Builtin.unreachable()
    }
  }
}

//===----------------------------------------------------------------------===//
// ${size} bit atomic exchanges
//===----------------------------------------------------------------------===//

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Atomically sets the current value to `desired` and returns the original
  /// value, applying the specified memory ordering.
  ///
  /// - Parameter desired: The desired new value.
  /// - Parameter ordering: The memory ordering to apply on this operation.
  /// - Returns: The original value.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func exchange(
    _ desired: consuming Value,
    ordering: AtomicUpdateOrdering
  ) -> Value {
    let desired = Value.encodeAtomicRepresentation(desired)._storage

    let result = switch ordering {
%   for (name, api, _, llvm, _) in updateOrderings:
    case .${name}:
      Builtin.atomicrmw_xchg_${llvm}_Int${size}(rawAddress, desired)
%   end

    default:
      Builtin.unreachable()
    }

    let atomicStorage = Value.AtomicRepresentation(result)

    return Value.decodeAtomicRepresentation(atomicStorage)
  }
}

//===----------------------------------------------------------------------===//
// ${size} bit atomic compare exchanges
//===----------------------------------------------------------------------===//

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Perform an atomic compare and exchange operation on the current value,
  /// applying the specified memory ordering.
  ///
  /// This operation performs the following algorithm as a single atomic
  /// transaction:
  ///
  ///     atomic(self) { currentValue in
  ///       let original = currentValue
  ///       guard original == expected else { return (false, original) }
  ///       currentValue = desired
  ///       return (true, original)
  ///     }
  ///
  /// - Note: This method implements a "strong" compare and exchange operation
  ///   that does not permit spurious failures.
  ///
  /// - Parameter expected: The expected current value.
  /// - Parameter desired: The desired new value.
  /// - Parameter ordering: The memory ordering to apply on this operation.
  /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if
  ///   the exchange was successful, and `original` is the original value.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func compareExchange(
    expected: consuming Value,
    desired: consuming Value,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: Value) {
    compareExchange(
      expected: expected,
      desired: desired,
      successOrdering: ordering,
      failureOrdering: ._failureOrdering(for: ordering)
    )
  }
}

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Perform an atomic compare and exchange operation on the current value,
  /// applying the specified success/failure memory orderings.
  ///
  /// This operation performs the following algorithm as a single atomic
  /// transaction:
  ///
  ///     atomic(self) { currentValue in
  ///       let original = currentValue
  ///       guard original == expected else { return (false, original) }
  ///       currentValue = desired
  ///       return (true, original)
  ///     }
  ///
  /// The `successOrdering` argument specifies the memory ordering to use when
  /// the operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  ///
  /// - Note: This method implements a "strong" compare and exchange operation
  ///   that does not permit spurious failures.
  ///
  /// - Parameter expected: The expected current value.
  /// - Parameter desired: The desired new value.
  /// - Parameter successOrdering: The memory ordering to apply if this
  ///    operation performs the exchange.
  /// - Parameter failureOrdering: The memory ordering to apply on this
  ///    operation does not perform the exchange.
  /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if
  ///   the exchange was successful, and `original` is the original value.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func compareExchange(
    expected: consuming Value,
    desired: consuming Value,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: Value) {
    let expected = Value.encodeAtomicRepresentation(expected)._storage
    let desired = Value.encodeAtomicRepresentation(desired)._storage

    let (ov, won) = switch (successOrdering, failureOrdering) {
%   for (successName, _, _, successLLVM, successFailure) in updateOrderings:
%     for (failureName, _, _, failureLLVM) in loadOrderings:
    case (.${successName}, .${failureName}):
      Builtin.cmpxchg_${actualOrders(successLLVM, failureLLVM)}_Int${size}(
        rawAddress,
        expected,
        desired
      )
%     end
%   end

    default:
      Builtin.unreachable()
    }

    return (
      exchanged: Bool(won),
      original: Value.decodeAtomicRepresentation(Value.AtomicRepresentation(ov))
    )
  }
}

//===----------------------------------------------------------------------===//
// ${size} bit weak compare exchanges
//===----------------------------------------------------------------------===//

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Perform an atomic weak compare and exchange operation on the current
  /// value, applying the memory ordering. This compare-exchange variant is
  /// allowed to spuriously fail; it is designed to be called in a loop until
  /// it indicates a successful exchange has happened.
  ///
  /// This operation performs the following algorithm as a single atomic
  /// transaction:
  ///
  ///     atomic(self) { currentValue in
  ///       let original = currentValue
  ///       guard original == expected else { return (false, original) }
  ///       currentValue = desired
  ///       return (true, original)
  ///     }
  ///
  /// - Note: The weakCompareExchange form may sometimes return false even when
  ///   the original and expected values are equal. (Such failures may happen
  ///   when some transient condition prevents the underlying operation from
  ///   succeeding -- such as an incoming interrupt during a
  ///   load-link/store-conditional instruction sequence.) This variant is
  ///   designed to be called in a loop that only exits when the exchange is
  ///   successful; in such loops, using weakCompareExchange may lead to a
  ///   performance improvement by eliminating a nested loop in the regular,
  ///   "strong", compareExchange variants.
  ///
  /// - Parameter expected: The expected current value.
  /// - Parameter desired: The desired new value.
  /// - Parameter ordering: The memory ordering to apply on this operation.
  /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if
  ///   the exchange was successful, and `original` is the original value.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func weakCompareExchange(
    expected: consuming Value,
    desired: consuming Value,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: Value) {
    weakCompareExchange(
      expected: expected,
      desired: desired,
      successOrdering: ordering,
      failureOrdering: ._failureOrdering(for: ordering)
    )
  }
}

@available(SwiftStdlib 6.0, *)
extension Atomic where Value.AtomicRepresentation == ${type} {
  /// Perform an atomic weak compare and exchange operation on the current
  /// value, applying the specified success/failure memory orderings. This
  /// compare-exchange variant is allowed to spuriously fail; it is designed to
  /// be called in a loop until it indicates a successful exchange has happened.
  ///
  ///
  /// This operation performs the following algorithm as a single atomic
  /// transaction:
  ///
  ///     atomic(self) { currentValue in
  ///       let original = currentValue
  ///       guard original == expected else { return (false, original) }
  ///       currentValue = desired
  ///       return (true, original)
  ///     }
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  ///
  /// - Note: The weakCompareExchange form may sometimes return false even when
  ///   the original and expected values are equal. (Such failures may happen
  ///   when some transient condition prevents the underlying operation from
  ///   succeeding -- such as an incoming interrupt during a
  ///   load-link/store-conditional instruction sequence.) This variant is
  ///   designed to be called in a loop that only exits when the exchange is
  ///   successful; in such loops, using weakCompareExchange may lead to a
  ///   performance improvement by eliminating a nested loop in the regular,
  ///   "strong", compareExchange variants.
  ///
  /// - Parameter expected: The expected current value.
  /// - Parameter desired: The desired new value.
  /// - Parameter successOrdering: The memory ordering to apply if this
  ///    operation performs the exchange.
  /// - Parameter failureOrdering: The memory ordering to apply on this
  ///    operation does not perform the exchange.
  /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if
  ///   the exchange was successful, and `original` is the original value.
  @available(SwiftStdlib 6.0, *)
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent
  public func weakCompareExchange(
    expected: consuming Value,
    desired: consuming Value,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: Value) {
    let expected = Value.encodeAtomicRepresentation(expected)._storage
    let desired = Value.encodeAtomicRepresentation(desired)._storage

    let (ov, won) = switch (successOrdering, failureOrdering) {
%   for (successName, _, _, successLLVM, _) in updateOrderings:
%     for (failureName, _, _, failureLLVM) in loadOrderings:
    case (.${successName}, .${failureName}):
      Builtin.cmpxchg_${actualOrders(successLLVM, failureLLVM)}_weak_Int${size}(
        rawAddress,
        expected,
        desired
      )
%     end
%   end

    default:
      Builtin.unreachable()
    }

    return (
      exchanged: Bool(won),
      original: Value.decodeAtomicRepresentation(Value.AtomicRepresentation(ov))
    )
  }
}

% end