[stdlib]: Set,Dictionary: Fix subtle cross-collection hash collision issue

When Set/Dictionary is nested in another Set, the boundaries of the nested collections weren’t correctly delineated in commutative hashing.

For example, these Sets all hashed the same:

[[1, 2], [3, 4]]
[[1, 3], [2, 4]]
[[1, 4], [2, 3]]

Hash collisions could thus be systematically generated.

To fix this, remove collection-level support for one-shot hashing and revert to the previous method of generating hash values. (Set is still able to support one-shot hashing for its members, though.)

Author: lorentey

stdlib/public/core/Dictionary.swift

@@ -1453,7 +1453,7 @@ extension Dictionary: Hashable where Value: Hashable {
   public func hash(into hasher: inout Hasher) {
     var commutativeHash = 0
     for (k, v) in self {
-      // Note that we use a copy of the outer hasher here. This makes collisions
+      // Note that we use a copy of our own hasher here. This makes hash values
       // dependent on its state, eliminating static collision patterns.
       var elementHasher = hasher
       elementHasher.combine(k)
@@ -1462,19 +1462,6 @@ extension Dictionary: Hashable where Value: Hashable {
     }
     hasher.combine(commutativeHash)
   }
-
-  @inlinable // FIXME(sil-serialize-all)
-  public func _rawHashValue(seed: (UInt64, UInt64)) -> Int {
-    var commutativeHash = 0
-    let hasher = Hasher(_seed: seed)
-    for (k, v) in self {
-      var elementHasher = hasher
-      elementHasher.combine(k)
-      elementHasher.combine(v)
-      commutativeHash ^= elementHasher._finalize()
-    }
-    return commutativeHash
-  }
 }
 
 extension Dictionary: CustomStringConvertible, CustomDebugStringConvertible {

stdlib/public/core/Set.swift

@@ -487,16 +487,12 @@ extension Set: Hashable {
   @inlinable // FIXME(sil-serialize-all)
   public func hash(into hasher: inout Hasher) {
     // FIXME(ABI)#177: <rdar://problem/18915294> Cache Set<T> hashValue
-    hasher.combine(_rawHashValue(seed: hasher._generateSeed()))
-  }
-
-  @inlinable // FIXME(sil-serialize-all)
-  public func _rawHashValue(seed: (UInt64, UInt64)) -> Int {
     var hash = 0
+    let seed = hasher._generateSeed()
     for member in self {
       hash ^= member._rawHashValue(seed: seed)
     }
-    return hash
+    hasher.combine(hash)
   }
 }
 

validation-test/stdlib/Dictionary4.swift

@@ -9,31 +9,47 @@ import StdlibCollectionUnittest
 var DictionaryTestSuite = TestSuite("Dictionary4")
 
 DictionaryTestSuite.test("Hashable") {
-  let d1: Dictionary<Int, String> = [1: "meow", 2: "meow", 3: "meow"]
-  let d2: Dictionary<Int, String> = [1: "meow", 2: "meow", 3: "mooo"]
-  let d3: Dictionary<Int, String> = [1: "meow", 2: "meow", 4: "meow"]
-  let d4: Dictionary<Int, String> = [1: "meow", 2: "meow", 4: "mooo"]
-  checkHashable([d1, d2, d3, d4], equalityOracle: { $0 == $1 })
-
-  let dd1: Dictionary<Int, Dictionary<Int, String>> = [1: [2: "meow"]]
-  let dd2: Dictionary<Int, Dictionary<Int, String>> = [2: [1: "meow"]]
-  let dd3: Dictionary<Int, Dictionary<Int, String>> = [2: [2: "meow"]]
-  let dd4: Dictionary<Int, Dictionary<Int, String>> = [1: [1: "meow"]]
-  let dd5: Dictionary<Int, Dictionary<Int, String>> = [2: [2: "mooo"]]
-  let dd6: Dictionary<Int, Dictionary<Int, String>> = [2: [:]]
-  let dd7: Dictionary<Int, Dictionary<Int, String>> = [:]
-  checkHashable(
-    [dd1, dd2, dd3, dd4, dd5, dd6, dd7],
-    equalityOracle: { $0 == $1 })
-
-  // Check that hash is equal even though dictionary is traversed differently
-  var d5: Dictionary<Int, String> =
-    [1: "meow", 2: "meow", 3: "mooo", 4: "woof", 5: "baah", 6: "mooo"]
-  let expected = d5.hashValue
-  for capacity in [4, 8, 16, 32, 64, 128, 256] {
-    d5.reserveCapacity(capacity)
-    expectEqual(d5.hashValue, expected)
+  let d1: [Dictionary<Int, String>] = [
+    [1: "meow", 2: "meow", 3: "meow"],
+    [1: "meow", 2: "meow", 3: "mooo"],
+    [1: "meow", 2: "meow", 4: "meow"],
+    [1: "meow", 2: "meow", 4: "mooo"]]
+  checkHashable(d1, equalityOracle: { $0 == $1 })
+
+  let d2: [Dictionary<Int, Dictionary<Int, String>>] = [
+    [1: [2: "meow"]],
+    [2: [1: "meow"]],
+    [2: [2: "meow"]],
+    [1: [1: "meow"]],
+    [2: [2: "mooo"]],
+    [2: [:]],
+    [:]]
+  checkHashable(d2, equalityOracle: { $0 == $1 })
+
+  // Dictionary should hash itself in a way that ensures instances get correctly
+  // delineated even when they are nested in other commutative collections.
+  // These are different Sets, so they should produce different hashes:
+  let remix: [Set<Dictionary<String, Int>>] = [
+    [["Blanche": 1, "Rose": 2], ["Dorothy": 3, "Sophia": 4]],
+    [["Blanche": 1, "Dorothy": 3], ["Rose": 2, "Sophia": 4]],
+    [["Blanche": 1, "Sophia": 4], ["Rose": 2, "Dorothy": 3]]
+  ]
+  checkHashable(remix, equalityOracle: { $0 == $1 })
+
+  // Dictionary ordering is not guaranteed to be consistent across equal
+  // instances. In particular, ordering is highly sensitive to the size of the
+  // allocated storage buffer. Generate a few copies of the same dictionary with
+  // different capacities, and verify that they compare and hash the same.
+  var variants: [Dictionary<String, Int>] = []
+  for i in 4 ..< 12 {
+    var set: Dictionary<String, Int> = [
+      "one": 1,   "two": 2,
+      "three": 3, "four": 4,
+      "five": 5,  "six": 6]
+    set.reserveCapacity(1 << i)
+    variants.append(set)
   }
+  checkHashable(variants, equalityOracle: { _, _ in true })
 }
 
 runAllTests()

validation-test/stdlib/Set.swift

@@ -3470,6 +3470,27 @@ SetTestSuite.test("Hashable") {
   let ss11 = Set([Set([2020] as [Int]), Set([3030] as [Int]), Set([2020] as [Int])])
   let ss2 = Set([Set([9090] as [Int])])
   checkHashable([ss1, ss11, ss2], equalityOracle: { $0 == $1 })
+
+  // Set should hash itself in a way that ensures instances get correctly
+  // delineated even when they are nested in other commutative collections.
+  // These are different Sets, so they should produce different hashes:
+  let remix: [Set<Set<Int>>] = [
+    [[1, 2], [3, 4]],
+    [[1, 3], [2, 4]],
+    [[1, 4], [2, 3]],
+  ]
+  checkHashable(remix, equalityOracle: { $0 == $1 })
+
+  // Set ordering is not guaranteed to be consistent across equal instances. In
+  // particular, ordering is highly sensitive to the size of the allocated
+  // storage buffer.
+  var variants: [Set<Int>] = []
+  for i in 4 ..< 12 {
+    var set: Set<Int> = [1, 2, 3, 4, 5, 6]
+    set.reserveCapacity(1 << i)
+    variants.append(set)
+  }
+  checkHashable(variants, equalityOracle: { _, _ in true })
 }
 
 //===---