Merge commit '71d1bdfee579' from swift/release/5.6 into stable/20210726

Author: git apple-llvm automerger

llvm/lib/Target/AArch64/AArch64InstrAtomics.td

@@ -430,7 +430,8 @@ def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
 
 let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
     mayLoad = 1, mayStore = 1 in {
-class cmp_swap_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32common:$scratch),
+class cmp_swap_128 : Pseudo<(outs GPR64common:$RdLo, GPR64common:$RdHi,
+                                  GPR32common:$scratch),
                            (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
                                 GPR64:$newLo, GPR64:$newHi), []>,
                      Sched<[WriteAtomic]>;

llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-128.mir

@@ -30,35 +30,37 @@ body:             |
     ; CHECK: RET_ReallyLR
     ; CHECK-NOLSE-LABEL: name: compare_swap_128
     ; CHECK-NOLSE: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4
-    ; CHECK-NOLSE: [[COPY:%[0-9]+]]:gpr64(p0) = COPY $x0
-    ; CHECK-NOLSE: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
-    ; CHECK-NOLSE: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
-    ; CHECK-NOLSE: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3
-    ; CHECK-NOLSE: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
-    ; CHECK-NOLSE: [[COPY5:%[0-9]+]]:gpr64(s64) = COPY [[COPY1]](s64)
-    ; CHECK-NOLSE: [[COPY6:%[0-9]+]]:gpr64(s64) = COPY [[COPY2]](s64)
-    ; CHECK-NOLSE: [[COPY7:%[0-9]+]]:gpr64(s64) = COPY [[COPY3]](s64)
-    ; CHECK-NOLSE: [[COPY8:%[0-9]+]]:gpr64(s64) = COPY [[COPY4]](s64)
-    ; CHECK-NOLSE: early-clobber %13:gpr64(s64), early-clobber %14:gpr64(s64), early-clobber %16:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire (s128))
-    ; CHECK-NOLSE: [[COPY9:%[0-9]+]]:gpr64 = COPY %16
-    ; CHECK-NOLSE: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %13(s64), %14(s64)
-    ; CHECK-NOLSE: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128))
-    ; CHECK-NOLSE: RET_ReallyLR
+    ; CHECK-NOLSE-NEXT: {{  $}}
+    ; CHECK-NOLSE-NEXT: [[COPY:%[0-9]+]]:gpr64(p0) = COPY $x0
+    ; CHECK-NOLSE-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NOLSE-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NOLSE-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NOLSE-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NOLSE-NEXT: [[COPY5:%[0-9]+]]:gpr64(s64) = COPY [[COPY1]](s64)
+    ; CHECK-NOLSE-NEXT: [[COPY6:%[0-9]+]]:gpr64(s64) = COPY [[COPY2]](s64)
+    ; CHECK-NOLSE-NEXT: [[COPY7:%[0-9]+]]:gpr64(s64) = COPY [[COPY3]](s64)
+    ; CHECK-NOLSE-NEXT: [[COPY8:%[0-9]+]]:gpr64(s64) = COPY [[COPY4]](s64)
+    ; CHECK-NOLSE-NEXT: early-clobber %13:gpr64common(s64), early-clobber %14:gpr64common(s64), early-clobber %16:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire (s128))
+    ; CHECK-NOLSE-NEXT: [[COPY9:%[0-9]+]]:gpr64 = COPY %16
+    ; CHECK-NOLSE-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %13(s64), %14(s64)
+    ; CHECK-NOLSE-NEXT: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128))
+    ; CHECK-NOLSE-NEXT: RET_ReallyLR
     ; CHECK-LSE-LABEL: name: compare_swap_128
     ; CHECK-LSE: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4
-    ; CHECK-LSE: [[COPY:%[0-9]+]]:gpr64sp(p0) = COPY $x0
-    ; CHECK-LSE: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
-    ; CHECK-LSE: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
-    ; CHECK-LSE: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3
-    ; CHECK-LSE: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
-    ; CHECK-LSE: [[REG_SEQUENCE:%[0-9]+]]:xseqpairsclass(s128) = REG_SEQUENCE [[COPY1]](s64), %subreg.sube64, [[COPY2]](s64), %subreg.subo64
-    ; CHECK-LSE: [[REG_SEQUENCE1:%[0-9]+]]:xseqpairsclass(s128) = REG_SEQUENCE [[COPY3]](s64), %subreg.sube64, [[COPY4]](s64), %subreg.subo64
-    ; CHECK-LSE: [[CASPAX:%[0-9]+]]:xseqpairsclass(s128) = CASPAX [[REG_SEQUENCE]](s128), [[REG_SEQUENCE1]](s128), [[COPY]](p0) :: (load store acquire acquire (s128))
-    ; CHECK-LSE: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[CASPAX]](s128), 0
-    ; CHECK-LSE: [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[CASPAX]](s128), 64
-    ; CHECK-LSE: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[EXTRACT]](s64), [[EXTRACT1]](s64)
-    ; CHECK-LSE: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128))
-    ; CHECK-LSE: RET_ReallyLR
+    ; CHECK-LSE-NEXT: {{  $}}
+    ; CHECK-LSE-NEXT: [[COPY:%[0-9]+]]:gpr64sp(p0) = COPY $x0
+    ; CHECK-LSE-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-LSE-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-LSE-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-LSE-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-LSE-NEXT: [[REG_SEQUENCE:%[0-9]+]]:xseqpairsclass(s128) = REG_SEQUENCE [[COPY1]](s64), %subreg.sube64, [[COPY2]](s64), %subreg.subo64
+    ; CHECK-LSE-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:xseqpairsclass(s128) = REG_SEQUENCE [[COPY3]](s64), %subreg.sube64, [[COPY4]](s64), %subreg.subo64
+    ; CHECK-LSE-NEXT: [[CASPAX:%[0-9]+]]:xseqpairsclass(s128) = CASPAX [[REG_SEQUENCE]](s128), [[REG_SEQUENCE1]](s128), [[COPY]](p0) :: (load store acquire acquire (s128))
+    ; CHECK-LSE-NEXT: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[CASPAX]](s128), 0
+    ; CHECK-LSE-NEXT: [[EXTRACT1:%[0-9]+]]:_(s64) = G_EXTRACT [[CASPAX]](s128), 64
+    ; CHECK-LSE-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[EXTRACT]](s64), [[EXTRACT1]](s64)
+    ; CHECK-LSE-NEXT: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128))
+    ; CHECK-LSE-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %3:_(s64) = COPY $x1
     %4:_(s64) = COPY $x2

llvm/test/CodeGen/AArch64/arm64-atomic-128.ll

@@ -474,3 +474,52 @@ define void @atomic_store_relaxed(i128 %in, i128* %p) {
    store atomic i128 %in, i128* %p unordered, align 16
    ret void
 }
+
+; Since we store the original value to ensure no tearing for the unsuccessful
+; case, the register used must not be xzr.
+define void @cmpxchg_dead(i128* %ptr, i128 %desired, i128 %new) {
+; NOOUTLINE-LABEL: cmpxchg_dead:
+; NOOUTLINE:       // %bb.0:
+; NOOUTLINE-NEXT:  .LBB17_1: // =>This Inner Loop Header: Depth=1
+; NOOUTLINE-NEXT:    ldxp x8, x9, [x0]
+; NOOUTLINE-NEXT:    cmp x8, x2
+; NOOUTLINE-NEXT:    cset w10, ne
+; NOOUTLINE-NEXT:    cmp x9, x3
+; NOOUTLINE-NEXT:    cinc w10, w10, ne
+; NOOUTLINE-NEXT:    cbz w10, .LBB17_3
+; NOOUTLINE-NEXT:  // %bb.2: // in Loop: Header=BB17_1 Depth=1
+; NOOUTLINE-NEXT:    stxp w10, x8, x9, [x0]
+; NOOUTLINE-NEXT:    cbnz w10, .LBB17_1
+; NOOUTLINE-NEXT:    b .LBB17_4
+; NOOUTLINE-NEXT:  .LBB17_3: // in Loop: Header=BB17_1 Depth=1
+; NOOUTLINE-NEXT:    stxp w10, x4, x5, [x0]
+; NOOUTLINE-NEXT:    cbnz w10, .LBB17_1
+; NOOUTLINE-NEXT:  .LBB17_4:
+; NOOUTLINE-NEXT:    ret
+;
+; OUTLINE-LABEL: cmpxchg_dead:
+; OUTLINE:       // %bb.0:
+; OUTLINE-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT:    .cfi_def_cfa_offset 16
+; OUTLINE-NEXT:    .cfi_offset w30, -16
+; OUTLINE-NEXT:    mov x1, x3
+; OUTLINE-NEXT:    mov x8, x0
+; OUTLINE-NEXT:    mov x0, x2
+; OUTLINE-NEXT:    mov x2, x4
+; OUTLINE-NEXT:    mov x3, x5
+; OUTLINE-NEXT:    mov x4, x8
+; OUTLINE-NEXT:    bl __aarch64_cas16_relax
+; OUTLINE-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT:    ret
+;
+; LSE-LABEL: cmpxchg_dead:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
+; LSE-NEXT:    // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
+; LSE-NEXT:    // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5
+; LSE-NEXT:    // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3
+; LSE-NEXT:    casp x2, x3, x4, x5, [x0]
+; LSE-NEXT:    ret
+  cmpxchg i128* %ptr, i128 %desired, i128 %new monotonic monotonic
+  ret void
+}