Skip to content

Commit 1df5aff

Browse files
borsbors
committed
Auto merge of #133984 - DaniPopes:scmp-ucmp, r=scottmcm
Lower BinOp::Cmp to llvm.{s,u}cmp.* intrinsics Lowers `mir::BinOp::Cmp` (`three_way_compare` intrinsic) to the corresponding LLVM `llvm.{s,u}cmp.i8.*` intrinsics. These are the intrinsics mentioned in #118310, which are now available in LLVM 19. I couldn't find any follow-up PRs/discussions about this, please let me know if I missed something. r? `@scottmcm`
2 parents f8c27df + 58c10c6 commit 1df5aff

File tree

8 files changed

+124
-42
lines changed

8 files changed

+124
-42
lines changed

‎compiler/rustc_codegen_llvm/src/builder.rs

+30
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use rustc_codegen_ssa::mir::place::PlaceRef;
1414
use rustc_codegen_ssa::traits::*;
1515
use rustc_data_structures::small_c_str::SmallCStr;
1616
use rustc_hir::def_id::DefId;
17+
use rustc_middle::bug;
1718
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
1819
use rustc_middle::ty::layout::{
1920
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
@@ -1074,6 +1075,35 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
10741075
unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
10751076
}
10761077

1078+
fn three_way_compare(
1079+
&mut self,
1080+
ty: Ty<'tcx>,
1081+
lhs: Self::Value,
1082+
rhs: Self::Value,
1083+
) -> Option<Self::Value> {
1084+
// FIXME: See comment on the definition of `three_way_compare`.
1085+
if crate::llvm_util::get_version() < (20, 0, 0) {
1086+
return None;
1087+
}
1088+
1089+
let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) {
1090+
(true, 8) => "llvm.scmp.i8.i8",
1091+
(true, 16) => "llvm.scmp.i8.i16",
1092+
(true, 32) => "llvm.scmp.i8.i32",
1093+
(true, 64) => "llvm.scmp.i8.i64",
1094+
(true, 128) => "llvm.scmp.i8.i128",
1095+
1096+
(false, 8) => "llvm.ucmp.i8.i8",
1097+
(false, 16) => "llvm.ucmp.i8.i16",
1098+
(false, 32) => "llvm.ucmp.i8.i32",
1099+
(false, 64) => "llvm.ucmp.i8.i64",
1100+
(false, 128) => "llvm.ucmp.i8.i128",
1101+
1102+
_ => bug!("three-way compare unsupported for type {ty:?}"),
1103+
};
1104+
Some(self.call_intrinsic(name, &[lhs, rhs]))
1105+
}
1106+
10771107
/* Miscellaneous instructions */
10781108
fn memcpy(
10791109
&mut self,

‎compiler/rustc_codegen_llvm/src/context.rs

+12
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,18 @@ impl<'ll> CodegenCx<'ll, '_> {
11271127
ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64);
11281128
ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128);
11291129

1130+
ifn!("llvm.scmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
1131+
ifn!("llvm.scmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
1132+
ifn!("llvm.scmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
1133+
ifn!("llvm.scmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
1134+
ifn!("llvm.scmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
1135+
1136+
ifn!("llvm.ucmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
1137+
ifn!("llvm.ucmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
1138+
ifn!("llvm.ucmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
1139+
ifn!("llvm.ucmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
1140+
ifn!("llvm.ucmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
1141+
11301142
ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void);
11311143
ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void);
11321144

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+3
Original file line numberDiff line numberDiff line change
@@ -988,6 +988,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
988988
mir::BinOp::Cmp => {
989989
use std::cmp::Ordering;
990990
assert!(!is_float);
991+
if let Some(value) = bx.three_way_compare(lhs_ty, lhs, rhs) {
992+
return value;
993+
}
991994
let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed);
992995
if bx.cx().tcx().sess.opts.optimize == OptLevel::No {
993996
// FIXME: This actually generates tighter assembly, and is a classic trick

‎compiler/rustc_codegen_ssa/src/traits/builder.rs

+12
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,18 @@ pub trait BuilderMethods<'a, 'tcx>:
397397
fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
398398
fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
399399

400+
/// Returns `-1` if `lhs < rhs`, `0` if `lhs == rhs`, and `1` if `lhs > rhs`.
401+
// FIXME: Move the default implementation from `codegen_scalar_binop` into this method and
402+
// remove the `Option` return once LLVM 20 is the minimum version.
403+
fn three_way_compare(
404+
&mut self,
405+
_ty: Ty<'tcx>,
406+
_lhs: Self::Value,
407+
_rhs: Self::Value,
408+
) -> Option<Self::Value> {
409+
None
410+
}
411+
400412
fn memcpy(
401413
&mut self,
402414
dst: Self::Value,

‎tests/assembly/x86_64-cmp.rs

+30-16
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
//@ revisions: DEBUG LLVM-PRE-20-OPTIM LLVM-20-OPTIM
2-
//@ [DEBUG] compile-flags: -C opt-level=0
1+
//@ revisions: LLVM-PRE-20-DEBUG LLVM-20-DEBUG LLVM-PRE-20-OPTIM LLVM-20-OPTIM
2+
//@ [LLVM-PRE-20-DEBUG] compile-flags: -C opt-level=0
3+
//@ [LLVM-PRE-20-DEBUG] max-llvm-major-version: 19
4+
//@ [LLVM-20-DEBUG] compile-flags: -C opt-level=0
5+
//@ [LLVM-20-DEBUG] min-llvm-version: 20
36
//@ [LLVM-PRE-20-OPTIM] compile-flags: -C opt-level=3
47
//@ [LLVM-PRE-20-OPTIM] max-llvm-major-version: 19
58
//@ [LLVM-20-OPTIM] compile-flags: -C opt-level=3
@@ -16,13 +19,19 @@ use std::intrinsics::three_way_compare;
1619
#[no_mangle]
1720
// CHECK-LABEL: signed_cmp:
1821
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
19-
// DEBUG: cmp
20-
// DEBUG: setg
21-
// DEBUG: and
22-
// DEBUG: cmp
23-
// DEBUG: setl
24-
// DEBUG: and
25-
// DEBUG: sub
22+
// LLVM-PRE-20-DEBUG: cmp
23+
// LLVM-PRE-20-DEBUG: setg
24+
// LLVM-PRE-20-DEBUG: and
25+
// LLVM-PRE-20-DEBUG: cmp
26+
// LLVM-PRE-20-DEBUG: setl
27+
// LLVM-PRE-20-DEBUG: and
28+
// LLVM-PRE-20-DEBUG: sub
29+
//
30+
// LLVM-20-DEBUG: sub
31+
// LLVM-20-DEBUG: setl
32+
// LLVM-20-DEBUG: setg
33+
// LLVM-20-DEBUG: sub
34+
// LLVM-20-DEBUG: ret
2635

2736
// LLVM-PRE-20-OPTIM: xor
2837
// LLVM-PRE-20-OPTIM: cmp
@@ -42,13 +51,18 @@ pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
4251
#[no_mangle]
4352
// CHECK-LABEL: unsigned_cmp:
4453
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
45-
// DEBUG: cmp
46-
// DEBUG: seta
47-
// DEBUG: and
48-
// DEBUG: cmp
49-
// DEBUG: setb
50-
// DEBUG: and
51-
// DEBUG: sub
54+
// LLVM-PRE-20-DEBUG: cmp
55+
// LLVM-PRE-20-DEBUG: seta
56+
// LLVM-PRE-20-DEBUG: and
57+
// LLVM-PRE-20-DEBUG: cmp
58+
// LLVM-PRE-20-DEBUG: setb
59+
// LLVM-PRE-20-DEBUG: and
60+
// LLVM-PRE-20-DEBUG: sub
61+
//
62+
// LLVM-20-DEBUG: sub
63+
// LLVM-20-DEBUG: seta
64+
// LLVM-20-DEBUG: sbb
65+
// LLVM-20-DEBUG: ret
5266

5367
// LLVM-PRE-20-OPTIM: xor
5468
// LLVM-PRE-20-OPTIM: cmp

‎tests/codegen/comparison-operators-2-tuple.rs

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
//@ compile-flags: -C opt-level=1 -Z merge-functions=disabled
2-
//@ only-x86_64
32
//@ min-llvm-version: 20
43

54
#![crate_type = "lib"]

‎tests/codegen/integer-cmp.rs

+32-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
//@ revisions: llvm-pre-20 llvm-20
55
//@ [llvm-20] min-llvm-version: 20
66
//@ [llvm-pre-20] max-llvm-major-version: 19
7-
//@ compile-flags: -C opt-level=3
7+
//@ compile-flags: -C opt-level=3 -Zmerge-functions=disabled
88

99
#![crate_type = "lib"]
1010

@@ -13,7 +13,7 @@ use std::cmp::Ordering;
1313
// CHECK-LABEL: @cmp_signed
1414
#[no_mangle]
1515
pub fn cmp_signed(a: i64, b: i64) -> Ordering {
16-
// llvm-20: @llvm.scmp.i8.i64
16+
// llvm-20: call{{.*}} i8 @llvm.scmp.i8.i64
1717
// llvm-pre-20: icmp slt
1818
// llvm-pre-20: icmp ne
1919
// llvm-pre-20: zext i1
@@ -24,10 +24,39 @@ pub fn cmp_signed(a: i64, b: i64) -> Ordering {
2424
// CHECK-LABEL: @cmp_unsigned
2525
#[no_mangle]
2626
pub fn cmp_unsigned(a: u32, b: u32) -> Ordering {
27-
// llvm-20: @llvm.ucmp.i8.i32
27+
// llvm-20: call{{.*}} i8 @llvm.ucmp.i8.i32
2828
// llvm-pre-20: icmp ult
2929
// llvm-pre-20: icmp ne
3030
// llvm-pre-20: zext i1
3131
// llvm-pre-20: select i1
3232
a.cmp(&b)
3333
}
34+
35+
// CHECK-LABEL: @cmp_char
36+
#[no_mangle]
37+
pub fn cmp_char(a: char, b: char) -> Ordering {
38+
// llvm-20: call{{.*}} i8 @llvm.ucmp.i8.i32
39+
// llvm-pre-20: icmp ult
40+
// llvm-pre-20: icmp ne
41+
// llvm-pre-20: zext i1
42+
// llvm-pre-20: select i1
43+
a.cmp(&b)
44+
}
45+
46+
// CHECK-LABEL: @cmp_tuple
47+
#[no_mangle]
48+
pub fn cmp_tuple(a: (i16, u16), b: (i16, u16)) -> Ordering {
49+
// llvm-20-DAG: call{{.*}} i8 @llvm.ucmp.i8.i16
50+
// llvm-20-DAG: call{{.*}} i8 @llvm.scmp.i8.i16
51+
// llvm-20: ret i8
52+
// llvm-pre-20: icmp slt
53+
// llvm-pre-20: icmp ne
54+
// llvm-pre-20: zext i1
55+
// llvm-pre-20: select i1
56+
// llvm-pre-20: icmp ult
57+
// llvm-pre-20: icmp ne
58+
// llvm-pre-20: zext i1
59+
// llvm-pre-20: select i1
60+
// llvm-pre-20: select i1
61+
a.cmp(&b)
62+
}

‎tests/codegen/intrinsics/three_way_compare.rs

+5-22
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
//@ [DEBUG] compile-flags: -C opt-level=0
33
//@ [OPTIM] compile-flags: -C opt-level=3
44
//@ compile-flags: -C no-prepopulate-passes
5+
//@ min-llvm-version: 20
56

67
#![crate_type = "lib"]
78
#![feature(core_intrinsics)]
@@ -12,34 +13,16 @@ use std::intrinsics::three_way_compare;
1213
// CHECK-LABEL: @signed_cmp
1314
// CHECK-SAME: (i16{{.*}} %a, i16{{.*}} %b)
1415
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
15-
// DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b
16-
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
17-
// DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b
18-
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
19-
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
20-
21-
// OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b
22-
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
23-
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
24-
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
25-
// OPTIM: ret i8 %[[CGEL]]
16+
// CHECK: %[[CMP:.+]] = call i8 @llvm.scmp.i8.i16(i16 %a, i16 %b)
17+
// CHECK-NEXT: ret i8 %[[CMP]]
2618
three_way_compare(a, b)
2719
}
2820

2921
#[no_mangle]
3022
// CHECK-LABEL: @unsigned_cmp
3123
// CHECK-SAME: (i16{{.*}} %a, i16{{.*}} %b)
3224
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
33-
// DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b
34-
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
35-
// DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b
36-
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
37-
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
38-
39-
// OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b
40-
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
41-
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
42-
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
43-
// OPTIM: ret i8 %[[CGEL]]
25+
// CHECK: %[[CMP:.+]] = call i8 @llvm.ucmp.i8.i16(i16 %a, i16 %b)
26+
// CHECK-NEXT: ret i8 %[[CMP]]
4427
three_way_compare(a, b)
4528
}

0 commit comments

Comments
 (0)