MS ABI: Implement /volatile:ms

The /volatile:ms semantics turn volatile loads and stores into atomic
acquire and release operations.  This distinction is important because
volatile memory operations do not form a happens-before relationship
with non-atomic memory.  This means that a volatile store is not
sufficient for implementing a mutex unlock routine.

Differential Revision: http://reviews.llvm.org/D7580

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@229082 91177308-0d34-0410-b5e6-96231b3b80d8

Author: majnemer

Diff for: include/clang/Driver/CLCompatOptions.td

@@ -157,6 +157,7 @@ def _SLASH_arch : CLCompileJoined<"arch:">,
   HelpText<"Set architecture for code generation">;
 
 def _SLASH_M_Group : OptionGroup<"</M group>">, Group<cl_compile_Group>;
+def _SLASH_volatile_Group : OptionGroup<"</volatile group>">, Group<cl_compile_Group>;
 
 def _SLASH_EH : CLJoined<"EH">, HelpText<"Exception handling model">;
 def _SLASH_EP : CLFlag<"EP">,
@@ -201,7 +202,10 @@ def _SLASH_TC : CLCompileFlag<"TC">, HelpText<"Treat all source files as C">;
 def _SLASH_Tp : CLCompileJoinedOrSeparate<"Tp">,
   HelpText<"Specify a C++ source file">, MetaVarName<"<filename>">;
 def _SLASH_TP : CLCompileFlag<"TP">, HelpText<"Treat all source files as C++">;
-
+def _SLASH_volatile_iso : Option<["/", "-"], "volatile:iso", KIND_FLAG>, Group<_SLASH_volatile_Group>,
+  Flags<[CLOption, DriverOption]>, HelpText<"Volatile loads and stores have standard semantics">;
+def _SLASH_volatile_ms  : Option<["/", "-"], "volatile:ms", KIND_FLAG>, Group<_SLASH_volatile_Group>,
+  Flags<[CLOption, DriverOption]>, HelpText<"Volatile loads and stores have acquire and release semantics">;
 
 // Ignored:
 
@@ -220,7 +224,6 @@ def _SLASH_Ob2 : CLIgnoredFlag<"Ob2">;
 def _SLASH_RTC : CLIgnoredJoined<"RTC">;
 def _SLASH_sdl : CLIgnoredFlag<"sdl">;
 def _SLASH_sdl_ : CLIgnoredFlag<"sdl-">;
-def _SLASH_volatile_iso : CLIgnoredFlag<"volatile:iso">;
 def _SLASH_w : CLIgnoredJoined<"w">;
 def _SLASH_Zc_auto : CLIgnoredFlag<"Zc:auto">;
 def _SLASH_Zc_forScope : CLIgnoredFlag<"Zc:forScope">;
@@ -281,7 +284,6 @@ def _SLASH_Qpar : CLFlag<"Qpar">;
 def _SLASH_Qvec_report : CLJoined<"Qvec-report">;
 def _SLASH_u : CLFlag<"u">;
 def _SLASH_V : CLFlag<"V">;
-def _SLASH_volatile_ms : CLFlag<"volatile:ms">;
 def _SLASH_WL : CLFlag<"WL">;
 def _SLASH_Wp64 : CLFlag<"Wp64">;
 def _SLASH_X : CLFlag<"X">;

Diff for: include/clang/Driver/Options.td

@@ -628,6 +628,7 @@ def fms_extensions : Flag<["-"], "fms-extensions">, Group<f_Group>, Flags<[CC1Op
   HelpText<"Accept some non-standard constructs supported by the Microsoft compiler">;
 def fms_compatibility : Flag<["-"], "fms-compatibility">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Enable full Microsoft Visual C++ compatibility">;
+def fms_volatile : Joined<["-"], "fms-volatile">, Group<f_Group>, Flags<[CC1Option]>;
 def fmsc_version : Joined<["-"], "fmsc-version=">, Group<f_Group>, Flags<[DriverOption, CoreOption]>,
   HelpText<"Microsoft compiler version number to report in _MSC_VER (0 = don't define it (default))">;
 def fms_compatibility_version

Diff for: include/clang/Frontend/CodeGenOptions.def

@@ -68,6 +68,7 @@ CODEGENOPT(LessPreciseFPMAD  , 1, 0) ///< Enable less precise MAD instructions t
                                      ///< be generated.
 CODEGENOPT(MergeAllConstants , 1, 1) ///< Merge identical constants.
 CODEGENOPT(MergeFunctions    , 1, 0) ///< Set when -fmerge-functions is enabled.
+CODEGENOPT(MSVolatile        , 1, 0) ///< Set when /volatile:ms is enabled.
 CODEGENOPT(NoCommon          , 1, 0) ///< Set when -fno-common or C++ is enabled.
 CODEGENOPT(NoDwarfDirectoryAsm , 1, 0) ///< Set when -fno-dwarf-directory-asm is
                                        ///< enabled.

Diff for: lib/CodeGen/CGAtomic.cpp

@@ -1006,9 +1006,45 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal,
   return convertTempToRValue(Temp, ResultSlot, Loc);
 }
 
+/// An LValue is a candidate for having its loads and stores be made atomic if
+/// we are operating under /volatile:ms *and* the LValue itself is volatile and
+/// performing such an operation can be performed without a libcall.
+bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
+  AtomicInfo AI(*this, LV);
+  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
+  // An atomic is inline if we don't need to use a libcall.
+  bool AtomicIsInline = !AI.shouldUseLibcall();
+  return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
+}
+
+/// An type is a candidate for having its loads and stores be made atomic if
+/// we are operating under /volatile:ms *and* we know the access is volatile and
+/// performing such an operation can be performed without a libcall.
+bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
+                                                    bool IsVolatile) const {
+  // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
+  bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
+      getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
+  return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
+}
+
+RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
+                                       AggValueSlot Slot) {
+  llvm::AtomicOrdering AO;
+  bool IsVolatile = LV.isVolatileQualified();
+  if (LV.getType()->isAtomicType()) {
+    AO = llvm::SequentiallyConsistent;
+  } else {
+    AO = llvm::Acquire;
+    IsVolatile = true;
+  }
+  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
+}
+
 /// Emit a load from an l-value of atomic type.  Note that the r-value
 /// we produce is an r-value of the atomic *value* type.
 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
+                                       llvm::AtomicOrdering AO, bool IsVolatile,
                                        AggValueSlot resultSlot) {
   AtomicInfo atomics(*this, src);
   LValue LVal = atomics.getAtomicLValue();
@@ -1060,11 +1096,11 @@ RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
   // Okay, we're doing this natively.
   llvm::Value *addr = atomics.emitCastToAtomicIntPointer(SrcAddr);
   llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
-  load->setAtomic(llvm::SequentiallyConsistent);
+  load->setAtomic(AO);
 
   // Other decoration.
   load->setAlignment(src.getAlignment().getQuantity());
-  if (src.isVolatileQualified())
+  if (IsVolatile)
     load->setVolatile(true);
   if (src.getTBAAInfo())
     CGM.DecorateInstruction(load, src.getTBAAInfo());
@@ -1161,12 +1197,27 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
                                        getAtomicAlignment().getQuantity());
 }
 
+void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
+                                      bool isInit) {
+  bool IsVolatile = lvalue.isVolatileQualified();
+  llvm::AtomicOrdering AO;
+  if (lvalue.getType()->isAtomicType()) {
+    AO = llvm::SequentiallyConsistent;
+  } else {
+    AO = llvm::Release;
+    IsVolatile = true;
+  }
+  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
+}
+
 /// Emit a store to an l-value of atomic type.
 ///
 /// Note that the r-value is expected to be an r-value *of the atomic
 /// type*; this means that for aggregate r-values, it should include
 /// storage for any padding that was necessary.
-void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
+void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
+                                      llvm::AtomicOrdering AO, bool IsVolatile,
+                                      bool isInit) {
   // If this is an aggregate r-value, it should agree in type except
   // maybe for address-space qualification.
   assert(!rvalue.isAggregate() ||
@@ -1209,11 +1260,11 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
   llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
 
   // Initializations don't need to be atomic.
-  if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
+  if (!isInit) store->setAtomic(AO);
 
   // Other decoration.
   store->setAlignment(dest.getAlignment().getQuantity());
-  if (dest.isVolatileQualified())
+  if (IsVolatile)
     store->setVolatile(true);
   if (dest.getTBAAInfo())
     CGM.DecorateInstruction(store, dest.getTBAAInfo());

Diff for: lib/CodeGen/CGExpr.cpp

@@ -1136,7 +1136,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
   }
 
   // Atomic operations have to be done on integral types.
-  if (Ty->isAtomicType()) {
+  if (Ty->isAtomicType() || typeIsSuitableForInlineAtomic(Ty, Volatile)) {
     LValue lvalue = LValue::MakeAddr(Addr, Ty,
                                      CharUnits::fromQuantity(Alignment),
                                      getContext(), TBAAInfo);
@@ -1255,7 +1255,8 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
 
   Value = EmitToMemory(Value, Ty);
 
-  if (Ty->isAtomicType()) {
+  if (Ty->isAtomicType() ||
+      (!isInit && typeIsSuitableForInlineAtomic(Ty, Volatile))) {
     EmitAtomicStore(RValue::get(Value),
                     LValue::MakeAddr(Addr, Ty,
                                      CharUnits::fromQuantity(Alignment),

Diff for: lib/CodeGen/CGExprAgg.cpp

@@ -212,7 +212,7 @@ void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
   LValue LV = CGF.EmitLValue(E);
 
   // If the type of the l-value is atomic, then do an atomic load.
-  if (LV.getType()->isAtomicType()) {
+  if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) {
     CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest);
     return;
   }
@@ -865,7 +865,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
     LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
 
     // That copy is an atomic copy if the LHS is atomic.
-    if (LHS.getType()->isAtomicType()) {
+    if (LHS.getType()->isAtomicType() ||
+        CGF.LValueIsSuitableForInlineAtomic(LHS)) {
       CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
       return;
     }
@@ -882,7 +883,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
 
   // If we have an atomic type, evaluate into the destination and then
   // do an atomic copy.
-  if (LHS.getType()->isAtomicType()) {
+  if (LHS.getType()->isAtomicType() ||
+      CGF.LValueIsSuitableForInlineAtomic(LHS)) {
     EnsureDest(E->getRHS()->getType());
     Visit(E->getRHS());
     CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);

Diff for: lib/CodeGen/CGExprComplex.cpp

@@ -336,7 +336,8 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
 /// specified value pointer.
 void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
                                             bool isInit) {
-  if (lvalue.getType()->isAtomicType())
+  if (lvalue.getType()->isAtomicType() ||
+      (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue)))
     return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit);
 
   llvm::Value *Ptr = lvalue.getAddress();

Diff for: lib/CodeGen/CGStmtOpenMP.cpp

@@ -829,8 +829,11 @@ static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
   assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
   LValue XLValue = CGF.EmitLValue(X);
   LValue VLValue = CGF.EmitLValue(V);
-  RValue Res = XLValue.isGlobalReg() ? CGF.EmitLoadOfLValue(XLValue, Loc)
-                                     : CGF.EmitAtomicLoad(XLValue, Loc);
+  RValue Res = XLValue.isGlobalReg()
+                   ? CGF.EmitLoadOfLValue(XLValue, Loc)
+                   : CGF.EmitAtomicLoad(XLValue, Loc,
+                                        IsSeqCst ? llvm::SequentiallyConsistent
+                                                 : llvm::Monotonic);
   // OpenMP, 2.12.6, atomic Construct
   // Any atomic construct with a seq_cst clause forces the atomically
   // performed operation to include an implicit flush operation without a

Diff for: lib/CodeGen/CodeGenFunction.h

@@ -2147,11 +2147,21 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   void EmitAtomicInit(Expr *E, LValue lvalue);
 
+  bool LValueIsSuitableForInlineAtomic(LValue Src);
+  bool typeIsSuitableForInlineAtomic(QualType Ty, bool IsVolatile) const;
+
+  RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
+                        AggValueSlot Slot = AggValueSlot::ignored());
+
   RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
+                        llvm::AtomicOrdering AO, bool IsVolatile = false,
                         AggValueSlot slot = AggValueSlot::ignored());
 
   void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
 
+  void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
+                       bool IsVolatile, bool isInit);
+
   std::pair<RValue, RValue> EmitAtomicCompareExchange(
       LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
       llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,

Diff for: lib/Driver/Tools.cpp

@@ -4888,6 +4888,19 @@ void Clang::AddClangCLArgs(const ArgList &Args, ArgStringList &CmdArgs) const {
     CmdArgs.push_back("-P");
   }
 
+  unsigned VolatileOptionID;
+  if (getToolChain().getTriple().getArch() == llvm::Triple::x86_64 ||
+      getToolChain().getTriple().getArch() == llvm::Triple::x86)
+    VolatileOptionID = options::OPT__SLASH_volatile_ms;
+  else
+    VolatileOptionID = options::OPT__SLASH_volatile_iso;
+
+  if (Arg *A = Args.getLastArg(options::OPT__SLASH_volatile_Group))
+    VolatileOptionID = A->getOption().getID();
+
+  if (VolatileOptionID == options::OPT__SLASH_volatile_ms)
+    CmdArgs.push_back("-fms-volatile");
+
   Arg *MostGeneralArg = Args.getLastArg(options::OPT__SLASH_vmg);
   Arg *BestCaseArg = Args.getLastArg(options::OPT__SLASH_vmb);
   if (MostGeneralArg && BestCaseArg)

Diff for: lib/Frontend/CompilerInvocation.cpp

@@ -477,6 +477,8 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
                                    OPT_fno_data_sections, false);
   Opts.MergeFunctions = Args.hasArg(OPT_fmerge_functions);
 
+  Opts.MSVolatile = Args.hasArg(OPT_fms_volatile);
+
   Opts.VectorizeBB = Args.hasArg(OPT_vectorize_slp_aggressive);
   Opts.VectorizeLoop = Args.hasArg(OPT_vectorize_loops);
   Opts.VectorizeSLP = Args.hasArg(OPT_vectorize_slp);

Diff for: test/CodeGen/ms-volatile.c

@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -triple i386-pc-win32 -emit-llvm -fms-volatile -o - < %s | FileCheck %s
+struct foo {
+  volatile int x;
+};
+struct bar {
+  int x;
+};
+typedef _Complex float __declspec(align(8)) baz;
+
+void test1(struct foo *p, struct foo *q) {
+  *p = *q;
+  // CHECK-LABEL: @test1
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test2(volatile int *p, volatile int *q) {
+  *p = *q;
+  // CHECK-LABEL: @test2
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test3(struct foo *p, struct foo *q) {
+  p->x = q->x;
+  // CHECK-LABEL: @test3
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test4(volatile struct foo *p, volatile struct foo *q) {
+  p->x = q->x;
+  // CHECK-LABEL: @test4
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test5(volatile struct foo *p, volatile struct foo *q) {
+  *p = *q;
+  // CHECK-LABEL: @test5
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test6(struct bar *p, struct bar *q) {
+  *p = *q;
+  // CHECK-LABEL: @test6
+  // CHECK-NOT: load atomic volatile {{.*}}
+  // CHECK-NOT: store atomic volatile {{.*}}, {{.*}}
+}
+void test7(volatile struct bar *p, volatile struct bar *q) {
+  *p = *q;
+  // CHECK-LABEL: @test7
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test8(volatile double *p, volatile double *q) {
+  *p = *q;
+  // CHECK-LABEL: @test8
+  // CHECK: load atomic volatile {{.*}} acquire
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}
+void test9(volatile baz *p, baz *q) {
+  *p = *q;
+  // CHECK-LABEL: @test9
+  // CHECK: store atomic volatile {{.*}}, {{.*}} release
+}

Diff for: test/Driver/cl-options.c

@@ -123,6 +123,12 @@
 // RUN: %clang_cl /vmg /vmm /vms -### -- %s 2>&1 | FileCheck -check-prefix=VMX %s
 // VMX: '/vms' not allowed with '/vmm'
 
+// RUN: %clang_cl /volatile:iso -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-ISO %s
+// VOLATILE-ISO-NOT: "-fms-volatile"
+
+// RUN: %clang_cl /volatile:ms -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-MS %s
+// VOLATILE-MS: "-fms-volatile"
+
 // RUN: %clang_cl /W0 -### -- %s 2>&1 | FileCheck -check-prefix=W0 %s
 // W0: -w