[+0-all-args] Enable +0 normal arguments.

rdar://34222540

Author: gottesmm

CMakeLists.txt

@@ -305,7 +305,7 @@ option(SWIFT_STDLIB_ENABLE_SIL_OWNERSHIP
 
 option(SWIFT_ENABLE_GUARANTEED_NORMAL_ARGUMENTS
   "Build the standard libraries, overlays, and runtime with normal arguments at +0"
-  FALSE)
+  TRUE)
 
 option(SWIFT_ENABLE_RUNTIME_FUNCTION_COUNTERS
   "Enable runtime function counters and expose the API."

include/swift/AST/SILOptions.h

@@ -114,7 +114,7 @@ class SILOptions {
   llvm::StringRef ExternalPassPipelineFilename;
 
   /// Emit normal function arguments using the +0 guaranteed convention.
-  bool EnableGuaranteedNormalArguments = false;
+  bool EnableGuaranteedNormalArguments = true;
 
   /// Don't generate code using partial_apply in SIL generation.
   bool DisableSILPartialApply = false;

lib/SIL/SILOwnershipVerifier.cpp

@@ -62,6 +62,12 @@ llvm::cl::opt<bool> IsSILOwnershipVerifierTestingEnabled(
                    "comment in SILOwnershipVerifier.cpp above option for more "
                    "information."));
 
+/// This is an option to turn off ownership verification on a specific file. We
+/// still emit code as if we are in ownership mode, but we do not verify. This
+/// is useful for temporarily turning off verification on tests.
+static llvm::cl::opt<bool>
+    DisableOwnershipVerification("disable-sil-ownership-verification");
+
 //===----------------------------------------------------------------------===//
 //                              Generalized User
 //===----------------------------------------------------------------------===//
@@ -2206,6 +2212,9 @@ bool SILValueOwnershipChecker::checkDataflow() {
 
 void SILInstruction::verifyOperandOwnership() const {
 #ifndef NDEBUG
+  if (DisableOwnershipVerification)
+    return;
+
   if (isStaticInitializerInst())
     return;
 
@@ -2251,6 +2260,9 @@ void SILInstruction::verifyOperandOwnership() const {
 
 void SILValue::verifyOwnership(SILModule &Mod, DeadEndBlocks *DEBlocks) const {
 #ifndef NDEBUG
+  if (DisableOwnershipVerification)
+    return;
+
   // If we are SILUndef, just bail. SILUndef can pair with anything. Any uses of
   // the SILUndef will make sure that the matching checks out.
   if (isa<SILUndef>(*this))

lib/SILGen/SILGenApply.cpp

@@ -1960,13 +1960,9 @@ class DelayedArgument {
     // Recurse.
     optValue = emitBindOptionals(SGF, optValue, bind->getSubExpr());
 
-    // Check whether the value is non-nil.
-    SGF.emitBindOptional(bind, optValue, bind->getDepth());
-
-    // Extract the non-optional value.
-    auto &optTL = SGF.getTypeLowering(optValue.getType());
-    auto value = SGF.emitUncheckedGetOptionalValueFrom(bind, optValue, optTL);
-    return value;
+    // Check whether the value is non-nil and if the value is not-nil, return
+    // the unwrapped value.
+    return SGF.emitBindOptional(bind, optValue, bind->getDepth());
   }
 
   std::pair<ManagedValue, ManagedValue>

lib/SILGen/SILGenBridging.cpp

@@ -306,6 +306,10 @@ static ManagedValue emitManagedParameter(SILGenFunction &SGF, SILLocation loc,
     return SGF.emitManagedRValueWithCleanup(value, valueTL);
 
   case ParameterConvention::Direct_Guaranteed:
+    // If we have a guaranteed parameter, the object should not need to be
+    // retained or have a cleanup.
+    return ManagedValue::forUnmanaged(value);
+
   case ParameterConvention::Direct_Unowned:
     // We need to independently retain the value.
     return SGF.emitManagedRetain(loc, value, valueTL);
@@ -315,9 +319,9 @@ static ManagedValue emitManagedParameter(SILGenFunction &SGF, SILLocation loc,
 
   case ParameterConvention::Indirect_In_Guaranteed:
     if (valueTL.isLoadable()) {
-      return SGF.emitLoad(loc, value, valueTL, SGFContext(), IsNotTake);
+      return SGF.B.createLoadBorrow(loc, ManagedValue::forUnmanaged(value));
     } else {
-      return SGF.emitManagedRetain(loc, value, valueTL);
+      return ManagedValue::forUnmanaged(value);
     }
 
   case ParameterConvention::Indirect_In:
@@ -1118,12 +1122,20 @@ ManagedValue SILGenFunction::emitBridgedToNativeError(SILLocation loc,
   assert(bridgeFnType->getResults()[0].getConvention()
          == ResultConvention::Owned);
   auto nativeErrorType = bridgeFnConv.getSILType(bridgeFnType->getResults()[0]);
-  assert(bridgeFnType->getParameters()[0].getConvention()
-           == ParameterConvention::Direct_Owned);
+
+  SILValue arg;
+  if (SGM.M.getOptions().EnableGuaranteedNormalArguments) {
+    assert(bridgeFnType->getParameters()[0].getConvention() ==
+           ParameterConvention::Direct_Guaranteed);
+    arg = bridgedError.getValue();
+  } else {
+    assert(bridgeFnType->getParameters()[0].getConvention() ==
+           ParameterConvention::Direct_Owned);
+    arg = bridgedError.forward(*this);
+  }
 
   SILValue nativeError = B.createApply(loc, bridgeFn, bridgeFn->getType(),
-                                       nativeErrorType, {},
-                                       bridgedError.forward(*this));
+                                       nativeErrorType, {}, arg);
   return emitManagedRValueWithCleanup(nativeError);
 }
 
@@ -1166,12 +1178,20 @@ ManagedValue SILGenFunction::emitNativeToBridgedError(SILLocation loc,
          == ResultConvention::Owned);
   auto loweredBridgedErrorType =
     bridgeFnConv.getSILType(bridgeFnType->getResults()[0]);
-  assert(bridgeFnType->getParameters()[0].getConvention()
-           == ParameterConvention::Direct_Owned);
+
+  SILValue arg;
+  if (SGM.M.getOptions().EnableGuaranteedNormalArguments) {
+    assert(bridgeFnType->getParameters()[0].getConvention() ==
+           ParameterConvention::Direct_Guaranteed);
+    arg = nativeError.getValue();
+  } else {
+    assert(bridgeFnType->getParameters()[0].getConvention() ==
+           ParameterConvention::Direct_Owned);
+    arg = nativeError.forward(*this);
+  }
 
   SILValue bridgedError = B.createApply(loc, bridgeFn, bridgeFn->getType(),
-                                        loweredBridgedErrorType, {},
-                                        nativeError.forward(*this));
+                                        loweredBridgedErrorType, {}, arg);
   return emitManagedRValueWithCleanup(bridgedError);
 }
 

lib/SILGen/SILGenBuilder.cpp

@@ -901,3 +901,22 @@ ManagedValue SILGenBuilder::createUncheckedAddrCast(SILLocation loc, ManagedValu
   SILValue cast = createUncheckedAddrCast(loc, op.forward(SGF), resultTy);
   return cloner.clone(cast);
 }
+
+ManagedValue SILGenBuilder::tryCreateUncheckedRefCast(SILLocation loc,
+                                                      ManagedValue original,
+                                                      SILType type) {
+  CleanupCloner cloner(*this, original);
+  SILValue result = tryCreateUncheckedRefCast(loc, original.getValue(), type);
+  if (!result)
+    return ManagedValue();
+  original.forward(SGF);
+  return cloner.clone(result);
+}
+
+ManagedValue SILGenBuilder::createUncheckedTrivialBitCast(SILLocation loc,
+                                                          ManagedValue original,
+                                                          SILType type) {
+  SILValue result =
+      SGF.B.createUncheckedTrivialBitCast(loc, original.getValue(), type);
+  return ManagedValue::forUnmanaged(result);
+}

lib/SILGen/SILGenBuilder.h

@@ -306,6 +306,15 @@ class SILGenBuilder : public SILBuilder {
   ManagedValue createUpcast(SILLocation loc, ManagedValue original,
                             SILType type);
 
+  using SILBuilder::tryCreateUncheckedRefCast;
+  ManagedValue tryCreateUncheckedRefCast(SILLocation loc, ManagedValue original,
+                                         SILType type);
+
+  using SILBuilder::createUncheckedTrivialBitCast;
+  ManagedValue createUncheckedTrivialBitCast(SILLocation loc,
+                                             ManagedValue original,
+                                             SILType type);
+
   using SILBuilder::createUncheckedRefCast;
   ManagedValue createUncheckedRefCast(SILLocation loc, ManagedValue original,
                                       SILType type);

lib/SILGen/SILGenBuiltin.cpp

@@ -141,7 +141,8 @@ static ManagedValue emitBuiltinUnpin(SILGenFunction &SGF,
 
   if (requireIsOptionalNativeObject(SGF, loc, subs[0].getReplacement())) {
     // Unpinning takes responsibility for the +1 handle.
-    SGF.B.createStrongUnpin(loc, args[0].forward(SGF), SGF.B.getDefaultAtomicity());
+    SGF.B.createStrongUnpin(loc, args[0].ensurePlusOne(SGF, loc).forward(SGF),
+                            SGF.B.getDefaultAtomicity());
   }
 
   return ManagedValue::forUnmanaged(SGF.emitEmptyTuple(loc));
@@ -293,7 +294,7 @@ static ManagedValue emitBuiltinInit(SILGenFunction &SGF,
 
   TemporaryInitialization init(addr, CleanupHandle::invalid());
   SGF.emitExprInto(args[0], &init);
-  
+
   return ManagedValue::forUnmanaged(SGF.emitEmptyTuple(loc));
 }
 
@@ -544,14 +545,16 @@ emitBuiltinCastReference(SILGenFunction &SGF,
   auto &toTL = SGF.getTypeLowering(toTy);
   assert(!fromTL.isTrivial() && !toTL.isTrivial() && "expected ref type");
 
-  if (!fromTL.isAddress() || !toTL.isAddress()) { 
-    if (auto refCast = SGF.B.tryCreateUncheckedRefCast(loc, args[0].getValue(),
+  // TODO: Fix this API.
+  if (!fromTL.isAddress() || !toTL.isAddress()) {
+    if (auto refCast = SGF.B.tryCreateUncheckedRefCast(loc, args[0],
                                                        toTL.getLoweredType())) {
       // Create a reference cast, forwarding the cleanup.
       // The cast takes the source reference.
-      return ManagedValue(refCast, args[0].getCleanup());
+      return refCast;
     }
   }
+
   // We are either casting between address-only types, or cannot promote to a
   // cast of reference values.
   //
@@ -563,7 +566,7 @@ emitBuiltinCastReference(SILGenFunction &SGF,
   // TODO: For now, we leave invalid casts in address form so that the runtime
   // will trap. We could emit a noreturn call here instead which would provide
   // more information to the optimizer.
-  SILValue srcVal = args[0].forward(SGF);
+  SILValue srcVal = args[0].ensurePlusOne(SGF, loc).forward(SGF);
   SILValue fromAddr;
   if (!fromTL.isAddress()) {
     // Move the loadable value into a "source temp".  Since the source and
@@ -636,16 +639,17 @@ static ManagedValue emitBuiltinReinterpretCast(SILGenFunction &SGF,
         });
   }
   // Create the appropriate bitcast based on the source and dest types.
-  auto &in = args[0];
-  SILValue out = SGF.B.createUncheckedBitCast(loc, in.getValue(),
-                                              toTL.getLoweredType());
+  ManagedValue in = args[0];
+  SILType resultTy = toTL.getLoweredType();
+  if (resultTy.isTrivial(SGF.getModule()))
+    return SGF.B.createUncheckedTrivialBitCast(loc, in, resultTy);
 
-  // If the cast reduces to unchecked_ref_cast, then the source and dest
-  // have identical cleanup, so just forward the cleanup as an optimization.
-  if (isa<UncheckedRefCastInst>(out))
-    return ManagedValue(out, in.getCleanup());
+  // If we can perform a ref cast, just return.
+  if (auto refCast = SGF.B.tryCreateUncheckedRefCast(loc, in, resultTy))
+    return refCast;
 
   // Otherwise leave the original cleanup and retain the cast value.
+  SILValue out = SGF.B.createUncheckedBitwiseCast(loc, in.getValue(), resultTy);
   return SGF.emitManagedRetain(loc, out, toTL);
 }
 

lib/SILGen/SILGenConvert.cpp

@@ -66,7 +66,8 @@ SILGenFunction::emitInjectOptional(SILLocation loc,
         TemporaryInitialization init(objectBuf, CleanupHandle::invalid());
         ManagedValue objectResult = generator(SGFContext(&init));
         if (!objectResult.isInContext()) {
-          objectResult.forwardInto(*this, loc, objectBuf);
+          objectResult.ensurePlusOne(*this, loc)
+              .forwardInto(*this, loc, objectBuf);
         }
 
         // Finalize the outer optional buffer.
@@ -190,15 +191,21 @@ SILGenFunction::emitPreconditionOptionalHasValue(SILLocation loc,
   bool hadCleanup = optional.hasCleanup();
   bool hadLValue = optional.isLValue();
 
-  auto noneDecl = getASTContext().getOptionalNoneDecl();
   auto someDecl = getASTContext().getOptionalSomeDecl();
+  auto noneDecl = getASTContext().getOptionalNoneDecl();
+
+  // If we have an object, make sure the object is at +1. All switch_enum of
+  // objects is done at +1.
   if (optional.getType().isAddress()) {
     // We forward in the creation routine for
     // unchecked_take_enum_data_addr. switch_enum_addr is a +0 operation.
     B.createSwitchEnumAddr(loc, optional.getValue(),
                            /*defaultDest*/ nullptr,
                            {{someDecl, contBB}, {noneDecl, failBB}});
   } else {
+    optional = optional.ensurePlusOne(*this, loc);
+    hadCleanup = true;
+    hadLValue = false;
     B.createSwitchEnum(loc, optional.forward(*this),
                        /*defaultDest*/ nullptr,
                        {{someDecl, contBB}, {noneDecl, failBB}});
@@ -775,7 +782,8 @@ ManagedValue SILGenFunction::emitExistentialErasure(
                                             *this));
           ManagedValue mv = F(SGFContext(init.get()));
           if (!mv.isInContext()) {
-            init->copyOrInitValueInto(*this, loc, mv, /*init*/ true);
+            init->copyOrInitValueInto(*this, loc, mv.ensurePlusOne(*this, loc),
+                                      /*init*/ true);
             init->finishInitialization(*this);
           }
         });