IRGen: Pass large explosions as indirect arguments.

It's not worth burning more than three registers on a parameter, and doing so causes code size issues for large structs and enums. Make it so that values with more than three explosion members get passed indirectly, just like they get returned indirectly.

This time, modify emitPartialApplyForwarder not to attempt to 'tail' call the original function when indirect arguments get alloca'ed on the stack, which is UB, and don't use "byval", as suggested by John.

Swift SVN r29032

Author: jckarter

lib/IRGen/CallEmission.h

@@ -77,7 +77,9 @@ class CallEmission {
   }
 
   /// Set the arguments to the function from an explosion.
-  void setArgs(Explosion &arg, WitnessMetadata *witnessMetadata = nullptr);
+  void setArgs(Explosion &arg,
+               ArrayRef<SILParameterInfo> params,
+               WitnessMetadata *witnessMetadata = nullptr);
 
   void addAttribute(unsigned Index, llvm::Attribute::AttrKind Attr);
 

lib/IRGen/Explosion.h

@@ -233,6 +233,7 @@ class ExplosionSchema {
     return Elements[index];
   }
 
+  bool requiresIndirectParameter(IRGenModule &IGM) const;
   bool requiresIndirectResult(IRGenModule &IGM) const;
 
   typedef SmallVectorImpl<Element>::iterator iterator;
@@ -258,6 +259,8 @@ class ExplosionSchema {
   /// Treating the types in this schema as potential arguments to a
   /// function call, add them to the end of the given vector of types.
   void addToArgTypes(IRGenModule &IGM,
+                     const TypeInfo &TI,
+                     llvm::AttributeSet &Attrs,
                      SmallVectorImpl<llvm::Type*> &types) const;
 };
 

lib/IRGen/GenFunc.cpp

@@ -121,6 +121,12 @@ bool ExplosionSchema::requiresIndirectResult(IRGenModule &IGM) const {
          size() > IGM.TargetInfo.MaxScalarsForDirectResult;
 }
 
+bool ExplosionSchema::requiresIndirectParameter(IRGenModule &IGM) const {
+  // For now, use the same condition as requiresIndirectSchema. We may want
+  // to diverge at some point.
+  return requiresIndirectResult(IGM);
+}
+
 llvm::Type *ExplosionSchema::getScalarResultType(IRGenModule &IGM) const {
   if (size() == 0) {
     return IGM.VoidTy;
@@ -134,7 +140,14 @@ llvm::Type *ExplosionSchema::getScalarResultType(IRGenModule &IGM) const {
 }
 
 void ExplosionSchema::addToArgTypes(IRGenModule &IGM,
+                                    const TypeInfo &TI,
+                                    llvm::AttributeSet &Attrs,
                                     SmallVectorImpl<llvm::Type*> &types) const {
+  // Pass indirect arguments as byvals.
+  if (requiresIndirectParameter(IGM)) {
+    types.push_back(TI.getStorageType()->getPointerTo());
+    return;
+  }
   for (auto &elt : *this) {
     if (elt.isAggregate())
       types.push_back(elt.getAggregateType()->getPointerTo());
@@ -1271,8 +1284,9 @@ void SignatureExpansion::expand(SILParameterInfo param) {
       return;
     }
     case SILFunctionLanguage::Swift: {
-      auto schema = IGM.getSchema(param.getSILType());
-      schema.addToArgTypes(IGM, ParamIRTypes);
+      auto &ti = IGM.getTypeInfo(param.getSILType());
+      auto schema = ti.getSchema();
+      schema.addToArgTypes(IGM, ti, Attrs, ParamIRTypes);
       return;
     }
     }
@@ -2824,17 +2838,54 @@ static void externalizeArguments(IRGenFunction &IGF, const Callee &callee,
   }
 }
 
+namespace {
+enum IsIndirectValueArgument_t: bool {
+  IsNotIndirectValueArgument = false,
+  IsIndirectValueArgument = true,
+};
+}
+
+static IsIndirectValueArgument_t addNativeArgument(IRGenFunction &IGF,
+                                                 Explosion &in,
+                                                 const TypeInfo &ti,
+                                                 ParameterConvention convention,
+                                                 Explosion &out) {
+  // Addresses consist of a single pointer argument.
+  if (isIndirectParameter(convention)) {
+    out.add(in.claimNext());
+    return IsNotIndirectValueArgument;
+  }
+
+  auto &loadableTI = cast<LoadableTypeInfo>(ti);
+  auto schema = ti.getSchema();
+  
+  if (schema.requiresIndirectParameter(IGF.IGM)) {
+    // Pass the argument indirectly.
+    auto buf = IGF.createAlloca(ti.getStorageType(),
+                                loadableTI.getFixedAlignment(), "");
+    loadableTI.initialize(IGF, in, buf);
+    out.add(buf.getAddress());
+    return IsIndirectValueArgument;
+  } else {
+    // Pass the argument explosion directly.
+    loadableTI.reexplode(IGF, in, out);
+    return IsNotIndirectValueArgument;
+  }
+}
+
 /// Add a new set of arguments to the function.
-void CallEmission::setArgs(Explosion &arg, WitnessMetadata *witnessMetadata) {
+void CallEmission::setArgs(Explosion &arg,
+                           ArrayRef<SILParameterInfo> params,
+                           WitnessMetadata *witnessMetadata) {
   // Convert arguments to a representation appropriate to the calling
   // convention.
+  Explosion adjustedArg;
+  
   switch (getCallee().getRepresentation()) {
   case SILFunctionTypeRepresentation::CFunctionPointer:
   case SILFunctionTypeRepresentation::ObjCMethod:
   case SILFunctionTypeRepresentation::Block: {
-    Explosion externalized;
-    externalizeArguments(IGF, getCallee(), arg, externalized);
-    arg = std::move(externalized);
+    externalizeArguments(IGF, getCallee(), arg, adjustedArg);
     break;
   }
 
@@ -2848,19 +2899,26 @@ void CallEmission::setArgs(Explosion &arg, WitnessMetadata *witnessMetadata) {
   case SILFunctionTypeRepresentation::Method:
   case SILFunctionTypeRepresentation::Thin:
   case SILFunctionTypeRepresentation::Thick:
-    // Nothing to do.
+    // Check for value arguments that need to be passed indirectly.
+    for (auto param : params) {
+      addNativeArgument(IGF, arg,
+                        IGF.getTypeInfoForLowered(param.getType()),
+                        param.getConvention(),
+                        adjustedArg);
+    }
+    adjustedArg.add(arg.claimAll());
     break;
   }
 
   // Add the given number of arguments.
-  assert(LastArgWritten >= arg.size());
+  assert(LastArgWritten >= adjustedArg.size());
 
-  size_t targetIndex = LastArgWritten - arg.size();
+  size_t targetIndex = LastArgWritten - adjustedArg.size();
   assert(targetIndex <= 1);
   LastArgWritten = targetIndex;
 
   auto argIterator = Args.begin() + targetIndex;
-  for (auto value : arg.claimAll()) {
+  for (auto value : adjustedArg.claimAll()) {
     *argIterator++ = value;
   }
 }
@@ -3180,7 +3238,23 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
 
     // Reemit the parameters as unsubstituted.
     for (unsigned i = 0; i < outType->getParameters().size(); ++i) {
-      emitApplyArgument(subIGF, origType->getParameters()[i],
+      Explosion arg;
+      auto origParamInfo = origType->getParameters()[i];
+      auto &ti = IGM.getTypeInfoForLowered(origParamInfo.getType());
+      auto schema = ti.getSchema();
+      
+      // Forward the address of indirect value params.
+      if (!isIndirectParameter(origParamInfo.getConvention())
+          && schema.requiresIndirectParameter(IGM)) {
+        auto addr = origParams.claimNext();
+        if (addr->getType() != ti.getStorageType()->getPointerTo())
+          addr = subIGF.Builder.CreateBitCast(addr,
+                                           ti.getStorageType()->getPointerTo());
+        args.add(addr);
+        continue;
+      }
+      
+      emitApplyArgument(subIGF, origParamInfo,
                         outType->getParameters()[i],
                         origParams, args);
     }
@@ -3195,6 +3269,7 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
 
   bool dependsOnContextLifetime = false;
   bool consumesContext;
+  bool needsAllocas = false;
 
   switch (outType->getCalleeConvention()) {
   case ParameterConvention::Direct_Owned:
@@ -3260,12 +3335,14 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
       auto fieldConvention = conventions[nextCapturedField];
       Address fieldAddr = fieldLayout.project(subIGF, data, offsets);
       auto &fieldTI = fieldLayout.getType();
+      auto fieldSchema = fieldTI.getSchema();
 
       Explosion param;
       switch (fieldConvention) {
       case ParameterConvention::Indirect_In: {
         // The +1 argument is passed indirectly, so we need to copy into a
         // temporary.
+        needsAllocas = true;
         auto caddr = fieldTI.allocateStack(subIGF, fieldTy, "arg.temp");
         fieldTI.initializeWithCopy(subIGF, caddr.getAddress(), fieldAddr,
                                    fieldTy);
@@ -3314,14 +3391,21 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
 
       // Reemit the capture params as unsubstituted.
       if (origParamI < origType->getParameters().size()) {
-        emitApplyArgument(subIGF,
-                          origType->getParameters()[origParamI],
+        Explosion origParam;
+        auto origParamInfo = origType->getParameters()[origParamI];
+        emitApplyArgument(subIGF, origParamInfo,
                           substType->getParameters()[origParamI],
-                          param, args);
+                          param, origParam);
+
+        needsAllocas |= addNativeArgument(subIGF, origParam,
+                            IGM.getTypeInfoForLowered(origParamInfo.getType()),
+                            origParamInfo.getConvention(),
+                            args);
         ++origParamI;
       } else {
         args.add(param.claimAll());
       }
+      
     }
 
     // If the parameters can live independent of the context, release it now
@@ -3407,7 +3491,8 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
     // "C" calling convention, but that may change.
     call->setAttributes(origAttrs);
   }
-  if (!consumesContext || !dependsOnContextLifetime)
+  if (addressesToDeallocate.empty() && !needsAllocas &&
+      (!consumesContext || !dependsOnContextLifetime))
     call->setTailCall();
 
   // Deallocate everything we allocated above.

lib/IRGen/GenObjC.cpp

@@ -726,7 +726,7 @@ llvm::Value *irgen::emitObjCAllocObjectCall(IRGenFunction &IGF,
     args.add(self);
     args.add(IGF.emitObjCSelectorRefLoad("allocWithZone:"));
     args.add(llvm::ConstantPointerNull::get(IGF.IGM.Int8PtrTy));
-    emission.setArgs(args);
+    emission.setArgs(args, {});
   }
 
   // Emit the call.
@@ -824,7 +824,7 @@ static llvm::Function *emitObjCPartialApplicationForwarder(IRGenModule &IGM,
   Explosion args;
   addObjCMethodCallImplicitArguments(subIGF, args, method, self, SILType());
   args.add(params.claimAll());
-  emission.setArgs(args);
+  emission.setArgs(args, {});
 
   // Cleanup that always has to occur after the function call.
   auto cleanup = [&]{

lib/IRGen/IRGenSIL.cpp

@@ -946,20 +946,31 @@ static void bindParameter(IRGenSILFunction &IGF,
   auto &paramTI = IGF.getTypeInfo(param->getType());
 
   // If the SIL parameter isn't passed indirectly, we need to map it
-  // to an explosion.  Fortunately, in this case we have a guarantee
-  // that it's passed directly in IR.
+  // to an explosion.
   if (param->getType().isObject()) {
     Explosion paramValues;
-    cast<LoadableTypeInfo>(paramTI).reexplode(IGF, allParamValues, paramValues);
+    auto &loadableTI = cast<LoadableTypeInfo>(paramTI);
+    // If the explosion must be passed indirectly, load the value from the
+    // indirect address.
+    if (loadableTI.getSchema().requiresIndirectParameter(IGF.IGM)) {
+      Address paramAddr
+        = loadableTI.getAddressForPointer(allParamValues.claimNext());
+      loadableTI.loadAsTake(IGF, paramAddr, paramValues);
+    } else {
+      // Otherwise, we can just take the exploded arguments.
+      // FIXME: It doesn't necessarily make sense to pass all types using their
+      // explosion schema.
+      loadableTI.reexplode(IGF, allParamValues, paramValues);
+    }
     IGF.setLoweredExplosion(SILValue(param, 0), paramValues);
     return;
   }
 
   // Okay, the type is passed indirectly in SIL, so we need to map
   // it to an address.
   // FIXME: that doesn't mean we should physically pass it
-  // indirectly at this explosion level, but SIL currently gives us
-  // no ability to distinguish between an l-value and a byval argument.
+  // indirectly at this resilience expansion. An @in or @in_guaranteed parameter
+  // could be passed by value in the right resilience domain.
   Address paramAddr
     = paramTI.getAddressForPointer(allParamValues.claimNext());
   IGF.setLoweredAddress(SILValue(param, 0), paramAddr);
@@ -1882,7 +1893,7 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
   }
 
   // Add all those arguments.
-  emission.setArgs(llArgs, &witnessMetadata);
+  emission.setArgs(llArgs, params, &witnessMetadata);
 
   SILInstruction *i = site.getInstruction();
 

test/IRGen/abitypes.swift

@@ -35,40 +35,40 @@ class Foo {
   }
 
 
-  // x86_64-macosx: define hidden double @_TFC8abitypes3Foo14getXFromNSRect{{.*}}(double, double, double, double, %C8abitypes3Foo*) {{.*}} {
+  // x86_64-macosx: define hidden double @_TFC8abitypes3Foo14getXFromNSRect{{.*}}(%VSC6CGRect*, %C8abitypes3Foo*) {{.*}} {
   // x86_64-macosx: define hidden double @_TToFC8abitypes3Foo14getXFromNSRect{{.*}}(i8*, i8*, %VSC6CGRect* byval align 8) unnamed_addr {{.*}} {
-  // armv7-ios: define hidden double @_TFC8abitypes3Foo14getXFromNSRect{{.*}}(float, float, float, float, %C8abitypes3Foo*) {{.*}} {
+  // armv7-ios: define hidden double @_TFC8abitypes3Foo14getXFromNSRect{{.*}}(%VSC6CGRect*, %C8abitypes3Foo*) {{.*}} {
   // armv7-ios: define hidden double @_TToFC8abitypes3Foo14getXFromNSRect{{.*}}(i8*, i8*, [4 x i32]) unnamed_addr {{.*}} {
   dynamic func getXFromNSRect(r: NSRect) -> Double {
     return Double(r.origin.x)
   }
 
-  // x86_64-macosx: define hidden float @_TFC8abitypes3Foo12getXFromRect{{.*}}(float, float, float, float, %C8abitypes3Foo*) {{.*}} {
+  // x86_64-macosx: define hidden float @_TFC8abitypes3Foo12getXFromRect{{.*}}(%VSC6MyRect*, %C8abitypes3Foo*) {{.*}} {
   // x86_64-macosx: define hidden float @_TToFC8abitypes3Foo12getXFromRect{{.*}}(i8*, i8*, <2 x float>, <2 x float>) unnamed_addr {{.*}} {
-  // armv7-ios: define hidden float @_TFC8abitypes3Foo12getXFromRect{{.*}}(float, float, float, float, %C8abitypes3Foo*) {{.*}} {
+  // armv7-ios: define hidden float @_TFC8abitypes3Foo12getXFromRect{{.*}}(%VSC6MyRect*, %C8abitypes3Foo*) {{.*}} {
   // armv7-ios: define hidden float @_TToFC8abitypes3Foo12getXFromRect{{.*}}(i8*, i8*, [4 x i32]) unnamed_addr {{.*}} {
   dynamic func getXFromRect(r: MyRect) -> Float {
     return r.x
   }
 
   // Call from Swift entrypoint with exploded Rect to @objc entrypoint
   // with unexploaded ABI-coerced type.
-  // x86_64-macosx: define hidden float @_TFC8abitypes3Foo17getXFromRectSwift{{.*}}(float, float, float, float, [[SELF:%.*]]*) {{.*}} {
+  // x86_64-macosx: define hidden float @_TFC8abitypes3Foo17getXFromRectSwift{{.*}}(%VSC6MyRect*, [[SELF:%.*]]*) {{.*}} {
   // x86_64-macosx: [[COERCED:%.*]] = alloca [[MYRECT:%.*MyRect.*]], align 4
   // x86_64-macosx: [[SEL:%.*]] = load i8*, i8** @"\01L_selector(getXFromRect:)", align 8
   // x86_64-macosx: [[CAST:%.*]] = bitcast [[MYRECT]]* [[COERCED]] to { <2 x float>, <2 x float> }*
   // x86_64-macosx: [[T0:%.*]] = getelementptr inbounds { <2 x float>, <2 x float> }, { <2 x float>, <2 x float> }* [[CAST]], i32 0, i32 0
   // x86_64-macosx: [[FIRST_HALF:%.*]] = load <2 x float>, <2 x float>* [[T0]]
   // x86_64-macosx: [[T0:%.*]] = getelementptr inbounds { <2 x float>, <2 x float> }, { <2 x float>, <2 x float> }* [[CAST]], i32 0, i32 1
   // x86_64-macosx: [[SECOND_HALF:%.*]] = load <2 x float>, <2 x float>* [[T0]]
-  // x86_64-macosx: [[SELFCAST:%.*]] = bitcast [[SELF]]* %4 to i8*
+  // x86_64-macosx: [[SELFCAST:%.*]] = bitcast [[SELF]]* %1 to i8*
   // x86_64-macosx: [[RESULT:%.*]] = call float bitcast (void ()* @objc_msgSend to float (i8*, i8*,  <2 x float>, <2 x float>)*)(i8* [[SELFCAST]], i8* [[SEL]], <2 x float> [[FIRST_HALF]], <2 x float> [[SECOND_HALF]])
-  // armv7-ios: define hidden float @_TFC8abitypes3Foo17getXFromRectSwift{{.*}}(float, float, float, float, [[SELF:%.*]]*) {{.*}} {
+  // armv7-ios: define hidden float @_TFC8abitypes3Foo17getXFromRectSwift{{.*}}(%VSC6MyRect*, [[SELF:%.*]]*) {{.*}} {
   // armv7-ios: [[COERCED:%.*]] = alloca [[MYRECT:%.*MyRect.*]], align 4
   // armv7-ios: [[SEL:%.*]] = load i8*, i8** @"\01L_selector(getXFromRect:)", align 4
   // armv7-ios: [[CAST:%.*]] = bitcast [[MYRECT]]* [[COERCED]] to [4 x i32]*
   // armv7-ios: [[LOADED:%.*]] = load [4 x i32], [4 x i32]* [[CAST]]
-  // armv7-ios: [[SELFCAST:%.*]] = bitcast [[SELF]]* %4 to i8*
+  // armv7-ios: [[SELFCAST:%.*]] = bitcast [[SELF]]* %1 to i8*
   // armv7-ios: [[RESULT:%.*]] = call float bitcast (void ()* @objc_msgSend to float (i8*, i8*, [4 x i32])*)(i8* [[SELFCAST]], i8* [[SEL]], [4 x i32] [[LOADED]])
   func getXFromRectSwift(r: MyRect) -> Float {
     return getXFromRect(r)
@@ -85,7 +85,7 @@ class Foo {
   }
 
   // Make sure the caller-side from Swift also uses indirect-byval for the argument
-  // x86_64-macosx: define hidden float @_TFC8abitypes3Foo25getXFromRectIndirectSwift{{.*}}(float, float, float, float, %C8abitypes3Foo*) {{.*}} {
+  // x86_64-macosx: define hidden float @_TFC8abitypes3Foo25getXFromRectIndirectSwift{{.*}}(%VSC6MyRect*, %C8abitypes3Foo*) {{.*}} {
   func getXFromRectIndirectSwift(r: MyRect) -> Float {
     let f : Float = 1.0;
     // x86_64-macosx: [[TEMP:%.*]] = alloca [[TEMPTYPE:%.*]], align 4
@@ -361,10 +361,10 @@ class Foo {
     x.alDente()
   }
 
-  // arm64-ios: define hidden void @_TFC8abitypes3Foo14callJustReturn{{.*}}(%VSC9BigStruct* noalias sret, %CSo13StructReturns*, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, %C8abitypes3Foo*) {{.*}} {
+  // arm64-ios: define hidden void @_TFC8abitypes3Foo14callJustReturn{{.*}}(%VSC9BigStruct* noalias sret, %CSo13StructReturns*, %VSC9BigStruct*, %C8abitypes3Foo*) {{.*}} {
   // arm64-ios: define hidden void @_TToFC8abitypes3Foo14callJustReturnfS0_FTCSo13StructReturns4withVSC9BigStruct_S2_(%VSC9BigStruct* noalias sret, i8*, i8*, [[OPAQUE:.*]]*, %VSC9BigStruct*) unnamed_addr {{.*}} {
   //
-  // arm64-tvos: define hidden void @_TFC8abitypes3Foo14callJustReturn{{.*}}(%VSC9BigStruct* noalias sret, %CSo13StructReturns*, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, %C8abitypes3Foo*) {{.*}} {
+  // arm64-tvos: define hidden void @_TFC8abitypes3Foo14callJustReturn{{.*}}(%VSC9BigStruct* noalias sret, %CSo13StructReturns*, %VSC9BigStruct*, %C8abitypes3Foo*) {{.*}} {
   // arm64-tvos: define hidden void @_TToFC8abitypes3Foo14callJustReturnfS0_FTCSo13StructReturns4withVSC9BigStruct_S2_(%VSC9BigStruct* noalias sret, i8*, i8*, [[OPAQUE:.*]]*, %VSC9BigStruct*) unnamed_addr {{.*}} {
   dynamic func callJustReturn(r: StructReturns, with v: BigStruct) -> BigStruct {
     return r.justReturn(v)
@@ -380,12 +380,12 @@ class Foo {
 // armv7-ios: define internal void @makeOne(%struct.One* noalias sret %agg.result, float %f, float %s)
 
 // rdar://17631440 - Expand direct arguments that are coerced to aggregates.
-// x86_64-macosx: define float @_TF8abitypes13testInlineAggFVSC6MyRectSf(float, float, float, float) {{.*}} {
+// x86_64-macosx: define float @_TF8abitypes13testInlineAggFVSC6MyRectSf(%VSC6MyRect*) {{.*}} {
 // x86_64-macosx: [[COERCED:%.*]] = alloca %VSC6MyRect, align 4
-// x86_64-macosx: store float %0,
-// x86_64-macosx: store float %1,
-// x86_64-macosx: store float %2,
-// x86_64-macosx: store float %3,
+// x86_64-macosx: store float %
+// x86_64-macosx: store float %
+// x86_64-macosx: store float %
+// x86_64-macosx: store float %
 // x86_64-macosx: [[CAST:%.*]] = bitcast %VSC6MyRect* [[COERCED]] to { <2 x float>, <2 x float> }*
 // x86_64-macosx: [[T0:%.*]] = getelementptr inbounds { <2 x float>, <2 x float> }, { <2 x float>, <2 x float> }* [[CAST]], i32 0, i32 0
 // x86_64-macosx: [[FIRST_HALF:%.*]] = load <2 x float>, <2 x float>* [[T0]], align 4