SILOptimizer: add a mandatory generic-specialization pass.

This pass is only used for functions with performance annotations (@_noLocks, @_noAllocation).
It runs in the mandatory pipeline and specializes all function calls in performance-annotated functions and functions which are called from such functions.
In addition, the pass also does some other related optimizations: devirtualization, constant-folding Builtin.canBeClass, inlining of transparent functions and memory access optimizations.

Author: eeckstein

include/swift/SILOptimizer/PassManager/Passes.def

@@ -237,6 +237,8 @@ PASS(RedundantLoadElimination, "redundant-load-elim",
      "Redundant Load Elimination")
 PASS(DeadStoreElimination, "dead-store-elim",
      "Dead Store Elimination")
+PASS(MandatoryGenericSpecializer, "mandatory-generic-specializer",
+     "Mandatory Generic Function Specialization on Static Types")
 PASS(GenericSpecializer, "generic-specializer",
      "Generic Function Specialization on Static Types")
 PASS(ExistentialSpecializer, "existential-specializer",

include/swift/SILOptimizer/Utils/Generics.h

@@ -45,7 +45,8 @@ void trySpecializeApplyOfGeneric(
     SILOptFunctionBuilder &FunctionBuilder,
     ApplySite Apply, DeadInstructionSet &DeadApplies,
     llvm::SmallVectorImpl<SILFunction *> &NewFunctions,
-    OptRemark::Emitter &ORE);
+    OptRemark::Emitter &ORE,
+    bool isMandatory);
 
 /// Helper class to describe re-abstraction of function parameters done during
 /// specialization.
@@ -336,11 +337,14 @@ class GenericFuncSpecializer {
   SubstitutionMap ContextSubs;
   std::string ClonedName;
 
+  bool isMandatory;
+
 public:
   GenericFuncSpecializer(SILOptFunctionBuilder &FuncBuilder,
                          SILFunction *GenericFunc,
                          SubstitutionMap ParamSubs,
-                         const ReabstractionInfo &ReInfo);
+                         const ReabstractionInfo &ReInfo,
+                         bool isMandatory = false);
 
   /// If we already have this specialization, reuse it.
   SILFunction *lookupSpecialization();

lib/SILOptimizer/PassManager/PassPipeline.cpp

@@ -165,6 +165,7 @@ static void addMandatoryDiagnosticOptPipeline(SILPassPipelinePlan &P) {
   P.addPredictableDeadAllocationElimination();
 
   P.addOptimizeHopToExecutor();
+  P.addMandatoryGenericSpecializer();
 
   P.addDiagnoseUnreachable();
   P.addDiagnoseInfiniteRecursion();

lib/SILOptimizer/Transforms/GenericSpecializer.cpp

@@ -21,36 +21,23 @@
 #include "swift/SIL/SILFunction.h"
 #include "swift/SIL/SILInstruction.h"
 #include "swift/SILOptimizer/PassManager/Transforms.h"
+#include "swift/SILOptimizer/Utils/BasicBlockOptUtils.h"
+#include "swift/SILOptimizer/Utils/ConstantFolding.h"
+#include "swift/SILOptimizer/Utils/Devirtualize.h"
 #include "swift/SILOptimizer/Utils/Generics.h"
 #include "swift/SILOptimizer/Utils/InstOptUtils.h"
 #include "swift/SILOptimizer/Utils/SILOptFunctionBuilder.h"
+#include "swift/SILOptimizer/Utils/SILInliner.h"
 #include "llvm/ADT/SmallVector.h"
 
 using namespace swift;
 
 namespace {
 
-class GenericSpecializer : public SILFunctionTransform {
-
-  bool specializeAppliesInFunction(SILFunction &F);
-
-  /// The entry point to the transformation.
-  void run() override {
-    SILFunction &F = *getFunction();
-
-    LLVM_DEBUG(llvm::dbgs() << "***** GenericSpecializer on function:"
-                            << F.getName() << " *****\n");
-
-    if (specializeAppliesInFunction(F))
-      invalidateAnalysis(SILAnalysis::InvalidationKind::Everything);
-  }
-
-};
-
-} // end anonymous namespace
-
-bool GenericSpecializer::specializeAppliesInFunction(SILFunction &F) {
-  SILOptFunctionBuilder FunctionBuilder(*this);
+static bool specializeAppliesInFunction(SILFunction &F,
+                                        SILTransform *transform,
+                                        bool isMandatory) {
+  SILOptFunctionBuilder FunctionBuilder(*transform);
   DeadInstructionSet DeadApplies;
   llvm::SmallSetVector<SILInstruction *, 8> Applies;
   OptRemark::Emitter ORE(DEBUG_TYPE, F);
@@ -97,14 +84,17 @@ bool GenericSpecializer::specializeAppliesInFunction(SILFunction &F) {
       SILFunction *Callee = Apply.getReferencedFunctionOrNull();
       assert(Callee && "Expected to have a known callee");
 
-      if (!Apply.canOptimize() || !Callee->shouldOptimize())
+      if (!Apply.canOptimize())
+        continue;
+
+      if (!isMandatory && !Callee->shouldOptimize())
         continue;
 
       // We have a call that can potentially be specialized, so
       // attempt to do so.
       llvm::SmallVector<SILFunction *, 2> NewFunctions;
       trySpecializeApplyOfGeneric(FunctionBuilder, Apply, DeadApplies,
-                                  NewFunctions, ORE);
+                                  NewFunctions, ORE, isMandatory);
 
       // Remove all the now-dead applies. We must do this immediately
       // rather than defer it in order to avoid problems with cloning
@@ -120,18 +110,181 @@ bool GenericSpecializer::specializeAppliesInFunction(SILFunction &F) {
         Changed = true;
       }
 
-      // If calling the specialization utility resulted in new functions
-      // (as opposed to returning a previous specialization), we need to notify
-      // the pass manager so that the new functions get optimized.
-      for (SILFunction *NewF : reverse(NewFunctions)) {
-        addFunctionToPassManagerWorklist(NewF, Callee);
+      if (auto *sft = dyn_cast<SILFunctionTransform>(transform)) {
+        // If calling the specialization utility resulted in new functions
+        // (as opposed to returning a previous specialization), we need to notify
+        // the pass manager so that the new functions get optimized.
+        for (SILFunction *NewF : reverse(NewFunctions)) {
+          sft->addFunctionToPassManagerWorklist(NewF, Callee);
+        }
       }
     }
   }
 
   return Changed;
 }
 
+/// The generic specializer, used in the optimization pipeline.
+class GenericSpecializer : public SILFunctionTransform {
+
+  /// The entry point to the transformation.
+  void run() override {
+    SILFunction &F = *getFunction();
+
+    LLVM_DEBUG(llvm::dbgs() << "***** GenericSpecializer on function:"
+                            << F.getName() << " *****\n");
+
+    if (specializeAppliesInFunction(F, this, /*isMandatory*/ false)) {
+      invalidateAnalysis(SILAnalysis::InvalidationKind::Everything);
+    }
+  }
+};
+
+/// The mandatory specializer, which runs in the mandatory pipeline.
+///
+/// It specializes functions, called from performance-annotated functions
+/// (@_noLocks, @_noAllocation).
+class MandatoryGenericSpecializer : public SILModuleTransform {
+
+  void run() override;
+
+  bool optimize(SILFunction *func, ClassHierarchyAnalysis *cha);
+
+  bool optimizeInst(SILInstruction *inst, SILOptFunctionBuilder &funcBuilder,
+                    InstructionDeleter &deleter, ClassHierarchyAnalysis *cha);
+};
+
+
+void MandatoryGenericSpecializer::run() {
+  SILModule *module = getModule();
+
+  if (!module->getOptions().EnablePerformanceAnnotations)
+    return;
+
+  ClassHierarchyAnalysis *cha = getAnalysis<ClassHierarchyAnalysis>();
+  
+  llvm::SmallVector<SILFunction *, 8> workList;
+  llvm::SmallPtrSet<SILFunction *, 16> visited;
+  
+  // Look for performance-annotated functions.
+  for (SILFunction &function : *module) {
+    if (function.getPerfConstraints() != PerformanceConstraints::None) {
+      workList.push_back(&function);
+      visited.insert(&function);
+    }
+  }
+  
+  while (!workList.empty()) {
+    SILFunction *func = workList.pop_back_val();
+    module->linkFunction(func, SILModule::LinkingMode::LinkAll);
+    if (!func->isDefinition())
+      continue;
+
+    // Perform generic specialization and other related optimzations.
+    bool changed = optimize(func, cha);
+
+    if (changed)
+      invalidateAnalysis(func, SILAnalysis::InvalidationKind::Everything);
+
+    // Continue specializing called functions.
+    for (SILBasicBlock &block : *func) {
+      for (SILInstruction &inst : block) {
+        if (auto as = ApplySite::isa(&inst)) {
+          if (SILFunction *callee = as.getReferencedFunctionOrNull()) {
+            if (visited.insert(callee).second)
+              workList.push_back(callee);
+          }
+        }
+      }
+    }
+  }
+}
+
+/// Specialize generic calls in \p func and do some other related optimizations:
+/// devirtualization and constant-folding of the Builtin.canBeClass.
+bool MandatoryGenericSpecializer::optimize(SILFunction *func,
+                                           ClassHierarchyAnalysis *cha) {
+  bool changed = false;
+  SILOptFunctionBuilder funcBuilder(*this);
+  InstructionDeleter deleter;
+  ReachingReturnBlocks rrBlocks(func);
+  NonErrorHandlingBlocks neBlocks(func);
+
+  // If this is a just specialized function, try to optimize copy_addr, etc.
+  // instructions.
+  if (optimizeMemoryAccesses(*func)) {
+    eliminateDeadAllocations(*func);
+    changed = true;
+  }
+
+  // Visiting blocks in reverse order avoids revisiting instructions after block
+  // splitting, which would be quadratic.
+  for (SILBasicBlock &block : llvm::reverse(*func)) {
+    // Only consider blocks which are not on a "throw" path.
+    if (!rrBlocks.reachesReturn(&block) || !neBlocks.isNonErrorHandling(&block))
+      continue;
+  
+    for (SILInstruction *inst : deleter.updatingReverseRange(&block)) {
+      changed |= optimizeInst(inst, funcBuilder, deleter, cha);
+    }
+  }
+  deleter.cleanupDeadInstructions();
+
+  if (specializeAppliesInFunction(*func, this, /*isMandatory*/ true))
+    changed = true;
+
+  return changed;
+}
+
+bool MandatoryGenericSpecializer::
+optimizeInst(SILInstruction *inst, SILOptFunctionBuilder &funcBuilder,
+             InstructionDeleter &deleter, ClassHierarchyAnalysis *cha) {
+  if (auto as = ApplySite::isa(inst)) {
+    // Specialization opens opportunities to devirtualize method calls.
+    ApplySite newAS = tryDevirtualizeApply(as, cha).first;
+    if (!newAS)
+      return false;
+    deleter.forceDelete(as.getInstruction());
+    auto newFAS = FullApplySite::isa(newAS.getInstruction());
+    if (!newFAS)
+      return true;
+      
+    SILFunction *callee = newFAS.getReferencedFunctionOrNull();
+    if (!callee || callee->isTransparent() == IsNotTransparent)
+      return true;
+
+    // If the de-virtualized callee is a transparent function, inline it.
+    SILInliner::inlineFullApply(newFAS, SILInliner::InlineKind::MandatoryInline,
+                                funcBuilder, deleter);
+    return true;
+  }
+  if (auto *bi = dyn_cast<BuiltinInst>(inst)) {
+    // Constant-fold the Builtin.canBeClass. This is essential for Array code.
+    if (bi->getBuiltinInfo().ID != BuiltinValueKind::CanBeObjCClass)
+      return false;
+
+    SILBuilderWithScope builder(bi);
+    IntegerLiteralInst *lit = optimizeBuiltinCanBeObjCClass(bi, builder);
+    if (!lit)
+      return false;
+
+    bi->replaceAllUsesWith(lit);
+    ConstantFolder constFolder(funcBuilder, getOptions().AssertConfig,
+                               /*EnableDiagnostics*/ false);
+    constFolder.addToWorklist(lit);
+    constFolder.processWorkList();
+    deleter.forceDelete(bi);
+    return true;
+  }
+  return false;
+}
+
+} // end anonymous namespace
+
 SILTransform *swift::createGenericSpecializer() {
   return new GenericSpecializer();
 }
+
+SILTransform *swift::createMandatoryGenericSpecializer() {
+  return new MandatoryGenericSpecializer();
+}

lib/SILOptimizer/Utils/Generics.cpp

@@ -1829,11 +1829,12 @@ ReabstractionInfo::ReabstractionInfo(ModuleDecl *targetModule,
 GenericFuncSpecializer::GenericFuncSpecializer(
     SILOptFunctionBuilder &FuncBuilder, SILFunction *GenericFunc,
     SubstitutionMap ParamSubs,
-    const ReabstractionInfo &ReInfo)
+    const ReabstractionInfo &ReInfo,
+    bool isMandatory)
     : FuncBuilder(FuncBuilder), M(GenericFunc->getModule()),
       GenericFunc(GenericFunc),
       ParamSubs(ParamSubs),
-      ReInfo(ReInfo) {
+      ReInfo(ReInfo), isMandatory(isMandatory) {
 
   assert((GenericFunc->isDefinition() || ReInfo.isPrespecialized()) &&
          "Expected definition or pre-specialized entry-point to specialize!");
@@ -1888,7 +1889,7 @@ void ReabstractionInfo::verify() const {
 SILFunction *
 GenericFuncSpecializer::tryCreateSpecialization(bool forcePrespecialization) {
   // Do not create any new specializations at Onone.
-  if (!GenericFunc->shouldOptimize() && !forcePrespecialization)
+  if (!GenericFunc->shouldOptimize() && !forcePrespecialization && !isMandatory)
     return nullptr;
 
   LLVM_DEBUG(llvm::dbgs() << "Creating a specialization: "
@@ -2564,7 +2565,8 @@ void swift::trySpecializeApplyOfGeneric(
     SILOptFunctionBuilder &FuncBuilder,
     ApplySite Apply, DeadInstructionSet &DeadApplies,
     SmallVectorImpl<SILFunction *> &NewFunctions,
-    OptRemark::Emitter &ORE) {
+    OptRemark::Emitter &ORE,
+    bool isMandatory) {
   assert(Apply.hasSubstitutions() && "Expected an apply with substitutions!");
   auto *F = Apply.getFunction();
   auto *RefF =
@@ -2683,7 +2685,7 @@ void swift::trySpecializeApplyOfGeneric(
 
   GenericFuncSpecializer FuncSpecializer(FuncBuilder,
                                          RefF, Apply.getSubstitutionMap(),
-                                         ReInfo);
+                                         ReInfo, isMandatory);
   SILFunction *SpecializedF = prespecializedF
                                   ? prespecializedF
                                   : FuncSpecializer.lookupSpecialization();