MoveOnlyUtils.cpp

//===--- MoveOnlyUtils.cpp ------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
///
/// Transformation utilities used by MoveOnlyObjectChecker,
/// MoveOnlyAddressChecker, and MoveOnlyBorrowToDestructureTransform.
///
/// The reason that this is being done is so that the borrow to destructure
/// transform can be tested separately from testing the relevant checkers.
///
/// TODO: This should eventually be in SILOptimizer/Utils.
///
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "sil-move-only-checker"

#include "MoveOnlyDiagnostics.h"
#include "MoveOnlyObjectChecker.h"

#include "swift/Basic/Defer.h"
#include "swift/SIL/SILBuilder.h"
#include "swift/SILOptimizer/Utils/CFGOptUtils.h"

#include "swift/SIL/OwnershipUtils.h"
#include "swift/SIL/SILInstruction.h"
#include "swift/SIL/SILValue.h"
#include "swift/SIL/SILLocation.h"
#include "llvm/Support/Debug.h"

using namespace swift;
using namespace swift::siloptimizer;

namespace {
using AvailableValues = BorrowToDestructureTransform::AvailableValues;
}

//===----------------------------------------------------------------------===//
//                              MARK: Utilities
//===----------------------------------------------------------------------===//

void AvailableValues::print(llvm::raw_ostream &os, const char *prefix) const {
  if (prefix)
    os << prefix;
  os << "Dumping AvailableValues!\n";
  for (auto pair : llvm::enumerate(values)) {
    if (prefix)
      os << prefix;
    os << "    values[" << pair.index() << "] = ";
    if (pair.value()) {
      os << *pair.value();
    } else {
      os << "None\n";
    }
  }
}

void AvailableValues::dump() const { print(llvm::dbgs(), nullptr); }

/// Return a loc that can be used regardless if \p inst is a terminator or not.
static SILLocation getSafeLoc(SILInstruction *inst) {
  if (isa<TermInst>(inst))
    return RegularLocation::getDiagnosticsOnlyLocation(inst->getLoc(),
                                                       inst->getModule());
  return inst->getLoc();
}

//===----------------------------------------------------------------------===//
//            MARK: Convert Borrow Extracts To Owned Destructures
//===----------------------------------------------------------------------===//

bool BorrowToDestructureTransform::gatherUses(
    StackList<BeginBorrowInst *> &borrowWorklist) {
  LLVM_DEBUG(llvm::dbgs() << "Gathering uses!\n");
  StackList<Operand *> useWorklist(mmci->getFunction());

  for (auto *borrow : borrowWorklist) {
    for (auto *use : borrow->getUses()) {
      useWorklist.push_back(use);
    }
  }

  while (!useWorklist.empty()) {
    auto *nextUse = useWorklist.pop_back_val();
    LLVM_DEBUG(llvm::dbgs() << "    NextUse: " << *nextUse->getUser());
    switch (nextUse->getOperandOwnership()) {
    case OperandOwnership::NonUse:
      continue;

    // Conservatively treat a conversion to an unowned value as a pointer
    // escape. If we see this in the SIL, fail and return false so we emit a
    // "compiler doesn't understand error".
    case OperandOwnership::ForwardingUnowned:
    case OperandOwnership::PointerEscape:
      LLVM_DEBUG(llvm::dbgs()
                 << "        Found forwarding unowned or pointer escape!\n");
      return false;

    // These might be uses that we need to perform a destructure or insert
    // struct_extracts for.
    case OperandOwnership::TrivialUse:
    case OperandOwnership::InstantaneousUse:
    case OperandOwnership::UnownedInstantaneousUse:
    case OperandOwnership::InteriorPointer:
    case OperandOwnership::BitwiseEscape: {
      // Look through copy_value of a move only value. We treat copy_value of
      // copyable values as normal uses.
      if (auto *cvi = dyn_cast<CopyValueInst>(nextUse->getUser())) {
        if (cvi->getOperand()->getType().isMoveOnly()) {
          LLVM_DEBUG(llvm::dbgs() << "        Found copy value of move only "
                                     "field... looking through!\n");
          for (auto *use : cvi->getUses())
            useWorklist.push_back(use);
          continue;
        }

        // If we don't have a copy of a move only type, we just reat this as a
        // normal use, so we fall through.
      }

      auto leafRange = TypeTreeLeafTypeRange::get(nextUse->get(), mmci);
      if (!leafRange) {
        LLVM_DEBUG(llvm::dbgs() << "        Failed to compute leaf range?!\n");
        return false;
      }

      LLVM_DEBUG(llvm::dbgs() << "        Found non lifetime ending use!\n");
      blocksToUses.insert(
          nextUse->getParentBlock(),
          {nextUse, {*leafRange, false /*is lifetime ending*/}});
      liveness.updateForUse(nextUse->getUser(), *leafRange,
                            false /*is lifetime ending*/);
      continue;
    }

    case OperandOwnership::ForwardingConsume:
    case OperandOwnership::DestroyingConsume: {
      // Ignore destroy_value, we are going to eliminate them.
      if (isa<DestroyValueInst>(nextUse->getUser())) {
        LLVM_DEBUG(llvm::dbgs() << "        Found destroy value!\n");
        continue;
      }

      auto leafRange = TypeTreeLeafTypeRange::get(nextUse->get(), mmci);
      if (!leafRange) {
        LLVM_DEBUG(llvm::dbgs() << "        Failed to compute leaf range?!\n");
        return false;
      }

      LLVM_DEBUG(llvm::dbgs() << "        Found lifetime ending use!\n");
      destructureNeedingUses.push_back(nextUse);
      blocksToUses.insert(nextUse->getParentBlock(),
                          {nextUse, {*leafRange, true /*is lifetime ending*/}});
      liveness.updateForUse(nextUse->getUser(), *leafRange,
                            true /*is lifetime ending*/);
      continue;
    }

    case OperandOwnership::GuaranteedForwarding:
      // Look through guaranteed forwarding.
      ForwardingOperand(nextUse).visitForwardedValues([&](SILValue value) {
        for (auto *use : value->getUses()) {
          useWorklist.push_back(use);
        }
        return true;
      });
      continue;

    case OperandOwnership::Borrow:
      LLVM_DEBUG(llvm::dbgs() << "        Found recursive borrow!\n");
      // Look through borrows.
      for (auto value : nextUse->getUser()->getResults()) {
        for (auto *use : value->getUses()) {
          useWorklist.push_back(use);
        }
      }
      continue;
    case OperandOwnership::EndBorrow:
      LLVM_DEBUG(llvm::dbgs() << "        Found end borrow!\n");
      continue;
    case OperandOwnership::Reborrow:
      llvm_unreachable("Unsupported for now?!");
    }
  }
  return true;
}

void BorrowToDestructureTransform::checkDestructureUsesOnBoundary() const {
  LLVM_DEBUG(llvm::dbgs() << "Checking destructure uses on boundary!\n");
  // Now that we have found all of our destructure needing uses and liveness
  // needing uses, make sure that none of our destructure needing uses are
  // within our boundary. If so, we have an automatic error.
  for (auto *use : destructureNeedingUses) {
    LLVM_DEBUG(llvm::dbgs()
               << "    DestructureNeedingUse: " << *use->getUser());
    auto destructureUse = *TypeTreeLeafTypeRange::get(use->get(), mmci);
    if (liveness.isWithinBoundary(use->getUser(), destructureUse)) {
      LLVM_DEBUG(llvm::dbgs() << "        Within boundary! Emitting error!\n");
      // Emit an error. We have a use after free.
      //
      // NOTE: Since we are going to emit an error here, we do the boundary
      // computation to ensure that we only do the boundary computation once:
      // when we emit an error or once we know we need to do rewriting.
      //
      // TODO: Fix diagnostic to use destructure needing use and boundary
      // uses.
      FieldSensitivePrunedLivenessBoundary boundary(
          liveness.getNumSubElements());
      liveness.computeBoundary(boundary);
      diagnosticEmitter.emitObjectDestructureNeededWithinBorrowBoundary(
          mmci, use->getUser(), destructureUse, boundary);
      return;
    } else {
      LLVM_DEBUG(llvm::dbgs() << "        On boundary! No error!\n");
    }
  }
}

bool BorrowToDestructureTransform::gatherBorrows(
    MarkMustCheckInst *mmci, StackList<BeginBorrowInst *> &borrowWorklist) {
  LLVM_DEBUG(llvm::dbgs() << "Searching for borrows for inst: " << *mmci);

  StackList<Operand *> worklist(mmci->getFunction());
  for (auto *op : mmci->getUses())
    worklist.push_back(op);

  while (!worklist.empty()) {
    auto *use = worklist.pop_back_val();
    switch (use->getOperandOwnership()) {
    case OperandOwnership::NonUse:
    case OperandOwnership::TrivialUse:
      continue;

    // Conservatively treat a conversion to an unowned value as a pointer
    // escape. Is it legal to canonicalize ForwardingUnowned?
    case OperandOwnership::ForwardingUnowned:
    case OperandOwnership::PointerEscape:
      return false;

    case OperandOwnership::InstantaneousUse:
    case OperandOwnership::UnownedInstantaneousUse:
    case OperandOwnership::BitwiseEscape:
      // We don't care about these types of uses.
      continue;

    case OperandOwnership::ForwardingConsume:
      // Skip if our type is not move only.
      if (!use->get()->getType().isMoveOnly())
        continue;

      // Search through forwarding consumes.
      ForwardingOperand(use).visitForwardedValues([&](SILValue value) -> bool {
        for (auto *use : value->getUses())
          worklist.push_back(use);
        return true;
      });
      continue;
    case OperandOwnership::DestroyingConsume:
      // We don't care about destroying consume.
      continue;
    case OperandOwnership::Borrow:
      if (auto *bbi = dyn_cast<BeginBorrowInst>(use->getUser())) {
        LLVM_DEBUG(llvm::dbgs() << "    Found borrow: " << *bbi);
        borrowWorklist.push_back(bbi);
      }
      continue;
    case OperandOwnership::InteriorPointer:
      // We don't care about these.
      continue;
    case OperandOwnership::GuaranteedForwarding:
    case OperandOwnership::EndBorrow:
    case OperandOwnership::Reborrow:
      llvm_unreachable("Visiting an owned value!\n");
    }
  }

  return true;
}

static StructDecl *getFullyReferenceableStruct(SILType ktypeTy) {
  auto structDecl = ktypeTy.getStructOrBoundGenericStruct();
  if (!structDecl || structDecl->hasUnreferenceableStorage())
    return nullptr;
  return structDecl;
}

namespace {

struct TypeOffsetSizePair {
  SubElementOffset startOffset = 0;
  TypeSubElementCount size = 0;

  TypeOffsetSizePair() : startOffset(0), size(0) {}
  TypeOffsetSizePair(SILType baseType, SILFunction *fn)
      : startOffset(0), size(baseType, fn) {}
  TypeOffsetSizePair(SubElementOffset offset, TypeSubElementCount size)
      : startOffset(offset), size(size) {}
  TypeOffsetSizePair(SILValue projection, SILValue base)
      : startOffset(*SubElementOffset::compute(projection, base)),
        size(TypeSubElementCount(projection)) {}

  IntRange<unsigned> getRange() const {
    return range(startOffset, getEndOffset());
  }

  SubElementOffset getEndOffset() const {
    return SubElementOffset(startOffset + size);
  }

  bool operator==(const TypeOffsetSizePair &other) const {
    return startOffset == other.startOffset && size == other.size;
  }

  bool operator!=(const TypeOffsetSizePair &other) const {
    return !(*this == other);
  }

  /// Given an ancestor offset \p ancestorOffset and a type called \p
  /// ancestorType, walk one level towards this current type which is assumed to
  /// be a child type of \p ancestorType.
  Optional<std::pair<TypeOffsetSizePair, SILType>>
  walkOneLevelTowardsChild(TypeOffsetSizePair ancestorOffsetSize,
                           SILType ancestorType, SILFunction *fn) const {
    assert(ancestorOffsetSize.size >= size &&
           "Too large to be a child of ancestorType");
    assert((ancestorOffsetSize.startOffset <= startOffset &&
            startOffset <
                (ancestorOffsetSize.startOffset + ancestorOffsetSize.size)) &&
           "Not within the offset range of ancestor");

    if (auto tupleType = ancestorType.getAs<TupleType>()) {
      // Before we do anything, see if we have a single element tuple. If we do,
      // just return that.
      if (tupleType->getNumElements() == 1) {
        return {{ancestorOffsetSize, ancestorType.getTupleElementType(0)}};
      }

      assert(ancestorOffsetSize.size > size &&
             "Too large to be a child of ancestorType");

      unsigned childOffset = ancestorOffsetSize.startOffset;

      for (auto index : indices(tupleType->getElementTypes())) {
        SILType newType = ancestorType.getTupleElementType(index);
        unsigned newSize = TypeSubElementCount(newType, fn);

        // childOffset + size(tupleChild) is the offset of the next tuple
        // element. If our target offset is less than that, then we know that
        // the target type must be a descendent of this tuple element type.
        if (childOffset + newSize > startOffset) {
          return {{{childOffset, newSize}, newType}};
        }

        // Otherwise, add the new size of this field to iterOffset so we visit
        // our sibling type next.
        childOffset += newSize;
      }

      // At this point, we know that our type is not a subtype of this
      // type. Some sort of logic error occured.
      llvm_unreachable("Not a child of this type?!");
    }

    if (auto *structDecl = getFullyReferenceableStruct(ancestorType)) {
      // Before we do anything, see if we have a single element struct. If we
      // do, just return that.
      auto storedProperties = structDecl->getStoredProperties();
      if (storedProperties.size() == 1) {
        return {{ancestorOffsetSize,
                 ancestorType.getFieldType(storedProperties[0], fn)}};
      }

      assert(ancestorOffsetSize.size > size &&
             "Too large to be a child of ancestorType");

      unsigned childOffset = ancestorOffsetSize.startOffset;
      for (auto *fieldDecl : storedProperties) {
        SILType newType = ancestorType.getFieldType(fieldDecl, fn);
        unsigned newSize = TypeSubElementCount(newType, fn);

        // iterOffset + size(tupleChild) is the offset of the next tuple
        // element. If our target offset is less than that, then we know that
        // the target type must be a child of this tuple element type.
        if (childOffset + newSize > startOffset) {
          return {{{childOffset, newSize}, newType}};
        }

        // Otherwise, add the new size of this field to iterOffset so we visit
        // our sibling type next.
        childOffset += newSize;
      }

      // At this point, we know that our type is not a subtype of this
      // type. Some sort of logic error occured.
      llvm_unreachable("Not a child of this type?!");
    }

    if (auto *enumDecl = ancestorType.getEnumOrBoundGenericEnum()) {
      llvm_unreachable("Cannot find child type of enum!\n");
    }

    llvm_unreachable("Hit a leaf type?! Should have handled it earlier");
  }

  /// Given an ancestor offset \p ancestorOffset and a type called \p
  /// ancestorType, walk one level towards this current type inserting on value,
  /// the relevant projection.
  Optional<std::pair<TypeOffsetSizePair, SILValue>>
  walkOneLevelTowardsChild(SILBuilderWithScope &builder, SILLocation loc,
                           TypeOffsetSizePair ancestorOffsetSize,
                           SILValue ancestorValue) const {
    auto *fn = ancestorValue->getFunction();
    SILType ancestorType = ancestorValue->getType();

    assert(ancestorOffsetSize.size >= size &&
           "Too large to be a child of ancestorType");
    assert((ancestorOffsetSize.startOffset <= startOffset &&
            startOffset <
                (ancestorOffsetSize.startOffset + ancestorOffsetSize.size)) &&
           "Not within the offset range of ancestor");
    if (auto tupleType = ancestorType.getAs<TupleType>()) {
      // Before we do anything, see if we have a single element tuple. If we do,
      // just return that.
      if (tupleType->getNumElements() == 1) {
        auto *newValue = builder.createTupleExtract(loc, ancestorValue, 0);
        return {{ancestorOffsetSize, newValue}};
      }

      assert(ancestorOffsetSize.size > size &&
             "Too large to be a child of ancestorType");

      unsigned childOffset = ancestorOffsetSize.startOffset;

      for (auto index : indices(tupleType->getElementTypes())) {
        SILType newType = ancestorType.getTupleElementType(index);
        unsigned newSize = TypeSubElementCount(newType, fn);

        // childOffset + size(tupleChild) is the offset of the next tuple
        // element. If our target offset is less than that, then we know that
        // the target type must be a descendent of this tuple element type.
        if (childOffset + newSize > startOffset) {
          auto *newValue =
              builder.createTupleExtract(loc, ancestorValue, index);
          return {{{childOffset, newSize}, newValue}};
        }

        // Otherwise, add the new size of this field to iterOffset so we visit
        // our sibling type next.
        childOffset += newSize;
      }

      // At this point, we know that our type is not a subtype of this
      // type. Some sort of logic error occured.
      llvm_unreachable("Not a child of this type?!");
    }

    if (auto *structDecl = getFullyReferenceableStruct(ancestorType)) {
      // Before we do anything, see if we have a single element struct. If we
      // do, just return that.
      auto storedProperties = structDecl->getStoredProperties();
      if (storedProperties.size() == 1) {
        auto *newValue = builder.createStructExtract(loc, ancestorValue,
                                                     storedProperties[0]);
        return {{ancestorOffsetSize, newValue}};
      }

      assert(ancestorOffsetSize.size > size &&
             "Too large to be a child of ancestorType");

      unsigned childOffset = ancestorOffsetSize.startOffset;
      for (auto *fieldDecl : structDecl->getStoredProperties()) {
        SILType newType = ancestorType.getFieldType(fieldDecl, fn);
        unsigned newSize = TypeSubElementCount(newType, fn);

        // iterOffset + size(tupleChild) is the offset of the next tuple
        // element. If our target offset is less than that, then we know that
        // the target type must be a child of this tuple element type.
        if (childOffset + newSize > startOffset) {
          auto *newValue =
              builder.createStructExtract(loc, ancestorValue, fieldDecl);
          return {{{childOffset, newSize}, newValue}};
        }

        // Otherwise, add the new size of this field to iterOffset so we visit
        // our sibling type next.
        childOffset += newSize;
      }

      // At this point, we know that our type is not a subtype of this
      // type. Some sort of logic error occured.
      llvm_unreachable("Not a child of this type?!");
    }

    if (auto *enumDecl = ancestorType.getEnumOrBoundGenericEnum()) {
      llvm_unreachable("Cannot find child type of enum!\n");
    }

    llvm_unreachable("Hit a leaf type?! Should have handled it earlier");
  }
};

llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
                              const TypeOffsetSizePair &other) {
  return os << "(startOffset: " << other.startOffset << ", size: " << other.size
            << ")";
}

} // anonymous namespace

#ifndef NDEBUG
static void dumpSmallestTypeAvailable(
    SmallVectorImpl<Optional<std::pair<TypeOffsetSizePair, SILType>>>
        &smallestTypeAvailable) {
  LLVM_DEBUG(llvm::dbgs() << "            Dumping smallest type available!\n");
  for (auto pair : llvm::enumerate(smallestTypeAvailable)) {
    LLVM_DEBUG(llvm::dbgs() << "            value[" << pair.index() << "] = ");
    if (!pair.value()) {
      LLVM_DEBUG(llvm::dbgs() << "None\n");
      continue;
    }

    auto value = *pair.value();
    LLVM_DEBUG(llvm::dbgs() << "Span: " << value.first
                            << ". Type: " << value.second << '\n');
  }
}
#endif

/// When we compute available values, we have a few constraints:
///
/// 1. We want to be sure that we destructure as /late/ as possible. This
/// ensures that we match at the source level the assumption by users that they
/// can use entire valid parts as late as possible. If we were to do it earlier
/// we would emit errors too early.
AvailableValues &
BorrowToDestructureTransform::computeAvailableValues(SILBasicBlock *block) {
  LLVM_DEBUG(llvm::dbgs() << "    Computing Available Values For bb"
                          << block->getDebugID() << '\n');

  // First grab our block. If we already have state for the block, just return
  // its available values. We already computed the available values and
  // potentially updated it with new destructured values for our block.
  auto pair = blockToAvailableValues->get(block);
  if (!pair.second) {
    LLVM_DEBUG(llvm::dbgs()
               << "        Already have values! Returning them!\n");
    LLVM_DEBUG(pair.first->print(llvm::dbgs(), "            "));
    return *pair.first;
  }

  LLVM_DEBUG(llvm::dbgs() << "        No values computed! Initializing!\n");
  auto &newValues = *pair.first;

  // Otherwise, we need to initialize our available values with predecessor
  // information.

  // First check if the block is the one associated with our mark must check
  // inst. If we are in this block, set all available value bits to our initial
  // value which is a copy_value of \p initial value. We add the copy_value to
  // ensure that from an OSSA perspective any any destructures we insert are
  // independent of any other copies. We assume that OSSA canonicalization will
  // remove the extra copy later after we run or emit an error if it can't.
  if (block == mmci->getParent()) {
    LLVM_DEBUG(llvm::dbgs()
               << "        In initial block, setting to initial value!\n");
    for (unsigned i : indices(newValues))
      newValues[i] = initialValue;
    LLVM_DEBUG(newValues.print(llvm::dbgs(), "        "));
    return newValues;
  }

  // Otherwise, we need to handle predecessors. Our strategy is to loop over all
  // predecessors and:
  //
  // 1. If we have the same value along all predecessors, for a specific bit, we
  // just let it through.
  //
  // 2. If we find values that describe the same set of set bits and they only
  // describe those bits, we phi them together.
  //
  // 3. If we find a value that is unavailable along one of the other paths but
  // /could/ be destructured such that we could phi the destructured value, we
  // destructure the value in the predecessor and use that for our phi.
  //
  // 4. We assume optimistically that loop back-edge predecessors always contain
  // all available values that come into the loop. The reason why this is true
  // is that we know that either:
  //
  //    a. Our value either begins within a loop meaning that we either never
  //    seen the back edge or the back edge block is where our mark must check
  //    inst is so we won't visit the back edge.
  //
  //    b. Our mark must check block is further up the loop nest than the loop
  //    back edge implying if we were to destructure in the loop, we would
  //    destructure multiple times. This would have then resulted in a liveness
  //    error in the liveness check we ran earlier, resulting in us not running
  //    this transformation.
  struct SkipBackEdgeFilter {
    unsigned targetBlockRPO;
    SILBasicBlock::pred_iterator pe;
    PostOrderFunctionInfo *pofi;

    SkipBackEdgeFilter(SILBasicBlock *block, PostOrderFunctionInfo *pofi)
        : targetBlockRPO(*pofi->getRPONumber(block)), pe(block->pred_end()),
          pofi(pofi) {}

    Optional<SILBasicBlock *> operator()(SILBasicBlock *predBlock) const {
      // If our predecessor block has a larger RPO number than our target block,
      // then their edge must be a backedge.
      if (targetBlockRPO < *pofi->getRPONumber(predBlock))
        return None;
      return predBlock;
    }
  };
  auto predsSkippingBackEdges = makeOptionalTransformRange(
      llvm::make_range(block->pred_begin(), block->pred_end()),
      SkipBackEdgeFilter(block, getPostOrderFunctionInfo()));

  // Loop over all available values for all predecessors and determine for each
  // sub-element number the smallest type over all available values if we have
  // an available value for each predecessor. This is implemented by storing an
  // Optional<TypeOffsetSizePair> in an array for each available value. If we
  // find any predecessor without an available value at all for that entry, we
  // set the Optional to be none. Otherwise, we intersect each
  // TypeOffsetSizePair derived from each available value by always taking the
  // smaller TypeOffsetSizePair. We know by construction that we always will
  // move down the type tree, not up the type tree (see NOTE 2 below).
  //
  // NOTE: Given a parent type and a child type which is the only child of the
  // parent type, we always mathematically take the top type. If we have to
  // later destructure an additional time to wire up a use, we do it at the
  // use site when we wire it up. When phi-ing/destructuring, we hide it from
  // the algorithm. This allows us to keep the invariant that our type in size
  // is always absolutely decreasing in size.
  //
  // NOTE 2: We can only move up the type tree by calling a constructor which is
  // always a +1 operation that is treated as a consuming operation end
  // point. In Swift at the source level, we never construct aggregates in a
  // forwarding guaranteed manner for move only types.
  LLVM_DEBUG(llvm::dbgs() << "        Computing smallest type available for "
                             "available values for block bb"
                          << block->getDebugID() << '\n');
  SmallVector<Optional<std::pair<TypeOffsetSizePair, SILType>>, 8>
      smallestTypeAvailable;
  {
    auto pi = predsSkippingBackEdges.begin();
    auto pe = predsSkippingBackEdges.end();
    assert(pi != pe && "If initial block then should have been the mark must "
                       "check inst block?!");

    {
      auto *bb = *pi;
      LLVM_DEBUG(llvm::dbgs() << "        Visiting first block bb"
                              << bb->getDebugID() << '\n');
      LLVM_DEBUG(llvm::dbgs()
                 << "        Recursively loading its available values to "
                    "compute initial smallest type available for block bb"
                 << block->getDebugID() << '\n');
      auto &predAvailableValues = computeAvailableValues(bb);
      LLVM_DEBUG(
          llvm::dbgs()
          << "        Computing initial smallest type available for block bb"
          << block->getDebugID() << '\n');
      for (unsigned i : range(predAvailableValues.size())) {
        if (predAvailableValues[i])
          smallestTypeAvailable.push_back(
              {{TypeOffsetSizePair(predAvailableValues[i], mmci),
                predAvailableValues[i]->getType()}});
        else
          smallestTypeAvailable.emplace_back(None);
      }
      LLVM_DEBUG(llvm::dbgs() << "        Finished computing initial smallest "
                                 "type available for block bb"
                              << block->getDebugID() << '\n';
                 dumpSmallestTypeAvailable(smallestTypeAvailable));
    }
    LLVM_DEBUG(llvm::dbgs()
               << "        Visiting rest of preds and intersecting for block bb"
               << block->getDebugID() << '\n');
    for (auto ppi = std::next(pi); ppi != pe; ++ppi) {
      auto *bb = *ppi;
      LLVM_DEBUG(llvm::dbgs() << "        Computing smallest type for bb"
                              << bb->getDebugID() << '\n');
      LLVM_DEBUG(llvm::dbgs()
                 << "        Recursively loading its available values!\n");
      auto &predAvailableValues = computeAvailableValues(bb);
      for (unsigned i : range(predAvailableValues.size())) {
        if (!smallestTypeAvailable[i].hasValue())
          continue;

        if (!predAvailableValues[i]) {
          smallestTypeAvailable[i] = None;
          continue;
        }

        // Since we assume all types in the type tree for our purposes are
        // absolutely monotonically decreasing in size from their parent (noting
        // the NOTE above), we know that if subElt has a smaller size than our
        // accumulator, then it must be further down the type tree from our
        // accumulator.
        auto offsetSize = TypeOffsetSizePair(predAvailableValues[i], mmci);
        if (smallestTypeAvailable[i]->first.size > offsetSize.size)
          smallestTypeAvailable[i] = {offsetSize,
                                      predAvailableValues[i]->getType()};
      }
      LLVM_DEBUG(llvm::dbgs() << "        Smallest type available after "
                                 "intersecting with block!\n");
      LLVM_DEBUG(dumpSmallestTypeAvailable(smallestTypeAvailable));
    }
  }

  // At this point, in smallestValueAvailable, we have for each phi slot the
  // smallest size element needed. Now we go through our predecessors again,
  // destructuring available values to match the smallest value needed. If we
  // destructure a larger value, we always update any other available values we
  // are propagating for it using an interval map over the type offsets.
  LLVM_DEBUG(
      llvm::dbgs()
      << "    Destructuring available values in preds to smallest size for bb"
      << block->getDebugID() << '\n');
  auto *fn = block->getFunction();
  IntervalMapAllocator::Map typeSpanToValue(allocator.get());
  for (auto *predBlock : predsSkippingBackEdges) {
    SWIFT_DEFER { typeSpanToValue.clear(); };

    auto &predAvailableValues = computeAvailableValues(predBlock);

    // First go through our available values and initialize our interval map. We
    // should never fail to insert. We want to insert /all/ available values so
    // we can update values that may not be available along other paths if we
    // destructure.
    for (unsigned i : range(predAvailableValues.size())) {
      if (auto value = predAvailableValues[i]) {
#if false
        auto iterOffsetSize = TypeOffsetSizePair(value, mmci);
        typeSpanToValue.insert(iterOffsetSize.startOffset,
                               iterOffsetSize.getEndOffset(),
                               value);
#else
        // We check later that we store entire values.
        typeSpanToValue.insert(i, i + 1, value);
#endif
      }
    }

    // Now walk through our available values and chop up the contents of our
    // interval map to fit our smallest offset size.
    for (unsigned i : range(predAvailableValues.size())) {
      // If we do not have an offset size for this available value, just
      // continue, we do not need to perform any destructuring.
      //
      // NOTE: If we do not have an available value for this element, then we
      // will already not have a smallest type available due to our earlier
      // work.
      auto smallestOffsetSize = smallestTypeAvailable[i];
      if (!smallestOffsetSize)
        continue;

      // Otherwise, compute the offsetSize for the value associated with this
      // offset in the interval map. If the value is already the correct size,
      // just continue, we do not need to perform any destructuring.
      auto iter = typeSpanToValue.find(i);
      assert(iter != typeSpanToValue.end());
      auto iterValue = iter.value();
      auto iterOffsetSize = TypeOffsetSizePair(iterValue, mmci);
      if (smallestOffsetSize->first.size == iterOffsetSize.size) {
        // Our value should already be in the interval map.
        assert(iter.start() == iterOffsetSize.startOffset &&
               iter.stop() == iterOffsetSize.getEndOffset() &&
               "We should always store entire values");
        continue;
      }

      // Otherwise, we need to destructure the value. Our overall plan is that
      // we walk down the type tree, destructuring as we go.
      //
      // NOTE: We do not actually update our available values here since a later
      // smallest offset size could result in further destructuring that an
      // earlier value required. Instead, we do a final loop afterwards using
      // the interval map to update each available value.
      auto iterType = iterValue->getType();
      auto loc = getSafeLoc(predBlock->getTerminator());
      SILBuilderWithScope builder(predBlock->getTerminator());

      while (smallestOffsetSize->first.size < iterOffsetSize.size) {
        TypeOffsetSizePair childOffsetSize;
        SILType childType;

        // We are returned an optional here and should never fail... so use a
        // force unwrap.
        std::tie(childOffsetSize, childType) =
            *iterOffsetSize.walkOneLevelTowardsChild(iterOffsetSize, iterType,
                                                     fn);

        // Before we destructure ourselves, erase our entire value from the
        // map. We do not need to consider the possibility of there being holes
        // in our range since we always store values whole to their entire
        // subelement range. If we lose a single bit of the value, we split it
        // until we again have whole values.
        {
          auto iter = typeSpanToValue.find(i);
          assert(iter.start() == iterOffsetSize.startOffset &&
                 iter.stop() == iterOffsetSize.getEndOffset() &&
                 "We should always store complete values");
          iter.erase();
        }

        // Then perform our destructuring.
        unsigned childOffsetIterator = iterOffsetSize.startOffset;
        builder.emitDestructureValueOperation(
            loc, predAvailableValues[i], [&](unsigned index, SILValue value) {
              // Now, wire up our new value to its span in the interval map.
              TypeSubElementCount childSize(value);
              typeSpanToValue.insert(childOffsetIterator, childSize, value);

              // Update childOffsetIterator so it points at our next child.
              childOffsetIterator += childSize;
            });
      }
    }

    LLVM_DEBUG(llvm::dbgs()
               << "    Updating available values for bb"
               << predBlock->getDebugID() << "\n    Before Update:\n");
    LLVM_DEBUG(predAvailableValues.print(llvm::dbgs(), "        "));

    // Now do one final loop updating our available values using the interval
    // map.
    for (unsigned i : range(predAvailableValues.size())) {
      auto iter = typeSpanToValue.find(i);
      if (iter == typeSpanToValue.end() || iter.start() > i ||
          iter.stop() <= i) {
        predAvailableValues[i] = SILValue();
      } else {
        predAvailableValues[i] = iter.value();
      }
    }
    LLVM_DEBUG(llvm::dbgs() << "    After Update:\n");
    LLVM_DEBUG(predAvailableValues.print(llvm::dbgs(), "        "));
  }

  LLVM_DEBUG(llvm::dbgs() << "    Inserting phis if needed for bb"
                          << block->getDebugID() << '\n');
  // At this point, all of our values should be the "appropriate size". Now we
  // need to perform the actual phi-ing.
  InstructionDeleter deleter;
  for (unsigned i = 0, e = smallestTypeAvailable.size(); i != e; ++i) {
    // If we don't have a smallest value computed for this, this is not a value
    // to phi. Just continue.
    if (!smallestTypeAvailable[i]) {
      continue;
    }

    // Do a quick check to see if all of the values are the same. In such a
    // case, we can quickly update and continue. Otherwise, we need to insert
    // phis.
    SILValue sameValue;
    for (auto *predBlock : predsSkippingBackEdges) {
      auto &predAvailableValues = computeAvailableValues(predBlock);
      if (!sameValue) {
        sameValue = predAvailableValues[i];
      } else if (sameValue != predAvailableValues[i]) {
        sameValue = SILValue();
      }
    }
    if (sameValue) {
      newValues[i] = sameValue;
      continue;
    }

    // Ok, we need to actually construct a phi.
    {
      SILType offsetType = smallestTypeAvailable[i]->second;
      auto *phi = block->createPhiArgument(offsetType, OwnershipKind::Owned);
      newValues[i] = phi;
      createdPhiArguments.push_back(phi);
    }

    for (auto *predBlock : predsSkippingBackEdges) {
      auto &predAvailableValues = computeAvailableValues(predBlock);
      addNewEdgeValueToBranch(predBlock->getTerminator(), block,
                              predAvailableValues[i], deleter);
    }

    // Then walk ahead until we find a type offset size that is not our type
    // offset size. These were all already handled by this value. Each of those
    // we need to assign the same value as in newValues.
    while (i + 1 != e) {
      // If our next available type does not have a smallest type available or
      // that smallest type available is not our smallest type available,
      // break. We will increment and handle this on the next iteration.
      if (!smallestTypeAvailable[i + 1] ||
          *smallestTypeAvailable[i] != *smallestTypeAvailable[i + 1]) {
        break;
      }

      // Otherwise, reuse our phied value for this available value and increment
      // our iterator.
      newValues[i + 1] = newValues[i];
      ++i;
    }
  }

  LLVM_DEBUG(llvm::dbgs() << "    Final available values for bb"
                          << block->getDebugID() << '\n');
  LLVM_DEBUG(newValues.print(llvm::dbgs(), "        "));
  return newValues;
}

#ifndef NDEBUG
static LLVM_ATTRIBUTE_USED void
dumpIntervalMap(BorrowToDestructureTransform::IntervalMapAllocator::Map &map) {
  llvm::dbgs() << "Dumping Interval Map!\n";
  for (auto bi = map.begin(), be = map.end(); bi != be; ++bi) {
    llvm::dbgs() << "Entry. Start: " << bi.start() << " End: " << bi.stop()
                 << "\n";
    llvm::dbgs() << "Value: " << *bi.value() << '\n';
  }
}
#endif

void BorrowToDestructureTransform::rewriteUses() {
  blocksToUses.setFrozen();

  LLVM_DEBUG(llvm::dbgs()
             << "Performing BorrowToDestructureTransform::rewriteUses()!\n");

  llvm::SmallPtrSet<Operand *, 8> seenOperands;
  SmallBitVector bitsNeededInBlock(liveness.getNumSubElements());
  IntervalMapAllocator::Map typeSpanToValue(allocator.get());

  auto *fn = mmci->getFunction();
  assert(!initialValue);
  {
    auto *next = mmci->getNextInstruction();
    SILBuilderWithScope builder(next);
    initialValue = builder.createCopyValue(getSafeLoc(next), mmci);
  }
  assert(initialValue);

  // Walking each block in RPO order.
  for (auto *block :
       getPostOrderFunctionInfo()->getReversePostOrder(mmci->getParent())) {
    SWIFT_DEFER {
      bitsNeededInBlock.reset();
      seenOperands.clear();
    };

    LLVM_DEBUG(llvm::dbgs()
               << "Visiting block bb" << block->getDebugID() << '\n');

    // See if we have any operands that we need to process...
    if (auto operandList = blocksToUses.find(block)) {
      // If we do, gather up the bits that we need.
      for (auto operand : *operandList) {
        auto &subEltSpan = operand.second.subEltSpan;
        LLVM_DEBUG(llvm::dbgs() << "    Found need operand "
                                << operand.first->getOperandNumber()
                                << " of inst: " << *operand.first->getUser()
                                << "    Needs bits: " << subEltSpan << '\n');
        bitsNeededInBlock.set(subEltSpan.startEltOffset,
                              subEltSpan.endEltOffset);
        seenOperands.insert(operand.first);
      }
    }

    // If we do not need any bits... just continue.
    if (bitsNeededInBlock.none()) {
      LLVM_DEBUG(llvm::dbgs() << "    No needed bits! Continuing!\n");
      continue;
    }

    // Ok, we need some bits in this block. Compute our available values in this
    // block.
    LLVM_DEBUG(llvm::dbgs()
               << "    Found needed bits! Propagating available values!\n");
    auto &availableValues = computeAvailableValues(block);
    LLVM_DEBUG(llvm::dbgs() << "    Computed available values for block bb"
                            << block->getDebugID() << '\n';
               availableValues.print(llvm::dbgs(), "        "));
    // Then walk from the top to the bottom of the block rewriting as we go.
    for (auto ii = block->begin(), ie = block->end(); ii != ie;) {
      auto *inst = &*ii;
      ++ii;

      for (auto &operand : inst->getAllOperands()) {
        if (!seenOperands.count(&operand))
          continue;

        auto span = *TypeTreeLeafTypeRange::get(operand.get(), mmci);

        // All available values in our span should have the same value
        // associated with it.
        SILValue first = availableValues[span.startEltOffset];
        assert(llvm::all_of(
            range(span.startEltOffset + 1, span.endEltOffset),
            [&](unsigned index) { return first == availableValues[index]; }));

        LLVM_DEBUG(llvm::dbgs()
                   << "    Rewriting Operand: " << operand.getOperandNumber()
                   << " of inst: " << *operand.getUser()
                   << "    Type Span: " << span << '\n'
                   << "    AvailableValue: " << *first);

        // Then see if first at a type level is equal to our operand's value
        // type. If so, we can just reuse it.
        if (first->getType().removingMoveOnlyWrapper() ==
            operand.get()->getType().removingMoveOnlyWrapper()) {
          LLVM_DEBUG(llvm::dbgs() << "    Found a value that completely covers "
                                     "the operand!\n    Value: "
                                  << *first);
          // If we have:
          //
          // 1. A consuming use.
          // 2. A value that is /not/ move only wrapped and an operand that is
          // non-consuming but can accept an owned value.
          //
          // Just use the owned value. In the case of 2, we need to use a borrow
          // so we can insert the moveonlywrapper_to_copyable [guaranteed] for
          // the use.
          if (operand.isConsuming() ||
              (operand.canAcceptKind(OwnershipKind::Owned) &&
               (first->getType().isMoveOnlyWrapped() ==
                operand.get()->getType().isMoveOnlyWrapped()))) {
            // If we get to this point and have a move only wrapped type and our
            // operand is not a move only wrapped type, then we need to insert
            // an owned moveonlywrapper_to_copyable. We know it must be owned
            // since we can only reach this point if we are consuming.
            if (first->getType().isMoveOnlyWrapped() &&
                !operand.get()->getType().isMoveOnlyWrapped()) {
              SILBuilderWithScope builder(inst);
              first = builder.createOwnedMoveOnlyWrapperToCopyableValue(
                  getSafeLoc(inst), first);
            }
            operand.set(first);
            continue;
          }

          // Otherwise, we need to insert a borrow.
          SILBuilderWithScope borrowBuilder(inst);
          SILValue borrow =
              borrowBuilder.createBeginBorrow(getSafeLoc(inst), first);
          SILValue innerValue = borrow;
          if (innerValue->getType().isMoveOnlyWrapped()) {
            innerValue =
                borrowBuilder.createGuaranteedMoveOnlyWrapperToCopyableValue(
                    getSafeLoc(inst), innerValue);
          }

          if (auto op = InteriorPointerOperand::get(&operand)) {
            op.visitBaseValueScopeEndingUses([&](Operand *endScope) -> bool {
              auto *endScopeInst = endScope->getUser();
              SILBuilderWithScope endBuilder(endScopeInst);
              endBuilder.createEndBorrow(getSafeLoc(endScopeInst), borrow);
              return true;
            });
          } else {
            auto *nextInst = inst->getNextInstruction();
            SILBuilderWithScope endBuilder(nextInst);
            endBuilder.createEndBorrow(getSafeLoc(nextInst), borrow);
          }

          // NOTE: This needs to be /after/the interior pointer operand usage
          // above so that we can use the end scope of our interior pointer base
          // value.
          operand.set(innerValue);
          continue;
        }

        // Compute the location in the type of first's type and operand.get()'s
        // type.
        TypeOffsetSizePair firstValueOffsetSize(first, mmci);
        TypeOffsetSizePair useOffsetSize(operand.get(), mmci);

        LLVM_DEBUG(llvm::dbgs() << "    FirstValueTypeOffsetSize: "
                                << firstValueOffsetSize << '\n');
        LLVM_DEBUG(llvm::dbgs()
                   << "    UseOffsetSize: " << useOffsetSize << '\n');

        // Make sure that useOffsetSize is within firstOffsetSize. If it isn't,
        // then lets emit an early error than erroring within our iteration
        // below.
        assert((firstValueOffsetSize.startOffset <= useOffsetSize.startOffset &&
                useOffsetSize.getEndOffset() <=
                    firstValueOffsetSize.getEndOffset()) &&
               "useOffsetSize not within firstOffsetSize?! "
               "operand.get()->getType() isn't a child type of "
               "first->getType()?!");

        // Otherwise, if we have a non-consuming use, we need to create a new
        // borrow scope and extract out the value. Our value should always be
        // fully available.
        if (!operand.isConsuming()) {
          LLVM_DEBUG(
              llvm::dbgs()
              << "    Non Consuming Operand! Extracting using borrows!\n");

          SILBuilderWithScope borrowBuilder(inst);
          auto loc = getSafeLoc(inst);
          auto *borrow = borrowBuilder.createBeginBorrow(loc, first);
          SILValue value = borrow;

          // First walk until we find the same size use as our element and our
          // type that also equals our type. The second part of the check allows
          // us to skip through single level types.
          SILType operandUnwrappedType =
              operand.get()->getType().removingMoveOnlyWrapper();
          while (operandUnwrappedType !=
                 value->getType().removingMoveOnlyWrapper()) {
            std::tie(firstValueOffsetSize, value) =
                *useOffsetSize.walkOneLevelTowardsChild(
                    borrowBuilder, loc, firstValueOffsetSize, value);
          }

          // At this point, we know we have a type of the same size and the same
          // type (modulo moveonlywrapped). If we need to wrap our gepped value,
          // do so now and then set operand to take this new value.
          if (!operand.get()->getType().isMoveOnlyWrapped() &&
              value->getType().isMoveOnlyWrapped()) {
            value =
                borrowBuilder.createGuaranteedMoveOnlyWrapperToCopyableValue(
                    loc, value);
          }
          operand.set(value);

          // If we have a terminator that is a trivial use (e.x.: we
          // struct_extract a trivial value). Just put the end_borrow before the
          // terminator.
          if (auto *ti = dyn_cast<TermInst>(inst)) {
            if (ti->isFunctionExiting() && operand.getOperandOwnership() == OperandOwnership::TrivialUse) {
              SILBuilderWithScope endBuilder(inst);
              endBuilder.createEndBorrow(getSafeLoc(inst), borrow);
              continue;
            }
          }

          auto *nextInst = inst->getNextInstruction();
          SILBuilderWithScope endBuilder(nextInst);
          endBuilder.createEndBorrow(getSafeLoc(nextInst), borrow);
          continue;
        }

        // If we do a consuming use though, we need to destructure and then
        // update our available value array.
        LLVM_DEBUG(
            llvm::dbgs()
            << "    Consuming Operand! Extracting using destructures!\n");
        SILBuilderWithScope consumeBuilder(inst, &createdDestructures);
        auto loc = getSafeLoc(inst);
        auto iterOffsetSize = firstValueOffsetSize;
        SILValue iterValue = first;
        SILType iterType = iterValue->getType();
        SWIFT_DEFER { typeSpanToValue.clear(); };

        SILType unwrappedOperandType =
            operand.get()->getType().removingMoveOnlyWrapper();
        while (unwrappedOperandType != iterType.removingMoveOnlyWrapper()) {
          // NOTE: We purposely do not erase our parent offset from the
          // typeSpanToValue. We never insert any element along our walk path
          // (including the initial value) into the interval map.
          auto parentOffsetSize = iterOffsetSize;

          // Then walk one level towards our target type.
          std::tie(iterOffsetSize, iterType) =
              *useOffsetSize.walkOneLevelTowardsChild(parentOffsetSize,
                                                      iterType, fn);

          unsigned start = parentOffsetSize.startOffset;
          consumeBuilder.emitDestructureValueOperation(
              loc, iterValue, [&](unsigned index, SILValue value) {
                unsigned childSize = TypeSubElementCount(value);
                // If we found our value, then stash it into iter value. We are
                // going to consume it directly here.
                if (start == iterOffsetSize.startOffset) {
                  iterValue = value;
                } else {
                  // Otherwise, add it to the type span to value array so we can
                  // update our available values as appropriate.
                  typeSpanToValue.insert(start, start + childSize, value);
                }
                start += childSize;
              });
        }

        // Now that we have finished destructuring, set operand to our iter
        // value... unwrapping iterValue if we need to do so.
        if (iterValue->getType().isMoveOnlyWrapped() &&
            !operand.get()->getType().isMoveOnlyWrapped()) {
          iterValue = consumeBuilder.createOwnedMoveOnlyWrapperToCopyableValue(
              loc, iterValue);
        }
        operand.set(iterValue);

        // Then go through our available values and use the interval map to
        // update them with the destructured values if we have one for it.
        for (unsigned i : firstValueOffsetSize.getRange()) {
          // NOTE: IntervalMap.find returns the first interval that ends /after/
          // i. This means we need to treat a found iterator that doesn't
          // contain i to be a fail.
          auto iter = typeSpanToValue.find(i);
          if (iter == typeSpanToValue.end() || iter.start() > i ||
              iter.stop() <= i)
            availableValues[i] = SILValue();
          else
            availableValues[i] = iter.value();
        }
        LLVM_DEBUG(llvm::dbgs()
                       << "    Available values after destructuring:\n";
                   availableValues.print(llvm::dbgs(), "        "));
      }
    }

    LLVM_DEBUG(llvm::dbgs() << "Finished visiting/rewriting uses for block bb"
                            << block->getDebugID() << '\n');
  }
}

static void addCompensatingDestroys(SSAPrunedLiveness &liveness,
                                    PrunedLivenessBoundary &boundary,
                                    SILValue value) {
  InstructionSet consumingInsts(value->getFunction());
  liveness.initializeDef(value);
  for (auto *use : value->getUses()) {
    if (use->isConsuming())
      consumingInsts.insert(use->getUser());
    liveness.updateForUse(use->getUser(), use->isConsuming());
    if (auto *bbi = dyn_cast<BeginBorrowInst>(use->getUser())) {
      for (auto *ebi : bbi->getEndBorrows()) {
        liveness.updateForUse(ebi, false /*use is consuming*/);
      }
    }
  }
  liveness.computeBoundary(boundary);
  for (auto *user : boundary.lastUsers) {
    // If this is a consuming inst, just continue.
    if (consumingInsts.contains(user))
      continue;
    // Otherwise, we need to insert a destroy_value afterwards.
    auto *next = user->getNextInstruction();
    SILBuilderWithScope builder(next);
    builder.createDestroyValue(getSafeLoc(next), value);
  }

  // Insert destroy_value along all boundary edges.
  for (auto *edge : boundary.boundaryEdges) {
    SILBuilderWithScope builder(edge->begin());
    builder.createDestroyValue(getSafeLoc(&*edge->begin()), value);
  }

  // If we have a dead def, insert the destroy_value immediately at the def.
  for (auto *deadDef : boundary.deadDefs) {
    SILInstruction *nextInst = nullptr;
    if (auto *inst = dyn_cast<SILInstruction>(deadDef)) {
      nextInst = inst->getNextInstruction();
    } else if (auto *arg = dyn_cast<SILArgument>(deadDef)) {
      nextInst = arg->getNextInstruction();
    } else {
      llvm_unreachable("Unhandled dead def?!");
    }
    SILBuilderWithScope builder(nextInst);
    builder.createDestroyValue(getSafeLoc(nextInst), value);
  }
}

void BorrowToDestructureTransform::cleanup(
    StackList<BeginBorrowInst *> &borrowWorklist) {
  // First clean up all of our borrows/copies/struct_extracts which no longer
  // have any uses...
  InstructionDeleter deleter;
  while (!borrowWorklist.empty()) {
    deleter.recursivelyForceDeleteUsersAndFixLifetimes(
        borrowWorklist.pop_back_val());
  }

  // Then add destroys for any destructure elements that we inserted that we did
  // not actually completely consume.
  auto *fn = mmci->getFunction();
  SmallVector<SILBasicBlock *, 8> discoveredBlocks;
  SSAPrunedLiveness liveness(&discoveredBlocks);
  PrunedLivenessBoundary boundary;
  while (!createdDestructures.empty()) {
    auto *inst = createdDestructures.pop_back_val();
    assert(isa<DestructureStructInst>(inst) || isa<DestructureTupleInst>(inst));
    for (auto result : inst->getResults()) {
      if (result->getType().isTrivial(*fn))
        continue;
      SWIFT_DEFER {
        liveness.clear();
        discoveredBlocks.clear();
        boundary.clear();
      };
      addCompensatingDestroys(liveness, boundary, result);
    }
  }

  // Then do this for our inserted phis.
  while (!createdPhiArguments.empty()) {
    auto *arg = createdPhiArguments.pop_back_val();

    // If we have a trivial argument, we do not ened to add any compensating
    // destroys.
    if (arg->getType().isTrivial(*fn))
      continue;

    SWIFT_DEFER {
      liveness.clear();
      discoveredBlocks.clear();
      boundary.clear();
    };
    addCompensatingDestroys(liveness, boundary, arg);
  }

  // And finally do the same thing for our initial copy_value.
  addCompensatingDestroys(liveness, boundary, initialValue);
}