@@ -472,7 +472,7 @@ class DestroyLocalVariable : public Cleanup {
472472 Var->print(llvm::errs());
473473 llvm::errs() << "\n";
474474 if (isActive()) {
475- auto loc = SGF.VarLocs[Var];
475+ auto & loc = SGF.VarLocs[Var];
476476 assert((loc.box || loc.value) && "One of box or value should be set");
477477 if (loc.box) {
478478 llvm::errs() << "Box: " << loc.box << "\n";
@@ -664,7 +664,8 @@ class LocalVariableInitialization : public SingleBufferInitialization {
664664 /// decl to.
665665 assert(SGF.VarLocs.count(decl) == 0 && "Already emitted the local?");
666666
667- SGF.VarLocs[decl] = SILGenFunction::VarLoc::get(Addr, Box);
667+ SGF.VarLocs[decl] = SILGenFunction::VarLoc(Addr,
668+ SILAccessEnforcement::Dynamic, Box);
668669
669670 SingleBufferInitialization::finishInitialization(SGF);
670671 assert(!DidFinish &&
@@ -677,6 +678,54 @@ class LocalVariableInitialization : public SingleBufferInitialization {
677678} // end anonymous namespace
678679
679680namespace {
681+
682+ static void deallocateAddressable(SILGenFunction &SGF,
683+ SILLocation l,
684+ const SILGenFunction::VarLoc::AddressableBuffer::State &state) {
685+ SGF.B.createEndBorrow(l, state.storeBorrow);
686+ SGF.B.createDeallocStack(l, state.allocStack);
687+ if (state.reabstraction) {
688+ SGF.B.createDestroyValue(l, state.reabstraction);
689+ }
690+ }
691+
692+ /// Cleanup to deallocate the addressable buffer for a parameter or let
693+ /// binding.
694+ class DeallocateLocalVariableAddressableBuffer : public Cleanup {
695+ ValueDecl *vd;
696+ public:
697+ DeallocateLocalVariableAddressableBuffer(ValueDecl *vd) : vd(vd) {}
698+
699+ void emit(SILGenFunction &SGF, CleanupLocation l,
700+ ForUnwind_t forUnwind) override {
701+ auto found = SGF.VarLocs.find(vd);
702+ if (found == SGF.VarLocs.end()) {
703+ return;
704+ }
705+ auto &loc = found->second;
706+
707+ if (auto &state = loc.addressableBuffer.state) {
708+ // The addressable buffer was forced, so clean it up now.
709+ deallocateAddressable(SGF, l, *state);
710+ } else {
711+ // Remember this insert location in case we need to force the addressable
712+ // buffer later.
713+ SILInstruction *marker = SGF.B.createTuple(l, {});
714+ loc.addressableBuffer.cleanupPoints.emplace_back(marker);
715+ }
716+ }
717+
718+ void dump(SILGenFunction &SGF) const override {
719+ #ifndef NDEBUG
720+ llvm::errs() << "DeallocateLocalVariableAddressableBuffer\n"
721+ << "State:" << getState() << "\n"
722+ << "Decl: ";
723+ vd->print(llvm::errs());
724+ llvm::errs() << "\n";
725+ #endif
726+ }
727+ };
728+
680729/// Initialize a writeback buffer that receives the value of a 'let'
681730/// declaration.
682731class LetValueInitialization : public Initialization {
@@ -755,7 +804,8 @@ class LetValueInitialization : public Initialization {
755804 if (isUninitialized)
756805 address = SGF.B.createMarkUninitializedVar(vd, address);
757806 DestroyCleanup = SGF.enterDormantTemporaryCleanup(address, *lowering);
758- SGF.VarLocs[vd] = SILGenFunction::VarLoc::get(address);
807+ SGF.VarLocs[vd] = SILGenFunction::VarLoc(address,
808+ SILAccessEnforcement::Unknown);
759809 }
760810 // Push a cleanup to destroy the let declaration. This has to be
761811 // inactive until the variable is initialized: if control flow exits the
@@ -766,6 +816,10 @@ class LetValueInitialization : public Initialization {
766816 SGF.Cleanups.pushCleanupInState<DestroyLocalVariable>(
767817 CleanupState::Dormant, vd);
768818 DestroyCleanup = SGF.Cleanups.getTopCleanup();
819+
820+ // If the binding has an addressable buffer forced, it should be cleaned
821+ // up here.
822+ SGF.enterLocalVariableAddressableBufferScope(vd);
769823 }
770824
771825 ~LetValueInitialization() override {
@@ -883,7 +937,8 @@ class LetValueInitialization : public Initialization {
883937 if (SGF.getASTContext().SILOpts.supportsLexicalLifetimes(SGF.getModule()))
884938 value = getValueForLexicalLifetimeBinding(SGF, loc, value, wasPlusOne);
885939
886- SGF.VarLocs[vd] = SILGenFunction::VarLoc::get(value);
940+ SGF.VarLocs[vd] = SILGenFunction::VarLoc(value,
941+ SILAccessEnforcement::Unknown);
887942
888943 // Emit a debug_value[_addr] instruction to record the start of this value's
889944 // lifetime, if permitted to do so.
@@ -1463,7 +1518,7 @@ SILGenFunction::emitInitializationForVarDecl(VarDecl *vd, bool forceImmutable,
14631518 assert(SILDebugClient && "Debugger client doesn't support SIL");
14641519 SILValue SV = SILDebugClient->emitLValueForVariable(vd, B);
14651520
1466- VarLocs[vd] = SILGenFunction:: VarLoc::get (SV);
1521+ VarLocs[vd] = VarLoc(SV, SILAccessEnforcement::Dynamic );
14671522 return InitializationPtr(new KnownAddressInitialization(SV));
14681523 }
14691524
@@ -1494,7 +1549,7 @@ SILGenFunction::emitInitializationForVarDecl(VarDecl *vd, bool forceImmutable,
14941549 if (isUninitialized)
14951550 addr = B.createMarkUninitializedVar(loc, addr);
14961551
1497- VarLocs[vd] = SILGenFunction:: VarLoc::get (addr);
1552+ VarLocs[vd] = VarLoc(addr, SILAccessEnforcement::Dynamic );
14981553 Result = InitializationPtr(new KnownAddressInitialization(addr));
14991554 } else {
15001555 std::optional<MarkUninitializedInst::Kind> uninitKind;
@@ -2309,11 +2364,9 @@ void SILGenFunction::destroyLocalVariable(SILLocation silLoc, VarDecl *vd) {
23092364 }
23102365 };
23112366
2312- auto loc = VarLocs[vd];
2313-
23142367 // For a heap variable, the box is responsible for the value. We just need
23152368 // to give up our retain count on it.
2316- if (auto boxValue = loc .box) {
2369+ if (auto boxValue = VarLocs[vd] .box) {
23172370 if (!getASTContext().SILOpts.supportsLexicalLifetimes(getModule())) {
23182371 emitDestroy(boxValue);
23192372 return;
@@ -2329,7 +2382,7 @@ void SILGenFunction::destroyLocalVariable(SILLocation silLoc, VarDecl *vd) {
23292382
23302383 // For 'let' bindings, we emit a release_value or destroy_addr, depending on
23312384 // whether we have an address or not.
2332- SILValue Val = loc .value;
2385+ SILValue Val = VarLocs[vd] .value;
23332386
23342387 if (Val->getType().isAddress()) {
23352388 B.createDestroyAddr(silLoc, Val);
@@ -2406,6 +2459,108 @@ void SILGenFunction::destroyLocalVariable(SILLocation silLoc, VarDecl *vd) {
24062459 llvm_unreachable("unhandled case");
24072460}
24082461
2462+ void
2463+ SILGenFunction::enterLocalVariableAddressableBufferScope(VarDecl *decl) {
2464+ Cleanups.pushCleanup<DeallocateLocalVariableAddressableBuffer>(decl);
2465+ }
2466+
2467+ SILValue
2468+ SILGenFunction::getLocalVariableAddressableBuffer(VarDecl *decl,
2469+ SILLocation curLoc,
2470+ ValueOwnership ownership) {
2471+ auto foundVarLoc = VarLocs.find(decl);
2472+ if (foundVarLoc == VarLocs.end()) {
2473+ return SILValue();
2474+ }
2475+
2476+ auto value = foundVarLoc->second.value;
2477+ auto access = foundVarLoc->second.access;
2478+ auto *state = foundVarLoc->second.addressableBuffer.state.get();
2479+
2480+ SILType fullyAbstractedTy = getLoweredType(AbstractionPattern::getOpaque(),
2481+ decl->getTypeInContext()->getRValueType());
2482+
2483+ // Check whether the bound value is inherently suitable for addressability.
2484+ // It must already be in memory and fully abstracted.
2485+ if (value->getType().isAddress()
2486+ && fullyAbstractedTy.getASTType() == value->getType().getASTType()) {
2487+ SILValue address = value;
2488+ // Begin an access if the address is mutable.
2489+ if (access != SILAccessEnforcement::Unknown) {
2490+ address = B.emitBeginAccess(curLoc, address,
2491+ ownership == ValueOwnership::InOut ? SILAccessKind::Modify
2492+ : SILAccessKind::Read,
2493+ access);
2494+ }
2495+ return address;
2496+ }
2497+
2498+ // We can't retroactively introduce a reabstracted representation for a
2499+ // mutable binding (since we would now have two mutable memory locations
2500+ // representing the same value).
2501+ if (access != SILAccessEnforcement::Unknown) {
2502+ return SILValue();
2503+ }
2504+
2505+ assert(ownership == ValueOwnership::Shared);
2506+
2507+ // Check whether the in-memory representation has already been forced.
2508+ if (state) {
2509+ return state->storeBorrow;
2510+ }
2511+
2512+ // Otherwise, force the addressable representation.
2513+ SILValue reabstraction, allocStack, storeBorrow;
2514+ {
2515+ SavedInsertionPointRAII save(B);
2516+ B.setInsertionPoint(value->getNextInstruction());
2517+ auto declarationLoc = value->getDefiningInsertionPoint()->getLoc();
2518+
2519+ // Reabstract if necessary.
2520+ auto newValue = value;
2521+ reabstraction = SILValue();
2522+ if (newValue->getType().getASTType() != fullyAbstractedTy.getASTType()){
2523+ auto reabstracted = emitSubstToOrigValue(curLoc,
2524+ ManagedValue::forBorrowedRValue(value),
2525+ AbstractionPattern::getOpaque(),
2526+ decl->getTypeInContext()->getCanonicalType(),
2527+ SGFContext());
2528+ reabstraction = reabstracted.forward(*this);
2529+ newValue = reabstraction;
2530+ }
2531+ // TODO: reabstract
2532+ allocStack = B.createAllocStack(declarationLoc, newValue->getType(),
2533+ std::nullopt,
2534+ DoesNotHaveDynamicLifetime,
2535+ IsNotLexical,
2536+ IsNotFromVarDecl,
2537+ DoesNotUseMoveableValueDebugInfo,
2538+ /*skipVarDeclAssert*/ true);
2539+ storeBorrow = B.createStoreBorrow(declarationLoc, newValue, allocStack);
2540+ }
2541+
2542+ // Record the addressable representation.
2543+ auto &addressableBuffer = VarLocs[decl].addressableBuffer;
2544+ addressableBuffer.state
2545+ = std::make_unique<VarLoc::AddressableBuffer::State>(reabstraction,
2546+ allocStack,
2547+ storeBorrow);
2548+ auto *newState = addressableBuffer.state.get();
2549+
2550+ // Emit cleanups on any paths where we previously would have cleaned up
2551+ // the addressable representation if it had been forced earlier.
2552+ decltype(addressableBuffer.cleanupPoints) cleanupPoints;
2553+ cleanupPoints.swap(addressableBuffer.cleanupPoints);
2554+
2555+ for (SILInstruction *cleanupPoint : cleanupPoints) {
2556+ SavedInsertionPointRAII insertCleanup(B, cleanupPoint);
2557+ deallocateAddressable(*this, cleanupPoint->getLoc(), *newState);
2558+ cleanupPoint->eraseFromParent();
2559+ }
2560+
2561+ return storeBorrow;
2562+ }
2563+
24092564void BlackHoleInitialization::performPackExpansionInitialization(
24102565 SILGenFunction &SGF,
24112566 SILLocation loc,
@@ -2437,3 +2592,9 @@ void BlackHoleInitialization::copyOrInitValueInto(SILGenFunction &SGF, SILLocati
24372592 value = SGF.B.createMoveValue(loc, value);
24382593 SGF.B.createIgnoredUse(loc, value.getValue());
24392594}
2595+
2596+ SILGenFunction::VarLoc::AddressableBuffer::~AddressableBuffer() {
2597+ for (auto cleanupPoint : cleanupPoints) {
2598+ cleanupPoint->eraseFromParent();
2599+ }
2600+ }
0 commit comments