[SourceKit] Support cancellation of requests while an AST is being built

This commit refactors the way ASTs are being built in SourceKit and how `SwiftASTConsumer`s are served by the built ASTs. `SwiftASTManager.h` should give an overview of the new design.

This commit does not change the cancellation paradigm in SourceKit (yet). That is, subsequent requests with the same `OncePerASTToken` still cancel previous requests with the same token. But while previously, we were only able to cancel requests that haven’t started an AST build yet, we can now also cancel the AST build of the to-be-cancelled requests.

With this change in place, we can start looking into explicit cancellation of requests or other cancellation paradigms.

Author: ahoppen

include/swift/Basic/LangOptions.h

@@ -599,7 +599,7 @@ namespace swift {
     /// typechecking should be aborted at the next possible opportunity.
     /// This is used by SourceKit to cancel requests for which the result is no
     /// longer of interest.
-    std::shared_ptr<bool> CancellationFlag = nullptr;
+    std::shared_ptr<std::atomic<bool>> CancellationFlag = nullptr;
 
     /// If non-zero, abort the switch statement exhaustiveness checker if
     /// the Space::minus function is called more than this many times.

include/swift/Sema/ConstraintSystem.h

@@ -5156,7 +5156,7 @@ class ConstraintSystem {
       return true;
 
     auto CancellationFlag = getASTContext().TypeCheckerOpts.CancellationFlag;
-    if (CancellationFlag && *CancellationFlag)
+    if (CancellationFlag && CancellationFlag->load(std::memory_order_relaxed))
       return true;
 
     auto used = getASTContext().getSolverMemory() + solutionMemory;

test/SourceKit/Misc/stats.swift

@@ -46,8 +46,8 @@ func foo() {}
 // SEMA_3: 1 {{.*}} source.statistic.num-ast-builds
 // SEMA_3: 1 {{.*}} source.statistic.num-asts-in-memory
 // SEMA_3: 1 {{.*}} source.statistic.max-asts-in-memory
-// SEMA_3: 0 {{.*}} source.statistic.num-ast-cache-hits
-// SEMA_3: 1 {{.*}} source.statistic.num-ast-snaphost-uses
+// SEMA_3: 1 {{.*}} source.statistic.num-ast-cache-hits
+// SEMA_3: 0 {{.*}} source.statistic.num-ast-snaphost-uses
 
 // RUN: %sourcekitd-test -req=sema %s -- -Xfrontend -disable-implicit-concurrency-module-import  %s == -req=related-idents -pos=1:6 %s -- -Xfrontend -disable-implicit-concurrency-module-import  %s == -req=stats | %FileCheck %s -check-prefix=SEMA_4
 
@@ -58,3 +58,14 @@ func foo() {}
 // SEMA_4: 1 {{.*}} source.statistic.max-asts-in-memory
 // SEMA_4: 1 {{.*}} source.statistic.num-ast-cache-hits
 // SEMA_4: 0 {{.*}} source.statistic.num-ast-snaphost-uses
+
+// Test that we can have two files open and don't need to rebuild an AST when doing the cursor info request on '%s' after opening '10bytes.swift'
+// RUN: %sourcekitd-test \
+// RUN: -req=sema %s -- -Xfrontend -disable-implicit-concurrency-module-import  %s == \
+// RUN: -req=sema %S/Inputs/10bytes.swift -- -Xfrontend -disable-implicit-concurrency-module-import %S/Inputs/10bytes.swift == \
+// RUN: -req=cursor -pos=1:6 %s -- -Xfrontend -disable-implicit-concurrency-module-import  %s == \
+// RUN: -req=stats | %FileCheck %s -check-prefix=OPEN_TWO_FILES
+
+// OPEN_TWO_FILES: 2 {{.*}} source.statistic.num-ast-builds
+// OPEN_TWO_FILES: 2 {{.*}} source.statistic.num-asts-in-memory
+// OPEN_TWO_FILES: 2 {{.*}} source.statistic.max-asts-in-memory