Skip to content

Commit 37f352f

Browse files
nkcsgexinkcsgexi
authored
sourcekitd: build Swift syntax tree more lazily than collecting parsed tokens. (#14578)
Before this patch, we have one flag (KeepSyntaxInfo) to turn on two syntax functionalities of parser: (1) collecting parsed tokens for coloring and (2) building syntax trees. Since sourcekitd is the only consumer of either of these functionalities, sourcekitd by default always enables such flag. However, empirical results show (2) is both heavier and less-frequently needed than (1). Therefore, separating the flag to two flags makes more sense, where CollectParsedToken controls (1) and BuildSyntaxTree controls (2). CollectingParsedToken is always enabled by sourcekitd because formatting and syntax-coloring need it; however BuildSyntaxTree should be explicitly switched on by sourcekitd clients. resolves: rdar://problem/37483076
1 parent cfca329 commit 37f352f

File tree

16 files changed

+64
-37
lines changed

16 files changed

+64
-37
lines changed

Diff for: ‎include/swift/AST/Module.h

+5-2
Original file line numberDiff line numberDiff line change
@@ -888,7 +888,8 @@ class SourceFile final : public FileUnit {
888888
ASTStage_t ASTStage = Parsing;
889889

890890
SourceFile(ModuleDecl &M, SourceFileKind K, Optional<unsigned> bufferID,
891-
ImplicitModuleImportKind ModImpKind, bool KeepSyntaxInfo);
891+
ImplicitModuleImportKind ModImpKind, bool KeepParsedTokens = false,
892+
bool KeepSyntaxTree = false);
892893

893894
void
894895
addImports(ArrayRef<std::pair<ModuleDecl::ImportedModule, ImportOptions>> IM);
@@ -1077,7 +1078,9 @@ class SourceFile final : public FileUnit {
10771078

10781079
ArrayRef<Token> getAllTokens() const;
10791080

1080-
bool shouldKeepSyntaxInfo() const;
1081+
bool shouldCollectToken() const;
1082+
1083+
bool shouldBuildSyntaxTree() const;
10811084

10821085
syntax::SourceFileSyntax getSyntaxRoot() const;
10831086
void setSyntaxRoot(syntax::SourceFileSyntax &&Root);

Diff for: ‎include/swift/Basic/LangOptions.h

+5-3
Original file line numberDiff line numberDiff line change
@@ -248,9 +248,11 @@ namespace swift {
248248
/// This is used to guard preemptive testing for the fix-it.
249249
bool FixStringToSubstringConversions = false;
250250

251-
/// Whether to create and keep track of a libSyntax tree associated with
252-
/// this source file.
253-
bool KeepSyntaxInfoInSourceFile = false;
251+
/// Whether collect tokens during parsing for syntax coloring.
252+
bool CollectParsedToken = false;
253+
254+
/// Whether to parse syntax tree.
255+
bool BuildSyntaxTree = false;
254256

255257
/// Whether to verify the parsed syntax tree and emit related diagnostics.
256258
bool VerifySyntaxTree = false;

Diff for: ‎include/swift/Parse/Parser.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -201,11 +201,11 @@ class Parser {
201201
Token Tok;
202202

203203
/// \brief leading trivias for \c Tok.
204-
/// Always empty if !SF.shouldKeepSyntaxInfo().
204+
/// Always empty if !SF.shouldBuildSyntaxTree().
205205
syntax::Trivia LeadingTrivia;
206206

207207
/// \brief trailing trivias for \c Tok.
208-
/// Always empty if !SF.shouldKeepSyntaxInfo().
208+
/// Always empty if !SF.shouldBuildSyntaxTree().
209209
syntax::Trivia TrailingTrivia;
210210

211211
/// \brief The receiver to collect all consumed tokens.

Diff for: ‎lib/AST/Module.cpp

+19-6
Original file line numberDiff line numberDiff line change
@@ -751,8 +751,10 @@ class SourceFile::Impl {
751751
};
752752

753753
struct SourceFile::SourceFileSyntaxInfo {
754+
const bool Enable;
754755
/// The root of the syntax tree representing the source file.
755756
Optional<syntax::SourceFileSyntax> SyntaxRoot;
757+
SourceFileSyntaxInfo(bool Enable): Enable(Enable) {}
756758
};
757759

758760
bool SourceFile::hasSyntaxRoot() const {
@@ -1328,10 +1330,10 @@ static void performAutoImport(
13281330
SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
13291331
Optional<unsigned> bufferID,
13301332
ImplicitModuleImportKind ModImpKind,
1331-
bool KeepSyntaxInfo)
1333+
bool KeepParsedTokens, bool BuildSyntaxTree)
13321334
: FileUnit(FileUnitKind::Source, M),
13331335
BufferID(bufferID ? *bufferID : -1),
1334-
Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo()) {
1336+
Kind(K), SyntaxInfo(*new SourceFileSyntaxInfo(BuildSyntaxTree)) {
13351337
M.getASTContext().addDestructorCleanup(*this);
13361338
performAutoImport(*this, ModImpKind);
13371339

@@ -1340,24 +1342,24 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
13401342
assert(!problem && "multiple main files?");
13411343
(void)problem;
13421344
}
1343-
if (KeepSyntaxInfo) {
1345+
if (KeepParsedTokens) {
13441346
AllCorrectedTokens = std::vector<Token>();
13451347
}
13461348
}
13471349

13481350
SourceFile::~SourceFile() { delete &SyntaxInfo; }
13491351

13501352
std::vector<Token> &SourceFile::getTokenVector() {
1351-
assert(shouldKeepSyntaxInfo() && "Disabled");
1353+
assert(shouldCollectToken() && "Disabled");
13521354
return *AllCorrectedTokens;
13531355
}
13541356

13551357
ArrayRef<Token> SourceFile::getAllTokens() const {
1356-
assert(shouldKeepSyntaxInfo() && "Disabled");
1358+
assert(shouldCollectToken() && "Disabled");
13571359
return *AllCorrectedTokens;
13581360
}
13591361

1360-
bool SourceFile::shouldKeepSyntaxInfo() const {
1362+
bool SourceFile::shouldCollectToken() const {
13611363
switch (Kind) {
13621364
case SourceFileKind::Library:
13631365
case SourceFileKind::Main:
@@ -1368,6 +1370,17 @@ bool SourceFile::shouldKeepSyntaxInfo() const {
13681370
}
13691371
}
13701372

1373+
bool SourceFile::shouldBuildSyntaxTree() const {
1374+
switch (Kind) {
1375+
case SourceFileKind::Library:
1376+
case SourceFileKind::Main:
1377+
return SyntaxInfo.Enable;
1378+
case SourceFileKind::REPL:
1379+
case SourceFileKind::SIL:
1380+
return false;
1381+
}
1382+
}
1383+
13711384
bool FileUnit::walk(ASTWalker &walker) {
13721385
SmallVector<Decl *, 64> Decls;
13731386
getTopLevelDecls(Decls);

Diff for: ‎lib/Frontend/Frontend.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -747,9 +747,10 @@ SourceFile *CompilerInstance::createSourceFileForMainModule(
747747
SourceFileKind fileKind, SourceFile::ImplicitModuleImportKind importKind,
748748
Optional<unsigned> bufferID) {
749749
ModuleDecl *mainModule = getMainModule();
750-
bool keepSyntaxInfo = Invocation.getLangOptions().KeepSyntaxInfoInSourceFile;
751750
SourceFile *inputFile = new (*Context)
752-
SourceFile(*mainModule, fileKind, bufferID, importKind, keepSyntaxInfo);
751+
SourceFile(*mainModule, fileKind, bufferID, importKind,
752+
Invocation.getLangOptions().CollectParsedToken,
753+
Invocation.getLangOptions().BuildSyntaxTree);
753754
MainModule->addFile(*inputFile);
754755

755756
if (bufferID && isPrimaryInput(*bufferID)) {

Diff for: ‎lib/FrontendTool/FrontendTool.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -838,7 +838,7 @@ static bool performCompile(CompilerInstance &Instance,
838838
FrontendOptions::ActionType Action = opts.RequestedAction;
839839

840840
if (Action == FrontendOptions::ActionType::EmitSyntax) {
841-
Instance.getASTContext().LangOpts.KeepSyntaxInfoInSourceFile = true;
841+
Instance.getASTContext().LangOpts.BuildSyntaxTree = true;
842842
Instance.getASTContext().LangOpts.VerifySyntaxTree = true;
843843
}
844844

Diff for: ‎lib/Parse/Parser.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
336336
SF.getASTContext().LangOpts.AttachCommentsToDecls
337337
? CommentRetentionMode::AttachToNextToken
338338
: CommentRetentionMode::None,
339-
SF.shouldKeepSyntaxInfo()
339+
SF.shouldBuildSyntaxTree()
340340
? TriviaRetentionMode::WithTrivia
341341
: TriviaRetentionMode::WithoutTrivia)),
342342
SF, SIL, PersistentState) {}
@@ -465,7 +465,7 @@ Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
465465
SIL(SIL),
466466
CurDeclContext(&SF),
467467
Context(SF.getASTContext()),
468-
TokReceiver(SF.shouldKeepSyntaxInfo() ?
468+
TokReceiver(SF.shouldCollectToken() ?
469469
new TokenRecorder(SF) :
470470
new ConsumeTokenReceiver()),
471471
SyntaxContext(new SyntaxParsingContext(SyntaxContext, SF,
@@ -982,7 +982,8 @@ struct ParserUnit::Implementation {
982982
*ModuleDecl::create(Ctx.getIdentifier(ModuleName), Ctx),
983983
SourceFileKind::Main, BufferID,
984984
SourceFile::ImplicitModuleImportKind::None,
985-
Opts.KeepSyntaxInfoInSourceFile)) {
985+
Opts.CollectParsedToken,
986+
Opts.BuildSyntaxTree)) {
986987
}
987988
};
988989

Diff for: ‎lib/Parse/SyntaxParsingContext.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ SyntaxParsingContext::SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder,
3939
BufferID)),
4040
CtxtHolder(CtxtHolder), Arena(SF.getASTContext().getSyntaxArena()),
4141
Storage(getRootData().Storage), Offset(0), Mode(AccumulationMode::Root),
42-
Enabled(SF.shouldKeepSyntaxInfo()) {
42+
Enabled(SF.shouldBuildSyntaxTree()) {
4343
CtxtHolder = this;
4444
Storage.reserve(128);
4545
}

Diff for: ‎lib/Sema/SourceLoader.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,8 @@ ModuleDecl *SourceLoader::loadModule(SourceLoc importLoc,
132132

133133
auto *importFile = new (Ctx) SourceFile(*importMod, SourceFileKind::Library,
134134
bufferID, implicitImportKind,
135-
Ctx.LangOpts.KeepSyntaxInfoInSourceFile);
135+
Ctx.LangOpts.CollectParsedToken,
136+
Ctx.LangOpts.BuildSyntaxTree);
136137
importMod->addFile(*importFile);
137138

138139
bool done;

Diff for: ‎tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ bool SwiftASTManager::initCompilerInvocation(CompilerInvocation &Invocation,
442442
Invocation.setSerializedDiagnosticsPath(StringRef());
443443
Invocation.getLangOptions().AttachCommentsToDecls = true;
444444
Invocation.getLangOptions().DiagnosticsEditorMode = true;
445-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
445+
Invocation.getLangOptions().CollectParsedToken = true;
446446
auto &FrontendOpts = Invocation.getFrontendOptions();
447447
if (FrontendOpts.PlaygroundTransform) {
448448
// The playground instrumenter changes the AST in ways that disrupt the
@@ -817,7 +817,7 @@ ASTUnitRef ASTProducer::createASTUnit(SwiftASTManager::Implementation &MgrImpl,
817817
InvokRef->Impl.Opts.applyToSubstitutingInputs(
818818
Invocation, convertFileContentsToInputs(Contents));
819819

820-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
820+
Invocation.getLangOptions().CollectParsedToken = true;
821821

822822
if (CompIns.setup(Invocation)) {
823823
// FIXME: Report the diagnostic.

Diff for: ‎tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

+6-6
Original file line numberDiff line numberDiff line change
@@ -1753,7 +1753,7 @@ void SwiftEditorDocument::updateSemaInfo() {
17531753
}
17541754

17551755
void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
1756-
SwiftLangSupport &Lang) {
1756+
SwiftLangSupport &Lang, bool BuildSyntexTree) {
17571757
llvm::sys::ScopedLock L(Impl.AccessMtx);
17581758

17591759
assert(Impl.SemanticInfo && "Impl.SemanticInfo must be set");
@@ -1773,7 +1773,7 @@ void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
17731773
Lang.getASTManager().
17741774
initCompilerInvocation(CompInv, Args, StringRef(), Error);
17751775
}
1776-
1776+
CompInv.getLangOptions().BuildSyntaxTree = BuildSyntexTree;
17771777
// Access to Impl.SyntaxInfo is guarded by Impl.AccessMtx
17781778
Impl.SyntaxInfo.reset(
17791779
new SwiftDocumentSyntaxInfo(CompInv, Snapshot, Args, Impl.FilePath));
@@ -2095,12 +2095,12 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
20952095
ArrayRef<const char *> Args) {
20962096

20972097
ImmutableTextSnapshotRef Snapshot = nullptr;
2098-
2098+
const bool BuildSyntaxTree = Consumer.syntaxTreeEnabled();
20992099
auto EditorDoc = EditorDocuments.getByUnresolvedName(Name);
21002100
if (!EditorDoc) {
21012101
EditorDoc = new SwiftEditorDocument(Name, *this);
21022102
Snapshot = EditorDoc->initializeText(Buf, Args);
2103-
EditorDoc->parse(Snapshot, *this);
2103+
EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
21042104
if (EditorDocuments.getOrUpdate(Name, *this, EditorDoc)) {
21052105
// Document already exists, re-initialize it. This should only happen
21062106
// if we get OPEN request while the previous document is not closed.
@@ -2113,7 +2113,7 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
21132113

21142114
if (!Snapshot) {
21152115
Snapshot = EditorDoc->initializeText(Buf, Args);
2116-
EditorDoc->parse(Snapshot, *this);
2116+
EditorDoc->parse(Snapshot, *this, BuildSyntaxTree);
21172117
}
21182118

21192119
if (Consumer.needsSemanticInfo()) {
@@ -2161,7 +2161,7 @@ void SwiftLangSupport::editorReplaceText(StringRef Name, llvm::MemoryBuffer *Buf
21612161
Snapshot = EditorDoc->replaceText(Offset, Length, Buf,
21622162
Consumer.needsSemanticInfo());
21632163
assert(Snapshot);
2164-
EditorDoc->parse(Snapshot, *this);
2164+
EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
21652165
EditorDoc->readSyntaxInfo(Consumer);
21662166
} else {
21672167
Snapshot = EditorDoc->getLatestSnapshot();

Diff for: ‎tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ class SwiftEditorDocument :
9191

9292
ImmutableTextSnapshotRef getLatestSnapshot() const;
9393

94-
void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang);
94+
void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang,
95+
bool BuildSyntaxTree);
9596
void readSyntaxInfo(EditorConsumer& consumer);
9697
void readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
9798
EditorConsumer& Consumer);

Diff for: ‎tools/driver/swift_format_main.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ class FormatterDocument {
5555
public:
5656
FormatterDocument(std::unique_ptr<llvm::MemoryBuffer> Buffer) {
5757
// Formatting logic requires tokens on source file.
58-
CompInv.getLangOptions().KeepSyntaxInfoInSourceFile = true;
58+
CompInv.getLangOptions().CollectParsedToken = true;
5959
updateCode(std::move(Buffer));
6060
}
6161

Diff for: ‎tools/swift-ide-test/swift-ide-test.cpp

+8-4
Original file line numberDiff line numberDiff line change
@@ -965,7 +965,8 @@ static int doSyntaxColoring(const CompilerInvocation &InitInvok,
965965
PrintingDiagnosticConsumer PrintDiags;
966966
CI.addDiagnosticConsumer(&PrintDiags);
967967
Invocation.getLangOptions().Playground = Playground;
968-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
968+
Invocation.getLangOptions().CollectParsedToken = true;
969+
Invocation.getLangOptions().BuildSyntaxTree = true;
969970
if (CI.setup(Invocation))
970971
return 1;
971972
if (!RunTypeChecker)
@@ -1178,7 +1179,8 @@ class StructureAnnotator : public ide::SyntaxModelWalker {
11781179
static int doStructureAnnotation(const CompilerInvocation &InitInvok,
11791180
StringRef SourceFilename) {
11801181
CompilerInvocation Invocation(InitInvok);
1181-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
1182+
Invocation.getLangOptions().BuildSyntaxTree = true;
1183+
Invocation.getLangOptions().CollectParsedToken = true;
11821184
Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(SourceFilename);
11831185

11841186
CompilerInstance CI;
@@ -2695,7 +2697,8 @@ static int doPrintRangeInfo(const CompilerInvocation &InitInvok,
26952697
CompilerInvocation Invocation(InitInvok);
26962698
Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(SourceFileName);
26972699
Invocation.getLangOptions().DisableAvailabilityChecking = false;
2698-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
2700+
Invocation.getLangOptions().BuildSyntaxTree = true;
2701+
Invocation.getLangOptions().CollectParsedToken = true;
26992702

27002703
CompilerInstance CI;
27012704

@@ -3017,7 +3020,8 @@ int main(int argc, char *argv[]) {
30173020
InitInvok.setModuleName(options::ModuleName);
30183021

30193022
InitInvok.setSDKPath(options::SDK);
3020-
InitInvok.getLangOptions().KeepSyntaxInfoInSourceFile = true;
3023+
InitInvok.getLangOptions().CollectParsedToken = true;
3024+
InitInvok.getLangOptions().BuildSyntaxTree = true;
30213025
if (options::DisableObjCInterop) {
30223026
InitInvok.getLangOptions().EnableObjCInterop = false;
30233027
} else if (options::EnableObjCInterop) {

Diff for: ‎tools/swift-refactor/swift-refactor.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,8 @@ int main(int argc, char *argv[]) {
236236
Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(
237237
options::SourceFilename);
238238
Invocation.getLangOptions().AttachCommentsToDecls = true;
239-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
239+
Invocation.getLangOptions().CollectParsedToken = true;
240+
Invocation.getLangOptions().BuildSyntaxTree = true;
240241

241242
for (auto FileName : options::InputFilenames)
242243
Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(FileName);

Diff for: ‎tools/swift-syntax-test/swift-syntax-test.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ SourceFile *getSourceFile(CompilerInstance &Instance,
144144
StringRef InputFileName,
145145
const char *MainExecutablePath) {
146146
CompilerInvocation Invocation;
147-
Invocation.getLangOptions().KeepSyntaxInfoInSourceFile = true;
147+
Invocation.getLangOptions().BuildSyntaxTree = true;
148148
Invocation.getLangOptions().VerifySyntaxTree = true;
149149
Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(InputFileName);
150150
Invocation.setMainExecutablePath(

0 commit comments

Comments
 (0)