Skip to content
This repository was archived by the owner on Nov 1, 2021. It is now read-only.

Commit 4419b67

Browse files
DougGregorDougGregor
committed
Improve the implementation of libclang's token-annotation logic for
entities in the preprocessing record. Previously, we would only end up getting the first token of a preprocessing record annotated correctly. For example, given #include "foo.h" we would only get the '#' annotated as an inclusion directive; the 'include' and '"foo.h"' tokens would be given the general 'processing directive' annotation. Now, we get proper annotations for entities in the preprocessing record. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@117001 91177308-0d34-0410-b5e6-96231b3b80d8
1 parent 13f8cbf commit 4419b67

File tree

4 files changed

+120
-66
lines changed

4 files changed

+120
-66
lines changed

‎test/Index/annotate-tokens-include.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22

33
// RUN: c-index-test -test-annotate-tokens=%s:1:1:2:1 %s | FileCheck %s
44
// CHECK: Punctuation: "#" [1:1 - 1:2] inclusion directive=annotate-tokens-include.h
5-
// CHECK: Identifier: "include" [1:2 - 1:9] preprocessing directive=
6-
// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] preprocessing directive=
5+
// CHECK: Identifier: "include" [1:2 - 1:9] inclusion directive=annotate-tokens-include.h
6+
// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] inclusion directive=annotate-tokens-include.h
77

‎test/Index/annotate-tokens-pp.c

+48-48
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,27 @@ void test() {
2929
// CHECK: Punctuation: "#" [2:1 - 2:2] preprocessing directive=
3030
// CHECK: Identifier: "define" [2:2 - 2:8] preprocessing directive=
3131
// CHECK: Identifier: "STILL_NOTHING" [2:9 - 2:22] macro definition=STILL_NOTHING
32-
// CHECK: Identifier: "NOTHING" [2:23 - 2:30] preprocessing directive=
33-
// CHECK: Punctuation: "(" [2:30 - 2:31] preprocessing directive=
34-
// CHECK: Identifier: "honk" [2:31 - 2:35] preprocessing directive=
35-
// CHECK: Punctuation: "," [2:35 - 2:36] preprocessing directive=
36-
// CHECK: Identifier: "warble" [2:36 - 2:42] preprocessing directive=
37-
// CHECK: Punctuation: ")" [2:42 - 2:43] preprocessing directive=
32+
// CHECK: Identifier: "NOTHING" [2:23 - 2:30] macro definition=STILL_NOTHING
33+
// CHECK: Punctuation: "(" [2:30 - 2:31] macro definition=STILL_NOTHING
34+
// CHECK: Identifier: "honk" [2:31 - 2:35] macro definition=STILL_NOTHING
35+
// CHECK: Punctuation: "," [2:35 - 2:36] macro definition=STILL_NOTHING
36+
// CHECK: Identifier: "warble" [2:36 - 2:42] macro definition=STILL_NOTHING
37+
// CHECK: Punctuation: ")" [2:42 - 2:43] macro definition=STILL_NOTHING
3838
// CHECK: Punctuation: "#" [3:1 - 3:2] preprocessing directive=
3939
// CHECK: Identifier: "define" [3:2 - 3:8] preprocessing directive=
4040
// CHECK: Identifier: "BAR" [3:9 - 3:12] macro definition=BAR
41-
// CHECK: Identifier: "baz" [3:13 - 3:16] preprocessing directive=
41+
// CHECK: Identifier: "baz" [3:13 - 3:16] macro definition=BAR
4242
// CHECK: Punctuation: "#" [4:1 - 4:2] preprocessing directive=
4343
// CHECK: Identifier: "define" [4:2 - 4:8] preprocessing directive=
4444
// CHECK: Identifier: "WIBBLE" [4:9 - 4:15] macro definition=WIBBLE
45-
// CHECK: Punctuation: "(" [4:15 - 4:16] preprocessing directive=
46-
// CHECK: Identifier: "X" [4:16 - 4:17] preprocessing directive=
47-
// CHECK: Punctuation: "," [4:17 - 4:18] preprocessing directive=
48-
// CHECK: Identifier: "Y" [4:19 - 4:20] preprocessing directive=
49-
// CHECK: Punctuation: ")" [4:20 - 4:21] preprocessing directive=
50-
// CHECK: Identifier: "X" [4:22 - 4:23] preprocessing directive=
51-
// CHECK: Punctuation: "##" [4:23 - 4:25] preprocessing directive=
52-
// CHECK: Identifier: "Y" [4:25 - 4:26] preprocessing directive=
45+
// CHECK: Punctuation: "(" [4:15 - 4:16] macro definition=WIBBLE
46+
// CHECK: Identifier: "X" [4:16 - 4:17] macro definition=WIBBLE
47+
// CHECK: Punctuation: "," [4:17 - 4:18] macro definition=WIBBLE
48+
// CHECK: Identifier: "Y" [4:19 - 4:20] macro definition=WIBBLE
49+
// CHECK: Punctuation: ")" [4:20 - 4:21] macro definition=WIBBLE
50+
// CHECK: Identifier: "X" [4:22 - 4:23] macro definition=WIBBLE
51+
// CHECK: Punctuation: "##" [4:23 - 4:25] macro definition=WIBBLE
52+
// CHECK: Identifier: "Y" [4:25 - 4:26] macro definition=WIBBLE
5353
// CHECK: Identifier: "NOTHING" [5:1 - 5:8] macro instantiation=NOTHING:1:9
5454
// CHECK: Punctuation: "(" [5:8 - 5:9]
5555
// CHECK: Identifier: "more" [5:9 - 5:13]
@@ -69,31 +69,31 @@ void test() {
6969
// CHECK: Identifier: "STILL_NOTHING" [6:9 - 6:22] macro instantiation=STILL_NOTHING:2:9
7070
// CHECK: Punctuation: ";" [6:22 - 6:23]
7171
// CHECK: Punctuation: "#" [7:1 - 7:2] inclusion directive=foo.h
72-
// CHECK: Identifier: "include" [7:2 - 7:9] preprocessing directive=
73-
// CHECK: Literal: ""foo.h"" [7:10 - 7:17] preprocessing directive=
72+
// CHECK: Identifier: "include" [7:2 - 7:9] inclusion directive=foo.h
73+
// CHECK: Literal: ""foo.h"" [7:10 - 7:17] inclusion directive=foo.h
7474
// CHECK: Punctuation: "#" [8:1 - 8:2] preprocessing directive=
7575
// CHECK: Identifier: "undef" [8:2 - 8:7] preprocessing directive=
7676
// CHECK: Identifier: "BAR" [8:8 - 8:11] preprocessing directive=
7777
// CHECK: Punctuation: "#" [10:1 - 10:2] preprocessing directive=
7878
// CHECK: Identifier: "define" [10:2 - 10:8] preprocessing directive=
7979
// CHECK: Identifier: "REVERSE_MACRO" [10:9 - 10:22] macro definition=REVERSE_MACRO
80-
// CHECK: Punctuation: "(" [10:22 - 10:23] preprocessing directive=
81-
// CHECK: Identifier: "x" [10:23 - 10:24] preprocessing directive=
82-
// CHECK: Punctuation: "," [10:24 - 10:25] preprocessing directive=
83-
// CHECK: Identifier: "y" [10:25 - 10:26] preprocessing directive=
84-
// CHECK: Punctuation: ")" [10:26 - 10:27] preprocessing directive=
85-
// CHECK: Identifier: "y" [10:28 - 10:29] preprocessing directive=
86-
// CHECK: Punctuation: "+" [10:30 - 10:31] preprocessing directive=
87-
// CHECK: Identifier: "x" [10:32 - 10:33] preprocessing directive=
80+
// CHECK: Punctuation: "(" [10:22 - 10:23] macro definition=REVERSE_MACRO
81+
// CHECK: Identifier: "x" [10:23 - 10:24] macro definition=REVERSE_MACRO
82+
// CHECK: Punctuation: "," [10:24 - 10:25] macro definition=REVERSE_MACRO
83+
// CHECK: Identifier: "y" [10:25 - 10:26] macro definition=REVERSE_MACRO
84+
// CHECK: Punctuation: ")" [10:26 - 10:27] macro definition=REVERSE_MACRO
85+
// CHECK: Identifier: "y" [10:28 - 10:29] macro definition=REVERSE_MACRO
86+
// CHECK: Punctuation: "+" [10:30 - 10:31] macro definition=REVERSE_MACRO
87+
// CHECK: Identifier: "x" [10:32 - 10:33] macro definition=REVERSE_MACRO
8888
// CHECK: Punctuation: "#" [11:1 - 11:2] preprocessing directive=
8989
// CHECK: Identifier: "define" [11:2 - 11:8] preprocessing directive=
9090
// CHECK: Identifier: "TWICE_MACRO" [11:9 - 11:20] macro definition=TWICE_MACRO
91-
// CHECK: Punctuation: "(" [11:20 - 11:21] preprocessing directive=
92-
// CHECK: Identifier: "y" [11:21 - 11:22] preprocessing directive=
93-
// CHECK: Punctuation: ")" [11:22 - 11:23] preprocessing directive=
94-
// CHECK: Identifier: "y" [11:24 - 11:25] preprocessing directive=
95-
// CHECK: Punctuation: "+" [11:26 - 11:27] preprocessing directive=
96-
// CHECK: Identifier: "y" [11:28 - 11:29] preprocessing directive=
91+
// CHECK: Punctuation: "(" [11:20 - 11:21] macro definition=TWICE_MACRO
92+
// CHECK: Identifier: "y" [11:21 - 11:22] macro definition=TWICE_MACRO
93+
// CHECK: Punctuation: ")" [11:22 - 11:23] macro definition=TWICE_MACRO
94+
// CHECK: Identifier: "y" [11:24 - 11:25] macro definition=TWICE_MACRO
95+
// CHECK: Punctuation: "+" [11:26 - 11:27] macro definition=TWICE_MACRO
96+
// CHECK: Identifier: "y" [11:28 - 11:29] macro definition=TWICE_MACRO
9797
// CHECK: Keyword: "void" [13:1 - 13:5] FunctionDecl=test_macro_args:13:6 (Definition)
9898
// CHECK: Identifier: "test_macro_args" [13:6 - 13:21] FunctionDecl=test_macro_args:13:6 (Definition)
9999
// CHECK: Punctuation: "(" [13:21 - 13:22] FunctionDecl=test_macro_args:13:6 (Definition)
@@ -140,23 +140,23 @@ void test() {
140140
// CHECK: Punctuation: "#" [21:1 - 21:2] preprocessing directive=
141141
// CHECK: Identifier: "define" [21:2 - 21:8] preprocessing directive=
142142
// CHECK: Identifier: "fun_with_macro_bodies" [21:9 - 21:30] macro definition=fun_with_macro_bodies
143-
// CHECK: Punctuation: "(" [21:30 - 21:31] preprocessing directive=
144-
// CHECK: Identifier: "x" [21:31 - 21:32] preprocessing directive=
145-
// CHECK: Punctuation: "," [21:32 - 21:33] preprocessing directive=
146-
// CHECK: Identifier: "y" [21:34 - 21:35] preprocessing directive=
147-
// CHECK: Punctuation: ")" [21:35 - 21:36] preprocessing directive=
148-
// CHECK: Keyword: "do" [21:37 - 21:39] preprocessing directive=
149-
// CHECK: Punctuation: "{" [21:40 - 21:41] preprocessing directive=
150-
// CHECK: Keyword: "if" [21:42 - 21:44] preprocessing directive=
151-
// CHECK: Punctuation: "(" [21:45 - 21:46] preprocessing directive=
152-
// CHECK: Identifier: "x" [21:46 - 21:47] preprocessing directive=
153-
// CHECK: Punctuation: ")" [21:47 - 21:48] preprocessing directive=
154-
// CHECK: Identifier: "y" [21:49 - 21:50] preprocessing directive=
155-
// CHECK: Punctuation: "}" [21:51 - 21:52] preprocessing directive=
156-
// CHECK: Keyword: "while" [21:53 - 21:58] preprocessing directive=
157-
// CHECK: Punctuation: "(" [21:59 - 21:60] preprocessing directive=
158-
// CHECK: Literal: "0" [21:60 - 21:61] preprocessing directive=
159-
// CHECK: Punctuation: ")" [21:61 - 21:62] preprocessing directive=
143+
// CHECK: Punctuation: "(" [21:30 - 21:31] macro definition=fun_with_macro_bodies
144+
// CHECK: Identifier: "x" [21:31 - 21:32] macro definition=fun_with_macro_bodies
145+
// CHECK: Punctuation: "," [21:32 - 21:33] macro definition=fun_with_macro_bodies
146+
// CHECK: Identifier: "y" [21:34 - 21:35] macro definition=fun_with_macro_bodies
147+
// CHECK: Punctuation: ")" [21:35 - 21:36] macro definition=fun_with_macro_bodies
148+
// CHECK: Keyword: "do" [21:37 - 21:39] macro definition=fun_with_macro_bodies
149+
// CHECK: Punctuation: "{" [21:40 - 21:41] macro definition=fun_with_macro_bodies
150+
// CHECK: Keyword: "if" [21:42 - 21:44] macro definition=fun_with_macro_bodies
151+
// CHECK: Punctuation: "(" [21:45 - 21:46] macro definition=fun_with_macro_bodies
152+
// CHECK: Identifier: "x" [21:46 - 21:47] macro definition=fun_with_macro_bodies
153+
// CHECK: Punctuation: ")" [21:47 - 21:48] macro definition=fun_with_macro_bodies
154+
// CHECK: Identifier: "y" [21:49 - 21:50] macro definition=fun_with_macro_bodies
155+
// CHECK: Punctuation: "}" [21:51 - 21:52] macro definition=fun_with_macro_bodies
156+
// CHECK: Keyword: "while" [21:53 - 21:58] macro definition=fun_with_macro_bodies
157+
// CHECK: Punctuation: "(" [21:59 - 21:60] macro definition=fun_with_macro_bodies
158+
// CHECK: Literal: "0" [21:60 - 21:61] macro definition=fun_with_macro_bodies
159+
// CHECK: Punctuation: ")" [21:61 - 21:62] macro definition=fun_with_macro_bodies
160160
// CHECK: Keyword: "void" [23:1 - 23:5] FunctionDecl=test:23:6 (Definition)
161161
// CHECK: Identifier: "test" [23:6 - 23:10] FunctionDecl=test:23:6 (Definition)
162162
// CHECK: Punctuation: "(" [23:10 - 23:11] FunctionDecl=test:23:6 (Definition)

‎test/Index/annotate-tokens.m

+1-1
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ void f() {
285285
// CHECK: Punctuation: "#" [63:1 - 63:2] preprocessing directive=
286286
// CHECK: Identifier: "define" [63:2 - 63:8] preprocessing directive=
287287
// CHECK: Identifier: "VAL" [63:9 - 63:12] macro definition=VAL
288-
// CHECK: Literal: "0" [63:13 - 63:14] preprocessing directive=
288+
// CHECK: Literal: "0" [63:13 - 63:14] macro definition=VAL
289289
// CHECK: Punctuation: "@" [65:1 - 65:2] ObjCInterfaceDecl=R7974151:65:12
290290
// CHECK: Keyword: "interface" [65:2 - 65:11] ObjCInterfaceDecl=R7974151:65:12
291291
// CHECK: Identifier: "R7974151" [65:12 - 65:20] ObjCInterfaceDecl=R7974151:65:12

‎tools/libclang/CIndex.cpp

+69-15
Original file line numberDiff line numberDiff line change
@@ -3657,6 +3657,7 @@ class AnnotateTokensWorker {
36573657
CXCursor *Cursors;
36583658
unsigned NumTokens;
36593659
unsigned TokIdx;
3660+
unsigned PreprocessingTokIdx;
36603661
CursorVisitor AnnotateVis;
36613662
SourceManager &SrcMgr;
36623663

@@ -3672,7 +3673,7 @@ class AnnotateTokensWorker {
36723673
CXToken *tokens, CXCursor *cursors, unsigned numTokens,
36733674
ASTUnit *CXXUnit, SourceRange RegionOfInterest)
36743675
: Annotated(annotated), Tokens(tokens), Cursors(cursors),
3675-
NumTokens(numTokens), TokIdx(0),
3676+
NumTokens(numTokens), TokIdx(0), PreprocessingTokIdx(0),
36763677
AnnotateVis(CXXUnit, AnnotateTokensVisitor, this,
36773678
Decl::MaxPCHLevel, RegionOfInterest),
36783679
SrcMgr(CXXUnit->getSourceManager()) {}
@@ -3690,7 +3691,9 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) {
36903691

36913692
for (unsigned I = 0 ; I < TokIdx ; ++I) {
36923693
AnnotateTokensData::iterator Pos = Annotated.find(Tokens[I].int_data[1]);
3693-
if (Pos != Annotated.end())
3694+
if (Pos != Annotated.end() &&
3695+
(clang_isInvalid(Cursors[I].kind) ||
3696+
Pos->second.kind != CXCursor_PreprocessingDirective))
36943697
Cursors[I] = Pos->second;
36953698
}
36963699

@@ -3706,16 +3709,66 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) {
37063709
}
37073710

37083711
enum CXChildVisitResult
3709-
AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
3712+
AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
37103713
CXSourceLocation Loc = clang_getCursorLocation(cursor);
3711-
// We can always annotate a preprocessing directive/macro instantiation.
3712-
if (clang_isPreprocessing(cursor.kind)) {
3713-
Annotated[Loc.int_data] = cursor;
3714+
SourceRange cursorRange = getRawCursorExtent(cursor);
3715+
3716+
if (clang_isPreprocessing(cursor.kind)) {
3717+
// For macro instantiations, just note where the beginning of the macro
3718+
// instantiation occurs.
3719+
if (cursor.kind == CXCursor_MacroInstantiation) {
3720+
Annotated[Loc.int_data] = cursor;
3721+
return CXChildVisit_Recurse;
3722+
}
3723+
3724+
if (cursorRange.isInvalid())
3725+
return CXChildVisit_Continue;
3726+
3727+
// Items in the preprocessing record are kept separate from items in
3728+
// declarations, so we keep a separate token index.
3729+
unsigned SavedTokIdx = TokIdx;
3730+
TokIdx = PreprocessingTokIdx;
3731+
3732+
// Skip tokens up until we catch up to the beginning of the preprocessing
3733+
// entry.
3734+
while (MoreTokens()) {
3735+
const unsigned I = NextToken();
3736+
SourceLocation TokLoc = GetTokenLoc(I);
3737+
switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) {
3738+
case RangeBefore:
3739+
AdvanceToken();
3740+
continue;
3741+
case RangeAfter:
3742+
case RangeOverlap:
3743+
break;
3744+
}
3745+
break;
3746+
}
3747+
3748+
// Look at all of the tokens within this range.
3749+
while (MoreTokens()) {
3750+
const unsigned I = NextToken();
3751+
SourceLocation TokLoc = GetTokenLoc(I);
3752+
switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) {
3753+
case RangeBefore:
3754+
assert(0 && "Infeasible");
3755+
case RangeAfter:
3756+
break;
3757+
case RangeOverlap:
3758+
Cursors[I] = cursor;
3759+
AdvanceToken();
3760+
continue;
3761+
}
3762+
break;
3763+
}
3764+
3765+
// Save the preprocessing token index; restore the non-preprocessing
3766+
// token index.
3767+
PreprocessingTokIdx = TokIdx;
3768+
TokIdx = SavedTokIdx;
37143769
return CXChildVisit_Recurse;
37153770
}
37163771

3717-
SourceRange cursorRange = getRawCursorExtent(cursor);
3718-
37193772
if (cursorRange.isInvalid())
37203773
return CXChildVisit_Continue;
37213774

@@ -3753,7 +3806,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
37533806
// directive. Here we assume that the default construction of CXCursor
37543807
// results in CXCursor.kind being an initialized value (i.e., 0). If
37553808
// this isn't the case, we can fix by doing lookup + insertion.
3756-
3809+
37573810
CXCursor &oldC = Annotated[rawEncoding];
37583811
if (!clang_isPreprocessing(oldC.kind))
37593812
oldC = cursor;
@@ -3815,6 +3868,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
38153868
for (unsigned I = BeforeChildren; I != AfterChildren; ++I) {
38163869
if (!clang_isInvalid(clang_getCursorKind(Cursors[I])))
38173870
break;
3871+
38183872
Cursors[I] = cursor;
38193873
}
38203874
// Scan the tokens that are at the end of the cursor, but are not captured
@@ -3841,14 +3895,14 @@ void clang_annotateTokens(CXTranslationUnit TU,
38413895
if (NumTokens == 0 || !Tokens || !Cursors)
38423896
return;
38433897

3898+
// Any token we don't specifically annotate will have a NULL cursor.
3899+
CXCursor C = clang_getNullCursor();
3900+
for (unsigned I = 0; I != NumTokens; ++I)
3901+
Cursors[I] = C;
3902+
38443903
ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
3845-
if (!CXXUnit) {
3846-
// Any token we don't specifically annotate will have a NULL cursor.
3847-
const CXCursor &C = clang_getNullCursor();
3848-
for (unsigned I = 0; I != NumTokens; ++I)
3849-
Cursors[I] = C;
3904+
if (!CXXUnit)
38503905
return;
3851-
}
38523906

38533907
ASTUnit::ConcurrencyCheck Check(*CXXUnit);
38543908

0 commit comments

Comments
 (0)