5757 */
5858class Tokenizer
5959{
60+ protected $ tokenCollection = array ();
61+
62+ protected $ contextPostfix = null ;
63+
6064 /**
6165 * Stack of token names to match curly brackets
6266 *
@@ -89,18 +93,10 @@ class Tokenizer
8993 'T_WHILE ' ,
9094 'T_CATCH ' ,
9195 'T_SWITCH ' ,
96+ 'T_ARRAY ' ,
9297 );
9398
94- // protected $curlyAfterRoundBrackets = array(
95- // 'T_FUNCTION_CLOSE_ROUND_BRACKET',
96- // 'T_IF_CLOSE_ROUND_BRACKET',
97- // 'T_ELSEIF_CLOSE_ROUND_BRACKET',
98- // 'T_FOR_CLOSE_ROUND_BRACKET',
99- // 'T_FOREACH_CLOSE_ROUND_BRACKET',
100- // 'T_WHILE_CLOSE_ROUND_BRACKET',
101- // 'T_CATCH_CLOSE_ROUND_BRACKET',
102- // 'T_SWITCH_CLOSE_ROUND_BRACKET',
103- // );
99+
104100
105101 /**
106102 * Stack of token names to match round brackets
@@ -116,7 +112,7 @@ class Tokenizer
116112 */
117113 public function tokenize ($ source )
118114 {
119- $ tokenCollection = array ();
115+ $ this -> tokenCollection = array ();
120116 $ this ->curlyBracketStack = array ();
121117 $ this ->roundBracketStack = array ();
122118 $ tokenizedSource = token_get_all ($ source );
@@ -128,23 +124,41 @@ public function tokenize($source)
128124 continue ;
129125 }
130126
131- $ parsedSourceToken = array ();
132-
133- if (is_array ($ sourceToken )) {
134- $ parsedSourceToken = $ sourceToken ;
135- $ parsedSourceToken [3 ] = token_name ($ sourceToken [0 ]);
136-
137- // $this->addTokenOntoStack($parsedSourceToken[3]);
138- } else {
139- $ parsedSourceToken = $ this ->addTokenData ($ sourceToken , $ previousToken );
140- }
141-
142- $ tokenObject = new Token ($ parsedSourceToken [3 ], $ parsedSourceToken [2 ], $ parsedSourceToken [1 ], $ previousToken );
143- $ tokenCollection [] = $ previousToken = $ tokenObject ;
144- $ this ->tokenCollection = $ tokenCollection ;
127+ $ tokenObject = $ this ->parseToken ($ sourceToken , $ previousToken );
128+ $ this ->tokenCollection [] = $ previousToken = $ tokenObject ;
129+ }
130+
131+ return $ this ->tokenCollection ;
132+ }
133+
134+ protected function parseToken ($ sourceToken , $ previousToken ) {
135+ $ parsedSourceToken = array ();
136+ if (is_array ($ sourceToken )) {
137+ $ parsedSourceToken = $ sourceToken ;
138+ $ parsedSourceToken [3 ] = token_name ($ sourceToken [0 ]);
139+ } else {
140+ $ parsedSourceToken = $ this ->addTokenData ($ sourceToken , $ previousToken );
141+ }
142+
143+ $ previousToken = $ this ->sanitizeCode ($ parsedSourceToken [3 ], $ previousToken );
144+
145+ $ tokenObject = new Token ($ parsedSourceToken [3 ], $ parsedSourceToken [2 ], $ parsedSourceToken [1 ], $ previousToken );
146+ $ this ->setContext ($ tokenObject ->getName ());
147+ return $ tokenObject ;
148+ }
149+
150+ protected function sanitizeCode ($ tokenName , $ previousToken )
151+ {
152+ switch ($ tokenName ) {
153+ case 'T_ARRAY_CLOSE_ROUND_BRACKET ' :
154+ if ('T_COMMA_FOR_ARRAY ' !== $ previousToken ->getName ()) {
155+ $ previousToken = $ this ->parseToken (', ' , $ previousToken );
156+ $ this ->tokenCollection [] = $ previousToken ;
157+ }
158+ break ;
145159 }
146160
147- return $ tokenCollection ;
161+ return $ previousToken ;
148162 }
149163
150164 /**
@@ -170,6 +184,7 @@ protected function addTokenOntoStack($tokenName)
170184 case 'T_FOR ' :
171185 case 'T_FOREACH ' :
172186 case 'T_WHILE ' :
187+ case 'T_ARRAY ' :
173188 array_unshift ($ this ->roundBracketStack , $ tokenName );
174189 break ;
175190 }
@@ -240,7 +255,7 @@ protected function addTokenData($sourceString, Token $previousToken)
240255// }
241256 break ;
242257 case ', ' :
243- $ tokenName = 'T_COMMA ' ;
258+ $ tokenName = 'T_COMMA ' . $ this -> contextPostfix ;
244259 break ;
245260 case '! ' :
246261 $ tokenName = 'T_EXCALMATION_MARK ' ;
@@ -252,7 +267,7 @@ protected function addTokenData($sourceString, Token $previousToken)
252267 $ tokenName = 'T_COLON ' ;
253268 break ;
254269 case '; ' :
255- $ tokenName = 'T_SEMICOLON ' ;
270+ $ tokenName = 'T_SEMICOLON ' . $ this -> contextPostfix ;
256271 break ;
257272 case '+ ' :
258273 $ tokenName = 'T_PLUS ' ;
@@ -312,10 +327,6 @@ protected function addTokenData($sourceString, Token $previousToken)
312327 $ tokenName = 'T_BACKTICK ' ;
313328 break ;
314329 default :
315- $ count = count ($ this ->tokenCollection );
316- var_dump ($ this ->tokenCollection [$ count -3 ]);
317- var_dump ($ this ->tokenCollection [$ count -2 ]);
318- var_dump ($ this ->tokenCollection [$ count -1 ]);
319330 var_dump ($ sourceString );
320331 die ("\nMissing behaviour \n" );
321332 }
@@ -353,4 +364,19 @@ protected function findPreviousControlTokenForRoundBracket(Token $token) {
353364 return null ;
354365 }
355366 }
367+
368+ protected function setContext ($ tokenName ) {
369+ switch ($ tokenName ) {
370+ case 'T_ARRAY_OPEN_ROUND_BRACKET ' :
371+ $ this ->contextPostfix = '_FOR_ARRAY ' ;
372+ break ;
373+ case 'T_FOR_OPEN_ROUND_BRACKET ' :
374+ $ this ->contextPostfix = '_IN_FOR ' ;
375+ break ;
376+ case 'T_SEMICOLON_FOR_ARRAY ' :
377+ case 'T_FOR_CLOSE_ROUND_BRACKET ' :
378+ $ this ->contextPostfix = null ;
379+ break ;
380+ }
381+ }
356382}
0 commit comments