fix: address duplicate token bug when syntax-highlighting

PR-URL: #2542
Reviewed-by: Athan Reines <kgryte@gmail.com>
Signed-off-by: Snehil Shah <snehilshah.989@gmail.com>

Author: Snehil-Shah

lib/node_modules/@stdlib/repl/lib/syntax_highlighter.js

@@ -26,6 +26,7 @@ var readline = require( 'readline' );
 var logger = require( 'debug' );
 var format = require( '@stdlib/string/format' );
 var setNonEnumerableReadOnly = require( '@stdlib/utils/define-nonenumerable-read-only-property' );
+var isEmptyArray = require( '@stdlib/assert/is-empty-array' );
 var objectKeys = require( '@stdlib/utils/keys' );
 var omit = require( '@stdlib/utils/omit' );
 var hasOwnProp = require( '@stdlib/assert/has-own-property' );
@@ -54,6 +55,26 @@ function tokenComparator( a, b ) {
 	return a.start - b.start;
 }
 
+/**
+* Removes duplicate tokens from a sorted tokens array.
+*
+* @private
+* @param {Array<Object>} tokens - sorted tokens array
+* @returns {Array<Object>} array with unique tokens
+*/
+function removeDuplicateTokens( tokens ) {
+	var out = [];
+	var i;
+
+	out.push( tokens[ 0 ] );
+	for ( i = 1; i < tokens.length; i++ ) {
+		if ( tokens[ i - 1 ].start !== tokens[ i ].start ) {
+			out.push( tokens[ i ] );
+		}
+	}
+	return out;
+}
+
 
 // MAIN //
 
@@ -125,8 +146,7 @@ setNonEnumerableReadOnly( SyntaxHighlighter.prototype, '_highlightLine', functio
 	var i;
 	var j;
 
-	// Sort and traverse the tokens...
-	tokens.sort( tokenComparator );
+	// Traverse the tokens...
 	for ( i = 0; i < tokens.length; i++ ) {
 		token = tokens[ i ];
 		color = theme[ token.type ];
@@ -309,11 +329,15 @@ setNonEnumerableReadOnly( SyntaxHighlighter.prototype, 'onKeypress', function on
 		// Tokenize:
 		debug( 'Line change detected. Tokenizing line: %s', this._rli.line );
 		tokens = tokenizer( this._rli.line, this._repl._context );
-		if ( !tokens ) {
+		if ( isEmptyArray( tokens ) ) {
 			debug( 'No tokens found. Skipping highlighting...' );
 			this._multilineHandler.updateLine( this._rli.line ); // save displayed line
 			return;
 		}
+		// Process tokens:
+		tokens.sort( tokenComparator );
+		tokens = removeDuplicateTokens( tokens );
+
 		// Highlight:
 		debug( '%d tokens found. Highlighting...', tokens.length );
 		this._line = this._rli.line; // updated line buffer

lib/node_modules/@stdlib/repl/test/integration/fixtures/syntax-highlighting/member_expressions.json

@@ -1,6 +1,6 @@
 {
-  "expression": "var a = { 'b': 'bar' }; var foo = { 'bar': { 'func': function() { return true; } } };\nfoo[a.b].func()",
-  "expected": "\u001b[31mfoo\u001b[0m[\u001b[31ma\u001b[0m.\u001b[32mb\u001b[0m].\u001b[33mfunc\u001b[0m()",
+  "expression": "var a = { 'b': { 'c': 'bar' } }; var foo = { 'bar': { 'func': function() { return true; } } };\nfoo[a.b.c].func()",
+  "expected": "\u001b[31mfoo\u001b[0m[\u001b[31ma\u001b[0m.\u001b[31mb\u001b[0m.\u001b[32mc\u001b[0m].\u001b[33mfunc\u001b[0m()",
   "theme": {
     "object": "red",
     "function": "yellow",