Add support for skipping rows according to a callback and update an option's default value

Author: kgryte

lib/node_modules/@stdlib/utils/dsv/base/parse/README.md

@@ -79,6 +79,8 @@ The constructor accepts the following `options`:
 
 -   **ltrim**: `boolean` indicating whether to trim leading whitepsace from field values. If `false`, the parser does not trim leading whitespace (e.g., `a, b, c` parses as `[ 'a', ' b', ' c' ]`).  If `true`, the parser trims leading whitespace (e.g., `a, b, c` parses as `[ 'a', 'b', 'c' ]`). Default: `false`.
 
+-   **maxRows**: maximum number of records to process (excluding skipped lines). By default, the maximum number of records is unlimited.
+
 -   **newline**: character sequence separating rows. Default: `'\r\n'` (see [RFC 4180][rfc-4180]).
 
 -   **onClose**: callback to be invoked upon closing the parser. If a parser has partially processed a record upon close, the callback is invoked with the following arguments:
@@ -92,6 +94,7 @@ The constructor accepts the following `options`:
     -   **field**: field value.
     -   **row**: row number (zero-based).
     -   **col**: field (column) number (zero-based).
+    -   **line**: line number (zero-based).
 
 -   **onComment**: callback to be invoked upon processing a commented line. The callback is invoked with the following arguments:
 
@@ -107,6 +110,7 @@ The constructor accepts the following `options`:
     -   **record**: an array-like object containing field values. If provided a `rowBuffer`, the `record` argument will be the **same** array-like object for each invocation.
     -   **row**: row number (zero-based).
     -   **ncols**: number of fields (columns).
+    -   **line**: line number (zero-based).
 
     If a parser is closed **before** fully processing the last record, the callback is invoked with field data for all fields which have been parsed. Any remaining field data is provided to the `onClose` callback. For example, if a parser has processed two fields and closes while attempting to process a third field, the parser invokes the `onRow` callback with field data for the first two fields and invokes the `onClose` callback with the partially processed data for the third field.
 
@@ -129,6 +133,17 @@ The constructor accepts the following `options`:
 
 -   **skip**: character sequence appearing at the beginning of a row which demarcates that the row content should be parsed as a skipped record. Default: `''`.
 
+-   **skipBlankRows**: `boolean` flag indicating whether to skip over rows which are either empty or containing only whitespace. Default: `false`.
+
+-   **skipRow**: callback whose return value indicates whether to skip over a row. The callback is invoked with the following arguments:
+
+    -   **nrows**: number of processed rows (equivalent to the current row number).
+    -   **line**: line number (zero-based).
+    
+    If the callback returns a truthy value, the parser skips the row; otherwise, the parser attempts to process the row.
+
+    Note, however, that, even if the callback returns a falsy value, a row may still be skipped depending on the presence of a `skip` character sequence.
+
 -   **strict**: `boolean` flag indicating whether to raise an exception upon encountering invalid DSV. When `false`, instead of throwing an `Error` or invoking the `onError` callback, the parser invokes an `onWarn` callback with an `Error` object specifying the encountered error. Default: `true`.
 
 -   **trimComment**: `boolean` flag indicating whether to trim leading whitespace in commented lines. Default: `true`.

lib/node_modules/@stdlib/utils/dsv/base/parse/lib/defaults.js

@@ -66,6 +66,9 @@ function defaults() {
 		// Flag indicating whether to trim leading whitespace from field values. If `false`, leading whitespace is not trimmed (e.g., `a, b, c` parses as `[ 'a', ' b', ' c' ]`). If `true`, leading whitespace is trimmed (e.g., `a, b, c` parses as `[ 'a', 'b', 'c' ]`).
 		'ltrim': false,
 
+		// Maximum number of records to process.
+		'maxRows': 1e308,
+
 		// Character sequence separating rows.
 		'newline': '\r\n',
 
@@ -105,6 +108,12 @@ function defaults() {
 		// Character sequence appearing at the beginning of a row which demarcates that the row content should be skipped.
 		'skip': '',
 
+		// Flag indicating whether to skip over rows which are either empty or containing only whitespace.
+		'skipBlankRows': false,
+
+		// Callback whose return value indicates whether to skip over a row.
+		'skipRow': null,
+
 		// Flag indicating whether to raise an exception upon encountering invalid DSV.
 		'strict': true,
 

lib/node_modules/@stdlib/utils/dsv/base/parse/lib/main.js

@@ -126,10 +126,7 @@ function Parser( options ) {
 	this._row = 0;
 	this._line = 0;
 
-	// Initialize the state flag:
-	this._state = INIT;
-
-	// Initialize flags indicating whether we're processing a commented line:
+	// Initialize flags indicating whether we're processing a commented/skipped line:
 	this._commented = false;
 	this._skipped = false;
 
@@ -142,11 +139,14 @@ function Parser( options ) {
 	this._doublequote = ( options.doublequote === void 0 ) ? opts.doublequote : options.doublequote;
 	this._escape = options.escape || opts.escape;
 	this._ltrim = ( options.ltrim === void 0 ) ? opts.ltrim : options.ltrim;
+	this._maxRows = ( options.maxRows === void 0 ) ? opts.maxRows : options.maxRows;
 	this._newline = options.newline || opts.newline;
 	this._quote = options.quote || opts.quote;
 	this._quoting = ( options.quoting === void 0 ) ? opts.quoting : options.quoting;
 	this._rtrim = ( options.rtrim === void 0 ) ? opts.rtrim : options.rtrim;
 	this._skip = options.skip || opts.skip;
+	this._skipBlankRows = ( options.skipBlankRows === void 0 ) ? opts.skipBlankRows : options.skipBlankRows;
+	this._skipRow = options.skipRow || opts.skipRow;
 	this._strict = ( options.strict === void 0 ) ? opts.strict : options.strict;
 	this._trimComment = ( options.trimComment === void 0 ) ? opts.trimComment : options.trimComment;
 	this._whitespace = options.whitespace || opts.whitespace;
@@ -178,6 +178,13 @@ function Parser( options ) {
 	this._skipLength = this._skip.length;
 	this._skipLastIndex = this._skipLength - 1;
 
+	// Initialize the state flag:
+	if ( this._skipRow && this._skipRow( 0, 0 ) ) {
+		this._state = SKIP;
+		this._skipped = true;
+	} else {
+		this._state = INIT;
+	}
 	// Initialize state processors...
 	this._states = states( this ); // NOTE: this should come after all other initialization!
 
@@ -268,12 +275,15 @@ setReadOnly( Parser.prototype, '_reset', function reset() {
 	this._qidx = -1;
 	this._eidx = -1;
 
-	// Reset the parser state to attempting to parse the first field of the next record:
-	this._state = INIT;
-
-	// Reset flags for commented lines:
+	// Reset the parser state...
+	if ( this._skipRow && this._skipRow( this._row, this._line ) ) {
+		this._state = SKIP;
+		this._skipped = true;
+	} else {
+		this._state = INIT;
+		this._skipped = false;
+	}
 	this._commented = false;
-	this._skipped = false;
 
 	// Reset the buffer:
 	this._cursor = -1;
@@ -419,7 +429,8 @@ setReadOnly( Parser.prototype, '_onField', function onField() {
 	this._setField( v, this._col );
 
 	// Invoke a callback for receiving field values:
-	this._onColumn( v, this._row, this._col );
+	this._onColumn( v, this._row, this._col, this._line );
+	debug( 'Field. Line: %d. Row: %d. Column: %d. Value: %s', this._line, this._row, this._col, v );
 
 	// Increment the field counter to record that we've moved on to the next field:
 	this._col += 1;
@@ -431,7 +442,6 @@ setReadOnly( Parser.prototype, '_onField', function onField() {
 	this._qidx = -1;
 	this._eidx = -1;
 
-	debug( 'New field. Line: %d. Field: %d. Value: %s', this._line+1, this._col, v );
 	return this;
 });
 
@@ -445,28 +455,38 @@ setReadOnly( Parser.prototype, '_onField', function onField() {
 * @returns {Parser} parser instance
 */
 setReadOnly( Parser.prototype, '_onRecord', function onRecord() {
-	// FIXME: check for whether `onEmpty` should be invoked and/or handle blank lines
+	var v;
 
 	// Extract the field value:
-	var v = this._getField( this._cidx, this._cursor );
+	v = this._getField( this._cidx, this._cursor );
+
+	// Check for a blank row (i.e., a row consisting only of whitespace):
+	if ( this._skipBlankRows ) {
+		// FIXME: check of whether this is col 0 and whether it consists of only whitespace
+	}
 
 	// Insert the field value into the row buffer:
 	this._setField( v, this._col );
 
 	// Invoke a callback for receiving field values:
-	this._onColumn( v, this._row, this._col );
+	this._onColumn( v, this._row, this._col, this._line );
 	this._col += 1;
 
 	// Invoke a callback for receiving rows:
-	this._onRow( this._getRow(), this._row, this._col );
+	this._onRow( this._getRow(), this._row, this._col, this._line );
+	debug( 'Record. Line: %d. Fields: %d.', this._line, this._col );
+
+	// Increment row and line counters to indicate that we've moved on to the next row/line:
 	this._row += 1;
 	this._line += 1;
 
-	debug( 'New record. Line: %d. Fields: %d.', this._line, this._col );
-
 	// Reset the parser:
 	this._reset();
 
+	// Check whether we have processed a desired number of rows...
+	if ( this._row >= this._maxRows ) {
+		this._changeState( CLOSED );
+	}
 	return this;
 });
 
@@ -489,7 +509,9 @@ setReadOnly( Parser.prototype, '_onCommentedRow', function onCommentedRow() {
 			// FIXME: trim comment; this may be better done character by character
 		}
 		this._onComment( v, this._line );
-		debug( 'New comment. Line: %d. Value: %s', this._line, v );
+		debug( 'Comment. Line: %d. Value: %s', this._line, v );
+	} else {
+		debug( 'Comment. Line: %d.', this._line );
 	}
 	// Increment the counter for how many lines have been processed:
 	this._line += 1;
@@ -516,7 +538,9 @@ setReadOnly( Parser.prototype, '_onSkippedRow', function onSkippedRow() {
 	if ( this._onSkip ) {
 		v = this._buffer.slice( 0, this._cursor+1 ).join( '' );
 		this._onSkip( v, this._line );
-		debug( 'New skipped row. Line: %d. Value: %s', this._line, v );
+		debug( 'Skipped row. Line: %d. Value: %s', this._line, v );
+	} else {
+		debug( 'Skipped row. Line: %d.', this._line );
 	}
 	// Increment the counter for how many lines have been processed:
 	this._line += 1;
@@ -572,13 +596,13 @@ setReadOnly( Parser.prototype, '_createException', function createException( nam
 
 	switch ( name ) {
 	case 'INVALID_CLOSING_QUOTE':
-		err = new Error( format( 'unexpected error. Encountered an invalid record. Field %d on line %d contains a closing quote which is not immediately followed by a delimiter or newline.', this._col+1, this._line+1 ) );
+		err = new Error( format( 'unexpected error. Encountered an invalid record. Field %d on line %d contains a closing quote which is not immediately followed by a delimiter or newline.', this._col, this._line ) );
 		break;
 	case 'INVALID_ESCAPE':
-		err = new Error( format( 'unexpected error. Encountered an invalid record. Field %d on line %d contains an escape sequence which is not immediately followed by a special character sequence.', this._col+1, this._line+1 ) );
+		err = new Error( format( 'unexpected error. Encountered an invalid record. Field %d on line %d contains an escape sequence which is not immediately followed by a special character sequence.', this._col, this._line ) );
 		break;
 	case 'INVALID_QUOTED_ESCAPE':
-		err = new Error( format( 'unexpected error. Encountered an invalid record. Field %d on line %d contains an escape sequence within a quoted field which is not immediately followed by a quote sequence.', this._col+1, this._line+1 ) );
+		err = new Error( format( 'unexpected error. Encountered an invalid record. Field %d on line %d contains an escape sequence within a quoted field which is not immediately followed by a quote sequence.', this._col, this._line ) );
 		break;
 	case 'CLOSED':
 		err = new Error( 'invalid operation. Parser is unable to parse new chunks, as the parser has been closed. To parse new chunks, create a new parser instance.' );
@@ -795,7 +819,7 @@ setReadOnly( Parser.prototype, 'next', function next( chunk ) {
 	states = this._states;
 	for ( i = 0; i < chunk.length; i++ ) {
 		states[ this._state ]( chunk[ i ] );
-		if ( this._state === ERROR ) {
+		if ( this._state === CLOSED || this._state === ERROR ) {
 			return this;
 		}
 	}