sv 0.4.2 → 0.5.0

package/index.js

@@ -1,29 +0,0 @@
-"use strict";
-var json_1 = require('streaming/json');
-var property_1 = require('streaming/property');
-var parser_1 = require('./parser');
-exports.Parser = parser_1.Parser;
-var stringifier_1 = require('./stringifier');
-exports.Stringifier = stringifier_1.Stringifier;
-function transform(input, parserConfig, stringifierConfig, callback) {
-    var transforms = [
-        parserConfig.json ? new json_1.Parser() : new parser_1.Parser(parserConfig),
-    ];
-    if (stringifierConfig.omit) {
-        transforms.push(new property_1.Omitter(stringifierConfig.omit.split(/,/g)));
-    }
-    if (stringifierConfig.filter) {
-        transforms.push(new property_1.Picker(stringifierConfig.filter.split(/,/g)));
-    }
-    var stringifier = stringifierConfig.json ? new json_1.Stringifier() : new stringifier_1.Stringifier(stringifierConfig);
-    transforms.push(stringifier);
-    var output = transforms.reduce(function (outputStream, transform) { return outputStream.pipe(transform); }, input).pipe(process.stdout);
-    output.on('finish', callback);
-    output.on('error', function (error) {
-        // panic! (lets us quit faster, actually)
-        input.unpipe();
-        // output.unpipe();
-        callback(error);
-    });
-}
-exports.transform = transform;

package/parser.d.ts

@@ -1,29 +0,0 @@
-import { Transform } from 'stream';
-import { Configuration } from './common';
-export interface ParserConfiguration extends Configuration {
-}
-export declare const defaultParserConfiguration: ParserConfiguration;
-/**
-returns a single char code (a byte) denoting the inferred delimiter.
-*/
-export declare function inferDelimiter(buffer: Buffer): number;
-/**
-- `byteBuffer` is a buffer (of bytes) that have yet to be processed (and sent to output).
-- `cellBuffer` is a list of strings that have yet to be processed (and sent to output).
-
-*/
-export declare class Parser extends Transform {
-    config: ParserConfiguration;
-    delimiterByte: number;
-    quoteByte: number;
-    quotequoteRegExp: RegExp;
-    escapeByte: number;
-    escapeQuoteRegExp: RegExp;
-    byteBuffer: Buffer;
-    cellBuffer: string[];
-    constructor(config?: ParserConfiguration);
-    protected writeRow(cells: string[]): void;
-    flush(callback: (error?: Error) => void, nonfinal: boolean): void;
-    _flush(callback: (error?: Error) => void): void;
-    _transform(chunk: Buffer, encoding: string, callback: (error?: Error) => void): void;
-}

package/parser.js

@@ -1,194 +0,0 @@
-"use strict";
-var __extends = (this && this.__extends) || function (d, b) {
-    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
-    function __() { this.constructor = d; }
-    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
-};
-var stream_1 = require('stream');
-var common_1 = require('./common');
-exports.defaultParserConfiguration = {
-    encoding: 'utf8',
-    missing: '',
-    newline: '\n',
-    // omit delimiter so that it gets inferred
-    quotechar: '"',
-    escape: '\\',
-};
-/**
-returns a single char code (a byte) denoting the inferred delimiter.
-*/
-function inferDelimiter(buffer) {
-    var counts = {};
-    // we look at the first newline or 256 chars, whichever is greater,
-    //   but without going through the whole file
-    var upto = Math.min(256, buffer.length);
-    for (var i = 0; i < upto && buffer[i] != 10 && buffer[i] != 13; i++) {
-        var charCode = buffer[i];
-        counts[charCode] = (counts[charCode] || 0) + 1;
-    }
-    // we'll go through, prioritizing characters that aren't likely to show
-    // up unless they are a delimiter.
-    var candidates = [
-        9,
-        59,
-        44,
-        32,
-    ];
-    // TODO: make this more robust (that's why I even counted them)
-    for (var candidate = void 0, j = 0; (candidate = candidates[j]); j++) {
-        if (counts[candidate] > 0) {
-            return candidate;
-        }
-    }
-}
-exports.inferDelimiter = inferDelimiter;
-/**
-- `byteBuffer` is a buffer (of bytes) that have yet to be processed (and sent to output).
-- `cellBuffer` is a list of strings that have yet to be processed (and sent to output).
-
-*/
-var Parser = (function (_super) {
-    __extends(Parser, _super);
-    function Parser(config) {
-        if (config === void 0) { config = {}; }
-        _super.call(this, {
-            decodeStrings: true,
-            readableObjectMode: true,
-            writableObjectMode: false,
-        });
-        this.byteBuffer = new Buffer(0);
-        this.cellBuffer = [];
-        // merge defaults
-        this.config = common_1.merge(config, exports.defaultParserConfiguration);
-        // special demarcating characters
-        // 1. delimiter
-        if (this.config.delimiter) {
-            this.delimiterByte = this.config.delimiter.charCodeAt(0);
-        }
-        // 2. quote
-        this.quoteByte = this.config.quotechar.charCodeAt(0);
-        this.quotequoteRegExp = new RegExp(this.config.quotechar + this.config.quotechar, 'g');
-        // 3. escape
-        this.escapeByte = this.config.escape.charCodeAt(0);
-        this.escapeQuoteRegExp = new RegExp('\\' + this.config.escape + this.config.quotechar, 'g');
-    }
-    Parser.prototype.writeRow = function (cells) {
-        if (!this.config.columns) {
-            // we don't emit the column names as data
-            this.config.columns = cells;
-        }
-        else {
-            this.push(common_1.zip(this.config.columns, cells, this.config.missing));
-        }
-    };
-    Parser.prototype.flush = function (callback, nonfinal) {
-        var buffer = this.byteBuffer;
-        var cells = this.cellBuffer;
-        if (!this.delimiterByte) {
-            // should we wait for some minimum amount of data?
-            this.delimiterByte = inferDelimiter(buffer);
-        }
-        var start = 0;
-        var end = buffer.length;
-        var inside_quote = false;
-        // outside_quote reminds us to remove the quotes later (in pushCell)
-        var outside_quote = false;
-        for (var i = 0; i < end; i++) {
-            var eos = !nonfinal && i + 1 == end;
-            // const snippet = buffer.toString('utf8', 0, i) +
-            //   '\x1b[7m' + buffer.toString('utf8', i, i + 1) + '\x1b[0m' +
-            //   buffer.toString('utf8', i + 1, end);
-            // console.error(snippet.replace(/\n/g, 'N').replace(/\t/g, 'T'), inside_quote ? 'inside_quote' : '');
-            // if we are on an escape char, simply skip over it (++) and the (default)
-            if (!eos && buffer[i] == this.escapeByte) {
-                // excel is bizarre. An escape before a quotebyte doesn't count,
-                //   so we only increment if the next character is not a quotebyte
-                // unless we are not inside quotes, in which case we do skip over it.
-                if (!inside_quote || buffer[i + 1] !== this.quoteByte) {
-                    i++;
-                }
-            }
-            else if (!eos && buffer[i] === this.quoteByte && inside_quote) {
-                // if we are inside, and on a "
-                // handle excel dialect: double quotebyte => single literal quotebyte
-                if (buffer[i + 1] === this.quoteByte) {
-                    // double quotebyte
-                    // we just advance over it for now, so that we can put this back on the buffer, if needed.
-                    i++;
-                }
-                else {
-                    // lone quotebyte -> don't assume that they're always followed by a delimiter.
-                    // they might be followed by a newline
-                    // and we advance so that buffer[i] skips over the delimiter
-                    inside_quote = false;
-                    outside_quote = true;
-                }
-            }
-            else if (!eos && buffer[i] === this.quoteByte && !inside_quote && i == start) {
-                // if we are not already inside, and on a "
-                inside_quote = true;
-            }
-            else if (
-            // if we are at the very end of the input and this is the final chunk (ignoring any sort of state)
-            eos ||
-                // OR, we push a new cell whenever we hit a delimiter (say, tab) and are not inside a quote
-                (!inside_quote && (buffer[i] == 13 || buffer[i] == 10 || buffer[i] == this.delimiterByte))) {
-                // this generally won't hurt, since it will only go to the end of the buffer anyway.
-                if (eos)
-                    i++;
-                // add the unprocessed buffer to our cells
-                // inside_quote might be true if the file ends on a quote
-                if (inside_quote || outside_quote) {
-                    var trimmed_cell = buffer.toString(this.config.encoding, start + 1, i - 1);
-                    if (this.quotequoteRegExp) {
-                        trimmed_cell = trimmed_cell.replace(this.quotequoteRegExp, this.config.quotechar);
-                    }
-                    // is this good enough?
-                    cells.push(trimmed_cell);
-                    outside_quote = inside_quote = false;
-                }
-                else {
-                    var cell = buffer.toString(this.config.encoding, start, i);
-                    if (this.escapeQuoteRegExp) {
-                        cell = cell.replace(this.escapeQuoteRegExp, this.config.quotechar);
-                    }
-                    cells.push(cell);
-                }
-                // handle \r, \r\n, or \n (but not \n\n) as one line break
-                // '\r' == 13, '\n' == 10
-                // we flush the row, also, if we are at the end and this is the final chunk
-                if (eos || (buffer[i] != this.delimiterByte)) {
-                    // add these cells to the emit queue
-                    this.writeRow(cells);
-                    // and reset them
-                    cells = [];
-                    // also consume a following \n, if this was \r, and there is one.
-                    if (buffer[i] == 13 && buffer[i + 1] == 10) {
-                        i++;
-                    }
-                }
-                start = i + 1;
-            }
-        }
-        // save whatever we have yet to process
-        this.byteBuffer = buffer.slice(start, end);
-        this.cellBuffer = cells;
-        // if there was a trailing newline, this._buffer.length = 0
-        callback();
-    };
-    Parser.prototype._flush = function (callback) {
-        return this.flush(callback, false);
-    };
-    Parser.prototype._transform = function (chunk, encoding, callback) {
-        // we'll assume that we always get chunks with the same encoding.
-        if (!this.config.encoding && encoding != 'buffer') {
-            this.config.encoding = encoding;
-        }
-        // collect unused buffer and new chunk into a single buffer
-        this.byteBuffer = this.byteBuffer.length ? Buffer.concat([this.byteBuffer, chunk]) : chunk;
-        // do all the processing
-        this.flush(callback, true);
-    };
-    return Parser;
-}(stream_1.Transform));
-exports.Parser = Parser;

package/stringifier.d.ts

@@ -1,42 +0,0 @@
-import { Transform } from 'stream';
-import { Configuration } from './common';
-export interface StringifierConfiguration extends Configuration {
-    peek?: number;
-}
-export declare const defaultStringifierConfiguration: {
-    encoding: string;
-    missing: string;
-    newline: string;
-    delimiter: string;
-    quotechar: string;
-    escape: string;
-    peek: number;
-};
-export declare function inferColumns(rows: string[][]): string[];
-/** Stringifier class
-  new Stringifier();
-  - `peek` is an integer (or undefined / null) describing how many rows we
-    should look at before inferring headers and flushing.
-  - `columns` is an array of strings once the headers have been inferred
-  - `encoding` is the encoding that the stream's read function will use.
-  - `missing` is the value we write for 'time' when we have
-    `columns = ['index', 'time']` and `write({index: 90})` is called
-
-  - `delimiter` is the field separator (defaults to a comma)
-  - `quotechar` is the character used to quote fields if they contain the
-    `delimiter` character (defaults to a double-quote)
-
-  - `_buffer` is an array of arrays or objects that need to be written
-*/
-export declare class Stringifier extends Transform {
-    /** should this even be 1? (ignored if opts.columns) */
-    config: StringifierConfiguration;
-    protected quotecharRegExp: RegExp;
-    protected rowBuffer: string[][];
-    constructor(config?: StringifierConfiguration);
-    protected writeObject(object: any): void;
-    protected writeObjects(objects: any[]): void;
-    flush(callback: (error?: Error) => void, nonfinal: boolean): void;
-    _flush(callback: (error?: Error) => void): void;
-    _transform(chunk: any, encoding: string, callback: (error?: Error) => void): void;
-}

package/stringifier.js

@@ -1,159 +0,0 @@
-"use strict";
-var __extends = (this && this.__extends) || function (d, b) {
-    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
-    function __() { this.constructor = d; }
-    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
-};
-var stream_1 = require('stream');
-var common_1 = require('./common');
-exports.defaultStringifierConfiguration = {
-    encoding: 'utf8',
-    missing: '',
-    newline: '\n',
-    delimiter: ',',
-    quotechar: '"',
-    escape: '\\',
-    peek: 1,
-};
-function inferColumns(rows) {
-    var columns = [];
-    var seen = {};
-    rows.forEach(function (row) {
-        // each object might be a string, array, or object, but only objects matter here.
-        if (typeof (row) !== 'string' && !Array.isArray(row)) {
-            Object.keys(row).forEach(function (key) {
-                // if (row.hasOwnProperty(key)) {
-                // maybe should also check that row[key] != null
-                if (!(key in seen)) {
-                    columns.push(key);
-                    seen[key] = 1;
-                }
-                // }
-            });
-        }
-    });
-    return columns;
-}
-exports.inferColumns = inferColumns;
-/** Stringifier class
-  new Stringifier();
-  - `peek` is an integer (or undefined / null) describing how many rows we
-    should look at before inferring headers and flushing.
-  - `columns` is an array of strings once the headers have been inferred
-  - `encoding` is the encoding that the stream's read function will use.
-  - `missing` is the value we write for 'time' when we have
-    `columns = ['index', 'time']` and `write({index: 90})` is called
-
-  - `delimiter` is the field separator (defaults to a comma)
-  - `quotechar` is the character used to quote fields if they contain the
-    `delimiter` character (defaults to a double-quote)
-
-  - `_buffer` is an array of arrays or objects that need to be written
-*/
-var Stringifier = (function (_super) {
-    __extends(Stringifier, _super);
-    function Stringifier(config) {
-        if (config === void 0) { config = {}; }
-        _super.call(this, { readableObjectMode: false, writableObjectMode: true });
-        this.rowBuffer = [];
-        this.config = common_1.merge(config, exports.defaultStringifierConfiguration);
-        this.quotecharRegExp = new RegExp(this.config.quotechar, 'ig');
-        if (this.config.columns) {
-            // maybe we should write the columns even if we don't get any data?
-            this.rowBuffer = [this.config.columns];
-        }
-        else {
-            this.rowBuffer = [];
-        }
-    }
-    Stringifier.prototype.writeObject = function (object) {
-        // _write is already a thing, so don't use it.
-        // this.columns must be set!
-        if (typeof (object) === 'string') {
-            // raw string
-            this.push(object + this.config.newline, this.config.encoding);
-        }
-        else {
-            // if object is an array, we ignore this.columns
-            var length_1 = object.length;
-            if (!Array.isArray(object)) {
-                // object
-                length_1 = this.config.columns.length;
-                // pull properties off the given object in proper column order
-                var list = new Array(length_1);
-                for (var i = 0; i < length_1; i++) {
-                    var column_value = object[this.config.columns[i]];
-                    list[i] = (column_value === undefined) ? this.config.missing : column_value;
-                }
-                object = list;
-            }
-            // obj is definitely an array now, but the fields aren't quoted.
-            for (var j = 0; j < length_1; j++) {
-                // assume minimal quoting (don't quote unless the cell contains the delimiter)
-                var value = String(object[j]);
-                var contains_newline = value.indexOf('\n') > -1 || value.indexOf('\r') > -1;
-                var contains_quotechar = value.indexOf(this.config.quotechar) > -1;
-                if (value.indexOf(this.config.delimiter) > -1 || contains_newline || contains_quotechar) {
-                    if (contains_quotechar) {
-                        // serialize into the excel dialect, currently
-                        value = value.replace(this.quotecharRegExp, this.config.quotechar + this.config.quotechar);
-                    }
-                    value = this.config.quotechar + value + this.config.quotechar;
-                }
-                object[j] = value;
-            }
-            this.push(object.join(this.config.delimiter) + this.config.newline, this.config.encoding);
-        }
-    };
-    Stringifier.prototype.writeObjects = function (objects) {
-        for (var i = 0, l = objects.length; i < l; i++) {
-            this.writeObject(objects[i]);
-        }
-    };
-    Stringifier.prototype.flush = function (callback, nonfinal) {
-        // called when we're done peeking (nonfinal = true) or when end() is
-        // called (nonfinal = false), in which case we are done peeking, but for a
-        // different reason. In either case, we need to flush the peeked columns.
-        if (!this.config.columns) {
-            // infer columns
-            this.config.columns = inferColumns(this.rowBuffer);
-            this.writeObject(this.config.columns);
-        }
-        if (this.rowBuffer) {
-            // flush the _buffer
-            this.writeObjects(this.rowBuffer);
-            // a null _buffer means we're done peeking and won't be buffering any more rows
-            this.rowBuffer = null;
-        }
-        // this.push(null); // inferred
-        callback();
-    };
-    // the docs decree that we shouldn't call _flush directly
-    Stringifier.prototype._flush = function (callback) {
-        return this.flush(callback, false);
-    };
-    Stringifier.prototype._transform = function (chunk, encoding, callback) {
-        // objectMode: true, so chunk is an object (and encoding is always 'utf8'?)
-        if (this.config.columns) {
-            // flush the _buffer, if needed
-            if (this.rowBuffer) {
-                this.writeObjects(this.rowBuffer);
-                this.rowBuffer = null;
-            }
-            this.writeObject(chunk);
-            callback();
-        }
-        else {
-            // if set {peek: 10}, column inference will be called when write(obj) is called the 10th time
-            this.rowBuffer.push(chunk);
-            if (this.rowBuffer.length >= this.config.peek) {
-                this.flush(callback, true);
-            }
-            else {
-                callback();
-            }
-        }
-    };
-    return Stringifier;
-}(stream_1.Transform));
-exports.Stringifier = Stringifier;