rbql 0.25.0 → 0.25.1

package/.eslintrc.json

@@ -0,0 +1,26 @@
+{
+    "env": {
+        "browser": false,
+        "commonjs": true,
+        "es6": true,
+        "node": true
+    },
+    "parserOptions": {
+        "ecmaFeatures": {
+            "jsx": true
+        },
+        "sourceType": "module",
+        "ecmaVersion": 2018
+    },
+    "rules": {
+        "no-const-assign": "warn",
+        "no-this-before-super": "warn",
+        "no-undef": "warn",
+        "semi": [2, "always"],
+        "no-unreachable": "warn",
+        "no-unused-vars": "warn",
+        "constructor-super": "warn",
+        "no-trailing-spaces": "error",
+        "valid-typeof": "warn"
+    }
+}

package/README.md

@@ -196,7 +196,7 @@ rbql.query_table(user_query, input_table, output_table, warnings).then(success_h
 
 #### Example of query_csv() usage:  
 ```
-const rbql_csv = require('rbql_csv');
+const rbql = require('rbql');
 let user_query = 'SELECT a1, parseInt(a2) % 1000 WHERE a3 != "USA" LIMIT 5';
 let error_handler = function(exception) {
     console.log('Error: ' + String(exception));
@@ -207,7 +207,7 @@ let success_handler = function() {
         console.log('warnings: ' + JSON.stringify(warnings));
     console.log('output table: output.csv');
 }
-rbql_csv.query_csv(user_query, 'input.csv', ',', 'quoted', 'output.csv', ',', 'quoted', 'utf-8', warnings).then(success_handler).catch(error_handler);
+rbql.query_csv(user_query, 'input.csv', ',', 'quoted', 'output.csv', ',', 'quoted', 'utf-8', warnings).then(success_handler).catch(error_handler);
 ```
 
 

package/csv_utils.js

@@ -39,6 +39,7 @@ function extract_next_field(src, dlm, preserve_quotes_and_whitespaces, allow_ext
 
 
 function split_quoted_str(src, dlm, preserve_quotes_and_whitespaces=false) {
+    // This function is newline-agnostic i.e. it can also split records with multiline fields.
     if (src.indexOf('"') == -1) // Optimization for most common case
         return [src.split(dlm), false];
     var result = [];
@@ -116,6 +117,41 @@ function smart_split(src, dlm, policy, preserve_quotes_and_whitespaces) {
 }
 
 
+class MultilineRecordAggregator {
+    constructor(comment_prefix) {
+        this.comment_prefix = comment_prefix;
+        this.reset();
+    }
+    add_line(line_text) {
+        if (this.has_full_record || this.has_comment_line) {
+            throw new Error('Invalid usage - record aggregator must be reset before adding new lines');
+        }
+        if (this.comment_prefix && this.rfc_line_buffer.length == 0 && line_text.startsWith(this.comment_prefix)) {
+            this.has_comment_line = true;
+            return false;
+        }
+        let match_list = line_text.match(/"/g);
+        let has_unbalanced_double_quote = match_list && match_list.length % 2 == 1;
+        this.rfc_line_buffer.push(line_text);
+        this.has_full_record = (!has_unbalanced_double_quote && this.rfc_line_buffer.length == 1) || (has_unbalanced_double_quote && this.rfc_line_buffer.length > 1);
+        return this.has_full_record;
+    }
+    is_inside_multiline_record() {
+        return this.rfc_line_buffer.length && !this.has_full_record;
+    }
+    get_full_line(line_separator) {
+        return this.rfc_line_buffer.join(line_separator);
+    }
+    get_num_lines_in_record() {
+        return this.rfc_line_buffer.length;
+    }
+    reset() {
+        this.rfc_line_buffer = [];
+        this.has_full_record = false;
+        this.has_comment_line = false;
+    }
+}
+
 
 module.exports.split_quoted_str = split_quoted_str;
 module.exports.split_whitespace_separated_str = split_whitespace_separated_str;
@@ -125,3 +161,4 @@ module.exports.rfc_quote_field = rfc_quote_field;
 module.exports.unquote_field = unquote_field;
 module.exports.unquote_fields = unquote_fields;
 module.exports.split_lines = split_lines;
+module.exports.MultilineRecordAggregator = MultilineRecordAggregator;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rbql",
-  "version": "0.25.0",
+  "version": "0.25.1",
   "description": "Rainbow Query Language",
   "keywords": ["CSV", "TSV", "spreadsheet", "SQL", "SQL-like", "transpiler", "CLI", "command-line", "library", "browser", "Node", "select", "update", "join"],
   "scripts": {

package/rbql.js

@@ -70,7 +70,7 @@ var query_context = null; // Needs to be global for MIN(), MAX(), etc functions.
 
 
 const wrong_aggregation_usage_error = 'Usage of RBQL aggregation functions inside JavaScript expressions is not allowed, see the docs';
-const RBQL_VERSION = '0.25.0';
+const RBQL_VERSION = '0.25.1';
 
 
 function check_if_brackets_match(opening_bracket, closing_bracket) {
@@ -1608,20 +1608,20 @@ function select_output_header(input_header, join_header, query_column_infos) {
 }
 
 
-function make_inconsistent_num_fields_warning(table_name, inconsistent_records_info) {
-    let keys = Object.keys(inconsistent_records_info);
-    let entries = [];
-    for (let i = 0; i < keys.length; i++) {
-        let key = keys[i];
-        let record_id = inconsistent_records_info[key];
-        entries.push([record_id, key]);
-    }
-    entries.sort(function(a, b) { return a[0] - b[0]; });
+function sample_first_two_inconsistent_records(inconsistent_records_info) {
+    let entries = Array.from(inconsistent_records_info.entries());
+    entries.sort(function(a, b) { return a[1] - b[1]; });
     assert(entries.length > 1);
-    let [record_1, num_fields_1] = entries[0];
-    let [record_2, num_fields_2] = entries[1];
+    let [num_fields_1, record_num_1] = entries[0];
+    let [num_fields_2, record_num_2] = entries[1];
+    return [record_num_1, num_fields_1, record_num_2, num_fields_2];
+}
+
+
+function make_inconsistent_num_fields_warning(table_name, inconsistent_records_info) {
+    let [record_num_1, num_fields_1, record_num_2, num_fields_2] = sample_first_two_inconsistent_records(inconsistent_records_info);
     let warn_msg = `Number of fields in "${table_name}" table is not consistent: `;
-    warn_msg += `e.g. record ${record_1} -> ${num_fields_1} fields, record ${record_2} -> ${num_fields_2} fields`;
+    warn_msg += `e.g. record ${record_num_1} -> ${num_fields_1} fields, record ${record_num_2} -> ${num_fields_2} fields`;
     return warn_msg;
 }
 
@@ -1691,7 +1691,7 @@ class TableIterator extends RBQLInputIterator {
         this.normalize_column_names = normalize_column_names;
         this.variable_prefix = variable_prefix;
         this.nr = 0;
-        this.fields_info = new Object();
+        this.fields_info = new Map();
         this.stopped = false;
     }
 
@@ -1727,13 +1727,13 @@ class TableIterator extends RBQLInputIterator {
         let record = this.table[this.nr];
         this.nr += 1;
         let num_fields = record.length;
-        if (!this.fields_info.hasOwnProperty(num_fields))
-            this.fields_info[num_fields] = this.nr;
+        if (!this.fields_info.has(num_fields))
+            this.fields_info.set(num_fields, this.nr);
         return record;
     };
 
     get_warnings() {
-        if (Object.keys(this.fields_info).length > 1)
+        if (this.fields_info.size > 1)
             return [make_inconsistent_num_fields_warning('input', this.fields_info)];
         return [];
     };
@@ -1840,6 +1840,9 @@ async function shallow_parse_input_query(query_text, input_iterator, join_tables
     if (rb_actions.hasOwnProperty(SELECT)) {
         query_context.top_count = find_top(rb_actions);
         if (rb_actions.hasOwnProperty(EXCEPT)) {
+            if (rb_actions.hasOwnProperty(JOIN)) {
+                throw new RbqlParsingError('EXCEPT and JOIN are not allowed in the same query');
+            }
             let [output_header, select_expression] = translate_except_expression(rb_actions[EXCEPT]['text'], input_variables_map, string_literals, input_header);
             query_context.select_expression = select_expression;
             query_context.writer.set_header(output_header);
@@ -1947,5 +1950,6 @@ exports.adhoc_parse_select_expression_to_column_infos = adhoc_parse_select_expre
 exports.replace_star_count = replace_star_count;
 exports.replace_star_vars_for_header_parsing = replace_star_vars_for_header_parsing;
 exports.select_output_header = select_output_header;
+exports.sample_first_two_inconsistent_records = sample_first_two_inconsistent_records;
 
 }(typeof exports === 'undefined' ? this.rbql = {} : exports));

package/rbql_csv.js

@@ -62,19 +62,9 @@ function remove_utf8_bom(line, assumed_source_encoding) {
 
 
 function make_inconsistent_num_fields_warning(table_name, inconsistent_records_info) {
-    let keys = Object.keys(inconsistent_records_info);
-    let entries = [];
-    for (let i = 0; i < keys.length; i++) {
-        let key = keys[i];
-        let record_id = inconsistent_records_info[key];
-        entries.push([record_id, key]);
-    }
-    entries.sort(function(a, b) { return a[0] - b[0]; });
-    assert(entries.length > 1);
-    let [record_1, num_fields_1] = entries[0];
-    let [record_2, num_fields_2] = entries[1];
+    let [record_num_1, num_fields_1, record_num_2, num_fields_2] = rbql.sample_first_two_inconsistent_records(inconsistent_records_info);
     let warn_msg = `Number of fields in "${table_name}" table is not consistent: `;
-    warn_msg += `e.g. record ${record_1} -> ${num_fields_1} fields, record ${record_2} -> ${num_fields_2} fields`;
+    warn_msg += `e.g. record ${record_num_1} -> ${num_fields_1} fields, record ${record_num_2} -> ${num_fields_2} fields`;
     return warn_msg;
 }
 
@@ -182,7 +172,7 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
 
         this.table_name = table_name;
         this.variable_prefix = variable_prefix;
-        this.comment_prefix = (comment_prefix !== null && comment_prefix.length) ? comment_prefix : null;
+        this.comment_prefix = comment_prefix;
 
         this.decoder = null;
         if (encoding == 'utf-8' && this.csv_path === null) {
@@ -204,11 +194,11 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         this.utf8_bom_removed = false; // BOM doesn't get automatically removed by the decoder when utf-8 file is treated as latin-1
         this.first_defective_line = null;
 
-        this.fields_info = new Object();
+        this.fields_info = new Map();
         this.NR = 0; // Record number
         this.NL = 0; // Line number (NL != NR when the CSV file has comments or multiline fields)
 
-        this.rfc_line_buffer = [];
+        this.line_aggregator = new csv_utils.MultilineRecordAggregator(comment_prefix);
 
         this.partially_decoded_line = '';
         this.partially_decoded_line_ends_with_cr = false;
@@ -222,7 +212,7 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
 
         this.produced_records_queue = new RecordQueue();
 
-        this.process_line_polymorphic = policy == 'quoted_rfc' ? this.process_partial_rfc_record_line : this.process_record_line;
+        this.process_line_polymorphic = policy == 'quoted_rfc' ? this.process_partial_rfc_record_line : this.process_record_line_simple;
     }
 
 
@@ -350,9 +340,14 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
     };
 
 
-    process_record_line(line) {
-        if (this.comment_prefix !== null && line.startsWith(this.comment_prefix))
+    process_record_line_simple(line) {
+        if (this.comment_prefix && line.startsWith(this.comment_prefix))
             return; // Just skip the line
+        this.process_record_line(line);
+    }
+
+
+    process_record_line(line) {
         this.NR += 1;
         var [record, warning] = csv_utils.smart_split(line, this.delim, this.policy, false);
         if (warning) {
@@ -363,29 +358,20 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
             }
         }
         let num_fields = record.length;
-        if (!this.fields_info.hasOwnProperty(num_fields))
-            this.fields_info[num_fields] = this.NR;
+        if (!this.fields_info.has(num_fields))
+            this.fields_info.set(num_fields, this.NR);
         this.produced_records_queue.enqueue(record);
         this.try_resolve_next_record();
     };
 
 
     process_partial_rfc_record_line(line) {
-        if (this.comment_prefix !== null && this.rfc_line_buffer.length == 0 && line.startsWith(this.comment_prefix))
-            return; // Just skip the line
-        let match_list = line.match(/"/g);
-        let has_unbalanced_double_quote = match_list && match_list.length % 2 == 1;
-        if (this.rfc_line_buffer.length == 0 && !has_unbalanced_double_quote) {
-            this.process_record_line(line);
-        } else if (this.rfc_line_buffer.length == 0 && has_unbalanced_double_quote) {
-            this.rfc_line_buffer.push(line);
-        } else if (!has_unbalanced_double_quote) {
-            this.rfc_line_buffer.push(line);
-        } else {
-            this.rfc_line_buffer.push(line);
-            let multiline_row = this.rfc_line_buffer.join('\n');
-            this.rfc_line_buffer = [];
-            this.process_record_line(multiline_row);
+        this.line_aggregator.add_line(line);
+        if (this.line_aggregator.has_comment_line) {
+            this.line_aggregator.reset();
+        } else if (this.line_aggregator.has_full_record) {
+            this.process_record_line(this.line_aggregator.get_full_line('\n'));
+            this.line_aggregator.reset();
         }
     };
 
@@ -432,13 +418,13 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
     };
 
 
-    process_data_bulk(data_chunk) {
-        let decoded_string = data_chunk.toString(this.encoding);
+    process_data_bulk(data_blob) {
+        let decoded_string = data_blob.toString(this.encoding);
         if (this.encoding == 'utf-8') {
             // Using hacky comparison method from here: https://stackoverflow.com/a/32279283/2898283
             // TODO get rid of this once TextDecoder is really fixed or when alternative method of reliable decoding appears
             let control_buffer = Buffer.from(decoded_string, 'utf-8');
-            if (Buffer.compare(data_chunk, control_buffer) != 0) {
+            if (Buffer.compare(data_blob, control_buffer) != 0) {
                 this.store_or_propagate_exception(new RbqlIOHandlingError(utf_decoding_error));
                 return;
             }
@@ -449,8 +435,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         for (let i = 0; i < lines.length; i++) {
             this.process_line(lines[i]);
         }
-        if (this.rfc_line_buffer.length > 0) {
-            this.process_record_line(this.rfc_line_buffer.join('\n'));
+        if (this.line_aggregator.is_inside_multiline_record()) {
+            this.process_record_line(this.line_aggregator.get_full_line('\n'));
         }
         this.input_exhausted = true;
         this.try_resolve_next_record(); // Should be a NOOP here?
@@ -464,8 +450,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
             this.partially_decoded_line = '';
             this.process_line(last_line);
         }
-        if (this.rfc_line_buffer.length > 0) {
-            this.process_record_line(this.rfc_line_buffer.join('\n'));
+        if (this.line_aggregator.is_inside_multiline_record()) {
+            this.process_record_line(this.line_aggregator.get_full_line('\n'));
         }
         this.try_resolve_next_record();
     };
@@ -487,11 +473,11 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         } else {
             let parent_iterator = this;
             return new Promise(function(resolve, reject) {
-                fs.readFile(parent_iterator.csv_path, (err, data_chunk) => {
+                fs.readFile(parent_iterator.csv_path, (err, data_blob) => {
                     if (err) {
                         reject(err);
                     } else {
-                        parent_iterator.process_data_bulk(data_chunk);
+                        parent_iterator.process_data_bulk(data_blob);
                         resolve();
                     }
                 });
@@ -506,8 +492,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
             result.push(`Inconsistent double quote escaping in ${this.table_name} table. E.g. at line ${this.first_defective_line}`);
         if (this.utf8_bom_removed)
             result.push(`UTF-8 Byte Order Mark (BOM) was found and skipped in ${this.table_name} table`);
-        if (Object.keys(this.fields_info).length > 1)
-            result.push(make_inconsistent_num_fields_warning('input', this.fields_info));
+        if (this.fields_info.size > 1)
+            result.push(make_inconsistent_num_fields_warning(this.table_name, this.fields_info));
         return result;
     };
 }
@@ -710,7 +696,6 @@ async function query_csv(query_text, input_path, input_delim, input_policy, outp
         input_stream = input_path === null ? process.stdin : fs.createReadStream(input_path);
     }
     let [output_stream, close_output_on_finish] = output_path === null ? [process.stdout, false] : [fs.createWriteStream(output_path), true];
-    // FIXME add on(error) handler to avoid async errors, see https://github.com/nodejs/node-v0.x-archive/issues/406
     if (input_delim == '"' && input_policy == 'quoted')
         throw new RbqlIOHandlingError('Double quote delimiter is incompatible with "quoted" policy');
     if (csv_encoding == 'latin-1')