rbql 0.29.0 → 0.30.0

package/csv_utils.js

@@ -106,14 +106,23 @@ function split_whitespace_separated_str(src, preserve_whitespaces=false) {
 }
 
 
+function get_polymorphic_split_function(dlm, policy, preserve_quotes_and_whitespaces) {
+    // TODO consider moving this function to rbql_csv.js
+    if (policy === 'simple') {
+        return (src) => [src.split(dlm), false];
+    } else if (policy === 'whitespace') {
+        return (src) => [split_whitespace_separated_str(src, preserve_quotes_and_whitespaces), false];
+    } else if (policy === 'monocolumn') {
+        return (src) => [[src], false];
+    } else if (policy === 'quoted' || policy === 'quoted_rfc') {
+        return (src) => split_quoted_str(src, dlm, preserve_quotes_and_whitespaces);
+    } else {
+        throw new Error(`Unsupported splitting policy: ${policy}`);
+    }
+}
+
 function smart_split(src, dlm, policy, preserve_quotes_and_whitespaces) {
-    if (policy === 'simple')
-        return [src.split(dlm), false];
-    if (policy === 'whitespace')
-        return [split_whitespace_separated_str(src, preserve_quotes_and_whitespaces), false];
-    if (policy === 'monocolumn')
-        return [[src], false];
-    return split_quoted_str(src, dlm, preserve_quotes_and_whitespaces);
+    return get_polymorphic_split_function(dlm, policy, preserve_quotes_and_whitespaces)(src);
 }
 
 
@@ -161,6 +170,7 @@ class MultilineRecordAggregator {
 module.exports.split_quoted_str = split_quoted_str;
 module.exports.split_whitespace_separated_str = split_whitespace_separated_str;
 module.exports.smart_split = smart_split;
+module.exports.get_polymorphic_split_function = get_polymorphic_split_function;
 module.exports.quote_field = quote_field;
 module.exports.rfc_quote_field = rfc_quote_field;
 module.exports.unquote_field = unquote_field;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rbql",
-  "version": "0.29.0",
+  "version": "0.30.0",
   "description": "Rainbow Query Language",
   "keywords": ["CSV", "TSV", "spreadsheet", "SQL", "SQL-like", "transpiler", "CLI", "command-line", "library", "browser", "Node", "select", "update", "join"],
   "scripts": {

package/rbql.js

@@ -70,7 +70,7 @@ var query_context = null; // Needs to be global for MIN(), MAX(), etc functions.
 
 
 const wrong_aggregation_usage_error = 'Usage of RBQL aggregation functions inside JavaScript expressions is not allowed, see the docs';
-const RBQL_VERSION = '0.29.0';
+const RBQL_VERSION = '0.30.0';
 
 
 function check_if_brackets_match(opening_bracket, closing_bracket) {

package/rbql_csv.js

@@ -14,9 +14,6 @@ class RbqlIOHandlingError extends Error {}
 class AssertionError extends Error {}
 
 
-// TODO performance improvement: replace smart_split() with polymorphic_split()
-
-
 function assert(condition, message=null) {
     if (!condition) {
         if (!message) {
@@ -215,6 +212,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         this.produced_records_queue = new RecordQueue();
 
         this.process_line_polymorphic = policy == 'quoted_rfc' ? this.process_partial_rfc_record_line : this.process_record_line_simple;
+
+        this.polymorphic_split = csv_utils.get_polymorphic_split_function(this.delim, this.policy, false);
     }
 
 
@@ -353,7 +352,7 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
 
     process_record_line(line) {
         this.NR += 1;
-        var [record, warning] = csv_utils.smart_split(line, this.delim, this.policy, false);
+        var [record, warning] = this.polymorphic_split(line);
         if (this.trim_whitespaces) {
             record = record.map((v) => v.trim());
         }
@@ -664,33 +663,34 @@ class FileSystemCSVRegistry extends rbql.RBQLTableRegistry {
         this.encoding = encoding;
         this.has_header = has_header;
         this.comment_prefix = comment_prefix;
-        this.stream = null;
-        this.record_iterator = null;
-
         this.options = options;
-        this.bulk_input_path = null;
-        this.table_path = null;
+        this.active_join_files = [];
     }
 
     get_iterator_by_table_id(table_id) {
-        this.table_path = find_table_path(this.input_file_dir, table_id);
-        if (this.table_path === null) {
+        let stream = null;
+        let table_path = find_table_path(this.input_file_dir, table_id);
+        if (table_path === null) {
             throw new RbqlIOHandlingError(`Unable to find join table "${table_id}"`);
         }
+        let bulk_input_path = null;
         if (this.options && this.options['bulk_read']) {
-            this.bulk_input_path = this.table_path;
+            bulk_input_path = table_path;
         } else {
-            this.stream = fs.createReadStream(this.table_path);
+            stream = fs.createReadStream(table_path);
         }
         let trim_whitespaces = this.options && this.options['trim_whitespaces'] ? true : false;
         let comment_regex = this.options && this.options.hasOwnProperty('comment_regex') ? this.options['comment_regex'] : null;
-        this.record_iterator = new CSVRecordIterator(this.stream, this.bulk_input_path, this.encoding, this.delim, this.policy, this.has_header, this.comment_prefix, table_id, 'b', trim_whitespaces, comment_regex);
-        return this.record_iterator;
+        let record_iterator = new CSVRecordIterator(stream, bulk_input_path, this.encoding, this.delim, this.policy, this.has_header, this.comment_prefix, table_id, 'b', trim_whitespaces, comment_regex);
+        this.active_join_files.push({'table_path': table_path, 'input_stream': stream, 'record_iterator': record_iterator});
+        return record_iterator;
     };
 
     get_warnings(output_warnings) {
-        if (this.record_iterator && this.has_header) {
-            output_warnings.push(`The first record in JOIN file ${path.basename(this.table_path)} was also treated as header (and skipped)`);
+        if (this.has_header) {
+            for (let active_join_file of this.active_join_files) {
+                output_warnings.push(`The first record in JOIN file ${path.basename(active_join_file.table_path)} was also treated as header (and skipped)`);
+            }
         }
     }
 }