rbql 0.25.0 → 0.26.0

package/.eslintrc.json

@@ -0,0 +1,26 @@
+{
+    "env": {
+        "browser": false,
+        "commonjs": true,
+        "es6": true,
+        "node": true
+    },
+    "parserOptions": {
+        "ecmaFeatures": {
+            "jsx": true
+        },
+        "sourceType": "module",
+        "ecmaVersion": 2018
+    },
+    "rules": {
+        "no-const-assign": "warn",
+        "no-this-before-super": "warn",
+        "no-undef": "warn",
+        "semi": [2, "always"],
+        "no-unreachable": "warn",
+        "no-unused-vars": "warn",
+        "constructor-super": "warn",
+        "no-trailing-spaces": "error",
+        "valid-typeof": "warn"
+    }
+}

package/README.md

@@ -196,7 +196,7 @@ rbql.query_table(user_query, input_table, output_table, warnings).then(success_h
 
 #### Example of query_csv() usage:  
 ```
-const rbql_csv = require('rbql_csv');
+const rbql = require('rbql');
 let user_query = 'SELECT a1, parseInt(a2) % 1000 WHERE a3 != "USA" LIMIT 5';
 let error_handler = function(exception) {
     console.log('Error: ' + String(exception));
@@ -207,7 +207,7 @@ let success_handler = function() {
         console.log('warnings: ' + JSON.stringify(warnings));
     console.log('output table: output.csv');
 }
-rbql_csv.query_csv(user_query, 'input.csv', ',', 'quoted', 'output.csv', ',', 'quoted', 'utf-8', warnings).then(success_handler).catch(error_handler);
+rbql.query_csv(user_query, 'input.csv', ',', 'quoted', 'output.csv', ',', 'quoted', 'utf-8', warnings).then(success_handler).catch(error_handler);
 ```
 
 
@@ -267,6 +267,7 @@ $ rbql-js --input input.csv --output result.csv
 * GROUP BY
 * TOP _N_
 * LIMIT _N_
+* AS
 
 All keywords have the same meaning as in SQL queries. You can check them [online](https://www.w3schools.com/sql/default.asp)  
 
@@ -306,7 +307,7 @@ _COUNT_, _ARRAY_AGG_, _MIN_, _MAX_, _SUM_, _AVG_, _VARIANCE_, _MEDIAN_
 Limitation: aggregate functions inside JavaScript expressions are not supported. Although you can use expressions inside aggregate functions.  
 E.g. `MAX(float(a1) / 1000)` - valid; `MAX(a1) / 1000` - invalid.  
 There is a workaround for the limitation above for _ARRAY_AGG_ function which supports an optional parameter - a callback function that can do something with the aggregated array. Example:  
-`select a2, ARRAY_AGG(a1, v => v.sort().slice(0, 5)) group by a2`
+`SELECT a2, ARRAY_AGG(a1, v => v.sort().slice(0, 5)) GROUP BY a2`
 
 
 ### JOIN statements
@@ -346,14 +347,15 @@ You can define custom functions and/or import libraries in a special file: `~/.r
 
 ## Examples of RBQL queries
 
-* `select top 100 a1, a2 * 10, a4.length where a1 == "Buy" order by parseInt(a2) desc`
-* `select * order by Math.random()` - random sort
-* `select top 20 a.vehicle_price.length / 10, a2 where parseInt(a.vehicle_price) < 500 && ["car", "plane", "boat"].indexOf(a['Vehicle type']) > -1 limit 20` - referencing columns by names from header
-* `update set a3 = 'NPC' where a3.indexOf('Non-playable character') != -1`
-* `select NR, *` - enumerate records, NR is 1-based
-* `select a1, b1, b2 inner join ./countries.txt on a2 == b1 order by a1, a3` - example of join query
-* `select MAX(a1), MIN(a1) where a.Name != 'John' group by a2, a3` - example of aggregate query
-* `select ...a1.split(':')` - Using JS "destructuring assignment" syntax to split one column into many. Do not try this with other SQL engines!
+* `SELECT TOP 100 a1, a2 * 10, a4.length WHERE a1 == "Buy" ORDER BY parseInt(a2) DESC`
+* `SELECT a.id, a.weight / 1000 AS weight_kg`
+* `SELECT * ORDER BY Math.random()` - random sort
+* `SELECT TOP 20 a.vehicle_price.length / 10, a2 WHERE parseInt(a.vehicle_price) < 500 && ["car", "plane", "boat"].indexOf(a['Vehicle type']) > -1 limit 20` - referencing columns by names from header
+* `UPDATE SET a3 = 'NPC' WHERE a3.indexOf('Non-playable character') != -1`
+* `SELECT NR, *` - enumerate records, NR is 1-based
+* `SELECT a1, b1, b2 INNER JOIN ./countries.txt ON a2 == b1 ORDER BY a1, a3` - example of join query
+* `SELECT MAX(a1), MIN(a1) WHERE a.Name != 'John' GROUP BY a2, a3` - example of aggregate query
+* `SELECT ...a1.split(':')` - Using JS "destructuring assignment" syntax to split one column into many. Do not try this with other SQL engines!
 
 
 ### References

package/csv_utils.js

@@ -39,6 +39,7 @@ function extract_next_field(src, dlm, preserve_quotes_and_whitespaces, allow_ext
 
 
 function split_quoted_str(src, dlm, preserve_quotes_and_whitespaces=false) {
+    // This function is newline-agnostic i.e. it can also split records with multiline fields.
     if (src.indexOf('"') == -1) // Optimization for most common case
         return [src.split(dlm), false];
     var result = [];
@@ -116,6 +117,41 @@ function smart_split(src, dlm, policy, preserve_quotes_and_whitespaces) {
 }
 
 
+class MultilineRecordAggregator {
+    constructor(comment_prefix) {
+        this.comment_prefix = comment_prefix;
+        this.reset();
+    }
+    add_line(line_text) {
+        if (this.has_full_record || this.has_comment_line) {
+            throw new Error('Invalid usage - record aggregator must be reset before adding new lines');
+        }
+        if (this.comment_prefix && this.rfc_line_buffer.length == 0 && line_text.startsWith(this.comment_prefix)) {
+            this.has_comment_line = true;
+            return false;
+        }
+        let match_list = line_text.match(/"/g);
+        let has_unbalanced_double_quote = match_list && match_list.length % 2 == 1;
+        this.rfc_line_buffer.push(line_text);
+        this.has_full_record = (!has_unbalanced_double_quote && this.rfc_line_buffer.length == 1) || (has_unbalanced_double_quote && this.rfc_line_buffer.length > 1);
+        return this.has_full_record;
+    }
+    is_inside_multiline_record() {
+        return this.rfc_line_buffer.length && !this.has_full_record;
+    }
+    get_full_line(line_separator) {
+        return this.rfc_line_buffer.join(line_separator);
+    }
+    get_num_lines_in_record() {
+        return this.rfc_line_buffer.length;
+    }
+    reset() {
+        this.rfc_line_buffer = [];
+        this.has_full_record = false;
+        this.has_comment_line = false;
+    }
+}
+
 
 module.exports.split_quoted_str = split_quoted_str;
 module.exports.split_whitespace_separated_str = split_whitespace_separated_str;
@@ -125,3 +161,4 @@ module.exports.rfc_quote_field = rfc_quote_field;
 module.exports.unquote_field = unquote_field;
 module.exports.unquote_fields = unquote_fields;
 module.exports.split_lines = split_lines;
+module.exports.MultilineRecordAggregator = MultilineRecordAggregator;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rbql",
-  "version": "0.25.0",
+  "version": "0.26.0",
   "description": "Rainbow Query Language",
   "keywords": ["CSV", "TSV", "spreadsheet", "SQL", "SQL-like", "transpiler", "CLI", "command-line", "library", "browser", "Node", "select", "update", "join"],
   "scripts": {

package/rbql.js

@@ -70,7 +70,7 @@ var query_context = null; // Needs to be global for MIN(), MAX(), etc functions.
 
 
 const wrong_aggregation_usage_error = 'Usage of RBQL aggregation functions inside JavaScript expressions is not allowed, see the docs';
-const RBQL_VERSION = '0.25.0';
+const RBQL_VERSION = '0.26.0';
 
 
 function check_if_brackets_match(opening_bracket, closing_bracket) {
@@ -128,28 +128,32 @@ function column_info_from_text_span(text_span, string_literals) {
     let attribute_match = /^([ab])\.([_a-zA-Z][_a-zA-Z0-9]*)$/.exec(text_span);
     let subscript_int_match = /^([ab])\[([0-9]+)\]$/.exec(text_span);
     let subscript_str_match = /^([ab])\[___RBQL_STRING_LITERAL([0-9]+)___\]$/.exec(text_span);
+    let as_alias_match = /^(.*) (as|AS) +([a-zA-Z][a-zA-Z0-9_]*) *$/.exec(text_span);
+    if (as_alias_match !== null) {
+        return {table_name: null, column_index: null, column_name: as_alias_match[3], is_star: false, is_alias: true};
+    }
     if (simple_var_match !== null) {
         if (text_span == rbql_star_marker)
-            return {table_name: null, column_index: null, column_name: null, is_star: true};
+            return {table_name: null, column_index: null, column_name: null, is_star: true, is_alias: false};
         if (text_span.startsWith('___RBQL_STRING_LITERAL'))
             return null;
         let match = /^([ab])([0-9]+)$/.exec(text_span);
         if (match !== null) {
-            return {table_name: match[1], column_index: parseInt(match[2]) - 1, column_name: null, is_star: false};
+            return {table_name: match[1], column_index: parseInt(match[2]) - 1, column_name: null, is_star: false, is_alias: false};
         }
         // Some examples for this branch: NR, NF
-        return {table_name: null, column_index: null, column_name: text_span, is_star: false};
+        return {table_name: null, column_index: null, column_name: text_span, is_star: false, is_alias: false};
     } else if (attribute_match !== null) {
         let table_name = attribute_match[1];
         let column_name = attribute_match[2];
         if (column_name == rbql_star_marker) {
-            return {table_name: table_name, column_index: null, column_name: null, is_star: true};
+            return {table_name: table_name, column_index: null, column_name: null, is_star: true, is_alias: false};
         }
-        return {table_name: null, column_index: null, column_name: column_name, is_star: false};
+        return {table_name: null, column_index: null, column_name: column_name, is_star: false, is_alias: false};
     } else if (subscript_int_match != null) {
         let table_name = subscript_int_match[1];
         let column_index = parseInt(subscript_int_match[2]) - 1;
-        return {table_name: table_name, column_index: column_index, column_name: null, is_star: false};
+        return {table_name: table_name, column_index: column_index, column_name: null, is_star: false, is_alias: false};
     } else if (subscript_str_match != null) {
         let table_name = subscript_str_match[1];
         let replaced_string_literal_id = subscript_str_match[2];
@@ -157,7 +161,7 @@ function column_info_from_text_span(text_span, string_literals) {
             let quoted_column_name = string_literals[replaced_string_literal_id];
             let unquoted_column_name = unquote_string(quoted_column_name);
             if (unquoted_column_name !== null && unquoted_column_name !== undefined) {
-                return {table_name: null, column_index: null, column_name: unquoted_column_name, is_star: false};
+                return {table_name: null, column_index: null, column_name: unquoted_column_name, is_star: false, is_alias: false};
             }
         }
     }
@@ -1328,9 +1332,11 @@ function translate_update_expression(update_expression, input_variables_map, str
 
 
 function translate_select_expression(select_expression) {
-    let expression_without_stars = replace_star_count(select_expression);
-    let translated = str_strip(replace_star_vars(expression_without_stars));
-    let translated_for_header = str_strip(replace_star_vars_for_header_parsing(expression_without_stars));
+    let as_alias_replacement_regexp = / +(AS|as) +([a-zA-Z][a-zA-Z0-9_]*) *(?=$|,)/g;
+    let expression_without_counting_stars = replace_star_count(select_expression);
+    let expression_without_as_column_alias = expression_without_counting_stars.replace(as_alias_replacement_regexp, '');
+    let translated = str_strip(replace_star_vars(expression_without_as_column_alias));
+    let translated_for_header = str_strip(replace_star_vars_for_header_parsing(expression_without_counting_stars));
     if (!translated.length)
         throw new RbqlParsingError('"SELECT" expression is empty');
     return [`[].concat([${translated}])`, translated_for_header];
@@ -1571,12 +1577,21 @@ function remove_redundant_table_name(query_text) {
 
 
 function select_output_header(input_header, join_header, query_column_infos) {
-    if (input_header === null && join_header === null)
+    if (input_header === null) {
+        assert(join_header === null);
+    }
+    if (input_header === null) {
+        for (let qci of query_column_infos) {
+            if (qci !== null && qci.is_alias) {
+                throw new RbqlParsingError(`Specifying column alias "AS ${qci.column_name}" is not allowed if input table has no header`);
+            }
+        }
         return null;
-    if (input_header === null)
-        input_header = [];
-    if (join_header === null)
+    }
+    if (join_header === null) {
+        // This means there is no JOIN table.
         join_header = [];
+    }
     let output_header = [];
     for (let qci of query_column_infos) {
         // TODO refactor this and python version: extract this code into a function instead to always return something
@@ -1608,20 +1623,20 @@ function select_output_header(input_header, join_header, query_column_infos) {
 }
 
 
-function make_inconsistent_num_fields_warning(table_name, inconsistent_records_info) {
-    let keys = Object.keys(inconsistent_records_info);
-    let entries = [];
-    for (let i = 0; i < keys.length; i++) {
-        let key = keys[i];
-        let record_id = inconsistent_records_info[key];
-        entries.push([record_id, key]);
-    }
-    entries.sort(function(a, b) { return a[0] - b[0]; });
+function sample_first_two_inconsistent_records(inconsistent_records_info) {
+    let entries = Array.from(inconsistent_records_info.entries());
+    entries.sort(function(a, b) { return a[1] - b[1]; });
     assert(entries.length > 1);
-    let [record_1, num_fields_1] = entries[0];
-    let [record_2, num_fields_2] = entries[1];
+    let [num_fields_1, record_num_1] = entries[0];
+    let [num_fields_2, record_num_2] = entries[1];
+    return [record_num_1, num_fields_1, record_num_2, num_fields_2];
+}
+
+
+function make_inconsistent_num_fields_warning(table_name, inconsistent_records_info) {
+    let [record_num_1, num_fields_1, record_num_2, num_fields_2] = sample_first_two_inconsistent_records(inconsistent_records_info);
     let warn_msg = `Number of fields in "${table_name}" table is not consistent: `;
-    warn_msg += `e.g. record ${record_1} -> ${num_fields_1} fields, record ${record_2} -> ${num_fields_2} fields`;
+    warn_msg += `e.g. record ${record_num_1} -> ${num_fields_1} fields, record ${record_num_2} -> ${num_fields_2} fields`;
     return warn_msg;
 }
 
@@ -1691,7 +1706,7 @@ class TableIterator extends RBQLInputIterator {
         this.normalize_column_names = normalize_column_names;
         this.variable_prefix = variable_prefix;
         this.nr = 0;
-        this.fields_info = new Object();
+        this.fields_info = new Map();
         this.stopped = false;
     }
 
@@ -1727,13 +1742,13 @@ class TableIterator extends RBQLInputIterator {
         let record = this.table[this.nr];
         this.nr += 1;
         let num_fields = record.length;
-        if (!this.fields_info.hasOwnProperty(num_fields))
-            this.fields_info[num_fields] = this.nr;
+        if (!this.fields_info.has(num_fields))
+            this.fields_info.set(num_fields, this.nr);
         return record;
     };
 
     get_warnings() {
-        if (Object.keys(this.fields_info).length > 1)
+        if (this.fields_info.size > 1)
             return [make_inconsistent_num_fields_warning('input', this.fields_info)];
         return [];
     };
@@ -1799,6 +1814,8 @@ async function shallow_parse_input_query(query_text, input_iterator, join_tables
         query_context.aggregation_key_expression = '[' + combine_string_literals(rb_actions[GROUP_BY]['text'], string_literals) + ']';
     }
 
+
+    let input_header = await input_iterator.get_header();
     let join_variables_map = null;
     let join_header = null;
     if (rb_actions.hasOwnProperty(JOIN)) {
@@ -1813,6 +1830,12 @@ async function shallow_parse_input_query(query_text, input_iterator, join_tables
         }
         join_variables_map = await join_record_iterator.get_variables_map(query_text);
         join_header = await join_record_iterator.get_header();
+        if (input_header === null && join_header !== null) {
+            throw new RbqlIOHandlingError('Inconsistent modes: Input table doesn\'t have a header while the Join table has a header');
+        }
+        if (input_header !== null && join_header === null) {
+            throw new RbqlIOHandlingError('Inconsistent modes: Input table has a header while the Join table doesn\'t have a header');
+        }
         let [lhs_variables, rhs_indices] = resolve_join_variables(input_variables_map, join_variables_map, variable_pairs, string_literals);
         let sql_join_type = {'JOIN': InnerJoiner, 'INNER JOIN': InnerJoiner, 'LEFT JOIN': LeftJoiner, 'LEFT OUTER JOIN': LeftJoiner, 'STRICT LEFT JOIN': StrictLeftJoiner}[rb_actions[JOIN]['join_subtype']];
         query_context.lhs_join_var_expression = lhs_variables.length == 1 ? lhs_variables[0] : 'JSON.stringify([' + lhs_variables.join(',') + '])';
@@ -1830,7 +1853,6 @@ async function shallow_parse_input_query(query_text, input_iterator, join_tables
         query_context.where_expression = combine_string_literals(where_expression, string_literals);
     }
 
-    let input_header = await input_iterator.get_header();
     if (rb_actions.hasOwnProperty(UPDATE)) {
         var update_expression = translate_update_expression(rb_actions[UPDATE]['text'], input_variables_map, string_literals, ' '.repeat(8));
         query_context.update_expressions = combine_string_literals(update_expression, string_literals);
@@ -1840,13 +1862,16 @@ async function shallow_parse_input_query(query_text, input_iterator, join_tables
     if (rb_actions.hasOwnProperty(SELECT)) {
         query_context.top_count = find_top(rb_actions);
         if (rb_actions.hasOwnProperty(EXCEPT)) {
+            if (rb_actions.hasOwnProperty(JOIN)) {
+                throw new RbqlParsingError('EXCEPT and JOIN are not allowed in the same query');
+            }
             let [output_header, select_expression] = translate_except_expression(rb_actions[EXCEPT]['text'], input_variables_map, string_literals, input_header);
             query_context.select_expression = select_expression;
             query_context.writer.set_header(output_header);
         } else {
-            let [select_expression, select_expression_for_ast] = translate_select_expression(rb_actions[SELECT]['text']);
+            let [select_expression, select_expression_for_header] = translate_select_expression(rb_actions[SELECT]['text']);
             query_context.select_expression = combine_string_literals(select_expression, string_literals);
-            let column_infos = adhoc_parse_select_expression_to_column_infos(select_expression_for_ast, string_literals);
+            let column_infos = adhoc_parse_select_expression_to_column_infos(select_expression_for_header, string_literals);
             let output_header = select_output_header(input_header, join_header, column_infos);
             query_context.writer.set_header(output_header);
         }
@@ -1947,5 +1972,6 @@ exports.adhoc_parse_select_expression_to_column_infos = adhoc_parse_select_expre
 exports.replace_star_count = replace_star_count;
 exports.replace_star_vars_for_header_parsing = replace_star_vars_for_header_parsing;
 exports.select_output_header = select_output_header;
+exports.sample_first_two_inconsistent_records = sample_first_two_inconsistent_records;
 
 }(typeof exports === 'undefined' ? this.rbql = {} : exports));

package/rbql_csv.js

@@ -62,19 +62,9 @@ function remove_utf8_bom(line, assumed_source_encoding) {
 
 
 function make_inconsistent_num_fields_warning(table_name, inconsistent_records_info) {
-    let keys = Object.keys(inconsistent_records_info);
-    let entries = [];
-    for (let i = 0; i < keys.length; i++) {
-        let key = keys[i];
-        let record_id = inconsistent_records_info[key];
-        entries.push([record_id, key]);
-    }
-    entries.sort(function(a, b) { return a[0] - b[0]; });
-    assert(entries.length > 1);
-    let [record_1, num_fields_1] = entries[0];
-    let [record_2, num_fields_2] = entries[1];
+    let [record_num_1, num_fields_1, record_num_2, num_fields_2] = rbql.sample_first_two_inconsistent_records(inconsistent_records_info);
     let warn_msg = `Number of fields in "${table_name}" table is not consistent: `;
-    warn_msg += `e.g. record ${record_1} -> ${num_fields_1} fields, record ${record_2} -> ${num_fields_2} fields`;
+    warn_msg += `e.g. record ${record_num_1} -> ${num_fields_1} fields, record ${record_num_2} -> ${num_fields_2} fields`;
     return warn_msg;
 }
 
@@ -182,7 +172,7 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
 
         this.table_name = table_name;
         this.variable_prefix = variable_prefix;
-        this.comment_prefix = (comment_prefix !== null && comment_prefix.length) ? comment_prefix : null;
+        this.comment_prefix = comment_prefix;
 
         this.decoder = null;
         if (encoding == 'utf-8' && this.csv_path === null) {
@@ -204,11 +194,11 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         this.utf8_bom_removed = false; // BOM doesn't get automatically removed by the decoder when utf-8 file is treated as latin-1
         this.first_defective_line = null;
 
-        this.fields_info = new Object();
+        this.fields_info = new Map();
         this.NR = 0; // Record number
         this.NL = 0; // Line number (NL != NR when the CSV file has comments or multiline fields)
 
-        this.rfc_line_buffer = [];
+        this.line_aggregator = new csv_utils.MultilineRecordAggregator(comment_prefix);
 
         this.partially_decoded_line = '';
         this.partially_decoded_line_ends_with_cr = false;
@@ -222,7 +212,7 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
 
         this.produced_records_queue = new RecordQueue();
 
-        this.process_line_polymorphic = policy == 'quoted_rfc' ? this.process_partial_rfc_record_line : this.process_record_line;
+        this.process_line_polymorphic = policy == 'quoted_rfc' ? this.process_partial_rfc_record_line : this.process_record_line_simple;
     }
 
 
@@ -350,9 +340,14 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
     };
 
 
-    process_record_line(line) {
-        if (this.comment_prefix !== null && line.startsWith(this.comment_prefix))
+    process_record_line_simple(line) {
+        if (this.comment_prefix && line.startsWith(this.comment_prefix))
             return; // Just skip the line
+        this.process_record_line(line);
+    }
+
+
+    process_record_line(line) {
         this.NR += 1;
         var [record, warning] = csv_utils.smart_split(line, this.delim, this.policy, false);
         if (warning) {
@@ -363,29 +358,20 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
             }
         }
         let num_fields = record.length;
-        if (!this.fields_info.hasOwnProperty(num_fields))
-            this.fields_info[num_fields] = this.NR;
+        if (!this.fields_info.has(num_fields))
+            this.fields_info.set(num_fields, this.NR);
         this.produced_records_queue.enqueue(record);
         this.try_resolve_next_record();
     };
 
 
     process_partial_rfc_record_line(line) {
-        if (this.comment_prefix !== null && this.rfc_line_buffer.length == 0 && line.startsWith(this.comment_prefix))
-            return; // Just skip the line
-        let match_list = line.match(/"/g);
-        let has_unbalanced_double_quote = match_list && match_list.length % 2 == 1;
-        if (this.rfc_line_buffer.length == 0 && !has_unbalanced_double_quote) {
-            this.process_record_line(line);
-        } else if (this.rfc_line_buffer.length == 0 && has_unbalanced_double_quote) {
-            this.rfc_line_buffer.push(line);
-        } else if (!has_unbalanced_double_quote) {
-            this.rfc_line_buffer.push(line);
-        } else {
-            this.rfc_line_buffer.push(line);
-            let multiline_row = this.rfc_line_buffer.join('\n');
-            this.rfc_line_buffer = [];
-            this.process_record_line(multiline_row);
+        this.line_aggregator.add_line(line);
+        if (this.line_aggregator.has_comment_line) {
+            this.line_aggregator.reset();
+        } else if (this.line_aggregator.has_full_record) {
+            this.process_record_line(this.line_aggregator.get_full_line('\n'));
+            this.line_aggregator.reset();
         }
     };
 
@@ -432,13 +418,13 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
     };
 
 
-    process_data_bulk(data_chunk) {
-        let decoded_string = data_chunk.toString(this.encoding);
+    process_data_bulk(data_blob) {
+        let decoded_string = data_blob.toString(this.encoding);
         if (this.encoding == 'utf-8') {
             // Using hacky comparison method from here: https://stackoverflow.com/a/32279283/2898283
             // TODO get rid of this once TextDecoder is really fixed or when alternative method of reliable decoding appears
             let control_buffer = Buffer.from(decoded_string, 'utf-8');
-            if (Buffer.compare(data_chunk, control_buffer) != 0) {
+            if (Buffer.compare(data_blob, control_buffer) != 0) {
                 this.store_or_propagate_exception(new RbqlIOHandlingError(utf_decoding_error));
                 return;
             }
@@ -449,8 +435,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         for (let i = 0; i < lines.length; i++) {
             this.process_line(lines[i]);
         }
-        if (this.rfc_line_buffer.length > 0) {
-            this.process_record_line(this.rfc_line_buffer.join('\n'));
+        if (this.line_aggregator.is_inside_multiline_record()) {
+            this.process_record_line(this.line_aggregator.get_full_line('\n'));
         }
         this.input_exhausted = true;
         this.try_resolve_next_record(); // Should be a NOOP here?
@@ -464,8 +450,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
             this.partially_decoded_line = '';
             this.process_line(last_line);
         }
-        if (this.rfc_line_buffer.length > 0) {
-            this.process_record_line(this.rfc_line_buffer.join('\n'));
+        if (this.line_aggregator.is_inside_multiline_record()) {
+            this.process_record_line(this.line_aggregator.get_full_line('\n'));
         }
         this.try_resolve_next_record();
     };
@@ -487,11 +473,11 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
         } else {
             let parent_iterator = this;
             return new Promise(function(resolve, reject) {
-                fs.readFile(parent_iterator.csv_path, (err, data_chunk) => {
+                fs.readFile(parent_iterator.csv_path, (err, data_blob) => {
                     if (err) {
                         reject(err);
                     } else {
-                        parent_iterator.process_data_bulk(data_chunk);
+                        parent_iterator.process_data_bulk(data_blob);
                         resolve();
                     }
                 });
@@ -506,8 +492,8 @@ class CSVRecordIterator extends rbql.RBQLInputIterator {
             result.push(`Inconsistent double quote escaping in ${this.table_name} table. E.g. at line ${this.first_defective_line}`);
         if (this.utf8_bom_removed)
             result.push(`UTF-8 Byte Order Mark (BOM) was found and skipped in ${this.table_name} table`);
-        if (Object.keys(this.fields_info).length > 1)
-            result.push(make_inconsistent_num_fields_warning('input', this.fields_info));
+        if (this.fields_info.size > 1)
+            result.push(make_inconsistent_num_fields_warning(this.table_name, this.fields_info));
         return result;
     };
 }
@@ -710,7 +696,6 @@ async function query_csv(query_text, input_path, input_delim, input_policy, outp
         input_stream = input_path === null ? process.stdin : fs.createReadStream(input_path);
     }
     let [output_stream, close_output_on_finish] = output_path === null ? [process.stdout, false] : [fs.createWriteStream(output_path), true];
-    // FIXME add on(error) handler to avoid async errors, see https://github.com/nodejs/node-v0.x-archive/issues/406
     if (input_delim == '"' && input_policy == 'quoted')
         throw new RbqlIOHandlingError('Double quote delimiter is incompatible with "quoted" policy');
     if (csv_encoding == 'latin-1')