rapydscript-ng 0.7.23 → 0.8.0

package/tools/{gettext.js → gettext.mjs}

@@ -1,17 +1,19 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
-"use strict";  /*jshint node:true */
+"use strict";
+
+import fs from 'fs';
+import path from 'path';
 
-var fs = require('fs');
-var RapydScript = (typeof create_rapydscript_compiler === 'function') ? create_rapydscript_compiler() : require('./compiler').create_compiler();
-var path = require('path');
+var _rapydscript_compiler = typeof create_rapydscript_compiler !== 'undefined' ? create_rapydscript_compiler : globalThis.create_rapydscript_compiler;
+var RapydScript = _rapydscript_compiler();
 
-function parse_file(code, filename) {
-    return RapydScript.parse(code, {
+async function parse_file(code, filename) {
+    return await RapydScript.parse(code, {
         filename: filename,
         basedir: path.dirname(filename),
         libdir: path.dirname(filename),
@@ -27,18 +29,19 @@ function detect_format(msgid) {
 
 function Gettext(catalog, filename) {
     this._visit = function (node, cont) {
-        if (node instanceof RapydScript.AST_Call && node.args && node.args.length && node.expression instanceof RapydScript.AST_Symbol) {
+        if (node instanceof RapydScript.AST_Call && node.args && node.args.args && node.args.args.length && node.expression instanceof RapydScript.AST_Symbol) {
             var name = node.expression.name;
             if (name === '_' || name === 'gettext' || name === 'ngettext') {
                 var nargs = (name === 'ngettext') ? 2 : 1;
                 var line = node.start.line;
+                var pargs = node.args.args;
                 for (var i = 0; i < nargs; i++) {
-                    if (!(node.args[i] instanceof RapydScript.AST_String)) {
+                    if (!(pargs[i].value instanceof RapydScript.AST_String)) {
                         console.error('Translation function: ' + name + ' does not have a string literal argument at line: ' + line + ' of ' + filename);
                         process.exit(1);
                     }
                 }
-                var msgid = node.args[0].value;
+                var msgid = pargs[0].value.value;
                 if (!Object.prototype.hasOwnProperty.call(catalog, msgid)) {
                     catalog[msgid] = {
                         'locations': [],
@@ -46,20 +49,20 @@ function Gettext(catalog, filename) {
                         'format': detect_format(msgid),
                     };
                 }
-                if (name === 'ngettext') catalog[msgid].plural = node.args[1].value;
+                if (name === 'ngettext') catalog[msgid].plural = pargs[1].value.value;
                 if (filename) catalog[msgid].locations.push(filename + ':' + line);
             }
-            
+
         }
         if (cont !== undefined) cont();
     };
 }
 
-function gettext(catalog, code, filename) {
+async function gettext(catalog, code, filename) {
     var toplevel;
 
     try {
-        toplevel = parse_file(code, filename);
+        toplevel = await parse_file(code, filename);
     } catch(e) {
         if (e instanceof RapydScript.SyntaxError) {
             console.error('Failed to parse: ' + filename + ' with error: ' + e.line + ':' + e.col + ':' + e.message);
@@ -125,61 +128,51 @@ function write_output(catalog, options, write) {
 
 // CLI {{{
 
-function read_whole_file(filename, cb) {
+async function read_whole_file(filename) {
     if (!filename) {
         var chunks = [];
         process.stdin.setEncoding('utf-8');
-        process.stdin.on('data', function (chunk) {
-            chunks.push(chunk);
-        }).on('end', function () {
-            cb(null, chunks.join(""));
+        await new Promise((resolve, reject) => {
+            process.stdin.on('data', chunk => chunks.push(chunk));
+            process.stdin.on('end', resolve);
+            process.stdin.on('error', reject);
         });
         process.openStdin();
+        return chunks.join('');
     } else {
-        fs.readFile(filename, "utf-8", cb);
+        return await fs.promises.readFile(filename, 'utf-8');
     }
 }
 
-module.exports.cli = function(argv, base_path, src_path, lib_path) {
+export async function cli(argv, base_path, src_path, lib_path) {
     var files = [];
-    var num_of_files = files.length || 1;
     var catalog = {};
 
-    function read_files(src) {
-        src.forEach(function(f) {
-            if (fs.lstatSync(f).isDirectory()) {
-                var children = [];
-                fs.readdirSync(f).forEach(function(x) { children.push(path.join(f, x)); });
-                read_files(children);
-            }
-            else files.push(f);
-        });
+    async function read_files(src) {
+        for (const f of src) {
+            var stat = await fs.promises.lstat(f);
+            if (stat.isDirectory()) {
+                var children = (await fs.promises.readdir(f)).map(x => path.join(f, x));
+                await read_files(children);
+            } else files.push(f);
+        }
     }
-    read_files(argv.files);
-
-    function process_single_file(err, code) {
-        if (err) {
-            console.error("ERROR: can't read file: " + files[0]);
+    await read_files(argv.files);
+
+    for (const f of (files.length ? files : [null])) {
+        var code;
+        try {
+            code = await read_whole_file(f);
+        } catch(e) {
+            console.error("ERROR: can't read file: " + f);
             process.exit(1);
         }
-
-        gettext(catalog, code, files[0]);
-
-        files = files.slice(1);
-        if (files.length) {
-            setImmediate(read_whole_file, files[0], process_single_file);
-            return;
-        } else {
-            write_output(catalog, argv);
-            process.exit(0);
-        }
+        await gettext(catalog, code, f);
     }
- 
-    setImmediate(read_whole_file, files[0], process_single_file);
 
-};
+    write_output(catalog, argv);
+    process.exit(0);
+}
 
-module.exports.gettext = gettext;
-module.exports.entry_to_string = entry_to_string;
-module.exports.write_output = write_output;
+export { gettext, entry_to_string, write_output };
 // }}}

package/tools/{ini.js → ini.mjs}

@@ -1,16 +1,15 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
-"use strict";  /*jshint node:true */
 
-var fs = require('fs');
-var path = require('path');
+import fs from 'fs';
+import path from 'path';
 
 function parse_ini_data(data) {
-    // Based on MIT licensed code from: 
+    // Based on MIT licensed code from:
     // https://github.com/shockie/node-iniparser/blob/master/lib/node-iniparser.js
     var ans = {}, match;
 	var lines = data.split(/\r\n|\r|\n/);
@@ -40,11 +39,11 @@ function parse_ini_data(data) {
 	return ans;
 }
 
-function find_cfg_file(toplevel_dir) {
+async function find_cfg_file(toplevel_dir) {
     var current_dir = toplevel_dir, previous_dir = toplevel_dir;
     do {
         try {
-            return fs.readFileSync(path.join(current_dir, 'setup.cfg'), 'utf-8');
+            return await fs.promises.readFile(path.join(current_dir, 'setup.cfg'), 'utf-8');
         } catch (e) {
             if (e.code !== 'ENOENT') throw e;
         }
@@ -55,11 +54,11 @@ function find_cfg_file(toplevel_dir) {
     return null;
 }
 
-function read_config(toplevel_dir) {
-    var data = find_cfg_file(toplevel_dir);
+async function read_config(toplevel_dir) {
+    var data = await find_cfg_file(toplevel_dir);
     if (!data) return {};
     return parse_ini_data(data);
 }
 
 
-exports.read_config = read_config;
+export { read_config };

package/tools/{lint.js → lint.mjs}

@@ -1,19 +1,23 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
-"use strict";  /*jshint node:true */
 
-var fs = require('fs');
-var RapydScript = require("./compiler").create_compiler();
-var path = require('path');
-var utils = require('./utils');
+import fs from 'fs';
+import path from 'path';
+import { read_config } from './ini.mjs';
+import * as utils from './utils.mjs';
+
+// Use the compiler instance pre-created by bin/rapydscript. It is set as a
+// synchronous global before any dynamic imports, so this module can be
+// require()'d without top-level await.
+const RapydScript = globalThis.create_rapydscript_compiler();
 var colored = utils.safe_colored;
 
-var WARN = 1, ERROR = 2;
-var MESSAGES = {
+export var WARN = 1, ERROR = 2;
+export var MESSAGES = {
     'undef': 'undefined symbol: "{name}"',
     'unused-import': '"{name}" is imported but not used',
     'unused-local' : '"{name}" is defined but not used',
@@ -29,8 +33,8 @@ var MESSAGES = {
 };
 
 var BUILTINS = Object.create(null);
-('this self window document chr ord iterator_symbol print len range dir' + 
- ' eval undefined arguments abs max min enumerate pow callable reversed sum' + 
+('this self window document chr ord iterator_symbol print len range dir' +
+ ' eval undefined arguments abs max min enumerate pow callable reversed sum' +
  ' getattr isFinite setattr hasattr parseInt parseFloat options_object' +
  ' isNaN JSON Math list set list_wrap ρσ_modules require bool int bin' +
  ' float iter Error EvalError set_wrap RangeError ReferenceError SyntaxError' +
@@ -39,7 +43,7 @@ var BUILTINS = Object.create(null);
  ' NodeList alert console Node Symbol NamedNodeMap ρσ_eslice ρσ_delslice Number' +
  ' Boolean encodeURIComponent decodeURIComponent setTimeout setInterval' +
  ' setImmediate clearTimeout clearInterval clearImmediate requestAnimationFrame' +
- ' id repr sorted __name__ equals get_module ρσ_str jstype divmod NaN'
+ ' id repr sorted __name__ equals get_module ρσ_str jstype divmod NaN no_prune'
  ).split(' ').forEach(function(x) { BUILTINS[x] = true; });
 
 Object.keys(RapydScript.NATIVE_CLASSES).forEach(function (name) { BUILTINS[name] = true; });
@@ -56,8 +60,8 @@ function cmp(a, b) {
     return (a < b) ? -1 : ((a > b) ? 1 : 0);
 }
 
-function parse_file(code, filename) {
-    return RapydScript.parse(code, {
+async function parse_file(code, filename) {
+    return await RapydScript.parse(code, {
         filename: filename,
         basedir: path.dirname(filename),
         libdir: path.dirname(filename),
@@ -70,7 +74,7 @@ function msg_from_node(filename, ident, name, node, level, line) {
     if (node instanceof RapydScript.AST_Lambda && node.name) name = node.name.name;
     var msg = MESSAGES[ident].replace('{name}', name || '').replace('{line}', line || '');
     return {
-        filename: filename, 
+        filename: filename,
         start_line: (node.start) ? node.start.line : undefined,
         start_col: (node.start) ? node.start.col : undefined,
         end_line: (node.end) ? node.end.line : undefined,
@@ -157,7 +161,7 @@ function Scope(is_toplevel, parent_scope, filename, is_class) {
             this.for_descendants(function (scope) {
                 if (has_prop(scope.undefined_references, name)) {
                     found = true;
-                    // Remove from childs' undefined references 
+                    // Remove from childs' undefined references
                     delete scope.undefined_references[name];
                 } else if (has_prop(scope.nonlocals, name) && has_prop(scope.bindings, name)) found = true;
             });
@@ -295,7 +299,7 @@ function Linter(toplevel, filename, code, options) {
         if (node.left instanceof RapydScript.AST_SymbolRef) {
             node.left.lint_visited = node.operator === '=';  // Could be compound assignment like: +=
             if (node.operator === '=') {
-                // Only create a binding if the operator is not 
+                // Only create a binding if the operator is not
                 // a compound assignment operator
                 this.current_node = node.left;
                 this.add_binding(node.left.name);
@@ -374,7 +378,7 @@ function Linter(toplevel, filename, code, options) {
                     }
                 }
             }
- 
+
         }
     };
 
@@ -419,7 +423,7 @@ function Linter(toplevel, filename, code, options) {
         (node.properties || []).forEach(function (prop) {
             if (prop.key instanceof RapydScript.AST_Constant) {
                 var val = prop.key.value;
-                if (has_prop(seen, val)) 
+                if (has_prop(seen, val))
                     this.messages.push(msg_from_node(filename, 'dup-key', val, prop));
                 seen[val] = true;
             }
@@ -483,7 +487,7 @@ function Linter(toplevel, filename, code, options) {
 
         if (node instanceof RapydScript.AST_Scope) {
             this.handle_scope();
-        } 
+        }
 
         if (cont !== undefined) cont();
 
@@ -498,11 +502,39 @@ function Linter(toplevel, filename, code, options) {
     this.resolve = function() {
         var messages = this.messages;
         var line_filters = {};
+        var in_multiline_str = false;
+        var ml_quote = null;
 
         code.split('\n').forEach(function(line, num) {
             line = line.trimRight();
             num++;
-            if (line[line.length - 1] === ';') {
+
+            // Track triple-quoted multiline string state so we can skip
+            // eol-semicolon checks for lines that are inside string literals.
+            var line_starts_in_multiline = in_multiline_str;
+            var pos = 0, close_pos;
+            while (pos < line.length) {
+                if (!in_multiline_str) {
+                    var dq = line.indexOf('"""', pos);
+                    var sq = line.indexOf("'''", pos);
+                    var first, quote;
+                    if (dq !== -1 && (sq === -1 || dq < sq)) { first = dq; quote = '"""'; }
+                    else if (sq !== -1) { first = sq; quote = "'''"; }
+                    else break;
+                    in_multiline_str = true;
+                    ml_quote = quote;
+                    pos = first + 3;
+                    close_pos = line.indexOf(ml_quote, pos);
+                    if (close_pos !== -1) { in_multiline_str = false; ml_quote = null; pos = close_pos + 3; }
+                    else break;
+                } else {
+                    close_pos = line.indexOf(ml_quote, pos);
+                    if (close_pos !== -1) { in_multiline_str = false; ml_quote = null; pos = close_pos + 3; }
+                    else break;
+                }
+            }
+
+            if (!line_starts_in_multiline && !in_multiline_str && line[line.length - 1] === ';') {
                 var ident = 'eol-semicolon';
                 messages.push({filename:filename, ident:ident, message:MESSAGES[ident],
                     level:WARN, name:';', start_line:num, start_col:line.lastIndexOf(';')});
@@ -535,7 +567,18 @@ function Linter(toplevel, filename, code, options) {
 
 }
 
-function lint_code(code, options) {
+// Run only the linter analysis over an already parsed toplevel. Used by both
+// lint_code and the LSP (which parses once, in error-recovery mode, and reuses
+// the resulting AST for all of its services).
+export function lint_parsed(toplevel, code, options) {
+    options = options || {};
+    var filename = options.filename || '<eval>';
+    var linter = new Linter(toplevel, filename, code, options);
+    toplevel.walk(linter);
+    return linter.resolve();
+}
+
+export async function lint_code(code, options) {
     options = options || {};
     var reportcb = {'json':cli_json_report, 'vim': cli_vim_report, 'undef': cli_undef_report}[options.errorformat] || (options.report || cli_report);
     var filename = options.filename || '<eval>';
@@ -543,7 +586,7 @@ function lint_code(code, options) {
     var lines = code.split('\n');  // Can be used (in the future) to display extract from code corresponding to error location
 
     try {
-        toplevel = parse_file(code, filename);
+        toplevel = await parse_file(code, filename);
     } catch(e) {
         if (e instanceof RapydScript.ImportError) {
             messages = [{filename:filename, start_line:e.line, start_col:e.col, level:ERROR, ident:'import-err', message:e.message}];
@@ -553,28 +596,29 @@ function lint_code(code, options) {
     }
 
     if (toplevel) {
-        var linter = new Linter(toplevel, filename, code, options);
-        toplevel.walk(linter);
-        messages = linter.resolve();
+        messages = lint_parsed(toplevel, code, options);
     }
     messages.forEach(function(msg, i) { msg.code_lines = lines; reportcb(msg, i, messages); });
     return messages;
 }
 
+export { BUILTINS };
+
 // CLI {{{
 
-function read_whole_file(filename, cb) {
+async function read_whole_file(filename) {
     if (!filename || filename === '-') {
         var chunks = [];
         process.stdin.setEncoding('utf-8');
-        process.stdin.on('data', function (chunk) {
-            chunks.push(chunk);
-        }).on('end', function () {
-            cb(null, chunks.join(""));
+        await new Promise((resolve, reject) => {
+            process.stdin.on('data', chunk => chunks.push(chunk));
+            process.stdin.on('end', resolve);
+            process.stdin.on('error', reject);
         });
         process.openStdin();
+        return chunks.join('');
     } else {
-        fs.readFile(filename, "utf-8", cb);
+        return await fs.promises.readFile(filename, 'utf-8');
     }
 }
 
@@ -619,17 +663,15 @@ function cli_vim_report(r) {
 
 var ini_cache = {};
 
-function get_ini(toplevel_dir) {
+async function get_ini(toplevel_dir) {
     if (has_prop(ini_cache, toplevel_dir)) return ini_cache[toplevel_dir];
-    var rl = require('./ini').read_config(toplevel_dir).rapydscript || {};
+    var rl = (await read_config(toplevel_dir)).rapydscript || {};
     ini_cache[toplevel_dir] = rl;
     return rl;
 }
 
-module.exports.cli = function(argv, base_path, src_path, lib_path) {
+export async function cli(argv, base_path, src_path, lib_path) {
     var files = argv.files.slice();
-    var num_of_files = files.length || 1;
-    var read_config = require('./ini');
 
     if (argv.noqa_list) {
         Object.keys(MESSAGES).forEach(function(ident) {
@@ -642,6 +684,23 @@ module.exports.cli = function(argv, base_path, src_path, lib_path) {
         process.exit(0);
     }
 
+    if (argv.input_list) {
+        // Check for conflict: --input-list=- and - as an input file
+        if (argv.input_list === '-' && files.indexOf('-') !== -1) {
+            console.error("ERROR: Cannot use - as both --input-list source and an input file");
+            process.exit(1);
+        }
+        var list_data;
+        try {
+            list_data = await read_whole_file(argv.input_list === '-' ? null : argv.input_list);
+        } catch(e) {
+            console.error("ERROR: can't read input list file: " + argv.input_list);
+            process.exit(1);
+        }
+        var list_files = list_data.split('\n').map(function(l) { return l.trim(); }).filter(function(l) { return l.length > 0; });
+        files = files.concat(list_files);
+    }
+
     if (files.filter(function(el){ return el === "-"; }).length > 1) {
         console.error("ERROR: Can read a single file from STDIN (two or more dashes specified)");
         process.exit(1);
@@ -657,15 +716,19 @@ module.exports.cli = function(argv, base_path, src_path, lib_path) {
         return x === '-' ? argv.stdin_filename : x;
     }
 
-    function lint_single_file(err, code) {
-        var output, final_builtins = merge(builtins), final_noqa = merge(noqa), rl;
-        if (err) {
-            console.error("ERROR: can't read file: " + files[0]);
+    for (const filename of (files.length ? files : [null])) {
+        var code;
+        try {
+            code = await read_whole_file(filename === '-' ? null : filename);
+        } catch(e) {
+            console.error("ERROR: can't read file: " + filename);
             process.exit(1);
         }
 
+        var final_builtins = merge(builtins), final_noqa = merge(noqa);
+
         // Read setup.cfg
-        rl = get_ini(path.dirname(path_for_filename(files[0])));
+        var rl = await get_ini(path.dirname(path_for_filename(filename || '-')));
         var g = {};
         (rl.globals || rl.builtins || '').split(',').forEach(function (x) { g[x.trim()] = true; });
         final_builtins = merge(final_builtins, g);
@@ -685,21 +748,9 @@ module.exports.cli = function(argv, base_path, src_path, lib_path) {
         });
 
         // Lint!
-        if (lint_code(code, {filename:path_for_filename(files[0]), builtins:final_builtins, noqa:final_noqa, errorformat:argv.errorformat || false}).length) all_ok = false;
-
-        files = files.slice(1);
-        if (files.length) {
-            setImmediate(read_whole_file, files[0], lint_single_file);
-            return;
-        } else process.exit((all_ok) ? 0 : 1);
+        if ((await lint_code(code, {filename:path_for_filename(filename || '-'), builtins:final_builtins, noqa:final_noqa, errorformat:argv.errorformat || false})).length) all_ok = false;
     }
- 
-    setImmediate(read_whole_file, files[0], lint_single_file);
 
-};
-
-module.exports.lint_code = lint_code;
-module.exports.WARN = WARN;
-module.exports.ERROR = ERROR;
-module.exports.MESSAGES = MESSAGES;
+    process.exit((all_ok) ? 0 : 1);
+}
 // }}}