rapydscript-ng 0.8.2 → 0.8.3

package/tools/lint.mjs

@@ -7,6 +7,7 @@
 
 import fs from 'fs';
 import path from 'path';
+import { fileURLToPath } from 'url';
 import { read_config } from './ini.mjs';
 import * as utils from './utils.mjs';
 
@@ -663,11 +664,12 @@ function cli_vim_report(r) {
 
 var ini_cache = {};
 
-async function get_ini(toplevel_dir) {
-    if (has_prop(ini_cache, toplevel_dir)) return ini_cache[toplevel_dir];
-    var rl = (await read_config(toplevel_dir)).rapydscript || {};
-    ini_cache[toplevel_dir] = rl;
-    return rl;
+function get_ini(toplevel_dir) {
+    if (!has_prop(ini_cache, toplevel_dir)) {
+        // Store the promise so concurrent callers for the same dir share one read.
+        ini_cache[toplevel_dir] = read_config(toplevel_dir).then(function(r) { return r.rapydscript || {}; });
+    }
+    return ini_cache[toplevel_dir];
 }
 
 export async function cli(argv, base_path, src_path, lib_path) {
@@ -716,7 +718,52 @@ export async function cli(argv, base_path, src_path, lib_path) {
         return x === '-' ? argv.stdin_filename : x;
     }
 
-    for (const filename of (files.length ? files : [null])) {
+    // reportcb used for final output (same logic as inside lint_code)
+    var reportcb = {'json': cli_json_report, 'vim': cli_vim_report, 'undef': cli_undef_report}[argv.errorformat] || cli_report;
+
+    // Expand directories to .pyj files recursively, stat-ing entries in parallel.
+    async function expand_dirs(inputs, toplevel) {
+        // Stat all entries concurrently, then process results in sorted order.
+        var entries = await Promise.all(inputs.map(async function(f) {
+            if (f === '-') return { f, type: 'stdin' };
+            var st;
+            try { st = await fs.promises.lstat(f); }
+            catch (e) {
+                if (toplevel) { console.error("ERROR: can't access: " + f); process.exit(1); }
+                return null;
+            }
+            if (st.isDirectory()) return { f, type: 'dir' };
+            if (st.isFile()) return { f, type: 'file' };
+            return null;
+        }));
+
+        var result = [];
+        for (var i = 0; i < entries.length; i++) {
+            var entry = entries[i];
+            if (!entry) continue;
+            if (entry.type === 'stdin') { result.push(entry.f); continue; }
+            if (entry.type === 'file') { result.push(entry.f); continue; }
+            // Directory: read children and recurse
+            var children;
+            try {
+                children = (await fs.promises.readdir(entry.f)).sort().map(function(x) { return path.join(entry.f, x); });
+            } catch(e) {
+                if (toplevel) { console.error("ERROR: can't read directory: " + entry.f); process.exit(1); }
+                continue;
+            }
+            var sub = await expand_dirs(children, false);
+            sub.forEach(function(x) { if (x.endsWith('.pyj')) result.push(x); });
+        }
+        return result;
+    }
+    if (files.length) files = await expand_dirs(files, true);
+
+    var all_files = files.length ? files : [null];
+    var has_stdin = all_files.indexOf('-') !== -1 || all_files.indexOf(null) !== -1;
+
+    // Process a single file and return its messages without printing them.
+    // Used when the worker pool is not active (stdin, or too few files).
+    async function process_file(filename) {
         var code;
         try {
             code = await read_whole_file(filename === '-' ? null : filename);
@@ -747,10 +794,82 @@ export async function cli(argv, base_path, src_path, lib_path) {
             }
         });
 
-        // Lint!
-        if ((await lint_code(code, {filename:path_for_filename(filename || '-'), builtins:final_builtins, noqa:final_noqa, errorformat:argv.errorformat || false})).length) all_ok = false;
+        var lines = code.split('\n');
+        var msgs = await lint_code(code, {filename:path_for_filename(filename || '-'), builtins:final_builtins, noqa:final_noqa, errorformat:argv.errorformat || false, report:function() {}});
+        msgs.forEach(function(msg) { msg.code_lines = lines; });
+        return msgs;
+    }
+
+    var all_results;
+    // Use a worker pool when there are enough files to justify the overhead.
+    // Workers own their own compiler instance so parsing runs in parallel across CPU cores.
+    var WORKER_THRESHOLD = 4;
+
+    if (!has_stdin && all_files.length >= WORKER_THRESHOLD) {
+        var { Worker } = await import('worker_threads');
+        var os_mod = await import('os');
+
+        var worker_path = path.join(path.dirname(fileURLToPath(import.meta.url)), 'lint-worker.mjs');
+        var num_workers = Math.min(os_mod.cpus().length, all_files.length, 8);
+
+        // Dispatch files to workers dynamically so fast files don't leave workers idle.
+        var ordered_results = new Array(all_files.length);
+        var next_idx = 0;
+        var pending = all_files.length;
+        var done_resolve;
+        var done_promise = new Promise(function(res) { done_resolve = res; });
+        if (pending === 0) done_resolve();
+
+        function dispatch(worker) {
+            if (next_idx >= all_files.length) return;
+            var idx = next_idx++;
+            worker.postMessage({
+                type: 'lint',
+                id: idx,
+                filename: all_files[idx],
+                base_builtins: builtins,
+                base_noqa: noqa,
+            });
+        }
+
+        // Create workers and dispatch eagerly as each becomes ready rather than
+        // waiting for all workers before starting any work.
+        var worker_promises = Array.from({length: num_workers}, function() {
+            return new Promise(function(resolve) {
+                var w = new Worker(worker_path);
+                w.once('message', function(m) {
+                    if (m.type !== 'ready') return;
+                    w.on('message', function(msg) {
+                        if (msg.type === 'result') {
+                            ordered_results[msg.id] = msg.messages;
+                        } else if (msg.type === 'error') {
+                            console.error("ERROR: lint failed for " + all_files[msg.id] + ": " + msg.error);
+                            ordered_results[msg.id] = [];
+                        }
+                        pending--;
+                        if (pending === 0) { done_resolve(); return; }
+                        dispatch(w);
+                    });
+                    dispatch(w);
+                    resolve(w);
+                });
+            });
+        });
+
+        await done_promise;
+        var workers = await Promise.all(worker_promises);
+        workers.forEach(function(w) { w.terminate(); });
+        all_results = ordered_results;
+    } else {
+        // Single-threaded path: process concurrently on the main thread.
+        all_results = await Promise.all(all_files.map(process_file));
     }
 
+    var all_messages = [];
+    all_results.forEach(function(msgs) { Array.prototype.push.apply(all_messages, msgs); });
+    if (all_messages.length) all_ok = false;
+    all_messages.forEach(function(msg, i) { reportcb(msg, i, all_messages); });
+
     process.exit((all_ok) ? 0 : 1);
 }
 // }}}

package/tree-sitter/grammar.js

@@ -57,6 +57,9 @@ module.exports = grammar({
     $._indent,
     $._dedent,
     $.regex,
+    $.fstring_start,
+    $.fstring_content,
+    $.fstring_end,
   ],
 
   extras: $ => [
@@ -589,6 +592,7 @@ module.exports = grammar({
       $.none,
       $.number,
       $.string,
+      $.f_string,
       $.concatenated_string,
       $.regex,
       $.verbatim,
@@ -924,20 +928,37 @@ module.exports = grammar({
     // ---- atoms / literals ------------------------------------------------
 
     concatenated_string: $ => prec.left(seq(
-      $.string,
-      repeat1($.string),
+      choice($.string, $.f_string),
+      repeat1(choice($.string, $.f_string)),
     )),
 
+    f_string: $ => seq(
+      $.fstring_start,
+      repeat(choice(
+        $.fstring_content,
+        $.interpolation,
+      )),
+      $.fstring_end,
+    ),
+
+    interpolation: $ => seq(
+      '{',
+      field('expression', $._expression),
+      optional(field('type_conversion', /![rsa]/)),
+      optional(seq(':', field('format_spec', $.fstring_content))),
+      '}',
+    ),
+
     this: _ => 'this',
     true: _ => 'True',
     false: _ => 'False',
     none: _ => 'None',
 
-    // string with optional r/u/f/b modifiers (the `v` modifier denotes a
-    // verbatim JavaScript literal and is handled separately). Interpolation
-    // and escapes are not sub-tokenised; the whole literal is a single node.
+    // string with optional r/u/b modifiers (the `v` modifier denotes a
+    // verbatim JavaScript literal, handled separately; the `f` modifier
+    // produces an f_string with structured interpolation nodes).
     string: _ => token(seq(
-      optional(/[rRuUfFbB]+/),
+      optional(/[rRuUbB]+/),
       choice(
         seq('"""', repeat(choice(/[^"\\]/, /\\(.|\n)/, /"[^"]/, /""[^"]/)), '"""'),
         seq("'''", repeat(choice(/[^'\\]/, /\\(.|\n)/, /'[^']/, /''[^']/)), "'''"),

package/tree-sitter/queries/highlights.scm

@@ -21,6 +21,16 @@
 (regex) @string.regexp
 (number) @number
 
+; f-strings: delimiters and literal content get @string, interpolation braces
+; get @punctuation.special so they stand out, and the expression inside is
+; transparent — its child nodes (identifiers, calls, etc.) are highlighted by
+; the normal rules below.
+(fstring_start) @string
+(fstring_content) @string
+(fstring_end) @string
+(interpolation "{" @punctuation.special)
+(interpolation "}" @punctuation.special)
+
 (true) @boolean
 (false) @boolean
 (none) @constant.builtin

package/tree-sitter/src/grammar.json

@@ -2474,6 +2474,10 @@
           "type": "SYMBOL",
           "name": "string"
         },
+        {
+          "type": "SYMBOL",
+          "name": "f_string"
+        },
         {
           "type": "SYMBOL",
           "name": "concatenated_string"
@@ -4493,19 +4497,128 @@
         "type": "SEQ",
         "members": [
           {
-            "type": "SYMBOL",
-            "name": "string"
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "string"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "f_string"
+              }
+            ]
           },
           {
             "type": "REPEAT1",
             "content": {
-              "type": "SYMBOL",
-              "name": "string"
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "string"
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "f_string"
+                }
+              ]
             }
           }
         ]
       }
     },
+    "f_string": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "fstring_start"
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "fstring_content"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "interpolation"
+              }
+            ]
+          }
+        },
+        {
+          "type": "SYMBOL",
+          "name": "fstring_end"
+        }
+      ]
+    },
+    "interpolation": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "{"
+        },
+        {
+          "type": "FIELD",
+          "name": "expression",
+          "content": {
+            "type": "SYMBOL",
+            "name": "_expression"
+          }
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "FIELD",
+              "name": "type_conversion",
+              "content": {
+                "type": "PATTERN",
+                "value": "![rsa]"
+              }
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "STRING",
+                  "value": ":"
+                },
+                {
+                  "type": "FIELD",
+                  "name": "format_spec",
+                  "content": {
+                    "type": "SYMBOL",
+                    "name": "fstring_content"
+                  }
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "STRING",
+          "value": "}"
+        }
+      ]
+    },
     "this": {
       "type": "STRING",
       "value": "this"
@@ -4532,7 +4645,7 @@
             "members": [
               {
                 "type": "PATTERN",
-                "value": "[rRuUfFbB]+"
+                "value": "[rRuUbB]+"
               },
               {
                 "type": "BLANK"
@@ -4963,6 +5076,18 @@
     {
       "type": "SYMBOL",
       "name": "regex"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "fstring_start"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "fstring_content"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "fstring_end"
     }
   ],
   "inline": [],

package/tree-sitter/src/node-types.json

@@ -131,6 +131,10 @@
         "type": "existential",
         "named": true
       },
+      {
+        "type": "f_string",
+        "named": true
+      },
       {
         "type": "false",
         "named": true
@@ -976,6 +980,10 @@
       "multiple": true,
       "required": true,
       "types": [
+        {
+          "type": "f_string",
+          "named": true
+        },
         {
           "type": "string",
           "named": true
@@ -1376,6 +1384,33 @@
       ]
     }
   },
+  {
+    "type": "f_string",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "fstring_content",
+          "named": true
+        },
+        {
+          "type": "fstring_end",
+          "named": true
+        },
+        {
+          "type": "fstring_start",
+          "named": true
+        },
+        {
+          "type": "interpolation",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "finally_clause",
     "named": true,
@@ -1702,6 +1737,32 @@
       }
     }
   },
+  {
+    "type": "interpolation",
+    "named": true,
+    "fields": {
+      "expression": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "_expression",
+            "named": true
+          }
+        ]
+      },
+      "format_spec": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "fstring_content",
+            "named": true
+          }
+        ]
+      }
+    }
+  },
   {
     "type": "keyword_argument",
     "named": true,
@@ -2770,6 +2831,18 @@
     "type": "from",
     "named": false
   },
+  {
+    "type": "fstring_content",
+    "named": true
+  },
+  {
+    "type": "fstring_end",
+    "named": true
+  },
+  {
+    "type": "fstring_start",
+    "named": true
+  },
   {
     "type": "global",
     "named": false