rapydscript-ng 0.7.24 → 0.8.1

package/tools/{utils.js → utils.mjs}

@@ -1,12 +1,14 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
-"use strict";  /*jshint node:true */
+"use strict";
+
+import fs from 'fs';
+import path from 'path';
 
-var comment_contents = /\/\*!?(?:\@preserve)?[ \t]*(?:\r\n|\n)([\s\S]*?)(?:\r\n|\n)[ \t]*\*\//;
 var colors = ['red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'];
 
 function ansi(code) {
@@ -14,13 +16,13 @@ function ansi(code) {
     return String.fromCharCode(27) + '[' + code + 'm';
 }
 
-function path_exists(path) {
-    var fs = require('fs');
+async function path_exists(p) {
     try {
-        fs.statSync(path);
+        await fs.promises.stat(p);
         return true;
     } catch(e) {
         if (e.code != 'ENOENT') throw e;
+        return false;
     }
 }
 
@@ -57,7 +59,7 @@ function supports_color(stdout) {
 
 }
 
-function safe_colored(string) {
+function passthrough_colored(string) {
     return string;
 }
 
@@ -82,11 +84,11 @@ function wrap(lines, width) {
 		line = prev + line;
 		prev = '';
 		if (line.length > width) {
-			prev = line.substr(width);
+			prev = line.substr(width - 1);
             if (prev) prev += ' ';
 			line = line.substr(0, width - 1);
-			if (line.substr(line.length - 1 !== ' ')) line += '-';
-		} 
+			if (line.substr(line.length - 1) !== ' ') line += '-';
+		}
 		ans.push(line);
 	});
 	if (prev) ans = ans.concat(wrap([prev]));
@@ -105,24 +107,25 @@ function merge() {
 }
 
 function get_import_dirs(paths_string, ignore_env) {
-    var path = require('path');
     var paths = [];
-    function merge(new_path) {
+    function add(new_path) {
         if (paths.indexOf(new_path) == -1) paths.push(new_path);
     }
     if (!ignore_env && process && process.env && process.env.RAPYDSCRIPT_IMPORT_PATH) {
-        process.env.RAPYDSCRIPT_IMPORT_PATH.split(path.delimiter).forEach(merge);
+        process.env.RAPYDSCRIPT_IMPORT_PATH.split(path.delimiter).forEach(add);
     }
-    if (paths_string) paths_string.split(path.delimiter).forEach(merge);
+    if (paths_string) paths_string.split(path.delimiter).forEach(add);
     return paths;
 }
 
-exports.comment_contents = comment_contents;
-exports.repeat = repeat;
-exports.wrap = wrap;
-exports.merge = merge;
-exports.colored = colored;
-exports.safe_colored = (supports_color()) ? colored : safe_colored;
-exports.generators_available = generators_available;
-exports.get_import_dirs = get_import_dirs;
-exports.path_exists = path_exists;
+export {
+    repeat,
+    wrap,
+    merge,
+    colored,
+    generators_available,
+    get_import_dirs,
+    path_exists,
+};
+
+export var safe_colored = (supports_color()) ? colored : passthrough_colored;

package/tools/{web_repl.js → web_repl.mjs}

@@ -1,18 +1,21 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
-"use strict";  /*jshint node:true */
-var vm = require('vm');
-var embedded_compiler = require('tools/embedded_compiler.js');
+"use strict";
+
+import vm from 'vm';
+import embedded_compiler from './embedded_compiler.mjs';
+import tree_shake from './treeshake.mjs';
+import { generate_source_map } from './sourcemap.mjs';
 
-module.exports = function(compiler, baselib) {
+export default async function(compiler, baselib) {
     var ctx = vm.createContext();
     var LINE_CONTINUATION_CHARS = ':\\';
     var find_completions = null;
-    var streaming_compiler = embedded_compiler(compiler, baselib, function(js) { return vm.runInContext(js, ctx); }, '__repl__');
+    var streaming_compiler = await embedded_compiler(compiler, baselib, function(js) { return vm.runInContext(js, ctx); }, '__repl__', tree_shake, generate_source_map);
 
     return {
         'in_block_mode': false,
@@ -20,13 +23,13 @@ module.exports = function(compiler, baselib) {
         'replace_print': function replace_print(write_line_func) {
             ctx.print = function() {
                 var parts = [];
-                for (var i = 0; i < arguments.length; i++) 
+                for (var i = 0; i < arguments.length; i++)
                     parts.push(ctx.ρσ_str(arguments[i]));
                 write_line_func(parts.join(' '));
             };
         },
 
-        'is_input_complete': function is_input_complete(source) {
+        'is_input_complete': async function is_input_complete(source) {
             if (!source || !source.trim()) return false;
             var lines = source.split('\n');
             var last_line = lines[lines.length - 1].trimRight();
@@ -45,7 +48,7 @@ module.exports = function(compiler, baselib) {
                 return false;
             }
             try {
-                compiler.parse(source, {'filename': '<repl>', 'basedir': '__stdlib__'});
+                await compiler.parse(source, {'filename': '<repl>', 'basedir': '__stdlib__'});
             } catch(e) {
                 if (e.is_eof && e.line === lines.length && e.col > 0) {
                     return false;
@@ -57,11 +60,11 @@ module.exports = function(compiler, baselib) {
             return true;
         },
 
-        'compile': function web_repl_compile(code, opts) {
+        'compile': async function web_repl_compile(code, opts) {
             opts = opts || {};
             opts.keep_docstrings = true;
             opts.filename = '<input>';
-            return streaming_compiler.compile(code, opts);
+            return await streaming_compiler.compile(code, opts);
         },
 
         'runjs': function runjs(code) {
@@ -83,5 +86,4 @@ module.exports = function(compiler, baselib) {
         },
 
     };
-};
-
+}

package/tools/web_repl_export.mjs

@@ -0,0 +1,97 @@
+/*
+ * web_repl_export.mjs
+ * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import * as utils from './utils.mjs';
+
+export default async function run_web_repl_export(base_path, lib_path, argv) {
+    if (globalThis.__rapydscript_embedded__) {
+        console.error('web-repl-export requires the full source repository and cannot run from a standalone binary.');
+        process.exit(1);
+    }
+    var output_dir = argv.files[0];
+    if (!output_dir) {
+        console.error('Usage: rapydscript web-repl-export /path/to/export/directory');
+        process.exit(1);
+    }
+
+    var meta = JSON.parse(await fs.promises.readFile(path.join(base_path, 'package.json'), 'utf-8'));
+    var commit_sha = (await fs.promises.readFile(path.join(base_path, '.git', 'refs', 'heads', 'master'), 'utf-8')).trim();
+
+    var manifest = {}, total = 0;
+    for (const x of ['compiler.js', 'baselib-plain-pretty.js']) {
+        manifest[x] = await fs.promises.readFile(path.join(lib_path, x), 'utf-8');
+        total += manifest[x].length;
+    }
+
+    for (const x of ['web_repl.js', 'embedded_compiler.js', 'treeshake.js', 'sourcemap.js', 'utils.js', 'completer.js', 'msgfmt.js', 'gettext.js', 'ast_serialize.mjs']) {
+        var mjs = x.replace(/\.js$/, '.mjs');
+        var mjs_path = path.join(base_path, 'tools', mjs);
+        var js_path = path.join(base_path, 'tools', x);
+        var use_mjs = await utils.path_exists(mjs_path);
+        var key = 'tools/' + (use_mjs ? mjs : x);
+        manifest[key] = await fs.promises.readFile(use_mjs ? mjs_path : js_path, 'utf-8');
+        total += manifest[key].length;
+    }
+
+    var stdlib = path.join(base_path, 'src', 'lib');
+
+    async function process_stdlib_dir(relpath) {
+        var fullpath = relpath ? path.join(stdlib, relpath) : stdlib;
+        var entries = await fs.promises.readdir(fullpath);
+        for (const x of entries) {
+            var q = path.join(fullpath, x);
+            var s = await fs.promises.stat(q);
+            if (s.isDirectory()) { await process_stdlib_dir(relpath + '/' + x); continue; }
+            if (!x.endsWith('.pyj')) continue;
+            var iname = path.normalize('__stdlib__' + '/' + relpath + '/' + x);
+            var raw = await fs.promises.readFile(q, 'utf-8');
+            manifest[iname] = raw;
+            total += s.size;
+        }
+    }
+    await process_stdlib_dir('');
+
+    var rs = '// vim:fileencoding=utf-8\n';
+    rs += '(function(external_namespace) {\n';
+    rs += '"use strict;"\n';
+    rs += 'var rs_version = ' + JSON.stringify(meta.version) + ';\n';
+    rs += 'var rs_commit_sha = ' + JSON.stringify(commit_sha) + ';\n';
+    rs += '\n// Embedded modules {{{\n';
+    rs += 'var data = ' + JSON.stringify(manifest) + ';\n\n';
+    rs += '// End embedded modules }}}\n\n';
+    rs += await fs.promises.readFile(path.join(base_path, 'web-repl', 'env.js'));
+    rs += '\n// Embedded sha1 implementation {{{\n';
+    rs += '(function() {\n';
+    rs += await fs.promises.readFile(path.join(base_path, 'web-repl', 'sha1.js'));
+    rs += '}).call(jsSHA);\n';
+    rs += '// End embedded sha1 implementation }}}\n\n';
+    rs += 'var exports = namespace;\n';
+    rs += await fs.promises.readFile(path.join(base_path, 'tools', 'export.js'), 'utf-8');
+    rs += 'external_namespace.RapydScript = namespace;\n';
+    rs += '})(this);\n';
+
+    var base_dir = path.normalize(path.resolve(output_dir));
+
+    try {
+        await fs.promises.mkdir(base_dir);
+    } catch(e) {
+        if (e.code !== 'EEXIST') throw e;
+    }
+
+    await fs.promises.writeFile(path.join(base_dir, 'rapydscript.js'), rs, 'utf-8');
+    var web_repl_dir = path.join(base_path, 'web-repl');
+    var web_repl_files = await fs.promises.readdir(web_repl_dir);
+    for (const x of web_repl_files) {
+        if (['sha1.js', 'env.js'].indexOf(x) !== -1) continue;
+        var file_data = await fs.promises.readFile(path.join(web_repl_dir, x), 'utf-8');
+        await fs.promises.writeFile(path.join(base_dir, x), file_data, 'utf-8');
+    }
+    console.log('RapydScript compiler (uncompressed) size: ' + (total/(1024)).toFixed(1) + ' KB');
+    console.log('web-repl exported to: ' + base_dir);
+}

package/tree-sitter/README.md

@@ -0,0 +1,101 @@
+# tree-sitter-rapydscript
+
+A [tree-sitter](https://tree-sitter.github.io/tree-sitter/) grammar for
+[RapydScript](../README.md) (`.pyj` files).
+
+RapydScript is a Python-like language that compiles to JavaScript. This grammar
+covers the language as implemented by `rapydscript-ng`, including the features
+that distinguish it from Python:
+
+- **multi-line anonymous functions** declared with `def(...)` / `async def(...)`
+  (RapydScript's answer to Python's single-expression `lambda`), both inline
+  (`f = def(a, b): return a + b`) and with an indented block body;
+- the **existential operator** `?` in all its forms — `a?` (existence check),
+  `a?.b` / `a?[i]` / `a?()` (safe access), and `a ? b` (default operator);
+- **verbatim JavaScript** literals via the `v` string modifier (`v'...'`) and
+  the native `for v'i=0; i<n; i++':` loop;
+- JavaScript **regular-expression literals**, including the verbose
+  `/// ... ///` form;
+- `do: ... .while cond` loops;
+- **scoped compiler flags** (`from __python__ import dict_literals, ...`).
+
+It deliberately does **not** accept constructs that RapydScript itself rejects,
+such as the walrus operator `:=`, `match`/`case`, or `*`-unpacking in assignment
+targets.
+
+It **does** handle the trickier corners of the language, including assignment as
+an expression (`if m = re.exec(x):`, `a[i -= 1]`), keyword-vs-assignment
+disambiguation in calls, named function *expressions* (`x = def named(a): ...`),
+typed `*args`/`**kwargs`, bare generator arguments (`sum(x for x in xs)`),
+multi-line anonymous-function arguments with the comma-on-the-next-line idiom,
+and comma-first dict literals whose values are multi-line functions.
+
+### Known limitations
+
+A few rarely-used constructs — found only in the RapydScript compiler's own
+source and standard library, not in ordinary programs — are not parsed. Every
+file under `test/` (representative user code) and every example in the language
+[README](../README.md) parses cleanly; the exceptions are:
+
+- the colon-less one-line `if`: `if (cond) do_something()`;
+- chained assignment whose value is a multi-line-block anonymous function:
+  `a = b = def(): <block>`;
+- a ternary used as the return value of an *inline* anonymous function that is
+  itself a call argument, with a trailing `;`.
+
+In these cases tree-sitter's error recovery still highlights the surrounding
+code.
+
+## Layout
+
+```
+tree-sitter/
+├── grammar.js              # the grammar
+├── tree-sitter.json        # grammar metadata (CLI)
+├── package.json            # npm package metadata
+├── src/
+│   ├── parser.c            # generated parser (checked in)
+│   ├── scanner.c           # external scanner: indentation + regex literals
+│   ├── grammar.json        # generated
+│   └── node-types.json     # generated
+├── queries/
+│   ├── highlights.scm      # syntax highlighting
+│   └── injections.scm      # embedded-language injections
+└── test/corpus/            # tree-sitter test suite
+```
+
+## Building & testing
+
+From this directory:
+
+```sh
+tree-sitter generate     # regenerate src/parser.c from grammar.js
+tree-sitter test         # run the corpus tests in test/corpus/
+tree-sitter parse FILE   # parse a .pyj file and print its syntax tree
+tree-sitter build        # compile the parser (+ scanner) into a shared object
+```
+
+## Using it in Neovim
+
+With [nvim-treesitter](https://github.com/nvim-treesitter/nvim-treesitter),
+register the parser and map the `.pyj` extension to it:
+
+```lua
+local rapyd_repo_path = vim.fn.expand("/path/to/rapydscript/tree-sitter")
+vim.api.nvim_create_autocmd('User', {
+    pattern = 'TSUpdate',
+    callback = function()
+        require("nvim-treesitter.parsers").rapydscript = {
+            install_info = {
+                path = rapyd_repo_path,
+                files = { "src/parser.c", "src/scanner.c" },
+            },
+            filetype = "rapydscript",
+        }
+    end
+})
+vim.opt.rtp:append(rapyd_repo_path)
+vim.filetype.add({ extension = { pyj = "rapydscript" } })
+```
+
+Then `:TSInstall rapydscript`.