rapydscript-ng 0.7.23 → 0.8.0

package/tools/compile.mjs

@@ -0,0 +1,201 @@
+/*
+ * compile.js
+ * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import vm from 'vm';
+import { createRequire } from 'module';
+import { create_compiler } from './compiler.mjs';
+import * as utils from './utils.mjs';
+import { generate_source_map } from './sourcemap.mjs';
+import tree_shake from './treeshake.mjs';
+
+const require = createRequire(import.meta.url);
+const RapydScript = await create_compiler();
+
+async function read_whole_file(filename) {
+    if (!filename) {
+        var chunks = [];
+        process.stdin.setEncoding('utf-8');
+        await new Promise((resolve, reject) => {
+            process.stdin.on('data', chunk => chunks.push(chunk));
+            process.stdin.on('end', resolve);
+            process.stdin.on('error', reject);
+        });
+        process.openStdin();
+        return chunks.join('');
+    } else {
+        return await fs.promises.readFile(filename, 'utf-8');
+    }
+}
+
+async function makedirs(dir) {
+    try {
+        await fs.promises.mkdir(dir);
+    } catch(e) {
+        if (e.code == 'EEXIST') return;
+        if (e.code == 'ENOENT') { await makedirs(path.dirname(dir)); await fs.promises.mkdir(dir); }
+        else throw e;
+    }
+}
+
+async function process_cache_dir(dir) {
+    dir = path.resolve(path.normalize(dir));
+    await makedirs(dir);
+    return dir;
+}
+
+export default async function(start_time, argv, base_path, src_path, lib_path) {
+    // configure settings for the output
+    var cache_dir = argv.cache_dir ? await process_cache_dir(argv.cache_dir) : '';
+    var OUTPUT_OPTIONS = {
+        beautify: !argv.uglify,
+        private_scope: !argv.bare,
+        omit_baselib: argv.omit_baselib,
+        js_version: parseInt(argv.js_version),
+        keep_docstrings: argv.keep_docstrings,
+        discard_asserts: argv.discard_asserts,
+        module_cache_dir: cache_dir,
+        source_map: !!argv.source_map,
+        source_map_line_offset: parseInt(argv.source_map_line_offset) || 0,
+    };
+
+    if (argv.comments) {
+        if (/^\//.test(argv.comments)) {
+            OUTPUT_OPTIONS.comments = new Function("return(" + argv.comments + ")")();  // jshint ignore:line
+        } else if (argv.comments == "all") {
+            OUTPUT_OPTIONS.comments = true;
+        } else {
+            OUTPUT_OPTIONS.comments = function(node, comment) {
+                var text = comment.value;
+                var type = comment.type;
+                if (type == "comment2") {
+                    // multiline comment
+                    return /@preserve|@license|@cc_on/i.test(text);
+                }
+            };
+        }
+    }
+
+    if (!argv.omit_baselib) {
+        var which = (OUTPUT_OPTIONS.beautify) ? 'pretty' : 'ugly';
+        OUTPUT_OPTIONS.baselib_plain = await fs.promises.readFile(path.join(lib_path, 'baselib-plain-' + which + '.js'), 'utf-8');
+    }
+
+    var files = argv.files.slice();
+    var STATS = {}, TOPLEVEL;
+    var num_of_files = files.length || 1;
+
+    if (files.filter(function(el){ return el == "-"; }).length > 1) {
+        console.error("ERROR: Can read a single file from STDIN (two or more dashes specified)");
+        process.exit(1);
+    }
+
+    async function parse_file(code, file, toplevel) {
+        return await RapydScript.parse(code, {
+            filename: file,
+            toplevel: toplevel,
+            basedir: (file !== '<stdin>') ? path.dirname(file) : undefined,
+            libdir: path.join(src_path, 'lib'),
+            import_dirs: utils.get_import_dirs(argv.import_path),
+            discard_asserts: argv.discard_asserts,
+            module_cache_dir: cache_dir,
+        });
+    }
+
+    async function write_output(js_output, output_stream) {
+        if (argv.source_map && output_stream) {
+            var segments = output_stream.get_source_map_segments();
+            var map_json = generate_source_map(segments, argv.output, '');
+            await fs.promises.writeFile(argv.source_map, map_json, 'utf8');
+            if (argv.output) {
+                var map_url = path.relative(path.dirname(path.resolve(argv.output)), path.resolve(argv.source_map));
+                js_output = js_output + '\n//# sourceMappingURL=' + map_url + '\n';
+            }
+        }
+        if (argv.output) {
+            // Node's filesystem module cannot write directly to /dev/stdout
+            if (argv.output == '/dev/stdout') console.log(js_output);
+            else if (argv.output == '/dev/stderr') console.error(js_output);
+            else await fs.promises.writeFile(argv.output, js_output, "utf8");
+        } else if (!argv.execute){
+            console.log(js_output);
+        }
+        if (argv.execute) {
+            vm.runInNewContext(js_output, {'console':console, 'require':require}, {'filename':files[0]});
+        }
+    }
+
+    async function time_it(name, cont) {
+        var t1 = new Date().getTime();
+        var ret = await cont();
+        if (argv.stats) {
+            var spent = new Date().getTime() - t1;
+            if (STATS[name]) STATS[name] += spent;
+            else STATS[name] = spent;
+        }
+        return ret;
+    }
+
+    var filenames = files.length ? files : [null];
+    for (var i = 0; i < filenames.length; i++) {
+        var filename = filenames[i];
+        var code;
+        try {
+            code = await read_whole_file(filename);
+        } catch(e) {
+            console.error("ERROR: can't read file: " + filename);
+            process.exit(1);
+        }
+
+        var output_stream;
+        await time_it("parse", async function(){
+            var file = filename || argv.filename_for_stdin || '<stdin>';
+            try {
+                TOPLEVEL = await parse_file(code, file, TOPLEVEL);
+            } catch (e) {
+                if (!(e instanceof RapydScript.SyntaxError)) throw e;
+                console.error(e.toString());
+                process.exit(1);
+            }
+        });
+
+        try {
+            output_stream = new RapydScript.OutputStream(OUTPUT_OPTIONS);
+        } catch(ex) {
+            if (ex instanceof RapydScript.DefaultsError) {
+                console.error(ex.message);
+                process.exit(1);
+            }
+            throw ex;
+        }
+
+        if (argv.tree_shaking) {
+            time_it("tree_shaking", function(){
+                TOPLEVEL = tree_shake(TOPLEVEL);
+            });
+        }
+
+        time_it("generate", function(){
+            TOPLEVEL.print(output_stream);
+        });
+
+        await write_output(output_stream.get(), output_stream);
+    }
+
+    if (argv.stats) {
+        console.error(RapydScript.string_template("Timing information (compressed {count} files):", {
+            count: num_of_files
+        }));
+        for (var i in STATS) if (Object.prototype.hasOwnProperty.call(STATS, i)) {
+            console.error(RapydScript.string_template("- {name}: {time}s", {
+                name: i,
+                time: (STATS[i] / 1000).toFixed(3)
+            }));
+        }
+    }
+}

package/tools/compiler.mjs

@@ -0,0 +1,127 @@
+/* vim:fileencoding=utf-8
+ *
+ * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license
+ */
+"use strict";
+
+// Thin wrapper around (release|dev)/compiler.js to setup some global facilities and
+// export the compiler's symbols safely.
+
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import vm from 'vm';
+import * as terser from 'terser';
+import { fileURLToPath } from 'url';
+import { createRequire } from 'module';
+import { generate_source_map } from './sourcemap.mjs';
+import embedded_compiler_factory from './embedded_compiler.mjs';
+import tree_shake from './treeshake.mjs';
+import { make_ast_serializer } from './ast_serialize.mjs';
+
+const _cjs_require = createRequire(import.meta.url);
+
+function sha1sum(data) {
+    var h = crypto.createHash('sha1');
+    h.update(data);
+    return h.digest('hex');
+}
+
+async function path_exists(p) {
+    try {
+        await fs.promises.stat(p);
+        return true;
+    } catch(e) {
+        if (e.code != 'ENOENT') throw e;
+        return false;
+    }
+}
+
+function uglify(code) {
+    var ans = terser.minify_sync(code);
+    if (ans.error) throw ans.error;
+    return ans.code;
+}
+
+
+async function find_compiler_dir() {
+    var base = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+    var compiler_dir = path.join(base, 'dev');
+    if (!await path_exists(path.join(compiler_dir, 'compiler.js'))) compiler_dir = path.join(base, 'release');
+    return { base, compiler_dir };
+}
+
+async function create_compiler(opts) {
+    opts = opts || {};
+    var vfs = opts.virtual_file_system;
+
+    const { base, compiler_dir } = await find_compiler_dir();
+
+    var readfile, writefile, stat_file;
+    if (vfs) {
+        var stdlib_dir = path.join(base, 'src', 'lib');
+        readfile = async (p, enc) => {
+            if (p.startsWith('__stdlib__/')) {
+                return fs.promises.readFile(path.join(stdlib_dir, p.slice('__stdlib__/'.length)), enc);
+            }
+            return vfs.read_file(p, enc);
+        };
+        writefile = async (p, data) => {
+            if (p.startsWith('__vfs__/')) {
+                if (typeof vfs.write_file === 'function') return vfs.write_file(p, data);
+                return;
+            }
+            if (p.startsWith('__stdlib__/')) return;
+            return fs.promises.writeFile(p, data);
+        };
+        // VFS has no stat; read the file to check existence and return the content
+        // so do_import can reuse it without a second read (mtimeMs: null disables
+        // the mtime fast-path, but the content avoids a double-read).
+        stat_file = async (p) => {
+            const content = await readfile(p, 'utf-8');
+            return { mtimeMs: null, content };
+        };
+    } else {
+        readfile = async (p, enc) => fs.promises.readFile(p, enc);
+        writefile = async (p, data) => fs.promises.writeFile(p, data);
+        stat_file = async (p) => {
+            const st = await fs.promises.stat(p);
+            return { mtimeMs: st.mtimeMs };
+        };
+    }
+
+    var compiler_exports = {};
+    var compiler_context = vm.createContext({
+        console       : console,
+        readfile      : readfile,
+        writefile     : writefile,
+        stat_file     : stat_file,
+        sha1sum       : sha1sum,
+        require       : _cjs_require,
+        exports       : compiler_exports,
+    });
+    var compiler_file = path.join(compiler_dir, 'compiler.js');
+    var compilerjs = await fs.promises.readFile(compiler_file, 'utf-8');
+    vm.runInContext(compilerjs, compiler_context, path.relative(base, compiler_file));
+    const { ast_to_json, ast_from_json, make_lazy_ast_module, encode_cache, decode_cache } = make_ast_serializer(compiler_exports);
+    compiler_exports.ast_to_json = ast_to_json;
+    compiler_exports.ast_from_json = ast_from_json;
+    compiler_exports.make_lazy_ast_module = make_lazy_ast_module;
+    compiler_exports.encode_cache = encode_cache;
+    compiler_exports.decode_cache = decode_cache;
+    // Inject into the VM context so parse.pyj compiled code can call them as globals.
+    compiler_context.ast_to_json = ast_to_json;
+    compiler_context.ast_from_json = ast_from_json;
+    compiler_context.make_lazy_ast_module = make_lazy_ast_module;
+    compiler_context.encode_cache = encode_cache;
+    compiler_context.decode_cache = decode_cache;
+    return compiler_exports;
+}
+
+async function create_embedded_compiler(compiler, baselib, runjs, name) {
+    return await embedded_compiler_factory(compiler || await create_compiler(), baselib, runjs, name, tree_shake, generate_source_map);
+}
+
+export { create_compiler, create_embedded_compiler, generate_source_map };

package/tools/{completer.js → completer.mjs}

@@ -1,15 +1,15 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
 
+import vm from 'vm';
 
-module.exports = function(compiler, options) {
+export default function(compiler, options) {
     "use strict";
     var all_keywords = compiler.ALL_KEYWORDS.split(' ');
-    var vm = require('vm');
     options = options || {};
     if (!options.enum_global) options.enum_global = "var global = Function('return this')(); Object.getOwnPropertyNames(global);";
 
@@ -19,7 +19,7 @@ module.exports = function(compiler, options) {
             ans = ans.concat(all_keywords);
             ans.sort();
             var seen = {};
-            ans.filter(function (item) { 
+            ans.filter(function (item) {
                 if (Object.prototype.hasOwnProperty.call(seen, item)) return false;
                 seen[item] = true;
                 return true;
@@ -45,7 +45,7 @@ module.exports = function(compiler, options) {
         if (typeof obj === 'object' || typeof obj === 'function') {
             add(obj);
             p = Object.getPrototypeOf(obj);
-        } else p = obj.constructor ? obj.constructor.prototype : null; 
+        } else p = obj.constructor ? obj.constructor.prototype : null;
 
         // Walk the prototype chain
         try {
@@ -128,4 +128,4 @@ module.exports = function(compiler, options) {
     }
 
     return find_completions;
-};
+}

package/tools/{embedded_compiler.js → embedded_compiler.mjs}

@@ -1,42 +1,47 @@
 /* vim:fileencoding=utf-8
- * 
+ *
  * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
  *
  * Distributed under terms of the BSD license
  */
-"use strict";  /*jshint node:true */
+"use strict";
 
 var has_prop = Object.prototype.hasOwnProperty.call.bind(Object.prototype.hasOwnProperty);
 
-module.exports = function(compiler, baselib, runjs, name) {
+export default async function(compiler, baselib, runjs, name, tree_shake, generate_source_map) {
     var LINE_CONTINUATION_CHARS = ':\\';
     runjs = runjs || eval;
-    runjs(print_ast(compiler.parse(''), true));
+    runjs(print_ast(await compiler.parse(''), true));
     runjs('var __name__ = "' + (name || '__embedded__') + '";');
 
-    function print_ast(ast, keep_baselib, keep_docstrings, js_version, private_scope, write_name) {
-        var output_options = {omit_baselib:!keep_baselib, write_name:!!write_name, private_scope:!!private_scope, beautify:true, js_version: (js_version || 6), keep_docstrings:keep_docstrings};
+    function print_ast(ast, keep_baselib, keep_docstrings, js_version, private_scope, write_name, source_map, source_map_line_offset) {
+        var output_options = {omit_baselib:!keep_baselib, write_name:!!write_name, private_scope:!!private_scope, beautify:true, js_version: (js_version || 6), keep_docstrings:keep_docstrings, source_map:!!source_map, source_map_line_offset: source_map_line_offset || 0};
         if (keep_baselib) output_options.baselib_plain = baselib;
         var output = new compiler.OutputStream(output_options);
         ast.print(output);
+        if (source_map && generate_source_map) return {code: output.get(), source_map: generate_source_map(output.get_source_map_segments(), '', '')};
         return output.get();
     }
 
     return {
         'toplevel': null,
 
-        'compile': function streaming_compile(code, opts) {
+        'compile': async function streaming_compile(code, opts) {
             opts = opts || {};
             var classes = (this.toplevel) ? this.toplevel.classes : undefined;
             var scoped_flags = (this.toplevel) ? this.toplevel.scoped_flags: undefined;
-            this.toplevel = compiler.parse(code, {
+            this.toplevel = await compiler.parse(code, {
                 'filename': opts.filename || '<embedded>',
-                'basedir': '__stdlib__',
+                'libdir': '__stdlib__',
+                'basedir': '__vfs__',
                 'classes': classes,
                 'scoped_flags': scoped_flags,
                 'discard_asserts': opts.discard_asserts,
             });
-            var ans = print_ast(this.toplevel, opts.keep_baselib, opts.keep_docstrings, opts.js_version, opts.private_scope, opts.write_name);
+            if (opts.tree_shaking && tree_shake) {
+                this.toplevel = tree_shake(this.toplevel);
+            }
+            var ans = print_ast(this.toplevel, opts.keep_baselib, opts.keep_docstrings, opts.js_version, opts.private_scope, opts.write_name, opts.source_map, opts.source_map_line_offset);
             if (classes) {
                 var exports = {};
                 var self = this;
@@ -47,10 +52,9 @@ module.exports = function(compiler, baselib, runjs, name) {
                 });
             }
             scoped_flags = this.toplevel.scoped_flags;
-    
+
             return ans;
         },
 
     };
-};
-
+}