rapydscript-ns 0.8.0

package/tools/embedded_compiler.js

@@ -0,0 +1,251 @@
+/* vim:fileencoding=utf-8
+ * 
+ * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license
+ */
+"use strict";  /*jshint node:true */
+
+var has_prop = Object.prototype.hasOwnProperty.call.bind(Object.prototype.hasOwnProperty);
+
+function build_scoped_flags(flags_str) {
+    var result = Object.create(null);
+    if (!flags_str) return result;
+    flags_str.split(',').forEach(function(flag) {
+        flag = flag.trim();
+        if (!flag) return;
+        var val = true;
+        if (flag.slice(0, 3) === 'no_') { val = false; flag = flag.slice(3); }
+        result[flag] = val;
+    });
+    return result;
+}
+
+module.exports = function(compiler, baselib, runjs, name, vf_context) {
+    var LINE_CONTINUATION_CHARS = ':\\';
+    runjs = runjs || eval;
+    runjs(print_ast(compiler.parse(''), true));
+    runjs('var __name__ = "' + (name || '__embedded__') + '";');
+
+    function print_ast(ast, keep_baselib, keep_docstrings, js_version, private_scope, write_name, omit_function_metadata, pythonize_strings) {
+        var output_options = {omit_baselib:!keep_baselib, write_name:!!write_name, private_scope:!!private_scope, beautify:true, js_version: (js_version || 6), keep_docstrings:keep_docstrings, omit_function_metadata:!!omit_function_metadata, pythonize_strings:!!pythonize_strings};
+        if (keep_baselib) output_options.baselib_plain = baselib;
+        var output = new compiler.OutputStream(output_options);
+        ast.print(output);
+        return output.get();
+    }
+
+    // --- Source map support ---
+
+    function vlq_encode(value) {
+        var BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+        var vlq = value < 0 ? ((-value) << 1) | 1 : value << 1;
+        var result = '';
+        do {
+            var digit = vlq & 31;
+            vlq >>>= 5;
+            if (vlq > 0) digit |= 32;
+            result += BASE64[digit];
+        } while (vlq > 0);
+        return result;
+    }
+
+    function build_source_map(raw_mappings, source_name, source_content) {
+        if (!raw_mappings.length) {
+            return JSON.stringify({version:3, sources:[source_name], sourcesContent:[source_content||null], names:[], mappings:''});
+        }
+        raw_mappings.sort(function(a, b) {
+            return a.gen_line !== b.gen_line ? a.gen_line - b.gen_line : a.gen_col - b.gen_col;
+        });
+        // Deduplicate same generated position (keep first)
+        var deduped = [];
+        var prev_key = null;
+        for (var i = 0; i < raw_mappings.length; i++) {
+            var m = raw_mappings[i];
+            var key = m.gen_line + ':' + m.gen_col;
+            if (key !== prev_key) { deduped.push(m); prev_key = key; }
+        }
+        var max_gen_line = deduped[deduped.length - 1].gen_line;
+        var by_line = Object.create(null);
+        for (var i = 0; i < deduped.length; i++) {
+            var m = deduped[i];
+            if (!by_line[m.gen_line]) by_line[m.gen_line] = [];
+            by_line[m.gen_line].push(m);
+        }
+        // prev_src_* track deltas across all generated lines (Source Map v3 spec)
+        var prev_src_line_0 = 0;
+        var prev_src_col = 0;
+        var lines_out = [];
+        for (var line = 1; line <= max_gen_line; line++) {
+            var prev_gen_col = 0;
+            var segs = [];
+            var lm = by_line[line] || [];
+            for (var j = 0; j < lm.length; j++) {
+                var m = lm[j];
+                var src_line_0 = m.src_line - 1;
+                segs.push(
+                    vlq_encode(m.gen_col - prev_gen_col) +
+                    vlq_encode(0) +
+                    vlq_encode(src_line_0 - prev_src_line_0) +
+                    vlq_encode(m.src_col - prev_src_col)
+                );
+                prev_gen_col = m.gen_col;
+                prev_src_line_0 = src_line_0;
+                prev_src_col = m.src_col;
+            }
+            lines_out.push(segs.join(','));
+        }
+        return JSON.stringify({
+            version: 3,
+            sources: [source_name],
+            sourcesContent: [source_content || null],
+            names: [],
+            mappings: lines_out.join(';'),
+        });
+    }
+
+    function print_ast_with_sourcemap(ast, keep_baselib, keep_docstrings, js_version, private_scope, write_name, omit_function_metadata, pythonize_strings, source_name, source_content) {
+        var output_options = {omit_baselib:!keep_baselib, write_name:!!write_name, private_scope:!!private_scope, beautify:true, js_version:(js_version||6), keep_docstrings:keep_docstrings, omit_function_metadata:!!omit_function_metadata, pythonize_strings:!!pythonize_strings};
+        if (keep_baselib) output_options.baselib_plain = baselib;
+        var raw_mappings = [];
+        var output = new compiler.OutputStream(output_options);
+        output.push_node = function(node) {
+            if (node && node.start && node.start.line) {
+                raw_mappings.push({
+                    gen_line: this.current_line,
+                    gen_col: this.current_col,
+                    src_line: node.start.line,
+                    src_col: node.start.col,
+                });
+            }
+            this._stack.push(node);
+        };
+        ast.print(output);
+        return {
+            code: output.get(),
+            sourceMap: build_source_map(raw_mappings, source_name, source_content),
+        };
+    }
+
+    return {
+        'toplevel': null,
+
+        'compile': function streaming_compile(code, opts) {
+            opts = opts || {};
+            var classes = (this.toplevel) ? this.toplevel.classes : undefined;
+            var inherited_flags = (this.toplevel) ? this.toplevel.scoped_flags : undefined;
+            var scoped_flags = Object.assign(
+                Object.create(null),
+                opts.python_flags ? build_scoped_flags(opts.python_flags) : {},
+                inherited_flags || {}
+            );
+            var vf = (opts.virtual_files && vf_context) ? opts.virtual_files : null;
+            var parse_opts = {
+                'filename': opts.filename || '<embedded>',
+                'basedir': '__stdlib__',
+                'classes': classes,
+                'scoped_flags': scoped_flags,
+                'discard_asserts': opts.discard_asserts,
+            };
+            if (vf) {
+                vf_context.set(vf);
+                parse_opts.import_dirs = ['__virtual__'];
+            }
+            try {
+                this.toplevel = compiler.parse(code, parse_opts);
+            } finally {
+                if (vf) vf_context.clear();
+            }
+            if (opts.tree_shake && compiler.tree_shake &&
+                    this.toplevel.imports && Object.keys(this.toplevel.imports).length) {
+                compiler.tree_shake(this.toplevel, {
+                    parse: compiler.parse,
+                    import_dirs: [],
+                    basedir: undefined,
+                    libdir: undefined,
+                });
+            }
+            var ans = print_ast(this.toplevel, opts.keep_baselib, opts.keep_docstrings, opts.js_version, opts.private_scope, opts.write_name, opts.omit_function_metadata, opts.pythonize_strings);
+            if (opts.export_main) {
+                ans = ans.replace(/^(function\smain)/gm, 'export $1')
+                    .replace(/^(async\sfunction\smain)/gm, 'export $1');
+            }
+            if (classes) {
+                var exports = {};
+                var self = this;
+                this.toplevel.exports.forEach(function (name) { exports[name] = true; });
+                Object.getOwnPropertyNames(classes).forEach(function (name) {
+                    if (!has_prop(exports, name) && !has_prop(self.toplevel.classes, name))
+                        self.toplevel.classes[name] = classes[name];
+                });
+            }
+            scoped_flags = this.toplevel.scoped_flags;
+
+            return ans;
+        },
+
+        'compile_with_sourcemap': function compile_with_sourcemap(code, opts) {
+            opts = opts || {};
+            var classes = (this.toplevel) ? this.toplevel.classes : undefined;
+            var inherited_flags = (this.toplevel) ? this.toplevel.scoped_flags : undefined;
+            var scoped_flags = Object.assign(
+                Object.create(null),
+                opts.python_flags ? build_scoped_flags(opts.python_flags) : {},
+                inherited_flags || {}
+            );
+            var vf = (opts.virtual_files && vf_context) ? opts.virtual_files : null;
+            var parse_opts = {
+                'filename': opts.filename || '<embedded>',
+                'basedir': '__stdlib__',
+                'classes': classes,
+                'scoped_flags': scoped_flags,
+                'discard_asserts': opts.discard_asserts,
+            };
+            if (vf) {
+                vf_context.set(vf);
+                parse_opts.import_dirs = ['__virtual__'];
+            }
+            try {
+                this.toplevel = compiler.parse(code, parse_opts);
+            } finally {
+                if (vf) vf_context.clear();
+            }
+            if (opts.tree_shake && compiler.tree_shake &&
+                    this.toplevel.imports && Object.keys(this.toplevel.imports).length) {
+                compiler.tree_shake(this.toplevel, {
+                    parse: compiler.parse,
+                    import_dirs: [],
+                    basedir: undefined,
+                    libdir: undefined,
+                });
+            }
+            var result = print_ast_with_sourcemap(
+                this.toplevel,
+                opts.keep_baselib, opts.keep_docstrings, opts.js_version,
+                opts.private_scope, opts.write_name, opts.omit_function_metadata,
+                opts.pythonize_strings,
+                opts.filename || '<input>',
+                code
+            );
+            var compiled_code = result.code;
+            if (opts.export_main) {
+                compiled_code = compiled_code
+                    .replace(/^(function\smain)/gm, 'export $1')
+                    .replace(/^(async\sfunction\smain)/gm, 'export $1');
+            }
+            if (classes) {
+                var exports_map = {};
+                var self_ref = this;
+                this.toplevel.exports.forEach(function(name) { exports_map[name] = true; });
+                Object.getOwnPropertyNames(classes).forEach(function(name) {
+                    if (!has_prop(exports_map, name) && !has_prop(self_ref.toplevel.classes, name))
+                        self_ref.toplevel.classes[name] = classes[name];
+                });
+            }
+            scoped_flags = this.toplevel.scoped_flags;
+            return { code: compiled_code, sourceMap: result.sourceMap };
+        },
+
+    };
+};
+

package/tools/export.js

@@ -0,0 +1,316 @@
+/* 
+ * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license
+ */
+
+var vm = require('vm');
+var native_require = require;
+
+function normalize_array(parts, allowAboveRoot) {
+  var res = [];
+  for (var i = 0; i < parts.length; i++) {
+    var p = parts[i];
+
+    // ignore empty parts
+    if (!p || p === '.')
+      continue;
+
+    if (p === '..') {
+      if (res.length && res[res.length - 1] !== '..') {
+        res.pop();
+      } else if (allowAboveRoot) {
+        res.push('..');
+      }
+    } else {
+      res.push(p);
+    }
+  }
+
+  return res;
+}
+
+function normalize(path) {
+    var is_abs = path && path[0] === '/';
+    var trailing_slash = path && path[path.length - 1] === '/';
+    path = normalize_array(path.split('/'), !is_abs).join('/');
+
+    if (!path && !is_abs) {
+        path = '.';
+    }
+    if (path && trailing_slash) {
+        path += '/';
+    }
+
+    return (is_abs ? '/' : '') + path;
+}
+
+function dirname(path) {
+    var idx = path.lastIndexOf('/');
+    if (idx != -1) path = path.slice(0, idx);
+    else path = '';
+    return path;
+}
+
+function basename(path) {
+    var idx = path.lastIndexOf('/');
+    if (idx != -1) path = path.slice(idx + 1);
+    return path;
+}
+
+var cache = {};
+
+function load(filepath) {
+    var cached = cache[filepath];
+    if (cached) return cached.exports;
+    var module = {'id':filepath, 'exports':{}};
+    cache[filepath] = module;
+
+    var content = data[filepath];
+    if (Array.isArray(content)) content = data[content[0]];
+    if (!content) throw 'Failed to load: ' + JSON.stringify(filepath);
+
+    if (filepath.slice(-5) == '.json') { module.exports = JSON.parse(content); return module.exports; }
+
+    var base = dirname(filepath);
+    function mrequire(x) {
+        return vrequire(x, base);
+    }
+    content = content.replace(/^\#\!.*/, '');
+    var wrapped = '(function(exports, require, module, __filename, __dirname, create_rapydscript_compiler) { ';
+    wrapped += content + '\n;})';
+    try {
+        vm.runInThisContext(wrapped, {'filename': filepath})(module.exports, mrequire, module, filepath, dirname(filepath), create_compiler);
+    } catch (e) {
+        console.error(e);
+        delete cache[filepath];
+        throw e;
+    }
+    return module.exports;
+}
+
+function has(x, y) { return Object.prototype.hasOwnProperty.call(x, y); }
+
+function try_files(candidate) {
+    if (has(data, candidate)) return candidate;
+    if (has(data, candidate + '.js')) return candidate + '.js';
+    if (has(data, candidate + '.json')) return candidate + '.json';
+    return null;
+}
+
+function find_in_modules_dir(name, base) {
+    var candidate = normalize(base + (base ? '/':'') + 'node_modules/' + name);
+    var q = try_files(candidate);
+    if (q) return q;
+
+    var pj = candidate + '/package.json';
+    if (has(data, pj)) {
+        var ans = normalize(candidate + '/' + JSON.parse(data[pj]).main);
+        if (has(data, ans)) return ans;
+    }
+    var index = candidate + '/index.js';
+    if (has(data, index)) return index;
+
+    var p = dirname(base);
+    if (p) return find_in_modules_dir(name, p);
+    return null;
+}
+
+function find_module(name, base) {
+    if (name[0] == '/') throw 'Cannot find absolute module: ' + name;
+    if (name.slice(0, 2) == './' || name.slice(0, 3) == '../') {
+        var candidate = normalize((base ? base + '/' : base) + name);
+        return try_files(candidate);
+    }
+    var q = try_files(name);
+    if (q) return q;
+    return find_in_modules_dir(name, base);
+}
+
+function vrequire(name, base) {
+    var exports = {};
+    var modpath = '';
+    base = base || '';
+    // console.log('vrequire', name, base);
+    if (!name) throw new Error('Cannot load a module from an empty name');
+
+    modpath = find_module(name, base);
+    if (!modpath && name && './'.indexOf(name[0]) === -1) {
+            try {
+                return native_require(name);
+            } catch (e) {}
+        }
+
+    if (!modpath) throw new Error("Failed to find module: " + JSON.stringify(name) + " with base: " + JSON.stringify(base));
+    return load(modpath);
+}
+
+var UglifyJS = null, regenerator = null;
+var crypto = null, fs = require('fs');
+
+var current_virtual_files = null;
+
+function readfile_wrapper(name, encoding) {
+    if (current_virtual_files && name.indexOf('__virtual__/') === 0) {
+        var rel = name.slice('__virtual__/'.length);
+        if (rel.slice(-11) === '.pyj-cached') rel = rel.slice(0, -11);
+        else if (rel.slice(-4) === '.pyj') rel = rel.slice(0, -4);
+        if (rel.slice(-9) === '/__init__') rel = rel.slice(0, -9);
+        if (Object.prototype.hasOwnProperty.call(current_virtual_files, rel)) {
+            return current_virtual_files[rel];
+        }
+    }
+    return fs.readFileSync(name, encoding);
+}
+
+function writefile_wrapper(name, content) {
+    // Silently discard cache writes for virtual files; __virtual__ is not a real directory.
+    if (current_virtual_files && name.indexOf('__virtual__/') === 0) return;
+    return fs.writeFileSync(name, content);
+}
+
+function uglify(x) {
+    if (!UglifyJS) UglifyJS = vrequire("uglify-js");
+    ans = UglifyJS.minify(x);
+    if (ans.error) throw ans.error;
+    return ans.code;
+}
+
+function regenerate(code, beautify) {
+    var orig = fs.readFileSync;
+    fs.readFileSync = function(name) { 
+        if (!has(data, name)) {
+            throw {message: "Failed to readfile from data: " + name};
+        }
+        return data[name]; 
+    };
+    if (!regenerator) regenerator = vrequire('regenerator');
+    var ans;
+    if (code) {
+        try {
+            ans = regenerator.compile(code).code;
+        } catch (e) {
+            console.error('regenerator failed for code: ' + code + 'with error stack:\n' + e.stack);
+            throw e;
+        }
+        if (!beautify) ans = uglify(ans);
+    } else {
+        // Return the runtime
+        ans = regenerator.compile('', {includeRuntime:true}).code;
+        start = ans.indexOf('=') + 1;
+        end = ans.lastIndexOf('typeof');
+        end = ans.lastIndexOf('}(', end);
+        ans = ans.slice(start + 1, end);
+        if (!beautify) {
+            var extra = '})()';
+            ans = uglify(ans + extra).slice(0, extra.length);
+        }
+    }
+    fs.readFileSync = orig;
+    return ans;
+}
+
+if (typeof this != 'object' || typeof this.sha1sum !== 'function') {
+    var sha1sum = function (data) { 
+        if (!crypto) crypto = require('crypto');
+        var h = crypto.createHash('sha1');
+        h.update(data);
+        return h.digest('hex');
+    };
+} else var sha1sum = this.sha1sum;
+
+function create_compiler() {
+    var compilerjs = data['compiler.js'];
+    var module = {'id':'compiler', 'exports':{}};
+    var wrapped = '(function(module, exports, readfile, writefile, sha1sum, regenerate) {' + data['compiler.js'] + ';\n})';
+    vm.runInThisContext(wrapped, {'filename': 'compiler.js'})(module, module.exports, readfile_wrapper, writefile_wrapper, sha1sum, regenerate);
+    return module.exports;
+}
+
+var RapydScript = null;
+
+function compile(code, filename, options) {
+    if (!RapydScript) RapydScript = create_compiler();
+    options = options || {};
+    var vf = options.virtual_files || null;
+    var prev_virtual_files = current_virtual_files;
+    var import_dirs = options.import_dirs || [];
+    if (vf) {
+        current_virtual_files = vf;
+        import_dirs = ['__virtual__'].concat(import_dirs);
+    }
+    try {
+        var ast = RapydScript.parse(code, {
+            filename: filename || '<eval>',
+            basedir: options.basedir || dirname(filename || ''),
+            libdir: options.libdir,
+            import_dirs: import_dirs,
+        });
+        var out_ops = {
+            beautify: (options.beautify === undefined ? true : options.beautify),
+            private_scope: !options.bare,
+            omit_baselib: !!options.omit_baselib,
+            js_version: options.js_version || 5,
+        };
+        if (!out_ops.omit_baselib) out_ops.baselib_plain = data['baselib-plain-' + (out_ops.beautify ? 'pretty' : 'ugly') + '.js'];
+        var out = new RapydScript.OutputStream(out_ops);
+        ast.print(out);
+        return out.get();
+    } finally {
+        current_virtual_files = prev_virtual_files;
+    }
+}
+
+function create_embedded_compiler(runjs) {
+    var c = vrequire('tools/embedded_compiler.js');
+    return c(create_compiler(), data['baselib-plain-pretty.js'], runjs);
+}
+
+function web_repl() {
+    var repl = vrequire('tools/web_repl.js');
+    var vf_context = {
+        set: function(vf) { current_virtual_files = vf; },
+        clear: function() { current_virtual_files = null; }
+    };
+    return repl(create_compiler(), data['baselib-plain-pretty.js'], vf_context);
+}
+
+function init_repl(options) {
+    var repl = vrequire('tools/repl.js');
+    options.baselib = data['baselib-plain-pretty.js'];
+    return repl(options);
+}
+
+function gettext_parse(catalog, code, filename) {
+    g = vrequire('tools/gettext.js');
+    g.gettext(catalog, code, filename);
+}
+
+function gettext_output(catalog, options, write) {
+    g = vrequire('tools/gettext.js');
+    g.write_output(catalog, options, write);
+}
+
+function msgfmt(data, options) {
+    m = vrequire('tools/msgfmt.js');
+    return m.build(data, options);
+}
+
+function completer(compiler, options) {
+    m = vrequire('tools/completer.js');
+    return m(compiler, options);
+}
+
+if (typeof exports === 'object') {
+    exports.compile = compile;
+    exports.create_embedded_compiler = create_embedded_compiler;
+    exports.web_repl = web_repl;
+    exports.init_repl = init_repl;
+    exports.gettext_parse = gettext_parse;
+    exports.gettext_output = gettext_output;
+    exports.msgfmt = msgfmt;
+    exports.rs_version = rs_version;
+    exports.file_data = data;
+    exports.completer = completer;
+    if (typeof rs_commit_sha === 'string') exports.rs_commit_sha = rs_commit_sha;
+}