rapydscript-ns 0.8.0

package/tools/compile.js

@@ -0,0 +1,219 @@
+/*
+ * compile.js
+ * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license.
+ */
+"use strict";  /*jshint node:true */
+
+var fs = require('fs');
+var path = require('path');
+var vm = require('vm');
+var RapydScript = require("./compiler").create_compiler();
+var utils = require('./utils');
+
+function read_whole_file(filename, cb) {
+    if (!filename) {
+        var chunks = [];
+        process.stdin.setEncoding('utf-8');
+        process.stdin.on('data', function (chunk) {
+            chunks.push(chunk);
+        }).on('end', function () {
+            cb(null, chunks.join(""));
+        });
+        process.openStdin();
+    } else {
+        fs.readFile(filename, "utf-8", cb);
+    }
+}
+
+function makedirs(dir) {
+    try {
+        fs.mkdirSync(dir);
+    } catch(e) {
+        if (e.code == 'EEXIST') return;
+        if (e.code == 'ENOENT') { makedirs(path.dirname(dir)); fs.mkdirSync(dir); }
+        throw e;
+    }
+}
+
+function process_cache_dir(dir) {
+    dir = path.resolve(path.normalize(dir));
+    makedirs(dir);
+    return dir;
+}
+
+function build_scoped_flags(flags_str) {
+    var result = Object.create(null);
+    if (!flags_str) return result;
+    flags_str.split(',').forEach(function(flag) {
+        flag = flag.trim();
+        if (!flag) return;
+        var val = true;
+        if (flag.slice(0, 3) === 'no_') { val = false; flag = flag.slice(3); }
+        result[flag] = val;
+    });
+    return result;
+}
+
+module.exports = function(start_time, argv, base_path, src_path, lib_path) {
+    // configure settings for the output
+    var cache_dir = argv.cache_dir ? process_cache_dir(argv.cache_dir) : '';
+    var OUTPUT_OPTIONS = {
+        beautify: !argv.uglify,
+        private_scope: !argv.bare && !argv.export_main,
+        omit_baselib: argv.omit_baselib,
+        js_version: parseInt(argv.js_version),
+        keep_docstrings: argv.keep_docstrings,
+        discard_asserts: argv.discard_asserts,
+        module_cache_dir: cache_dir,
+        pythonize_strings: argv.pythonize_strings,
+    };
+
+    var files = argv.files.slice();
+    var STATS = {}, TOPLEVEL;
+    var num_of_files = files.length || 1;
+
+    var global_scoped_flags = build_scoped_flags(argv.python_flags);
+
+    function parse_file(code, file, toplevel) {
+        return RapydScript.parse(code, {
+            filename: file,
+            toplevel: toplevel,
+            basedir: (file !== '<stdin>') ? path.dirname(file) : undefined,
+            libdir: path.join(src_path, 'lib'),
+            import_dirs: utils.get_import_dirs(argv.import_path),
+            discard_asserts: argv.discard_asserts,
+            module_cache_dir: cache_dir,
+            scoped_flags: global_scoped_flags,
+        });
+    }
+
+    function write_output(output) {
+        if (argv.output) {
+            // Node's filesystem module cannot write directly to /dev/stdout
+            if (argv.output == '/dev/stdout') console.log(output);
+            else if (argv.output == '/dev/stderr') console.error(output);
+            else fs.writeFileSync(argv.output, output, "utf8");
+        } else if (!argv.execute){
+            console.log(output);
+        }
+        if (argv.execute) {
+            vm.runInNewContext(output, {'console':console, 'require':require}, {'filename':files[0]});
+        }
+    }
+
+    function time_it(name, cont) {
+        var t1 = new Date().getTime();
+        var ret = cont();
+        if (argv.stats) {
+            var spent = new Date().getTime() - t1;
+            if (STATS[name]) STATS[name] += spent;
+            else STATS[name] = spent;
+        }
+        return ret;
+    }
+
+    function compile_single_file(err, code) {
+        var output;
+        if (err) {
+            console.error("ERROR: can't read file: " + files[0]);
+            process.exit(1);
+        }
+        time_it("parse", function(){
+            var file = files[0] || argv.filename_for_stdin || '<stdin>';
+            try {
+                TOPLEVEL = parse_file(code, file, TOPLEVEL);
+            } catch (e) {
+                if (!(e instanceof RapydScript.SyntaxError)) throw e;
+                console.error(e.toString());
+                process.exit(1);
+            }
+        });
+
+        if (TOPLEVEL.imports && Object.keys(TOPLEVEL.imports).length) {
+            time_it("treeshake", function() {
+                RapydScript.tree_shake(TOPLEVEL, {
+                    parse: RapydScript.parse,
+                    import_dirs: utils.get_import_dirs(argv.import_path),
+                    basedir: (files[0] !== '<stdin>') ? path.dirname(files[0]) : undefined,
+                    libdir: path.join(src_path, 'lib'),
+                    discard_asserts: argv.discard_asserts,
+                    module_cache_dir: cache_dir,
+                });
+            });
+        }
+
+        try {
+            output = new RapydScript.OutputStream(OUTPUT_OPTIONS);
+        } catch(ex) {
+            if (ex instanceof RapydScript.DefaultsError) {
+                console.error(ex.message);
+                process.exit(1);
+            }
+            throw ex;
+        }
+
+        time_it("generate", function(){
+            TOPLEVEL.print(output);
+        });
+
+        output = output.get();
+
+        if (argv.export_main) {
+            output = output.replace(/^(function\smain)/gm, 'export $1')
+                .replace(/^(async\sfunction\smain)/gm, 'export $1');
+        }
+
+        write_output(output);
+
+        files = files.slice(1);
+        if (files.length) {
+            setImmediate(read_whole_file, files[0], compile_single_file);
+            return;
+        }
+        if (argv.stats) {
+            console.error(RapydScript.string_template("Timing information (compressed {count} files):", {
+                count: num_of_files
+            }));
+            for (var i in STATS) if (Object.prototype.hasOwnProperty.call(STATS, i)) {
+                console.error(RapydScript.string_template("- {name}: {time}s", {
+                    name: i,
+                    time: (STATS[i] / 1000).toFixed(3)
+                }));
+            }
+        }
+    }
+
+
+    if (argv.comments) {
+        if (/^\//.test(argv.comments)) {
+            OUTPUT_OPTIONS.comments = new Function("return(" + argv.comments + ")")();  // jshint ignore:line
+        } else if (argv.comments == "all") {
+            OUTPUT_OPTIONS.comments = true;
+        } else {
+            OUTPUT_OPTIONS.comments = function(node, comment) {
+                var text = comment.value;
+                var type = comment.type;
+                if (type == "comment2") {
+                    // multiline comment
+                    return /@preserve|@license|@cc_on/i.test(text);
+                }
+            };
+        }
+    }
+
+    if (!argv.omit_baselib) {
+        var which = (OUTPUT_OPTIONS.beautify) ? 'pretty' : 'ugly';
+        OUTPUT_OPTIONS.baselib_plain = fs.readFileSync(path.join(lib_path, 'baselib-plain-' + which + '.js'), 'utf-8');
+    }
+
+    if (files.filter(function(el){ return el == "-"; }).length > 1) {
+        console.error("ERROR: Can read a single file from STDIN (two or more dashes specified)");
+        process.exit(1);
+    }
+
+    setImmediate(read_whole_file, files[0], compile_single_file);
+
+};
+

package/tools/compiler.js

@@ -0,0 +1,108 @@
+/* vim:fileencoding=utf-8
+ * 
+ * Copyright (C) 2015 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license
+ */
+"use strict";  /*jshint node:true */
+
+// Thin wrapper around (release|dev)/compiler.js to setup some global facilities and
+// export the compiler's symbols safely.
+
+var path = require("path");
+var fs = require("fs");
+var crypto = require('crypto');
+var vm = require("vm");
+var regenerator = require('regenerator');
+var UglifyJS = require("uglify-js");
+
+function sha1sum(data) { 
+    var h = crypto.createHash('sha1');
+    h.update(data);
+    return h.digest('hex');
+}
+
+function path_exists(path) {
+    try {
+        fs.statSync(path);
+        return true;
+    } catch(e) {
+        if (e.code != 'ENOENT') throw e;
+    }
+}
+
+function uglify(code) {
+    var ans = UglifyJS.minify(code);
+    if (ans.error) throw ans.error;
+    return ans.code;
+}
+
+
+function regenerate(code, beautify) {
+    var ans, start, end;
+    if (code) {
+        ans = regenerator.compile(code).code;
+        if (!beautify) {
+            ans = uglify(ans);
+        }
+    } else {
+        // Return the runtime
+        ans = regenerator.compile('', {includeRuntime:true}).code;
+        start = ans.indexOf('=') + 1;
+        end = ans.lastIndexOf('typeof');
+        end = ans.lastIndexOf('}(', end);
+        ans = ans.slice(start + 1, end);
+        if (!beautify) {
+            var extra = '})()';
+            ans = uglify(ans + extra).slice(0, extra.length);
+        }
+    }
+    return ans;
+}
+
+var _current_virtual_files = null;
+
+function virtual_readfile(name, encoding) {
+    if (_current_virtual_files && name.indexOf('__virtual__/') === 0) {
+        var rel = name.slice('__virtual__/'.length);
+        if (rel.slice(-11) === '.pyj-cached') rel = rel.slice(0, -11);
+        else if (rel.slice(-4) === '.pyj') rel = rel.slice(0, -4);
+        if (rel.slice(-9) === '/__init__') rel = rel.slice(0, -9);
+        if (Object.prototype.hasOwnProperty.call(_current_virtual_files, rel)) {
+            return _current_virtual_files[rel];
+        }
+    }
+    return fs.readFileSync(name, encoding);
+}
+
+function virtual_writefile(name, content) {
+    // Silently discard cache writes for virtual files; __virtual__ is not a real
+    // directory, and the compiled `except Error` guard fails across vm contexts.
+    if (name.indexOf('__virtual__/') === 0) return;
+    return fs.writeFileSync(name, content);
+}
+
+function create_compiler() {
+    var compiler_exports = {};
+    var compiler_context = vm.createContext({
+        console       : console,
+        readfile      : virtual_readfile,
+        writefile     : virtual_writefile,
+        sha1sum       : sha1sum,
+        require       : require,
+        regenerate    : regenerate,
+        exports       : compiler_exports,
+    });
+
+    var base = path.dirname(path.dirname(module.filename));
+    var compiler_dir = path.join(base, 'dev');
+    if (!path_exists(path.join(compiler_dir, 'compiler.js'))) compiler_dir = path.join(base, 'release');
+    var compiler_file = path.join(compiler_dir, 'compiler.js');
+    var compilerjs = fs.readFileSync(compiler_file, 'utf-8');
+    vm.runInContext(compilerjs, compiler_context, path.relative(base, compiler_file));
+    return compiler_exports;
+}
+
+exports.create_compiler = create_compiler;
+exports.set_virtual_files = function(vf) { _current_virtual_files = vf; };
+exports.clear_virtual_files = function() { _current_virtual_files = null; };

package/tools/completer.js

@@ -0,0 +1,131 @@
+/* vim:fileencoding=utf-8
+ * 
+ * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the BSD license
+ */
+
+
+module.exports = function(compiler, options) {
+    "use strict";
+    var all_keywords = compiler.ALL_KEYWORDS.split(' ');
+    var vm = require('vm');
+    options = options || {};
+    if (!options.enum_global) options.enum_global = "var global = Function('return this')(); Object.getOwnPropertyNames(global);";
+
+    function global_names(ctx) {
+        try {
+            var ans = vm.runInContext(options.enum_global, ctx);
+            ans = ans.concat(all_keywords);
+            ans.sort();
+            var seen = {};
+            ans.filter(function (item) { 
+                if (Object.prototype.hasOwnProperty.call(seen, item)) return false;
+                seen[item] = true;
+                return true;
+            });
+            return ans;
+        } catch(e) {
+            console.log(e.stack || e.toString());
+        }
+        return [];
+    }
+
+    function object_names(obj, prefix) {
+        if (obj === null || obj === undefined) return [];
+        var groups = [], prefix_len = prefix.length, p;
+
+        function prefix_filter(name) { return (prefix_len) ? (name.substr(0, prefix_len) === prefix) : true; }
+
+        function add(o) {
+            var items = Object.getOwnPropertyNames(o).filter(prefix_filter);
+            if (items.length) groups.push(items);
+        }
+
+        if (typeof obj === 'object' || typeof obj === 'function') {
+            add(obj);
+            p = Object.getPrototypeOf(obj);
+        } else p = obj.constructor ? obj.constructor.prototype : null; 
+
+        // Walk the prototype chain
+        try {
+            var sentinel = 5;
+            while (p !== null && sentinel > 0) {
+                add(p);
+                p = Object.getPrototypeOf(p);
+                // Circular refs possible? Let's guard against that.
+                sentinel--;
+            }
+        } catch (e) {
+            // console.error("completion error walking prototype chain:" + e);
+        }
+        if (!groups.length) return [];
+        var seen = {}, ans = [];
+        function uniq(name) {
+            if (Object.prototype.hasOwnProperty.call(seen, name)) return false;
+            seen[name] = true;
+            return true;
+        }
+        for (var i = 0; i < groups.length; i++) {
+            var group = groups[i];
+            group.sort();
+            ans = ans.concat(group.filter(uniq));
+            ans.push('');  // group separator
+
+        }
+        while (ans.length && ans[ans.length - 1] === '') ans.pop();
+        return ans;
+    }
+
+    function prefix_matches(prefix, items) {
+        var len = prefix.length;
+        var ans = items.filter(function(item) { return item.substr(0, len) === prefix; });
+        ans.sort();
+        return ans;
+    }
+
+    function find_completions(line, ctx) {
+        var t;
+        try {
+            t = compiler.tokenizer(line, '<repl>');
+        } catch(e) { return []; }
+        var tokens = [], token;
+        while (true) {
+            try {
+                token = t();
+            } catch (e) { return []; }
+            if (token.type === 'eof') break;
+            if (token.type === 'punc' && '(){},;:'.indexOf(token.value) > -1)
+                tokens = [];
+            tokens.push(token);
+        }
+        if (!tokens.length) {
+            // New line or trailing space
+            return [global_names(ctx), ''];
+        }
+        var last_tok = tokens[tokens.length - 1];
+        if (last_tok.value === '.' || (last_tok.type === 'name' && compiler.IDENTIFIER_PAT.test(last_tok.value))) {
+            last_tok = last_tok.value;
+            if (last_tok === '.') {
+                tokens.push({'value':''});
+                last_tok = '';
+            }
+            if (tokens.length > 1 && tokens[tokens.length - 2].value === '.') {
+                // A compound expression
+                var prefix = '', result;
+                tokens.slice(0, tokens.length - 2).forEach(function (tok) { prefix += tok.value; });
+                if (prefix) {
+                    try {
+                        result = vm.runInContext(prefix, ctx, {'displayErrors':false});
+                    } catch(e) { return []; }
+                    return [object_names(result, last_tok), last_tok];
+                }
+            } else {
+                return [prefix_matches(last_tok, global_names(ctx)), last_tok];
+            }
+        }
+        return [];
+    }
+
+    return find_completions;
+};