rapydscript-ns 0.8.0

package/src/output/stream.pyj

@@ -0,0 +1,309 @@
+# vim:fileencoding=utf-8
+# License: BSD Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+# globals:regenerate
+from __python__ import hash_literals
+
+from utils import make_predicate, defaults, repeat_string
+from tokenizer import is_identifier_char
+
+DANGEROUS = /[\u0000\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g
+
+
+def as_hex(code, sz):
+    val = code.toString(16)
+    if val.length < sz:
+        val = '0'.repeat(sz - val.length) + val
+    return val
+
+
+def to_ascii(str_, identifier):
+    return str_.replace(/[\u0080-\uffff]/g, def(ch):
+        code = ch.charCodeAt(0).toString(16)
+        if code.length <= 2 and not identifier:
+            return "\\x" + as_hex(code, 2)
+        else:
+            return '\\u' + as_hex(code, 4)
+
+    )
+
+def encode_string(str_):
+    return JSON.stringify(str_).replace(DANGEROUS, def(a):
+            return '\\u' + as_hex(a.charCodeAt(0), 4)
+    )
+require_semi_colon_chars = make_predicate("( [ + * / - , .")
+
+output_stream_defaults = {
+    'indent_start': 0,
+    'indent_level': 4,
+    'quote_keys': False,
+    'space_colon': True,
+    'ascii_only': False,
+    'width': 80,
+    'max_line_len': 32000,
+    'ie_proof': True,
+    'beautify': False,
+    'source_map': None,
+    'bracketize': False,
+    'semicolons': True,
+    'comments': False,
+    'preserve_line': False,
+    'omit_baselib': False,
+    'baselib_plain': None,
+    'private_scope': True,
+    'keep_docstrings': False,
+    'discard_asserts': False,
+    'module_cache_dir': '',
+    'js_version':5,
+    'write_name': True,
+    'omit_function_metadata': False,
+    'pythonize_strings': False,
+}
+
+class OutputStream:
+
+    def __init__(self, options):
+        self.options = defaults(options, output_stream_defaults, True)
+        self._indentation = 0
+        self.current_col = 0
+        self.current_line = 1
+        self.current_pos = 0
+        self.OUTPUT = ""
+        self.might_need_space = False
+        self.might_need_semicolon = False
+        self._last = None
+        self._stack = []
+        self.index_counter = 0
+        self.with_counter = 0
+        self.try_else_counter = 0
+        self.match_counter = 0
+
+    def new_try_else_counter(self):
+        self.try_else_counter += 1
+        return 'ρσ_try_else_' + self.try_else_counter
+
+    def make_name(self, name):
+        name = name.toString()
+        if self.options.ascii_only:
+            name = to_ascii(name, True)
+
+        return name
+
+    def print_name(self, name):
+        self.print(self.make_name(name))
+
+    def make_indent(self, back):
+        return repeat_string(" ", self.options.indent_start + self._indentation - back * self.options.indent_level)
+
+    # -----[ beautification/minification ]-----
+    def last_char(self):
+        return self._last.charAt(self._last.length - 1)
+
+    def maybe_newline(self):
+        if self.options.max_line_len and self.current_col > self.options.max_line_len:
+            self.print("\n")
+
+    def print(self, str_):
+        str_ = v"String(str_)"
+        ch = str_.charAt(0)
+        if self.might_need_semicolon:
+            if (not ch or ";}".indexOf(ch) < 0) and not /[;]$/.test(self._last):
+                if self.options.semicolons or require_semi_colon_chars[ch]:
+                    self.OUTPUT += ";"
+                    self.current_col += 1
+                    self.current_pos += 1
+                else:
+                    self.OUTPUT += "\n"
+                    self.current_pos += 1
+                    self.current_line += 1
+                    self.current_col = 0
+
+                if not self.options.beautify:
+                    self.might_need_space = False
+
+
+            self.might_need_semicolon = False
+            self.maybe_newline()
+
+        if not self.options.beautify and self.options.preserve_line and self._stack[self._stack.length - 1]:
+            target_line = self._stack[self._stack.length - 1].start.line
+            while self.current_line < target_line:
+                self.OUTPUT += "\n"
+                self.current_pos += 1
+                self.current_line += 1
+                self.current_col = 0
+                self.might_need_space = False
+
+
+        if self.might_need_space:
+            prev = self.last_char()
+            if is_identifier_char(prev) and (is_identifier_char(ch) or ch is "\\")
+            or /^[\+\-\/]$/.test(ch) and ch is prev:
+                self.OUTPUT += " "
+                self.current_col += 1
+                self.current_pos += 1
+
+            self.might_need_space = False
+
+        a = str_.split(/\r?\n/)
+        n = a.length - 1
+        self.current_line += n
+        if n is 0:
+            self.current_col += a[n].length
+        else:
+            self.current_col = a[n].length
+
+        self.current_pos += str_.length
+        self._last = str_
+        self.OUTPUT += str_
+
+    def space(self):
+        if self.options.beautify:
+            self.print(' ')
+        else:
+            self.might_need_space = True
+
+    def indent(self, half):
+        if self.options.beautify:
+            self.print(self.make_indent((0.5 if half else 0)))
+
+    def with_indent(self, col, proceed):
+        if self.options.beautify:
+            if col is True:
+                col = self.next_indent()
+
+            save_indentation = self._indentation
+            self._indentation = col
+            ret = proceed()
+            self._indentation = save_indentation
+            return ret
+        else:
+            return proceed()
+
+    def indentation(self):
+        return self._indentation
+
+    def set_indentation(self, val):
+        if self.options.beautify:
+            self._indentation = val
+
+    def newline(self):
+        if self.options.beautify:
+            self.print("\n")
+
+    def semicolon(self):
+        if self.options.beautify:
+            self.print(";")
+        else:
+            self.might_need_semicolon = True
+
+    def force_semicolon(self):
+        self.might_need_semicolon = False
+        self.print(";")
+
+    def next_indent(self):
+        return self._indentation + self.options.indent_level
+
+    def spaced(self):
+        for v'var i=0; i < arguments.length; i++':
+            if i > 0:
+                self.space()
+            if jstype(arguments[i].print) is 'function':
+                arguments[i].print(self)
+            else:
+                self.print(arguments[i])
+
+    def end_statement(self):
+        self.semicolon()
+        self.newline()
+
+    def with_block(self, cont):
+        ret = None
+        self.print("{")
+        self.newline()
+        self.with_indent(self.next_indent(), def():
+            nonlocal ret
+            ret = cont()
+        )
+        self.indent()
+        self.print("}")
+        return ret
+
+    def with_parens(self, cont):
+        self.print("(")
+        ret = cont()
+        self.print(")")
+        return ret
+
+    def with_square(self, cont):
+        self.print("[")
+        ret = cont()
+        self.print("]")
+        return ret
+
+    def comma(self):
+        self.print(",")
+        self.space()
+
+    def colon(self):
+        self.print(":")
+        if self.options.space_colon:
+            self.space()
+
+    def dump_yield(self):
+        self.indent()
+        self.spaced('var', 'ρσ_regenerator', '=', '{}')
+        self.end_statement()
+        code = 'ρσ_regenerator.regeneratorRuntime = ' + regenerate(False, self.options.beautify)
+        if self.options.beautify:
+            code = code.replace(/\/\/.*$/mg, '\n').replace(/^\s*$/gm, '')  # strip comments
+            ci = self.make_indent(0)
+            code = [ci + x for x in code.split('\n')].join('\n')
+        self.print(code + '})(ρσ_regenerator)')
+        self.end_statement()
+
+    def get(self):
+        return self.OUTPUT
+    toString = get
+
+    def assign(self, name):
+        # generates: '[name] = '
+        if jstype(name) is "string":
+            self.print(name)
+        else:
+            name.print(self)
+        self.space()
+        self.print("=")
+        self.space()
+
+    def current_width(self):
+        return self.current_col - self._indentation
+
+    def should_break(self):
+        return self.options.width and self.current_width() >= self.options.width
+
+    def last(self):
+        return self._last
+
+    def print_string(self, str_):
+        self.print(encode_string(str_))
+
+    def line(self):
+        return self.current_line
+
+    def col(self):
+        return self.current_col
+
+    def pos(self):
+        return self.current_pos
+
+    def push_node(self, node):
+        self._stack.push(node)
+
+    def pop_node(self):
+        return self._stack.pop()
+
+    def stack(self):
+        return self._stack
+
+    def parent(self, n):
+        return self._stack[self._stack.length - 2 - (n or 0)]

package/src/output/treeshake.pyj

@@ -0,0 +1,182 @@
+# vim:fileencoding=utf-8
+# License: BSD
+from __python__ import hash_literals
+
+from ast import (
+    AST_Function, AST_Class, AST_SimpleStatement, AST_Assign,
+    AST_SymbolRef, AST_Dot, AST_Sub, AST_Imports, TreeWalker, is_node_type
+)
+from utils import has_prop
+
+
+def get_top_level_name(stmt):
+    if is_node_type(stmt, AST_Function) or is_node_type(stmt, AST_Class):
+        if stmt.name:
+            return stmt.name.name
+        return None
+    if is_node_type(stmt, AST_SimpleStatement):
+        body = stmt.body
+        if is_node_type(body, AST_Assign):
+            lhs = body.left
+            if is_node_type(lhs, AST_SymbolRef):
+                return lhs.name
+    return None
+
+
+def collect_refs_in_node(stmt, top_level_set, refs):
+    def visit_fn(node, descend):
+        if is_node_type(node, AST_SymbolRef):
+            if has_prop(top_level_set, node.name):
+                refs[node.name] = True
+    stmt.walk(TreeWalker(visit_fn))
+
+
+def compute_transitive_closure(body, direct_names, nonlocalvars):
+    nonlocal_set = {}
+    if nonlocalvars:
+        for nv in nonlocalvars:
+            nonlocal_set[nv] = True
+
+    name_map = {}
+    unnamed_stmts = []
+    for stmt in body:
+        name = get_top_level_name(stmt)
+        if name is not None:
+            name_map[name] = stmt
+        else:
+            unnamed_stmts.push(stmt)
+
+    top_level_set = {}
+    for name in Object.keys(name_map):
+        top_level_set[name] = True
+
+    needed = {}
+    queue = []
+
+    # Always include top-level assignments to nonlocal vars — they affect the
+    # global JavaScript scope and must never be filtered out.
+    for name in Object.keys(name_map):
+        if has_prop(nonlocal_set, name):
+            needed[name] = True
+            queue.push(name)
+
+    # Add directly imported names
+    for name in Object.keys(direct_names):
+        if not has_prop(needed, name):
+            needed[name] = True
+            queue.push(name)
+
+    # Unnamed statements (imports, if-blocks, bare expressions) are always
+    # included in output, so their references to named top-level items must
+    # also be included transitively.
+    always_refs = {}
+    for stmt in unnamed_stmts:
+        collect_refs_in_node(stmt, top_level_set, always_refs)
+    for ref_name in Object.keys(always_refs):
+        if not has_prop(needed, ref_name):
+            needed[ref_name] = True
+            queue.push(ref_name)
+
+    while queue.length > 0:
+        current = queue.shift()
+        if not has_prop(name_map, current):
+            continue
+        refs = {}
+        collect_refs_in_node(name_map[current], top_level_set, refs)
+        for ref_name in Object.keys(refs):
+            if not has_prop(needed, ref_name):
+                needed[ref_name] = True
+                queue.push(ref_name)
+
+    return needed
+
+
+def check_module_attr_access(main_body, info, alias_set):
+    def visit_fn(node, descend):
+        if is_node_type(node, AST_Dot):
+            expr = node.expression
+            if is_node_type(expr, AST_SymbolRef) and has_prop(alias_set, expr.name):
+                info.direct_names[node.property] = True
+                return True
+        if is_node_type(node, AST_Sub):
+            expr = node.expression
+            if is_node_type(expr, AST_SymbolRef) and has_prop(alias_set, expr.name):
+                info.can_tree_shake = False
+    for stmt in main_body:
+        stmt.walk(TreeWalker(visit_fn))
+
+
+def analyze_imports(main_body):
+    result = {}
+
+    # First pass: walk the entire AST recursively to collect all from-imports
+    # (including those nested inside functions or other scopes).
+    def visit_from_imports(node, descend):
+        if is_node_type(node, AST_Imports):
+            for imp in node.imports:
+                if imp.argnames:
+                    key = imp.key
+                    if not has_prop(result, key):
+                        result[key] = {'direct_names': {}, 'can_tree_shake': True}
+                    for argname in imp.argnames:
+                        result[key].direct_names[argname.name] = True
+    for stmt in main_body:
+        stmt.walk(TreeWalker(visit_from_imports))
+
+    # Second pass: handle top-level plain imports (import X as Y, import X).
+    # Attribute tracking via check_module_attr_access applies to main_body scope.
+    for stmt in main_body:
+        if not is_node_type(stmt, AST_Imports):
+            continue
+        for imp in stmt.imports:
+            if imp.argnames:
+                continue  # already handled in first pass
+            key = imp.key
+            if not has_prop(result, key):
+                result[key] = {'direct_names': {}, 'can_tree_shake': True}
+            info = result[key]
+            if imp.alias:
+                alias_set = {}
+                alias_set[imp.alias.name] = True
+                check_module_attr_access(main_body, info, alias_set)
+            else:
+                parts = key.split('.')
+                if parts.length > 1:
+                    # import X.Y with no alias: user accesses via X.Y.attr which
+                    # requires two-level dot traversal — disable tree-shaking safely
+                    info.can_tree_shake = False
+                else:
+                    alias_set = {}
+                    alias_set[parts[0]] = True
+                    check_module_attr_access(main_body, info, alias_set)
+
+    return result
+
+
+def tree_shake(ast, context):
+    import_infos = analyze_imports(ast.body)
+    for mod_key in Object.keys(import_infos):
+        info = import_infos[mod_key]
+        if not info.can_tree_shake:
+            continue
+        if not has_prop(ast.imports, mod_key):
+            continue
+        mod = ast.imports[mod_key]
+        # If body is missing (cached module), re-parse to get it
+        if not mod.body and mod.src_code:
+            parsed = context.parse(mod.src_code, {
+                'filename': mod.filename,
+                'module_id': mod_key,
+                'libdir': context.libdir,
+                'import_dirs': context.import_dirs or [],
+                'discard_asserts': context.discard_asserts,
+                'for_linting': True,
+            })
+            mod.body = parsed.body
+            mod.localvars = parsed.localvars
+            if not mod.nonlocalvars:
+                mod.nonlocalvars = parsed.nonlocalvars
+        if not mod.body:
+            continue
+        needed = compute_transitive_closure(mod.body, info.direct_names, mod.nonlocalvars)
+        mod.needed_names = needed

package/src/output/utils.pyj

@@ -0,0 +1,72 @@
+# vim:fileencoding=utf-8
+# License: BSD Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+from __python__ import hash_literals
+
+from ast import AST_BlockStatement, is_node_type
+
+def best_of(a):
+    best = a[0]
+    len_ = best.length
+    for i in range(1, a.length):
+        if a[i].length < len_:
+            best = a[i]
+            len_ = best.length
+    return best
+
+def make_num(num):
+    str_ = num.toString(10)
+    a = [ str_.replace(/^0\./, ".").replace("e+", "e") ]
+    m = None
+
+    if Math.floor(num) is num:
+        if num >= 0:
+            a.push("0x" + num.toString(16).toLowerCase(), # probably pointless
+            "0" + num.toString(8))
+        else:
+            a.push("-0x" + (-num).toString(16).toLowerCase(), # probably pointless
+            "-0" + (-num).toString(8))
+
+        if m = /^(.*?)(0+)$/.exec(num):
+            a.push(m[1] + "e" + m[2].length)
+
+    elif m = /^0?\.(0+)(.*)$/.exec(num):
+        a.push(m[2] + "e-" + (m[1].length + m[2].length), str_.substr(str_.indexOf(".")))
+
+    return best_of(a)
+
+def make_block(stmt, output):
+    if is_node_type(stmt, AST_BlockStatement):
+        stmt.print(output)
+        return
+
+    output.with_block(def():
+        output.indent()
+        stmt.print(output)
+        output.newline()
+    )
+
+def create_doctring(docstrings):
+    ans = v'[]'
+    for ds in docstrings:
+        ds = str.rstrip(ds.value)
+        lines = v'[]'
+        min_leading_whitespace = ''
+        for line in ds.split(/$/gm):
+            r = /^\s+/.exec(line)
+            leading_whitespace = ''
+            if r:
+                leading_whitespace = r[0].replace(/[\n\r]/g, '') if r else ''
+                line = line[r[0].length:]
+            if not str.strip(line):
+                lines.push(v'["", ""]')
+            else:
+                leading_whitespace = leading_whitespace.replace(/\t/g, '    ')
+                if leading_whitespace and (not min_leading_whitespace or leading_whitespace.length < min_leading_whitespace.length):
+                    min_leading_whitespace = leading_whitespace
+                lines.push(v'[leading_whitespace, line]')
+        for lw, l in lines:
+            if min_leading_whitespace:
+                lw = lw[min_leading_whitespace.length:]
+            ans.push(lw + l)
+        ans.push('')
+    return str.rstrip(ans.join('\n'))