rapydscript-ng 0.7.24 → 0.8.0

package/src/tokenizer.pyj

@@ -81,16 +81,16 @@ PUNC_BEFORE_EXPRESSION = make_predicate(characters("[{(,.;:"))
 
 PUNC_CHARS = make_predicate(characters("[]{}(),;:?"))
 
-KEYWORDS = "as assert break class continue def del do elif else except finally for from global if import in is new nonlocal pass raise return yield try while with or and not"
+KEYWORDS = "as assert async await break class continue def del do elif else except finally for from global if import in is new nonlocal pass raise return yield try while with or and not"
 
 KEYWORDS_ATOM = "False None True"
 
 # see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar
 RESERVED_WORDS = ("break case class catch const continue debugger default delete do else export extends"
 " finally for function if import in instanceof new return super switch this throw try typeof var void"
-" while with yield enum implements static private package let public protected interface await null true false" )
+" while with yield enum implements static private package let public protected interface null true false" )
 
-KEYWORDS_BEFORE_EXPRESSION = "return yield new del raise elif else if"
+KEYWORDS_BEFORE_EXPRESSION = "return yield new del raise elif else if await"
 
 ALL_KEYWORDS = KEYWORDS + " " + KEYWORDS_ATOM
 
@@ -153,10 +153,12 @@ def is_token(token, type, val):
 
 EX_EOF = {}
 
-def tokenizer(raw_text, filename):
+def tokenizer(raw_text, filename, recover_errors):
     S = {
         'text': raw_text.replace(/\r\n?|[\n\u2028\u2029]/g, "\n").replace(/\uFEFF/g, ""),
         'filename': filename,
+        'recover_errors': not not recover_errors,  # LSP error-recovery mode (see parse())
+        'recovered_errors': v'[]',  # collected errors when recover_errors is on
         'pos': 0,
         'tokpos': 0,
         'line': 1,
@@ -703,6 +705,16 @@ def tokenizer(raw_text, filename):
                 tok.type = stok.type
             return tok
 
+        if S.recover_errors:
+            # In error-recovery mode (used by the LSP) an unrecognized character
+            # must not raise, because re-tokenizing from the same position would
+            # loop forever (parse_error is raised before the character is
+            # consumed). Record it and skip the single offending character so the
+            # tokenizer always makes forward progress.
+            S.recovered_errors.push({'message': "Unexpected character «" + ch + "»",
+                                     'line': S.tokline, 'col': S.tokcol, 'pos': S.tokpos, 'is_eof': False})
+            next()
+            return next_token()
         parse_error("Unexpected character «" + ch + "»")
 
     next_token.context = def(nc):

package/src/utils.pyj

@@ -188,5 +188,5 @@ def make_predicate(words):
 def cache_file_name(src, cache_dir):
     if cache_dir:
         src = str.replace(src, '\\', '/')
-        return cache_dir + '/' + str.lstrip(str.replace(src, '/', '-') + '.json', '-')
+        return cache_dir + '/' + str.lstrip(str.replace(src, '/', '-') + '.bin', '-')
     return src + '-cached'

package/test/_treeshake_mod.pyj

@@ -0,0 +1,30 @@
+# Helper module for tree shaking tests
+
+def func_used():
+    return 'used'
+
+def func_unused():
+    return 'unused'
+
+class ClassUsed:
+    def __init__(self):
+        self.val = 'used_class'
+
+class ClassUnused:
+    def __init__(self):
+        self.val = 'unused_class'
+
+module_var = 'module_var_value'
+
+def helper_a():
+    return 'a'
+
+def helper_b():
+    return helper_a()
+
+@no_prune
+def pinned_in_mod():
+    return 'pinned'
+
+def not_pinned_in_mod():
+    return 'not_pinned'

package/test/_treeshake_side_effect.pyj

@@ -0,0 +1,9 @@
+# vim:fileencoding=utf-8
+# Helper module for tree shaking side-effect import tests.
+# Mirrors the calibre initialize.pyj pattern: a module-level function call
+# that patches a global object (here String.prototype) as a side effect.
+
+def apply_patch():
+    v'String.prototype.__treeshake_test = function() { return "side_effect_ran"; }'
+
+apply_patch()

package/test/ast_serialization.pyj

@@ -0,0 +1,392 @@
+# vim:fileencoding=utf-8
+# Test AST serialization/deserialization round-trip.
+# Verifies that ast_to_json → ast_from_json produces an AST that generates
+# identical JavaScript output to the original parsed AST.
+# globals: assrt, RapydScript, fs, compiler_dir
+
+# ─── Helpers ──────────────────────────────────────────────────────────────────
+
+baselib = fs.readFileSync(compiler_dir + '/baselib-plain-pretty.js', {'encoding': 'utf-8'})
+
+def make_output():
+    return new RapydScript.OutputStream({
+        'baselib_plain': baselib,
+        'beautify': True,
+        'js_version': 6,
+        'write_name': False,
+        'private_scope': False,
+        'keep_docstrings': False,
+    })
+
+def ast_to_js(ast):
+    out = make_output()
+    ast.print(out)
+    return out.toString()
+
+async def round_trip(src, label):
+    """Parse src, generate js1, serialize+deserialize AST, generate js2, assert equal."""
+    ast1 = await RapydScript.parse(src, {'filename': label})
+    js1 = ast_to_js(ast1)
+
+    serialized = RapydScript.ast_to_json(ast1)
+    assrt.ok(serialized and jstype(serialized) is 'object', label + ': ast_to_json returned an object')
+
+    ast2 = RapydScript.ast_from_json(serialized)
+    js2 = ast_to_js(ast2)
+
+    assrt.equal(js2, js1, label + ': round-trip JS output mismatch')
+
+async def assert_json_valid(src, label):
+    """Verify ast_to_json produces a well-formed AST object."""
+    ast = await RapydScript.parse(src, {'filename': label})
+    data = RapydScript.ast_to_json(ast)
+    assrt.equal(data.v, 1, label + ': version field')
+    assrt.ok(Array.isArray(data.nodes), label + ': nodes is array')
+    assrt.equal(data.root, 0, label + ': root is 0')
+    assrt.ok(data.nodes.length > 0, label + ': pool is non-empty')
+
+async def run_tests():
+    # ─── Test 1: Basic literals and atoms ─────────────────────────────────────────
+    # Exercises: AST_Number, AST_String, AST_True, AST_False, AST_Null,
+    #            AST_NaN, AST_Infinity, AST_Undefined, AST_SimpleStatement, AST_Assign
+
+    await round_trip("""
+a = None
+b = True
+c = False
+d = 42
+e = 3.14
+f = "hello world"
+g = ''
+""", 'literals')
+
+    # ─── Test 2: RegExp literal (special serialization) ───────────────────────────
+    # Exercises: AST_RegExp with source and flags
+
+    await round_trip("""
+r1 = /[a-z]+/
+r2 = /[A-Z]+/gi
+r3 = /^\\d{3}-\\d{4}$/m
+""", 'regexp')
+
+    # ─── Test 3: Operators ─────────────────────────────────────────────────────────
+    # Exercises: AST_Binary, AST_UnaryPrefix, AST_Assign, AST_Conditional
+
+    await round_trip("""
+x = 1 + 2 * 3 - 4 / 5
+y = x ** 2
+z = x // 3
+a = True and False or not True
+b = 1 if a else 2
+c = x > 0 and x < 10
+""", 'operators')
+
+    # ─── Test 4: Collections ──────────────────────────────────────────────────────
+    # Exercises: AST_Array, AST_Object, AST_ObjectKeyVal, AST_Set, AST_SetItem
+
+    await round_trip("""
+arr = [1, 2, 3, "four"]
+obj = {'a': 1, 'b': True, 'c': None}
+st = {1, 2, 3}
+nested = [[1, 2], [3, 4]]
+""", 'collections')
+
+    # ─── Test 5: Function definitions ─────────────────────────────────────────────
+    # Exercises: AST_Function, AST_ArgsDef (simple + defaults + *args + **kwargs),
+    #            AST_Return, AST_SymbolFunarg, AST_SymbolDefun, AST_Decorator
+
+    await round_trip("""
+def simple(x):
+    return x
+
+def with_defaults(a, b=10, c="hi"):
+    return a + b
+
+def with_star(a, *args, **kwargs):
+    return args
+
+def annotated(x, y=0):
+    pass
+
+def no_args():
+    return None
+
+def decorator_fn(f):
+    return f
+
+@decorator_fn
+def decorated(x):
+    return x * 2
+""", 'functions')
+
+    # ─── Test 6: Classes ──────────────────────────────────────────────────────────
+    # Exercises: AST_Class, AST_Method (static/getter/setter), AST_SymbolDeclaration,
+    #            AST_This, dynamic_properties, classvars
+
+    await round_trip("""
+class Animal:
+    count = 0
+
+    def __init__(self, name):
+        self.name = name
+        Animal.count += 1
+
+    def speak(self):
+        return "..."
+
+    @staticmethod
+    def create(name):
+        return Animal(name)
+
+class Dog(Animal):
+    def __init__(self, name, breed):
+        self.breed = breed
+
+    def speak(self):
+        return "Woof"
+
+class Cat(Animal):
+    pass
+""", 'classes')
+
+    # ─── Test 7: Control flow ─────────────────────────────────────────────────────
+    # Exercises: AST_If, AST_While, AST_ForIn, AST_Break, AST_Continue,
+    #            AST_EmptyStatement, AST_Do
+
+    await round_trip("""
+x = 5
+
+if x > 10:
+    x = 10
+elif x < 0:
+    x = 0
+else:
+    pass
+
+i = 0
+while i < 5:
+    if i == 3:
+        break
+    i += 1
+
+for item in [1, 2, 3]:
+    if item == 2:
+        continue
+    x += item
+""", 'control_flow')
+
+    # ─── Test 8: Try/except ───────────────────────────────────────────────────────
+    # Exercises: AST_Try, AST_Catch, AST_Except, AST_Finally, AST_Else,
+    #            AST_Throw, AST_SymbolCatch
+
+    await round_trip("""
+def risky(x):
+    try:
+        if x < 0:
+            raise ValueError("negative")
+        return x
+    except ValueError as e:
+        return -1
+    except TypeError as e:
+        return -2
+    else:
+        pass
+    finally:
+        pass
+""", 'try_except')
+
+    # ─── Test 9: Comprehensions ───────────────────────────────────────────────────
+    # Exercises: AST_ListComprehension, AST_DictComprehension, AST_SetComprehension,
+    #            AST_GeneratorComprehension
+
+    await round_trip("""
+nums = [1, 2, 3, 4, 5]
+squares = [x**2 for x in nums if x > 1]
+even_sq = {x: x**2 for x in nums if x % 2 == 0}
+sq_set = {x**2 for x in nums}
+""", 'comprehensions')
+
+    # ─── Test 10: Property access and calls ───────────────────────────────────────
+    # Exercises: AST_Dot, AST_Sub, AST_Call, AST_CallArgs, AST_New,
+    #            AST_SymbolRef
+
+    await round_trip("""
+obj = {'a': {'b': [1, 2, 3]}}
+v = obj['a']['b'][0]
+s = "hello".upper()
+n = len([1, 2, 3])
+""", 'access_and_calls')
+
+    # ─── Test 11: Assert ──────────────────────────────────────────────────────────
+    # Exercises: AST_Assert
+
+    await round_trip("""
+x = 42
+assert x > 0
+assert x < 100, "x must be less than 100"
+""", 'assert')
+
+    # ─── Test 12: Generator function ──────────────────────────────────────────────
+    # Exercises: AST_Yield (is_yield_from=False and True)
+
+    await round_trip("""
+def counter(n):
+    for i in range(n):
+        yield i
+
+def chained(n):
+    yield from counter(n)
+    yield 99
+""", 'generators')
+
+    # ─── Test 13: Verbatim JS ─────────────────────────────────────────────────────
+    # Exercises: AST_Verbatim
+
+    await round_trip("""
+raw = v'typeof window !== "undefined"'
+""", 'verbatim')
+
+    # ─── Test 14: With statement ──────────────────────────────────────────────────
+    # Exercises: AST_With, AST_WithClause, AST_SymbolAlias
+
+    await round_trip("""
+class CM:
+    def __enter__(self):
+        return self
+    def __exit__(self, t, v, tb):
+        pass
+
+with CM() as ctx:
+    pass
+""", 'with_statement')
+
+    # ─── Test 15: Nested scopes / closures ────────────────────────────────────────
+    # Exercises: nested AST_Function, nonlocal
+
+    await round_trip("""
+def outer(x):
+    y = x * 2
+    def inner(z):
+        nonlocal y
+        y += z
+        return y
+    return inner
+""", 'closures')
+
+    # ─── Test 16: JSON format sanity checks ───────────────────────────────────────
+    # Verify the serialized JSON structure is well-formed
+
+    await assert_json_valid("x = 1 + 2", 'json_format_simple')
+    await assert_json_valid("r = /[a-z]+/gi", 'json_format_regexp')
+    await assert_json_valid("""
+def f(a, b=1):
+    return a + b
+""", 'json_format_function')
+
+    # ─── Test 17: RegExp round-trip value check ───────────────────────────────────
+    # Verify the deserialized RegExp has the correct source and flags
+
+    src17 = "r = /^hello\\sworld$/im"
+    ast17 = await RapydScript.parse(src17, {'filename': 'regexp_check'})
+    serialized17 = RapydScript.ast_to_json(ast17)
+    ast17b = RapydScript.ast_from_json(serialized17)
+
+    # Navigate to the AST_RegExp node: Toplevel → SimpleStatement → Assign → right
+    re_node = ast17b.body[0].body.right
+    assrt.ok(re_node.constructor.name is 'AST_RegExp', 'deserialized node is AST_RegExp')
+    assrt.ok(v'Object.prototype.toString.call(re_node.value) === "[object RegExp]"', 'value is a RegExp object')
+    assrt.equal(re_node.value.source, '^hello\\sworld$', 'regexp source matches')
+    assrt.ok(re_node.value.ignoreCase, 'regexp ignoreCase flag set')
+    assrt.ok(re_node.value.multiline, 'regexp multiline flag set')
+
+    # ─── Test 18: Shared reference integrity ──────────────────────────────────────
+    # A node appearing in multiple places (e.g. AST_Class.init also in body)
+    # should deserialize to the SAME JS object (same reference).
+
+    src18 = """
+class Foo:
+    def __init__(self):
+        self.x = 1
+    def bar(self):
+        return self.x
+"""
+    ast18 = await RapydScript.parse(src18, {'filename': 'shared_ref'})
+    data18 = RapydScript.ast_to_json(ast18)
+
+    # Check that the JSON pool has no duplicate entries for the same node:
+    # Every index should appear at most once as a _n value in the pool.
+    # (Shared refs should point to the same pool index, not copy the node.)
+    index_uses = {}
+    def count_refs(val):
+        if Array.isArray(val):
+            for el in val:
+                count_refs(el)
+        elif val and jstype(val) is 'object':
+            if Object.keys(val).length is 1 and val._n is not None and jstype(val._n) is 'number':
+                index_uses[val._n] = (index_uses[val._n] or 0) + 1
+            else:
+                for k in Object.keys(val):
+                    count_refs(val[k])
+
+    for node_entry in data18.nodes:
+        if node_entry:
+            count_refs(node_entry.p)
+
+    # init in AST_Class.p should reference the same index as the __init__ method in body.
+    # We just verify that the deserialized AST still round-trips correctly.
+    ast18b = RapydScript.ast_from_json(data18)
+    assrt.equal(ast_to_js(ast18b), ast_to_js(ast18), 'shared ref round-trip')
+
+    # ─── Test 19: Splat args (*arg) round-trip ────────────────────────────────────
+    # Exercises: is_array flag on call-site *arg nodes.
+    # Without fix: is_array is lost in serialization, causing func(*arg) to
+    # generate func.apply(self, [arg]) instead of func.apply(self, arg).
+
+    await round_trip("""
+def call_it(f, args, a, b):
+    f(*args)
+    f(a, *args)
+    f(a, b, *args)
+""", 'splat_args')
+
+    # ─── Test 20: Large/complex combined source ───────────────────────────────────
+    # Full round-trip of a more complex program
+
+    await round_trip("""
+def fib(n):
+    if n <= 1:
+        return n
+    return fib(n - 1) + fib(n - 2)
+
+class Stack:
+    def __init__(self):
+        self._items = []
+
+    def push(self, item):
+        self._items.append(item)
+
+    def pop(self):
+        if len(self._items) == 0:
+            raise IndexError("pop from empty stack")
+        return self._items.pop()
+
+    @property
+    def size(self):
+        return len(self._items)
+
+def process(items, transform=None, filter_fn=None):
+    result = []
+    for item in items:
+        if filter_fn and not filter_fn(item):
+            continue
+        if transform:
+            item = transform(item)
+        result.append(item)
+    return result
+
+numbers = [fib(i) for i in range(10)]
+evens = [x for x in numbers if x % 2 == 0]
+mapping = {x: x**2 for x in evens}
+""", 'complex_program')
+
+__test_async_done__ = run_tests()

package/test/async.pyj

@@ -0,0 +1,95 @@
+# Tests for async/await support
+
+async def async_identity(x):
+    return x
+
+async def async_add(x, y):
+    a = await async_identity(x)
+    b = await async_identity(y)
+    return a + b
+
+async def async_raises():
+    raise TypeError("async error")
+
+async def async_chain(n):
+    if n <= 0:
+        return 0
+    prev = await async_chain(n - 1)
+    return prev + n
+
+class AsyncCounter:
+    def __init__(self):
+        self.count = 0
+
+    async def increment(self, amount):
+        val = await async_identity(amount)
+        self.count += val
+        return self.count
+
+    async def reset(self):
+        self.count = 0
+        return self.count
+
+    async def get(self):
+        return await async_identity(self.count)
+
+async def run_tests():
+    # Basic async function returning a value
+    result = await async_identity(42)
+    assrt.equal(result, 42, "basic await identity")
+
+    # Multiple awaits in sequence
+    result = await async_add(3, 4)
+    assrt.equal(result, 7, "sequential awaits")
+
+    # Calling an async function without await returns a Promise
+    p = async_identity(10)
+    assrt.ok(v"p instanceof Promise", "async fn returns Promise")
+    resolved = await p
+    assrt.equal(resolved, 10, "awaiting stored Promise")
+
+    # Recursive async functions
+    result = await async_chain(5)
+    assrt.equal(result, 15, "recursive async (1+2+3+4+5)")
+
+    # Async class methods
+    counter = AsyncCounter()
+    result = await counter.increment(5)
+    assrt.equal(result, 5, "async method first increment")
+    result = await counter.increment(3)
+    assrt.equal(result, 8, "async method second increment")
+    result = await counter.get()
+    assrt.equal(result, 8, "async getter method")
+    result = await counter.reset()
+    assrt.equal(result, 0, "async reset method")
+
+    # Exception propagation through async boundaries
+    caught = False
+    try:
+        await async_raises()
+    except TypeError as e:
+        caught = True
+        assrt.ok(str(e).indexOf("async error") >= 0, "exception message preserved")
+    assrt.ok(caught, "exception propagated through await")
+
+    # async def as an expression (assigned to variable)
+    fn = async def(x): return x * 2
+    result = await fn(7)
+    assrt.equal(result, 14, "async lambda expression")
+
+    # Await in a nested async function
+    async def outer(x):
+        async def inner(y):
+            return await async_identity(y + 1)
+        return await inner(x)
+    result = await outer(9)
+    assrt.equal(result, 10, "nested async functions")
+
+    # Promise.all equivalent via verbatim JS
+    p1 = async_identity(100)
+    p2 = async_identity(200)
+    results = await v"Promise.all([p1, p2])"
+    assrt.equal(results[0], 100, "Promise.all first")
+    assrt.equal(results[1], 200, "Promise.all second")
+
+__test_async_done__ = run_tests()

package/test/baselib.pyj

@@ -13,7 +13,7 @@ class CustomIterable:
 t = []
 q = [1,2,3]
 for x in CustomIterable(q):
-    t.push(x)
+    t.append(x)
 assrt.deepEqual(q, t)
 
 assrt.deepEqual(['a', 'b'], list('ab'))
@@ -137,7 +137,6 @@ a.clear()
 assrt.equal(a.length, 0)
 assrt.deepEqual(a.as_array(), a)
 assrt.ok(a is not a.as_array())
-assrt.ok(a.as_array().extend == undefined)
 a = [1, 2, 1]
 assrt.equal(a.count(1), 2)
 a = [3, 2, 4, 1]

package/test/embedded_compiler.pyj

@@ -0,0 +1,57 @@
+# globals: assrt, rs_create_embedded_compiler, fs
+
+async def run_tests():
+    # Test that create_embedded_compiler() works with no arguments
+    compiler = await rs_create_embedded_compiler()
+
+    # compile must be callable and return non-empty JS
+    js = await compiler.compile('x = 1 + 2\n')
+    assrt.ok(js and js.length > 0, 'compile must return non-empty JS')
+
+    # Test compiling a class definition
+    js2 = await compiler.compile('class Foo:\n    def __init__(self, v):\n        self.v = v\n')
+    assrt.ok(js2 and js2.length > 0, 'compile must handle class definitions')
+
+    # Test that successive compile calls accumulate class knowledge (streaming mode)
+    compiler2 = await rs_create_embedded_compiler()
+    await compiler2.compile('class Bar:\n    def __init__(self, v):\n        self.v = v\n')
+    js3 = await compiler2.compile('b = Bar(42)\n')
+    assrt.ok(js3.indexOf('Bar') >= 0, 'subsequent compile should know about previously defined class Bar')
+
+    # Test virtual_file_system: compile code that imports from a virtual module
+    vfs_modules = {
+        '__vfs__/testmod.pyj': 'def add_one(x):\n    return x + 1\n'
+    }
+    def vfs_read(path, enc):
+        if vfs_modules[path]:
+            return Promise.resolve(vfs_modules[path])
+        return fs.promises.readFile(path, enc)
+
+    vfs_compiler = await rs_create_embedded_compiler({
+        'virtual_file_system': {'read_file': vfs_read}
+    })
+    js4 = await vfs_compiler.compile('from testmod import add_one\nresult = add_one(5)\n')
+    assrt.ok(js4 and js4.length > 0, 'compile with vfs import must return JS')
+    assrt.ok(js4.indexOf('add_one') >= 0, 'compiled JS must reference the imported function')
+
+    # Test that stdlib imports are served from the bundled stdlib, not from VFS.
+    # The strict VFS below throws ENOENT for any path it does not own, so if the
+    # compiler were to route a stdlib read through VFS the import would fail.
+    strict_vfs_modules = {
+        '__vfs__/testmod.pyj': 'def add_one(x):\n    return x + 1\n'
+    }
+    def strict_vfs_read(path, enc):
+        if strict_vfs_modules[path]:
+            return Promise.resolve(strict_vfs_modules[path])
+        err = new Error('ENOENT: ' + path)
+        err.code = 'ENOENT'
+        return Promise.reject(err)
+
+    strict_vfs_compiler = await rs_create_embedded_compiler({
+        'virtual_file_system': {'read_file': strict_vfs_read}
+    })
+    js5 = await strict_vfs_compiler.compile('from math import sqrt\nresult = sqrt(4)\n')
+    assrt.ok(js5 and js5.length > 0, 'stdlib import must succeed when VFS is provided (served from bundled cache)')
+    assrt.ok(js5.indexOf('sqrt') >= 0, 'compiled JS must reference the stdlib sqrt function')
+
+__test_async_done__ = run_tests()