rapydscript-ns 0.9.2 → 0.9.4

package/src/parse.pyj

@@ -6,7 +6,7 @@ from __python__ import hash_literals
 from utils import make_predicate, array_to_hash, defaults, has_prop, cache_file_name
 from errors import SyntaxError, ImportError
 from ast import (
-AST_Array, AST_Assign, AST_Binary, AST_BlockStatement, AST_Break,
+AST_Array, AST_Assign, AST_BigInt, AST_Binary, AST_BlockStatement, AST_Break,
 AST_Call, AST_Catch, AST_Class, AST_ClassCall, AST_Conditional,
 AST_Constant, AST_Continue, AST_DWLoop, AST_Debugger, AST_Decorator,
 AST_Definitions, AST_DictComprehension, AST_Directive, AST_Do, AST_Dot,
@@ -31,7 +31,7 @@ TreeWalker
 from tokenizer import tokenizer, is_token, RESERVED_WORDS
 
 COMPILER_VERSION = '__COMPILER_VERSION__'
-PYTHON_FLAGS = {'dict_literals':True, 'overload_getitem':True, 'bound_methods':True, 'hash_literals':True, 'overload_operators':True, 'truthiness':True, 'jsx':True, 'strict_arithmetic':True}
+PYTHON_FLAGS = {'dict_literals':True, 'overload_getitem':True, 'bound_methods':True, 'hash_literals':True, 'overload_operators':True, 'truthiness':True, 'jsx':True, 'strict_arithmetic':True, 'python_modulo':True, 'type_enforcement':True}
 
 
 def get_compiler_version():
@@ -144,7 +144,7 @@ PRECEDENCE = (def(a, ret):
 
 STATEMENTS_WITH_LABELS = array_to_hash([ "for", "do", "while", "switch" ])
 
-ATOMIC_START_TOKEN = array_to_hash([ "atom", "num", "imaginary", "string", "bytes_literal", "regexp", "name", "js" ])
+ATOMIC_START_TOKEN = array_to_hash([ "atom", "num", "bigint", "imaginary", "string", "bytes_literal", "regexp", "name", "js" ])
 
 compile_time_decorators = ['staticmethod', 'classmethod', 'external', 'property']
 
@@ -520,7 +520,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             return new AST_Directive({
                 'value': tmp_
             })
-        elif tmp_ is "num" or tmp_ is "regexp" or tmp_ is "operator" or tmp_ is "atom" or tmp_ is "js":
+        elif tmp_ is "num" or tmp_ is "bigint" or tmp_ is "regexp" or tmp_ is "operator" or tmp_ is "atom" or tmp_ is "js":
             return simple_statement()
         elif tmp_ is "punc":
             tmp_ = S.token.value
@@ -620,8 +620,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     })
             elif tmp_ is "async":
                 start = prev()
+                if is_("keyword", "for"):
+                    # async for: consume an async iterable using JS `for await ... of`
+                    next()
+                    forstmt = for_()
+                    forstmt.is_async = True
+                    forstmt.start = start
+                    return forstmt
                 if not is_("keyword", "def"):
-                    croak("Expected 'def' after 'async'")
+                    croak("Expected 'def' or 'for' after 'async'")
                 next()
                 func = function_(S.in_class[-1], False, True)
                 func.start = start
@@ -708,21 +715,70 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
     def with_():
         clauses = v'[]'
         start = S.token
-        while True:
-            if is_('eof'):
-                unexpected()
-            expr = expression()
-            alias = None
-            if is_('keyword', 'as'):
-                next()
-                alias = as_symbol(AST_SymbolAlias)
-            clauses.push(new AST_WithClause({'expression':expr, 'alias':alias}))
-            if is_('punc', ','):
-                next()
-                continue
-            if not is_('punc', ':'):
-                unexpected()
-            break
+
+        # Detect Python 3.10+ parenthesized form: with (cm1 as a, cm2 as b):
+        # vs traditional: with cm as a:  or  with (expr) as a:
+        # Lookahead: scan tokens inside the `(` for an `as` keyword at depth 0
+        # before the matching `)`. If found, it's the parenthesized form.
+        is_parenthesized_with = False
+        if is_('punc', '('):
+            depth = 0
+            i = 0
+            while True:
+                while S.peeked.length <= i:
+                    S.peeked.push(S.input())
+                tok = S.peeked[i]
+                if tok.type is 'eof':
+                    break
+                if tok.type is 'punc':
+                    if tok.value is '(' or tok.value is '[' or tok.value is '{':
+                        depth += 1
+                    elif tok.value is ')' or tok.value is ']' or tok.value is '}':
+                        if depth is 0:
+                            break
+                        depth -= 1
+                elif tok.type is 'keyword' and tok.value is 'as' and depth is 0:
+                    is_parenthesized_with = True
+                    break
+                i += 1
+
+        if is_parenthesized_with:
+            next()  # consume `(`
+            while True:
+                if is_('eof'):
+                    unexpected()
+                if is_('punc', ')'):
+                    next()  # trailing comma before closing `)`
+                    break
+                expr = expression()
+                alias = None
+                if is_('keyword', 'as'):
+                    next()
+                    alias = as_symbol(AST_SymbolAlias)
+                clauses.push(new AST_WithClause({'expression': expr, 'alias': alias}))
+                if is_('punc', ','):
+                    next()
+                    continue
+                if not is_('punc', ')'):
+                    unexpected()
+                next()  # consume `)`
+                break
+        else:
+            while True:
+                if is_('eof'):
+                    unexpected()
+                expr = expression()
+                alias = None
+                if is_('keyword', 'as'):
+                    next()
+                    alias = as_symbol(AST_SymbolAlias)
+                clauses.push(new AST_WithClause({'expression':expr, 'alias':alias}))
+                if is_('punc', ','):
+                    next()
+                    continue
+                if not is_('punc', ':'):
+                    unexpected()
+                break
 
         if not clauses.length:
             token_error(start, 'with statement must have at least one clause')
@@ -741,6 +797,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             return new AST_String({'start': tok, 'value': tok.value, 'end': tok})
         elif tok.type is 'num':
             return new AST_Number({'start': tok, 'value': tok.value, 'end': tok})
+        elif tok.type is 'bigint':
+            return new AST_BigInt({'start': tok, 'value': tok.value, 'end': tok})
         elif tok.type is 'atom':
             if tok.value is 'True':
                 return new AST_True({'start': tok, 'end': tok})
@@ -754,15 +812,18 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         """Parse a single atomic/closed pattern (no OR, no AS)."""
         start = S.token
 
-        # Negative number literal: -42
-        if is_('operator', '-') and peek().type is 'num':
+        # Negative number literal: -42 or -42n
+        if is_('operator', '-') and (peek().type is 'num' or peek().type is 'bigint'):
             next()  # consume '-'
-            val = -S.token.value
             tok = S.token
+            if tok.type is 'bigint':
+                val_node = new AST_BigInt({'start': start, 'value': '-' + tok.value, 'end': tok})
+            else:
+                val_node = new AST_Number({'start': start, 'value': -tok.value, 'end': tok})
             next()
             return new AST_MatchLiteral({
                 'start': start,
-                'value': new AST_Number({'start': start, 'value': val, 'end': tok}),
+                'value': val_node,
                 'end': prev()
             })
 
@@ -776,6 +837,16 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'end': prev()
             })
 
+        # BigInt literal
+        if is_('bigint'):
+            tok = S.token
+            next()
+            return new AST_MatchLiteral({
+                'start': tok,
+                'value': new AST_BigInt({'start': tok, 'value': tok.value, 'end': tok}),
+                'end': prev()
+            })
+
         # String literal
         if is_('string'):
             tok = S.token
@@ -1396,6 +1467,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     argnames.push(aname)
                     if is_('punc', ','):
                         next()
+                        if bracketed and is_('punc', ')'):
+                            next()
+                            break
                     else:
                         if bracketed:
                             if is_('punc', ')'):
@@ -1824,6 +1898,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             )(S.in_loop, S.labels)
         })
         definition.return_annotation = return_annotation
+        definition.type_enforce = S.scoped_flags.get('type_enforcement', False)
         definition.is_generator = is_generator[0]
         definition.is_async = is_async
         if is_node_type(definition, AST_Method):
@@ -2185,6 +2260,12 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'end': tok,
                 'value': tok.value
             })
+        elif tmp_ is "bigint":
+            return new AST_BigInt({
+                'start': tok,
+                'end': tok,
+                'value': tok.value
+            })
         elif tmp_ is "imaginary":
             return new AST_Call({
                 'start': tok,
@@ -3242,6 +3323,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'end': right.end,
                 'overloaded': S.scoped_flags.get('overload_operators', False),
                 'strict_arith': S.scoped_flags.get('overload_operators', False) and S.scoped_flags.get('strict_arithmetic', True),
+                'python_mod': (op is '%') and S.scoped_flags.get('python_modulo', True),
                 'python_truthiness': S.scoped_flags.get('truthiness', False) and (op is '&&' or op is '||')
             })
             return expr_op(ret, min_prec, no_in)
@@ -3324,6 +3406,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if S.scoped_flags.get('overload_operators', False) and val is not '=':
                 asgn.overloaded = True
                 asgn.strict_arith = S.scoped_flags.get('strict_arithmetic', True)
+            if val is '%=':
+                asgn.python_mod = S.scoped_flags.get('python_modulo', True)
             return asgn
         return left
 

package/src/tokenizer.pyj

@@ -334,6 +334,19 @@ def tokenizer(raw_text, filename):
             valid = parseInt(num, 2)
             if isNaN(valid):
                 parse_error('Invalid syntax for a binary number')
+            if peek() is 'n':
+                next()
+                return token('bigint', '0b' + num)
+            return token('num', valid)
+        if not prefix and peek() is '0' and S.text.charAt(S.pos + 1) is 'o':
+            next(), next()
+            num = read_while(def(ch): return ch >= '0' and ch <= '7';)
+            valid = parseInt(num, 8)
+            if isNaN(valid):
+                parse_error('Invalid syntax for an octal number')
+            if peek() is 'n':
+                next()
+                return token('bigint', '0o' + num)
             return token('num', valid)
         seen = v'[]'
         num = read_while(def(ch, i):
@@ -363,8 +376,8 @@ def tokenizer(raw_text, filename):
                 return False
             elif ch is '.':
                 return (has_dot = True) if not has_dot and not has_x and not has_e else False
-            elif ch is 'j' or ch is 'J':
-                return False  # imaginary suffix — stop here; handled after loop
+            elif ch is 'j' or ch is 'J' or ch is 'n':
+                return False  # imaginary/bigint suffix — stop here; handled after loop
             return is_alphanumeric_char(ch.charCodeAt(0))
         )
         if prefix:
@@ -375,6 +388,9 @@ def tokenizer(raw_text, filename):
             if peek() is 'j' or peek() is 'J':
                 next()
                 return token("imaginary", valid)
+            if peek() is 'n':
+                next()
+                return token("bigint", num)
             return token("num", valid)
         else:
             parse_error("Invalid syntax: " + num)
@@ -706,7 +722,7 @@ def tokenizer(raw_text, filename):
             if (peek() is "'" or peek() is '"') and is_string_modifier(tok.value):
                 mods = tok.value.toLowerCase()
                 start_pos_for_string = S.tokpos
-                stok = read_string(mods.indexOf('r') is not -1, mods.indexOf('v') is not -1)
+                stok = read_string(mods.indexOf('r') is not -1 or mods.indexOf('v') is not -1, mods.indexOf('v') is not -1)
                 tok.endpos = stok.endpos
                 if stok.type is not 'js' and mods.indexOf('f') is not -1:
                     tok.col += start_pos_for_string - tok.pos

package/test/async_generators.pyj

@@ -0,0 +1,144 @@
+# globals: assrt
+# vim:fileencoding=utf-8
+#
+# async_generators.pyj
+# Tests for `async def` functions with `yield` (async generators).
+#
+# An async generator is a coroutine that produces values via `yield`. Calling
+# it returns an async iterator whose `.next()` / `.asend()` return Promises.
+#
+# `vm.runInNewContext` (the test runner) is synchronous and does not drain the
+# microtask queue, so these tests assert *synchronously observable* properties:
+# the iterator shape, that .next() is thenable, that the compiled code runs
+# without throwing, and that single-microtask resolution works via
+# Promise.then chains observed through a shared mutable list. End-to-end
+# async-for execution is covered by the web-repl bundle tests.
+
+ae  = assrt.equal
+ade = assrt.deepEqual
+ok  = assrt.ok
+
+# ── 1. Calling an async generator returns an iterator with the right shape ───
+
+async def basic_gen():
+    yield 1
+    yield 2
+    yield 3
+
+it = basic_gen()
+ae(jstype(it.next), 'function')
+ae(jstype(it.send), 'function')   # Python compatibility alias (== .next)
+ae(jstype(it.asend), 'function')  # async-generator-specific alias (== .next)
+
+# .next() must return a thenable Promise (sync generators return {value,done}).
+p = it.next()
+ae(jstype(p.then), 'function')
+
+# ── 2. The wrapper is a sync function — no `await` needed to obtain it ───────
+# An accidental `async function* aiter` (without our wrapper) would return a
+# Promise here instead of an iterator.
+
+ae(jstype(it[v'Symbol.asyncIterator']), 'function')
+
+# ── 3. Promise resolution observable via .then ───────────────────────────────
+# We collect each yielded {value, done} synchronously by walking the iterator
+# manually. Each .then(...) chain runs as a microtask after this script ends,
+# but we can still capture the *number* of pending links and the result list
+# by chaining via Promise.all on the final .next.
+
+results = []
+
+def push_value(r):
+    results.push(r.value)
+
+# Drive three steps; the last assertion runs in the final .then. If any link
+# rejects, the final ok() in the assertion chain is skipped and the test
+# process exits with a non-zero status because of the unhandled rejection.
+chain = it.next()
+v"""
+chain = chain.then(function(r){ push_value(r); return basic_gen().next(); });
+chain.then(function(r){ push_value(r); });
+"""
+
+# ── 4. Empty async generator: a `yield` that never executes still makes the
+# function an async generator (not a regular coroutine). ─────────────────────
+
+async def empty_gen():
+    if False:
+        yield 0
+
+empty_it = empty_gen()
+ae(jstype(empty_it.next), 'function')
+ae(jstype(empty_it[v'Symbol.asyncIterator']), 'function')
+
+# ── 5. `await` is allowed before/between yields inside async generators ──────
+
+def deferred(v):
+    return v'Promise.resolve(v)'
+
+async def awaited_gen():
+    a = await deferred(10)
+    yield a
+    b = await deferred(20)
+    yield a + b
+
+aw_it = awaited_gen()
+ae(jstype(aw_it.next), 'function')
+# .next() returns a thenable even though the body awaits before yielding
+ae(jstype(aw_it.next().then), 'function')
+
+# ── 6. `async for` parses and compiles inside an async function ──────────────
+# We don't drive the loop here (microtasks won't run), but we verify the
+# coroutine object is produced and is thenable.
+
+async def consume():
+    out = []
+    async for x in basic_gen():
+        out.append(x)
+    return out
+
+p = consume()
+ae(jstype(p.then), 'function')
+
+# ── 7. Class methods can be async generators ─────────────────────────────────
+
+class Counter:
+    def __init__(self, limit):
+        self.limit = limit
+
+    async def values(self):
+        i = 0
+        while i < self.limit:
+            yield i
+            i += 1
+
+c_it = Counter(3).values()
+ae(jstype(c_it.next), 'function')
+ae(jstype(c_it[v'Symbol.asyncIterator']), 'function')
+
+# ── 8. Yielded values match Python semantics for the simple sync-yield case ──
+# Microtask-driven: the .then below runs after all top-level code, so we
+# accumulate into `results` and assert via the final chained .then. If the
+# assertion throws, Node exits non-zero (unhandled rejection).
+
+drain = []
+
+def collect_one(r):
+    if not r.done:
+        drain.push(r.value)
+
+def assert_drain():
+    # Order of resolution is sequential because each .then awaits the prior.
+    # Expected: yields 1, 2, 3 from basic_gen()
+    ade(drain, [1, 2, 3])
+
+g = basic_gen()
+v"""
+g.next().then(function(r){ collect_one(r); return g.next(); })
+       .then(function(r){ collect_one(r); return g.next(); })
+       .then(function(r){ collect_one(r); return g.next(); })
+       .then(function(r){
+           // r.done should be true now; assert the accumulated values.
+           assert_drain();
+       });
+"""