rapydscript-ns 0.9.1 → 0.9.3

package/src/parse.pyj

@@ -31,7 +31,7 @@ TreeWalker
 from tokenizer import tokenizer, is_token, RESERVED_WORDS
 
 COMPILER_VERSION = '__COMPILER_VERSION__'
-PYTHON_FLAGS = {'dict_literals':True, 'overload_getitem':True, 'bound_methods':True, 'hash_literals':True, 'overload_operators':True, 'truthiness':True, 'jsx':True, 'strict_arithmetic':True}
+PYTHON_FLAGS = {'dict_literals':True, 'overload_getitem':True, 'bound_methods':True, 'hash_literals':True, 'overload_operators':True, 'truthiness':True, 'jsx':True, 'strict_arithmetic':True, 'python_modulo':True, 'type_enforcement':True}
 
 
 def get_compiler_version():
@@ -620,8 +620,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     })
             elif tmp_ is "async":
                 start = prev()
+                if is_("keyword", "for"):
+                    # async for: consume an async iterable using JS `for await ... of`
+                    next()
+                    forstmt = for_()
+                    forstmt.is_async = True
+                    forstmt.start = start
+                    return forstmt
                 if not is_("keyword", "def"):
-                    croak("Expected 'def' after 'async'")
+                    croak("Expected 'def' or 'for' after 'async'")
                 next()
                 func = function_(S.in_class[-1], False, True)
                 func.start = start
@@ -708,21 +715,70 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
     def with_():
         clauses = v'[]'
         start = S.token
-        while True:
-            if is_('eof'):
-                unexpected()
-            expr = expression()
-            alias = None
-            if is_('keyword', 'as'):
-                next()
-                alias = as_symbol(AST_SymbolAlias)
-            clauses.push(new AST_WithClause({'expression':expr, 'alias':alias}))
-            if is_('punc', ','):
-                next()
-                continue
-            if not is_('punc', ':'):
-                unexpected()
-            break
+
+        # Detect Python 3.10+ parenthesized form: with (cm1 as a, cm2 as b):
+        # vs traditional: with cm as a:  or  with (expr) as a:
+        # Lookahead: scan tokens inside the `(` for an `as` keyword at depth 0
+        # before the matching `)`. If found, it's the parenthesized form.
+        is_parenthesized_with = False
+        if is_('punc', '('):
+            depth = 0
+            i = 0
+            while True:
+                while S.peeked.length <= i:
+                    S.peeked.push(S.input())
+                tok = S.peeked[i]
+                if tok.type is 'eof':
+                    break
+                if tok.type is 'punc':
+                    if tok.value is '(' or tok.value is '[' or tok.value is '{':
+                        depth += 1
+                    elif tok.value is ')' or tok.value is ']' or tok.value is '}':
+                        if depth is 0:
+                            break
+                        depth -= 1
+                elif tok.type is 'keyword' and tok.value is 'as' and depth is 0:
+                    is_parenthesized_with = True
+                    break
+                i += 1
+
+        if is_parenthesized_with:
+            next()  # consume `(`
+            while True:
+                if is_('eof'):
+                    unexpected()
+                if is_('punc', ')'):
+                    next()  # trailing comma before closing `)`
+                    break
+                expr = expression()
+                alias = None
+                if is_('keyword', 'as'):
+                    next()
+                    alias = as_symbol(AST_SymbolAlias)
+                clauses.push(new AST_WithClause({'expression': expr, 'alias': alias}))
+                if is_('punc', ','):
+                    next()
+                    continue
+                if not is_('punc', ')'):
+                    unexpected()
+                next()  # consume `)`
+                break
+        else:
+            while True:
+                if is_('eof'):
+                    unexpected()
+                expr = expression()
+                alias = None
+                if is_('keyword', 'as'):
+                    next()
+                    alias = as_symbol(AST_SymbolAlias)
+                clauses.push(new AST_WithClause({'expression':expr, 'alias':alias}))
+                if is_('punc', ','):
+                    next()
+                    continue
+                if not is_('punc', ':'):
+                    unexpected()
+                break
 
         if not clauses.length:
             token_error(start, 'with statement must have at least one clause')
@@ -1396,6 +1452,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     argnames.push(aname)
                     if is_('punc', ','):
                         next()
+                        if bracketed and is_('punc', ')'):
+                            next()
+                            break
                     else:
                         if bracketed:
                             if is_('punc', ')'):
@@ -1824,6 +1883,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             )(S.in_loop, S.labels)
         })
         definition.return_annotation = return_annotation
+        definition.type_enforce = S.scoped_flags.get('type_enforcement', False)
         definition.is_generator = is_generator[0]
         definition.is_async = is_async
         if is_node_type(definition, AST_Method):
@@ -3242,6 +3302,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'end': right.end,
                 'overloaded': S.scoped_flags.get('overload_operators', False),
                 'strict_arith': S.scoped_flags.get('overload_operators', False) and S.scoped_flags.get('strict_arithmetic', True),
+                'python_mod': (op is '%') and S.scoped_flags.get('python_modulo', True),
                 'python_truthiness': S.scoped_flags.get('truthiness', False) and (op is '&&' or op is '||')
             })
             return expr_op(ret, min_prec, no_in)
@@ -3324,6 +3385,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if S.scoped_flags.get('overload_operators', False) and val is not '=':
                 asgn.overloaded = True
                 asgn.strict_arith = S.scoped_flags.get('strict_arithmetic', True)
+            if val is '%=':
+                asgn.python_mod = S.scoped_flags.get('python_modulo', True)
             return asgn
         return left
 

package/src/tokenizer.pyj

@@ -706,7 +706,7 @@ def tokenizer(raw_text, filename):
             if (peek() is "'" or peek() is '"') and is_string_modifier(tok.value):
                 mods = tok.value.toLowerCase()
                 start_pos_for_string = S.tokpos
-                stok = read_string(mods.indexOf('r') is not -1, mods.indexOf('v') is not -1)
+                stok = read_string(mods.indexOf('r') is not -1 or mods.indexOf('v') is not -1, mods.indexOf('v') is not -1)
                 tok.endpos = stok.endpos
                 if stok.type is not 'js' and mods.indexOf('f') is not -1:
                     tok.col += start_pos_for_string - tok.pos

package/test/async_generators.pyj

@@ -0,0 +1,144 @@
+# globals: assrt
+# vim:fileencoding=utf-8
+#
+# async_generators.pyj
+# Tests for `async def` functions with `yield` (async generators).
+#
+# An async generator is a coroutine that produces values via `yield`. Calling
+# it returns an async iterator whose `.next()` / `.asend()` return Promises.
+#
+# `vm.runInNewContext` (the test runner) is synchronous and does not drain the
+# microtask queue, so these tests assert *synchronously observable* properties:
+# the iterator shape, that .next() is thenable, that the compiled code runs
+# without throwing, and that single-microtask resolution works via
+# Promise.then chains observed through a shared mutable list. End-to-end
+# async-for execution is covered by the web-repl bundle tests.
+
+ae  = assrt.equal
+ade = assrt.deepEqual
+ok  = assrt.ok
+
+# ── 1. Calling an async generator returns an iterator with the right shape ───
+
+async def basic_gen():
+    yield 1
+    yield 2
+    yield 3
+
+it = basic_gen()
+ae(jstype(it.next), 'function')
+ae(jstype(it.send), 'function')   # Python compatibility alias (== .next)
+ae(jstype(it.asend), 'function')  # async-generator-specific alias (== .next)
+
+# .next() must return a thenable Promise (sync generators return {value,done}).
+p = it.next()
+ae(jstype(p.then), 'function')
+
+# ── 2. The wrapper is a sync function — no `await` needed to obtain it ───────
+# An accidental `async function* aiter` (without our wrapper) would return a
+# Promise here instead of an iterator.
+
+ae(jstype(it[v'Symbol.asyncIterator']), 'function')
+
+# ── 3. Promise resolution observable via .then ───────────────────────────────
+# We collect each yielded {value, done} synchronously by walking the iterator
+# manually. Each .then(...) chain runs as a microtask after this script ends,
+# but we can still capture the *number* of pending links and the result list
+# by chaining via Promise.all on the final .next.
+
+results = []
+
+def push_value(r):
+    results.push(r.value)
+
+# Drive three steps; the last assertion runs in the final .then. If any link
+# rejects, the final ok() in the assertion chain is skipped and the test
+# process exits with a non-zero status because of the unhandled rejection.
+chain = it.next()
+v"""
+chain = chain.then(function(r){ push_value(r); return basic_gen().next(); });
+chain.then(function(r){ push_value(r); });
+"""
+
+# ── 4. Empty async generator: a `yield` that never executes still makes the
+# function an async generator (not a regular coroutine). ─────────────────────
+
+async def empty_gen():
+    if False:
+        yield 0
+
+empty_it = empty_gen()
+ae(jstype(empty_it.next), 'function')
+ae(jstype(empty_it[v'Symbol.asyncIterator']), 'function')
+
+# ── 5. `await` is allowed before/between yields inside async generators ──────
+
+def deferred(v):
+    return v'Promise.resolve(v)'
+
+async def awaited_gen():
+    a = await deferred(10)
+    yield a
+    b = await deferred(20)
+    yield a + b
+
+aw_it = awaited_gen()
+ae(jstype(aw_it.next), 'function')
+# .next() returns a thenable even though the body awaits before yielding
+ae(jstype(aw_it.next().then), 'function')
+
+# ── 6. `async for` parses and compiles inside an async function ──────────────
+# We don't drive the loop here (microtasks won't run), but we verify the
+# coroutine object is produced and is thenable.
+
+async def consume():
+    out = []
+    async for x in basic_gen():
+        out.append(x)
+    return out
+
+p = consume()
+ae(jstype(p.then), 'function')
+
+# ── 7. Class methods can be async generators ─────────────────────────────────
+
+class Counter:
+    def __init__(self, limit):
+        self.limit = limit
+
+    async def values(self):
+        i = 0
+        while i < self.limit:
+            yield i
+            i += 1
+
+c_it = Counter(3).values()
+ae(jstype(c_it.next), 'function')
+ae(jstype(c_it[v'Symbol.asyncIterator']), 'function')
+
+# ── 8. Yielded values match Python semantics for the simple sync-yield case ──
+# Microtask-driven: the .then below runs after all top-level code, so we
+# accumulate into `results` and assert via the final chained .then. If the
+# assertion throws, Node exits non-zero (unhandled rejection).
+
+drain = []
+
+def collect_one(r):
+    if not r.done:
+        drain.push(r.value)
+
+def assert_drain():
+    # Order of resolution is sequential because each .then awaits the prior.
+    # Expected: yields 1, 2, 3 from basic_gen()
+    ade(drain, [1, 2, 3])
+
+g = basic_gen()
+v"""
+g.next().then(function(r){ collect_one(r); return g.next(); })
+       .then(function(r){ collect_one(r); return g.next(); })
+       .then(function(r){ collect_one(r); return g.next(); })
+       .then(function(r){
+           // r.done should be true now; assert the accumulated values.
+           assert_drain();
+       });
+"""

package/test/asyncio.pyj

@@ -0,0 +1,307 @@
+# globals: assrt
+# vim:fileencoding=utf-8
+#
+# asyncio.pyj
+# Tests for the asyncio standard library module.
+#
+# Most tests verify synchronous-observable behaviour (exceptions, queue state,
+# lock/event/semaphore flags, iscoroutine/iscoroutinefunction) because the
+# vm.runInNewContext test runner is synchronous.  Async coroutine execution
+# is tested by verifying that async functions compile correctly and return
+# thenable Promises.
+
+from asyncio import (
+    sleep, gather, create_task, ensure_future, run,
+    shield, wait_for, iscoroutine, iscoroutinefunction,
+    current_task, all_tasks, get_event_loop, get_running_loop, new_event_loop,
+    CancelledError, TimeoutError, InvalidStateError, RuntimeError,
+    QueueEmpty, QueueFull,
+    Lock, Event, Semaphore, BoundedSemaphore,
+    Queue, LifoQueue, PriorityQueue
+)
+
+ae  = assrt.equal
+ade = assrt.deepEqual
+ok  = assrt.ok
+
+# ── 1. Exception classes ─────────────────────────────────────────────────────
+
+try:
+    raise CancelledError('cancelled')
+    ok(False)
+except CancelledError as e:
+    ae(e.message, 'cancelled')
+
+try:
+    raise TimeoutError('timed out')
+    ok(False)
+except TimeoutError as e:
+    ae(e.message, 'timed out')
+
+try:
+    raise InvalidStateError('bad state')
+    ok(False)
+except InvalidStateError as e:
+    ae(e.message, 'bad state')
+
+try:
+    raise QueueEmpty('empty')
+    ok(False)
+except QueueEmpty as e:
+    ae(e.message, 'empty')
+
+try:
+    raise QueueFull('full')
+    ok(False)
+except QueueFull as e:
+    ae(e.message, 'full')
+
+# ── 2. sleep / gather return Promises ────────────────────────────────────────
+
+ok(iscoroutine(sleep(0)))
+ok(iscoroutine(sleep(0.1)))
+ok(iscoroutine(gather()))
+ok(iscoroutine(gather(Promise.resolve(1), Promise.resolve(2))))
+
+# ── 3. create_task / ensure_future / run / shield pass through Promises ──────
+
+p = Promise.resolve(42)
+ok(iscoroutine(create_task(p)))
+ok(iscoroutine(ensure_future(p)))
+ok(iscoroutine(run(p)))
+ok(iscoroutine(shield(p)))
+
+# ── 4. iscoroutine / iscoroutinefunction ─────────────────────────────────────
+
+ok(not iscoroutine(42))
+ok(not iscoroutine(None))
+ok(not iscoroutine('string'))
+ok(not iscoroutine([]))
+
+async def _async_fn():
+    return 1
+
+ok(iscoroutinefunction(_async_fn))
+
+def _sync_fn():
+    return 1
+
+ok(not iscoroutinefunction(_sync_fn))
+ok(not iscoroutinefunction(42))
+ok(not iscoroutinefunction(None))
+
+# Async functions return Promises
+p2 = _async_fn()
+ok(iscoroutine(p2))
+
+# ── 5. current_task / all_tasks ───────────────────────────────────────────────
+
+ok(current_task() is None)
+ok(len(all_tasks()) == 0)
+
+# ── 6. Event loop stubs ───────────────────────────────────────────────────────
+
+loop = get_event_loop()
+ok(loop is not None)
+ok(loop is get_running_loop())
+
+loop2 = new_event_loop()
+ok(loop2 is not None)
+ok(not loop2.is_closed())
+ok(loop2.is_running())
+
+p3 = Promise.resolve(99)
+ok(iscoroutine(loop.run_until_complete(p3)))
+ok(iscoroutine(loop.create_task(p3)))
+
+# ── 7. Lock — synchronous state ──────────────────────────────────────────────
+
+lock = Lock()
+ok(not lock.locked())
+
+# acquire() returns a Promise (since it's async def)
+acq = lock.acquire()
+ok(iscoroutine(acq))
+
+# After acquiring asynchronously lock should become locked.
+# We can test the synchronous fast-path by observing the side-effect in a
+# chained then (fire-and-forget style; the VM won't await it but we verify
+# the structure is correct):
+lock2 = Lock()
+ok(not lock2.locked())
+
+try:
+    lock2.release()
+    ok(False)
+except RuntimeError:
+    ok(True)
+
+# ── 8. Event — synchronous set/clear/is_set ──────────────────────────────────
+
+ev = Event()
+ok(not ev.is_set())
+
+ev.set()
+ok(ev.is_set())
+
+ev.clear()
+ok(not ev.is_set())
+
+# wait() returns a Promise
+ok(iscoroutine(ev.wait()))
+
+# If event is already set, wait() still returns a thenable
+ev.set()
+ok(iscoroutine(ev.wait()))
+
+# ── 9. Semaphore — synchronous state ─────────────────────────────────────────
+
+sem = Semaphore(2)
+ok(not sem.locked())
+
+sem.release()
+ok(not sem.locked())
+
+sem.release()
+ok(not sem.locked())
+
+# acquire() returns a Promise
+ok(iscoroutine(sem.acquire()))
+
+try:
+    Semaphore(-1)
+    ok(False)
+except ValueError:
+    ok(True)
+
+# ── 10. BoundedSemaphore ─────────────────────────────────────────────────────
+
+# BoundedSemaphore(n) starts at value=n (same as bound).
+# Releasing without a prior acquire immediately exceeds the bound.
+bsem = BoundedSemaphore(1)
+ok(not bsem.locked())
+try:
+    bsem.release()
+    ok(False)
+except ValueError:
+    ok(True)
+
+# BoundedSemaphore(0) is always locked and cannot be released at all.
+bsem0 = BoundedSemaphore(0)
+ok(bsem0.locked())
+try:
+    bsem0.release()
+    ok(False)
+except ValueError:
+    ok(True)
+
+# ── 11. Queue — synchronous put_nowait / get_nowait ──────────────────────────
+
+q = Queue()
+ok(q.empty())
+ok(q.qsize() == 0)
+ok(not q.full())
+
+q.put_nowait('a')
+q.put_nowait('b')
+q.put_nowait('c')
+ok(q.qsize() == 3)
+ok(not q.empty())
+
+ae(q.get_nowait(), 'a')
+q.task_done()
+ae(q.get_nowait(), 'b')
+q.task_done()
+ae(q.get_nowait(), 'c')
+q.task_done()
+ok(q.empty())
+
+# QueueFull with bounded queue
+q2 = Queue(2)
+ok(not q2.full())
+q2.put_nowait(1)
+q2.put_nowait(2)
+ok(q2.full())
+try:
+    q2.put_nowait(3)
+    ok(False)
+except QueueFull:
+    ok(True)
+
+# QueueEmpty
+q3 = Queue()
+try:
+    q3.get_nowait()
+    ok(False)
+except QueueEmpty:
+    ok(True)
+
+# task_done() overflow check
+q4 = Queue()
+q4.put_nowait('x')
+ae(q4.get_nowait(), 'x')
+q4.task_done()
+try:
+    q4.task_done()
+    ok(False)
+except ValueError:
+    ok(True)
+
+# put() and get() return Promises (async methods) — test on fresh queues
+q5 = Queue()
+put_p = q5.put('async-item')  # no waiters, goes to internal queue
+ok(iscoroutine(put_p))
+ae(q5.get_nowait(), 'async-item')
+q5.task_done()
+
+q6 = Queue()
+get_p = q6.get()  # empty queue → adds a getter waiter
+ok(iscoroutine(get_p))
+
+# join() returns a Promise
+q7 = Queue()
+ok(iscoroutine(q7.join()))
+
+# ── 12. LifoQueue — last-in first-out ordering ───────────────────────────────
+
+lq = LifoQueue()
+lq.put_nowait(1)
+lq.put_nowait(2)
+lq.put_nowait(3)
+ae(lq.get_nowait(), 3)
+ae(lq.get_nowait(), 2)
+ae(lq.get_nowait(), 1)
+
+# ── 13. PriorityQueue — lowest value dequeued first ──────────────────────────
+
+pq = PriorityQueue()
+pq.put_nowait(30)
+pq.put_nowait(10)
+pq.put_nowait(20)
+ae(pq.get_nowait(), 10)
+ae(pq.get_nowait(), 20)
+ae(pq.get_nowait(), 30)
+
+# ── 14. async def / await compiles correctly ─────────────────────────────────
+
+async def _add_async(a, b):
+    return a + b
+
+ok(iscoroutinefunction(_add_async))
+p4 = _add_async(3, 4)
+ok(iscoroutine(p4))
+
+async def _chain():
+    x = await Promise.resolve(10)
+    y = await Promise.resolve(20)
+    return x + y
+
+p5 = _chain()
+ok(iscoroutine(p5))
+
+async def _gather_test():
+    results = await gather(Promise.resolve(1), Promise.resolve(2), Promise.resolve(3))
+    return results
+
+p6 = _gather_test()
+ok(iscoroutine(p6))