rapydscript-ng 0.7.24 → 0.8.1

package/src/parse.pyj

@@ -1,13 +1,11 @@
 # vim:fileencoding=utf-8
 # License: BSD Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-# globals: readfile
+# globals: readfile, writefile, stat_file, sha1sum, ast_from_json, ast_to_json, make_lazy_ast_module, decode_cache, encode_cache
 from __python__ import hash_literals
 
-from utils import make_predicate, array_to_hash, defaults, has_prop, cache_file_name
-from errors import SyntaxError, ImportError
 from ast import (
-AST_Array, AST_Assign, AST_Binary, AST_BlockStatement, AST_Break,
-AST_Call, AST_Catch, AST_Class, AST_ClassCall, AST_Conditional,
+AST_ArgsDef, AST_Array, AST_Assign, AST_Binary, AST_BlockStatement, AST_Break,
+AST_Call, AST_CallArg, AST_CallArgs, AST_Catch, AST_Class, AST_ClassCall, AST_Conditional,
 AST_Constant, AST_Continue, AST_DWLoop, AST_Debugger, AST_Decorator,
 AST_Definitions, AST_DictComprehension, AST_Directive, AST_Do, AST_Dot,
 AST_Else, AST_EmptyStatement, AST_Except, AST_ExpressiveObject, AST_False, AST_Finally,
@@ -21,13 +19,16 @@ AST_SymbolCatch, AST_SymbolDefun, AST_SymbolFunarg,
 AST_SymbolLambda, AST_SymbolNonlocal, AST_SymbolRef, AST_SymbolVar, AST_This,
 AST_Throw, AST_Toplevel, AST_True, AST_Try, AST_UnaryPrefix,
 AST_Undefined, AST_Var, AST_VarDef, AST_Verbatim, AST_While, AST_With, AST_WithClause,
-AST_Yield, AST_Assert, AST_Existential, is_node_type
+AST_Yield, AST_Await, AST_Assert, AST_Existential, is_node_type, TreeWalker
 )
+from errors import SyntaxError, ImportError
 from tokenizer import tokenizer, is_token, RESERVED_WORDS
+from utils import make_predicate, array_to_hash, defaults, has_prop, cache_file_name
 
 
 COMPILER_VERSION = '__COMPILER_VERSION__'
-PYTHON_FLAGS = {'dict_literals':True, 'overload_getitem':True, 'bound_methods':True, 'hash_literals':True}
+PYTHON_FLAGS = {'dict_literals': True, 'overload_getitem': True, 'bound_methods': True, 'hash_literals': True}
+CACHE_VERSION = 1
 
 
 def get_compiler_version():
@@ -35,7 +36,7 @@ def get_compiler_version():
 
 
 def static_predicate(names):
-    return {k:True for k in names.split(' ')}
+    return {k: True for k in names.split(' ')}
 
 NATIVE_CLASSES = {
     'Image': {},
@@ -59,14 +60,14 @@ NATIVE_CLASSES = {
         )
     },
     'String': {
-        'static': static_predicate("fromCharCode")
+        'static': static_predicate('fromCharCode')
     },
     'Array': {
-        'static': static_predicate("isArray from of")
+        'static': static_predicate('isArray from of')
     },
     'Function': {},
     'Date': {
-        'static': static_predicate("UTC now parse")
+        'static': static_predicate('UTC now parse')
     },
     'ArrayBuffer': {
         'static': static_predicate('isView transfer')
@@ -116,34 +117,35 @@ UNARY_PREFIX = make_predicate('typeof void delete ~ - + ! @')
 
 ASSIGNMENT = make_predicate('= += -= /= //= *= %= >>= <<= >>>= |= ^= &=')
 
-PRECEDENCE = (def(a, ret):
+PRECEDENCE = (def (a, ret):
     for i in range(a.length):
         b = a[i]
         for j in range(b.length):
-            ret[b[j]] = i+1
+            ret[b[j]] = i + 1
     return ret
 )([
     # lowest precedence
-    [ "||" ],
-    [ "&&" ],
-    [ "|" ],
-    [ "^" ],
-    [ "&" ],
-    [ "==", "===", "!=", "!==" ],
-    [ "<", ">", "<=", ">=", "in", "nin", "instanceof" ],
-    [ ">>", "<<", ">>>" ],
-    [ "+", "-" ],
-    [ "*", "/", "//", "%" ],
-    [ "**" ]
+    ['||'],
+    ['&&'],
+    ['|'],
+    ['^'],
+    ['&'],
+    ['==', '===', '!=', '!=='],
+    ['<', '>', '<=', '>=', 'in', 'nin', 'instanceof'],
+    ['>>', '<<', '>>>'],
+    ['+', '-'],
+    ['*', '/', '//', '%'],
+    ['**']
     # highest precedence
 ], {})
 
-STATEMENTS_WITH_LABELS = array_to_hash([ "for", "do", "while", "switch" ])
+STATEMENTS_WITH_LABELS = array_to_hash(['for', 'do', 'while', 'switch'])
 
-ATOMIC_START_TOKEN = array_to_hash([ "atom", "num", "string", "regexp", "name", "js" ])
+ATOMIC_START_TOKEN = array_to_hash(['atom', 'num', 'string', 'regexp', 'name', 'js'])
 
 compile_time_decorators = ['staticmethod', 'external', 'property']
 
+
 def has_simple_decorator(decorators, name):
     remove = v'[]'
     for v'var i = 0; i < decorators.length; i++':
@@ -157,6 +159,7 @@ def has_simple_decorator(decorators, name):
         return True
     return False
 
+
 def has_setter_decorator(decorators, name):
     remove = v'[]'
     for v'var i = 0; i < decorators.length; i++':
@@ -170,9 +173,9 @@ def has_setter_decorator(decorators, name):
         return True
     return False
 
+
 # -----[ Parser ]-----
 def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_ids, imported_modules, importing_modules, options):
-
     def next():
         S.prev = S.token
         if S.peeked.length:
@@ -206,26 +209,61 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
     def unexpected(token):
         if token is undefined:
             token = S.token
-        token_error(token, "Unexpected token: " + token.type + " «" + token.value + "»")
+        token_error(token, 'Unexpected token: ' + token.type + ' «' + token.value + '»')
+
+    # --- Error recovery (only active when options.recover_errors is True) --------
+    # These helpers let the LSP obtain a usable AST from a file that contains
+    # syntax errors. They are never invoked on the normal compile path, so they
+    # cannot introduce a performance or behavior regression there.
+
+    def is_recoverable_error(e):
+        return v'e instanceof SyntaxError'
+
+    def record_recovery(e):
+        S.recovered_errors.push({
+            'message': e.message, 'line': e.line, 'col': e.col, 'pos': e.pos,
+            'endpos': e.endpos, 'is_eof': e.is_eof,
+        })
+
+    async def skip_to_recovery_point():
+        # Resynchronize after a syntax error: always consume at least one token
+        # (guaranteeing forward progress) then stop at the start of the next
+        # logical line, a block dedent, or eof. next() may itself raise a
+        # tokenizer level SyntaxError (e.g. an unterminated string on the skipped
+        # line); such errors are recorded and skipping continues. The tokenizer
+        # advances its position for every such error (and never raises for an
+        # unexpected character in recover mode), so this loop always terminates.
+        while not is_('eof'):
+            try:
+                next()
+            except as e:
+                if is_recoverable_error(e):
+                    record_recovery(e)
+                    if is_('eof'):
+                        return
+                    continue
+                raise
+            if is_('eof') or is_('punc', '}') or S.token.nlb:
+                return
 
     def expect_token(type, val):
         if is_(type, val):
             return next()
-        token_error(S.token, "Unexpected token " + S.token.type + " «" + S.token.value + "»" +
-                    ", expected " + type + " «" + val + "»")
+        token_error(S.token, 'Unexpected token ' + S.token.type + ' «' + S.token.value + '»' +
+                    ', expected ' + type + ' «' + val + '»')
 
     def expect(punc):
-        return expect_token("punc", punc)
+        return expect_token('punc', punc)
 
     def semicolon():
-        if is_("punc", ";"):
+        if is_('punc', ';'):
             next()
             S.token.nlb = True
 
     def embed_tokens(parser):
-        def with_embedded_tokens():
+        async def with_embedded_tokens():
             start = S.token
-            expr = parser()
+            expr = await parser()
             if expr is undefined:
                 unexpected()
             end = prev()
@@ -243,7 +281,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     if obj.name:
                         ans.push(obj.name.name)
                     else:
-                        token_error(obj.start, "Top-level functions must have names")
+                        token_error(obj.start, 'Top-level functions must have names')
                 else:
                     # skip inner scopes
                     if is_node_type(obj, AST_Scope):
@@ -262,7 +300,6 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if body.alternative:
                 ans = ans.concat(scan_for_top_level_callables(body.alternative))
 
-
         return ans
 
     def scan_for_classes(body):
@@ -305,10 +342,10 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
         def add_assign_lhs(lhs):
             if is_node_type(lhs, AST_Seq):
-                lhs = new AST_Array({'elements':lhs.to_array()})
+                lhs = new AST_Array({'elements': lhs.to_array()})
             if is_node_type(lhs, AST_Array):
                 # assignment to an implicit tuple
-                push("ρσ_unpack")
+                push('ρσ_unpack')
                 scan_in_array(lhs.elements)
             elif lhs.name:
                 # assignment to a single variable
@@ -317,7 +354,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         def add_for_in(stmt):
             if is_node_type(stmt.init, AST_Array):
                 # iteration via implicit tuple
-                push("ρσ_unpack")
+                push('ρσ_unpack')
                 scan_in_array(stmt.init.elements)
             else:
                 # iteration via a single variable
@@ -395,30 +432,28 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     if opt:
                         vars = vars.concat(scan_for_nonlocal_defs(opt))
 
-
         elif body.body:
             vars = vars.concat(scan_for_nonlocal_defs(body.body))
             if body.alternative:
                 vars = vars.concat(scan_for_nonlocal_defs(body.alternative))
 
-
         return vars
 
-    def return_():
+    async def return_():
         if is_('punc', ';'):
             semicolon()
             value = None
         else:
-            is_end_of_statement = S.token.nlb or is_("eof") or is_("punc", "}")
+            is_end_of_statement = S.token.nlb or is_('eof') or is_('punc', '}')
             if is_end_of_statement:
                 value = None
             else:
-                value = expression(True)
+                value = await expression(True)
                 semicolon()
         return value
 
     @embed_tokens
-    def statement():
+    async def statement():
         # From Kovid: The next three lines were a hack to try to support statements
         # starting with a regexp literal. However, it did not work, for example:
         # echo 'f=1\n/asd/.test()' | rs -> parse error
@@ -433,83 +468,104 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         p = prev()
         if p and not S.token.nlb and ATOMIC_START_TOKEN[p.type] and not is_('punc', ':'):
             unexpected()
-        if tmp_ is "string":
-            return simple_statement()
-        elif tmp_ is "shebang":
+        if tmp_ is 'string':
+            return await simple_statement()
+        elif tmp_ is 'shebang':
             tmp_ = S.token.value
             next()
             return new AST_Directive({
                 'value': tmp_
             })
-        elif tmp_ is "num" or tmp_ is "regexp" or tmp_ is "operator" or tmp_ is "atom" or tmp_ is "js":
-            return simple_statement()
-        elif tmp_ is "punc":
+        elif tmp_ is 'num' or tmp_ is 'regexp' or tmp_ is 'operator' or tmp_ is 'atom' or tmp_ is 'js':
+            return await simple_statement()
+        elif tmp_ is 'punc':
             tmp_ = S.token.value
-            if tmp_ is ":":
+            if tmp_ is ':':
+                bs_body = await block_()
                 return new AST_BlockStatement({
                     'start': S.token,
-                    'body': block_(),
+                    'body': bs_body,
                     'end': prev()
                 })
-            elif tmp_ is "{" or tmp_ is "[" or tmp_ is "(":
-                return simple_statement()
-            elif tmp_ is ";":
+            elif tmp_ is '{' or tmp_ is '[' or tmp_ is '(':
+                return await simple_statement()
+            elif tmp_ is ';':
                 next()
-                return new AST_EmptyStatement({'stype':';', 'start':prev(), 'end':prev()})
+                return new AST_EmptyStatement({'stype': ';', 'start': prev(), 'end': prev()})
             else:
                 unexpected()
-        elif tmp_ is "name":
+        elif tmp_ is 'name':
             if (is_token(peek(), 'punc', ':')) token_error(peek(), 'invalid syntax, colon not allowed here')
-            return simple_statement()
-        elif tmp_ is "keyword":
+            return await simple_statement()
+        elif tmp_ is 'keyword':
             tmp_ = S.token.value
             next()
-            if tmp_ is "break":
+            if tmp_ is 'break':
                 return break_cont(AST_Break)
-            elif tmp_ is "continue":
+            elif tmp_ is 'continue':
                 return break_cont(AST_Continue)
-            elif tmp_ is "debugger":
+            elif tmp_ is 'debugger':
                 semicolon()
                 return new AST_Debugger()
-            elif tmp_ is "do":
+            elif tmp_ is 'do':
+                do_body = await in_loop(statement)
+                do_cond = await (async def ():
+                    expect('.')
+                    expect_token('keyword', 'while')
+                    tmp = await expression(True)
+                    if is_node_type(tmp, AST_Assign):
+                        croak('Assignments in do loop conditions are not allowed')
+                    semicolon()
+                    return tmp
+                )()
                 return new AST_Do({
-                    'body': in_loop(statement),
-                    'condition': (def():
-                        expect(".")
-                        expect_token("keyword", "while")
-                        tmp = expression(True)
-                        if is_node_type(tmp, AST_Assign):
-                            croak('Assignments in do loop conditions are not allowed')
-                        semicolon()
-                        return tmp
-                    )()
+                    'body': do_body,
+                    'condition': do_cond,
                 })
-            elif tmp_ is "while":
-                while_cond = expression(True)
+            elif tmp_ is 'while':
+                while_cond = await expression(True)
                 if is_node_type(while_cond, AST_Assign):
                     croak('Assignments in while loop conditions are not allowed')
                 if not is_('punc', ':'):
                     croak('Expected a colon after the while statement')
+                while_body = await in_loop(statement)
                 return new AST_While({
                     'condition': while_cond,
-                    'body': in_loop(statement)
+                    'body': while_body
                 })
-            elif tmp_ is "for":
+            elif tmp_ is 'for':
                 if is_('js'):
-                    return for_js()
-                return for_()
-            elif tmp_ is "from":
-                return import_(True)
-            elif tmp_ is "import":
-                return import_(False)
-            elif tmp_ is "class":
-                return class_()
-            elif tmp_ is "def":
+                    return await for_js()
+                return await for_()
+            elif tmp_ is 'from':
+                return await import_(True)
+            elif tmp_ is 'import':
+                return await import_(False)
+            elif tmp_ is 'class':
+                return await class_()
+            elif tmp_ is 'async':
+                if not is_('keyword', 'def'):
+                    croak("Expected 'def' after 'async'")
+                next()
+                start = prev()
+                func = await function_(S.in_class[-1], False, True)
+                func.start = start
+                func.end = prev()
+                chain = await subscripts(func, True)
+                if chain is func:
+                    return func
+                else:
+                    return new AST_SimpleStatement({
+                        'start': start,
+                        'body': chain,
+                        'end': prev()
+                    })
+            elif tmp_ is 'def':
                 start = prev()
-                func = function_(S.in_class[-1], False)
+                func = await function_(S.in_class[-1], False)
                 func.start = start
                 func.end = prev()
-                chain = subscripts(func, True)
+                chain = await subscripts(func, True)
                 if chain is func:
                     return func
                 else:
@@ -520,67 +576,75 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     })
             elif tmp_ is 'assert':
                 start = prev()
-                cond = expression(False)
+                cond = await expression(False)
                 msg = None
                 if is_('punc', ','):
                     next()
-                    msg = expression(False)
-                return new AST_Assert({'start': start, 'condition':cond, 'message':msg, 'end':prev()})
-            elif tmp_ is "if":
-                return if_()
-            elif tmp_ is "pass":
+                    msg = await expression(False)
+                return new AST_Assert({'start': start, 'condition': cond, 'message': msg, 'end': prev()})
+            elif tmp_ is 'if':
+                return await if_()
+            elif tmp_ is 'pass':
                 semicolon()
-                return new AST_EmptyStatement({'stype':'pass', 'start':prev(), 'end':prev()})
-            elif tmp_ is "return":
+                return new AST_EmptyStatement({'stype': 'pass', 'start': prev(), 'end': prev()})
+            elif tmp_ is 'return':
                 if S.in_function is 0:
                     croak("'return' outside of function")
                 if S.functions[-1].is_generator:
                     croak("'return' not allowed in a function with yield")
                 S.functions[-1].is_generator = False
 
-                return new AST_Return({'value':return_()})
-            elif tmp_ is "yield":
-                return yield_()
-            elif tmp_ is "raise":
+                ret_val = await return_()
+                return new AST_Return({'value': ret_val})
+            elif tmp_ is 'await':
+                if S.in_function is 0 or not S.functions[-1].is_async:
+                    croak("'await' outside of async function")
+                value = await expr_atom(True)
+                node = new AST_Await({'value': value})
+                semicolon()
+                return new AST_SimpleStatement({'body': node})
+            elif tmp_ is 'yield':
+                return await yield_()
+            elif tmp_ is 'raise':
                 if S.token.nlb:
                     return new AST_Throw({
                         'value': new AST_SymbolCatch({
-                            'name': "ρσ_Exception"
+                            'name': 'ρσ_Exception'
                         })
                     })
 
-                tmp = expression(True)
+                tmp = await expression(True)
                 semicolon()
                 return new AST_Throw({
                     'value': tmp
                 })
-            elif tmp_ is "try":
-                return try_()
-            elif tmp_ is "nonlocal":
-                tmp = nonlocal_()
+            elif tmp_ is 'try':
+                return await try_()
+            elif tmp_ is 'nonlocal':
+                tmp = await nonlocal_()
                 semicolon()
                 return tmp
             elif tmp_ is 'global':
-                tmp = nonlocal_(True)
+                tmp = await nonlocal_(True)
                 semicolon()
                 return tmp
-            elif tmp_ is "with":
-                return with_()
+            elif tmp_ is 'with':
+                return await with_()
             else:
                 unexpected()
 
-    def with_():
+    async def with_():
         clauses = v'[]'
         start = S.token
         while True:
             if is_('eof'):
                 unexpected()
-            expr = expression()
+            expr = await expression()
             alias = None
             if is_('keyword', 'as'):
                 next()
                 alias = as_symbol(AST_SymbolAlias)
-            clauses.push(new AST_WithClause({'expression':expr, 'alias':alias}))
+            clauses.push(new AST_WithClause({'expression': expr, 'alias': alias}))
             if is_('punc', ','):
                 next()
                 continue
@@ -590,15 +654,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
         if not clauses.length:
             token_error(start, 'with statement must have at least one clause')
-        body = statement()
+        body = await statement()
 
         return new AST_With({
             'clauses': clauses,
             'body': body
         })
 
-    def simple_statement(tmp):
-        tmp = expression(True)
+    async def simple_statement(tmp):
+        tmp = await expression(True)
         semicolon()
         return new AST_SimpleStatement({
             'body': tmp
@@ -606,26 +670,29 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
     def break_cont(t):
         if S.in_loop is 0:
-            croak(t.name.slice(4) + " not inside a loop or switch")
+            croak(t.name.slice(4) + ' not inside a loop or switch')
         semicolon()
         return new t()
 
-    def yield_():
+    async def yield_():
         if S.in_function is 0:
             croak("'yield' outside of function")
+        if S.functions[-1].is_async:
+            croak("'yield' not allowed in an async function")
         if S.functions[-1].is_generator is False:
             croak("'yield' not allowed in a function with return")
         S.functions[-1].is_generator = True
         is_yield_from = is_('keyword', 'from')
         if is_yield_from:
             next()
-        return new AST_Yield({'is_yield_from':is_yield_from, 'value': return_()})
+        yld_val = await return_()
+        return new AST_Yield({'is_yield_from': is_yield_from, 'value': yld_val})
 
-    def for_(list_comp):
+    async def for_(list_comp):
         #        expect("(")
         init = None
-        if not is_("punc", ";"):
-            init = expression(True, True)
+        if not is_('punc', ';'):
+            init = await expression(True, True)
             # standardize AST_Seq into array now for consistency
             if is_node_type(init, AST_Seq):
                 if is_node_type(init.car, AST_SymbolRef) and is_node_type(init.cdr, AST_SymbolRef):
@@ -639,17 +706,17 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     'end': init.end
                 })
 
-            if is_("operator", "in"):
+            if is_('operator', 'in'):
                 if is_node_type(init, AST_Var) and init.definitions.length > 1:
-                    croak("Only one variable declaration allowed in for..in loop")
+                    croak('Only one variable declaration allowed in for..in loop')
                 next()
-                return for_in(init, list_comp)
+                return await for_in(init, list_comp)
 
         unexpected()
 
-    def for_in(init, list_comp):
+    async def for_in(init, list_comp):
         lhs = init.definitions[0].name if is_node_type(init, AST_Var) else None
-        obj = expression(True)
+        obj = await expression(True)
         #        expect(")")
         if list_comp:
             return {
@@ -658,19 +725,21 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'object': obj
             }
 
+        fi_body = await in_loop(statement)
         return new AST_ForIn({
             'init': init,
             'name': lhs,
             'object': obj,
-            'body': in_loop(statement)
+            'body': fi_body
         })
 
     # A native JavaScript for loop - for v"var i=0; i<5000; i++":
-    def for_js():
+    async def for_js():
         condition = as_atom_node()
+        fj_body = await in_loop(statement)
         return new AST_ForJS({
             'condition': condition,
-            'body': in_loop(statement)
+            'body': fj_body
         })
 
     # scan function/class body for nested class declarations
@@ -688,7 +757,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 return ERROR_CLASSES[expr.name]
 
             # traverse in reverse to check local variables first
-            for s in range(S.classes.length-1, -1, -1):
+            for s in range(S.classes.length - 1, -1, -1):
                 if has_prop(S.classes[s], expr.name):
                     return S.classes[s][expr.name]
 
@@ -703,7 +772,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 # now 'referenced_path' should contain the full path of potential class
                 if len(referenced_path) > 1:
                     class_name = referenced_path.join('.')
-                    for s in range(S.classes.length-1, -1, -1):
+                    for s in range(S.classes.length - 1, -1, -1):
                         if has_prop(S.classes[s], class_name):
                             return S.classes[s][class_name]
         return False
@@ -712,72 +781,145 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         ctx = S.input.context()
         raise new ImportError(message, ctx.filename, ctx.tokline, ctx.tokcol, ctx.tokpos)
 
-    def do_import(key):
+    async def do_import(key):
         if has_prop(imported_modules, key):
             return
+        # Circular import check must come BEFORE loading_promises so we detect cycles
+        # even when the same module is being loaded by a parallel chain.
         if has_prop(importing_modules, key) and importing_modules[key]:
             import_error('Detected a recursive import of: ' + key + ' while importing: ' + module_id)
+        # If a parallel chain is already loading this module, wait for it instead of
+        # starting a duplicate load.  loading_promises is shared across the entire
+        # compilation via options.loading_promises.
+        loading_promises = options.loading_promises
+        if has_prop(loading_promises, key):
+            await loading_promises[key]
+            return
 
-        # Ensure that the package containing this module is also imported
-        package_module_id = key.split('.')[:-1].join('.')
-        if len(package_module_id) > 0:
-            do_import(package_module_id)
+        # Register this load BEFORE the first await so that any parallel caller
+        # that reaches this point sees the in-progress entry and waits.
+        # Use a single-element list to capture the resolve function from the Promise
+        # executor without needing nonlocal (mutating a container, not rebinding).
+        _rfn = v'[null]'
 
-        if options.for_linting:
-            imported_modules[key] = {'is_cached':True, 'classes':{}, 'module_id':key, 'exports':[],
-                                     'nonlocalvars':[], 'baselib':{}, 'outputs':{}, 'discard_asserts':options.discard_asserts}
-            return
+        def _capture_resolve(resolve):
+            _rfn[0] = resolve
+        loading_promises[key] = new Promise(_capture_resolve)
 
-        def safe_read(base_path):
-            for i, path in enumerate([base_path + '.pyj', base_path + '/__init__.pyj']):
-                try:
-                    return [readfile(path, "utf-8"), path]  # noqa:undef
-                except as e:
-                    if e.code is 'ENOENT' or e.code is 'EPERM' or e.code is 'EACCESS':
+        try:
+            # Ensure that the package containing this module is also imported
+            package_module_id = key.split('.')[:-1].join('.')
+            if len(package_module_id) > 0:
+                await do_import(package_module_id)
+
+            if options.for_linting:
+                imported_modules[key] = {'is_cached': True, 'classes': {}, 'module_id': key, 'exports': [],
+                                         'nonlocalvars': [], 'baselib': {}, 'outputs': {}, 'discard_asserts': options.discard_asserts}
+                return
+
+            # stat_file returns {mtimeMs, content?}.  For real filesystems mtimeMs is a
+            # number and content is absent; for virtual filesystems content holds the
+            # already-read source to avoid a double read.
+            async def safe_stat(base_path):  # noqa
+                for i, path in enumerate([base_path + '.pyj', base_path + '/__init__.pyj']):
+                    try:
+                        st = await stat_file(path)
+                        return [st.mtimeMs, path, st.content if has_prop(st, 'content') else None]
+                    except as e:
+                        if e.code is 'ENOENT' or e.code is 'EPERM' or e.code is 'EACCESS':
+                            if i is 1:
+                                return None, None, None
                         if i is 1:
-                            return None, None
-                    if i is 1:
-                        raise
+                            raise
 
-        src_code = filename = None
-        modpath = key.replace(/\./g, '/')
+            file_mtime = filename = prefetched_src = None
+            modpath = key.replace(/\./g, '/')
 
-        for location in import_dirs:
-            if location:
-                data, filename = safe_read(location + '/' + modpath)
-                if data is not None:
-                    src_code = data
-                    break
-        if src_code is None:
-            import_error("Failed Import: '" + key + "' module doesn't exist in any of the import directories: " + import_dirs.join(':'))
+            for location in import_dirs:
+                if location:
+                    file_mtime, filename, prefetched_src = await safe_stat(location + '/' + modpath)
+                    if filename is not None:
+                        break
+            if filename is None:
+                import_error("Failed Import: '" + key + "' module doesn't exist in any of the import directories: " + import_dirs.join(':'))
 
-        try:
-            cached = JSON.parse(readfile(cache_file_name(filename, options.module_cache_dir), 'utf-8'))
-        except:
-            cached = None
+            # Read the cache file.  I/O starts here; any CPU work done before the
+            # await can overlap with the disk read.
+            cache_read = readfile(cache_file_name(filename, options.module_cache_dir))
 
-        srchash = sha1sum(src_code)  # noqa:undef
-        if cached and cached.version is COMPILER_VERSION and cached.signature is srchash and cached.discard_asserts is v'!!options.discard_asserts':
-            for ikey in cached.imported_module_ids:
-                do_import(ikey)  # Ensure all modules imported by the cached module are also imported
-            imported_modules[key] = {
-                'is_cached':True, 'classes':cached.classes, 'outputs':cached.outputs, 'module_id':key, 'import_order':Object.keys(imported_modules).length,
-                'nonlocalvars':cached.nonlocalvars, 'baselib':cached.baselib, 'exports':cached.exports, 'discard_asserts':options.discard_asserts,
-                'imported_module_ids':cached.imported_module_ids,
-            }
-        else:
-            parse(src_code, {
-                'filename': filename,
-                'toplevel': None,
-                'basedir': options.basedir,
-                'libdir': options.libdir,
-                'import_dirs': options.import_dirs,
-                'module_id': key,
-                'imported_modules': imported_modules,
-                'importing_modules': importing_modules,
-                'discard_asserts': options.discard_asserts,
-                'module_cache_dir': options.module_cache_dir,
-            })  # This function will add the module to imported_modules itself
+            cached = None
+            ast_json_str = None
+            try:
+                _split = decode_cache(await cache_read)
+                if _split is not None:
+                    cached = JSON.parse(_split.meta)
+                    ast_json_str = _split.ast
+            except:
+                cached = None
+
+            srchash = src_code = None
+            cache_ok = cached and cached.v is CACHE_VERSION and cached.compiler_version is COMPILER_VERSION
+            if cache_ok and file_mtime is not None and cached.mtime is not None and cached.mtime is file_mtime:
+                # Fast path: mtime matches — source file is unchanged, skip reading and hashing it
+                srchash = cached.signature
+            else:
+                # Need source content to validate with sha1sum (or to re-parse on miss)
+                src_code = prefetched_src or await readfile(filename, 'utf-8')
+                srchash = sha1sum(src_code)
+                if not cache_ok or cached.signature is not srchash:
+                    cached = None
+
+            if cached is not None:
+                # Load all deps in parallel — cached module deps have been pre-validated as
+                # non-circular so Promise.all is safe.  The loading_promises dict ensures
+                # that if two parallel chains try to load the same dep, only one actually
+                # does the work; the other waits.
+                pending = v'[]'
+                for ikey in cached.imported_module_ids:
+                    if not has_prop(imported_modules, ikey):
+                        pending.push(do_import(ikey))
+                if pending.length > 0:
+                    await Promise.all(pending)
+                # Reconstruct classes dict from the simplified cache (no full AST nodes).
+                classes = {}
+                for cname in Object.keys(cached.classes or {}):
+                    ci = cached.classes[cname]
+                    classes[cname] = {'name': {'name': cname}, 'static': ci.static or {}, 'bound': ci.bound or [], 'classvars': ci.classvars or {}}
+                meta = {
+                    'module_id': key,
+                    'filename': filename,
+                    'import_order': Object.keys(imported_modules).length,
+                    'imports': imported_modules,
+                    'exports': [{'name': n} for n in cached.exported_names],
+                    'baselib': cached.baselib or {},
+                    'classes': classes,
+                }
+                if cached.has_side_effects is not undefined:
+                    meta['_has_side_effects'] = cached.has_side_effects
+                if cached.index is not undefined:
+                    meta['_cache_index'] = cached.index
+                module = make_lazy_ast_module(ast_json_str, meta)
+                imported_modules[key] = module
+            else:
+                await parse(src_code, {
+                    'filename': filename,
+                    'toplevel': None,
+                    'basedir': options.basedir,
+                    'libdir': options.libdir,
+                    'import_dirs': options.import_dirs,
+                    'module_id': key,
+                    'imported_modules': imported_modules,
+                    'importing_modules': importing_modules,
+                    'discard_asserts': options.discard_asserts,
+                    'module_cache_dir': options.module_cache_dir,
+                    '_src_hash': srchash,
+                    '_file_mtime': file_mtime,
+                    'loading_promises': loading_promises,
+                })  # This function will add the module to imported_modules itself
+        finally:
+            del loading_promises[key]
+            if _rfn[0]:
+                _rfn[0](None)
 
         imported_modules[key].srchash = srchash
 
@@ -785,7 +927,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             baselib_items[bitem] = True
 
     def read_python_flags():
-        expect_token("keyword", "import")
+        expect_token('keyword', 'import')
         bracketed = is_('punc', '(')
         if bracketed:
             next()
@@ -809,15 +951,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     else:
                         continue
                 break
-        return new AST_EmptyStatement({'stype':'scoped_flags', 'start':prev(), 'end':prev()})
+        return new AST_EmptyStatement({'stype': 'scoped_flags', 'start': prev(), 'end': prev()})
 
-    def import_(from_import):
-        ans = new AST_Imports({'imports':[]})
+    async def import_(from_import):
+        ans = new AST_Imports({'imports': []})
         while True:
-            tok = tmp = name = last_tok = expression(False)
+            tok = tmp = name = last_tok = await expression(False)
             key = ''
             while is_node_type(tmp, AST_Dot):
-                key = "." + tmp.property + key
+                key = '.' + tmp.property + key
                 tmp = last_tok = tmp.expression
             key = tmp.name + key
             if from_import and key is '__python__':
@@ -830,8 +972,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'module': name,
                 'key': key,
                 'alias': alias,
-                'argnames':None,
-                'body':def():
+                'argnames': None,
+                'body': def ():
                     return imported_modules[key]
             })
             aimp.start, aimp.end = tok.start, last_tok.end
@@ -844,12 +986,12 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 break
 
         for imp in ans['imports']:
-            do_import(imp.key)
+            await do_import(imp.key)
             if imported_module_ids.indexOf(imp.key) is -1:
                 imported_module_ids.push(imp.key)
             classes = imported_modules[key].classes
             if from_import:
-                expect_token("keyword", "import")
+                expect_token('keyword', 'import')
                 imp.argnames = argnames = []
                 bracketed = is_('punc', '(')
                 if bracketed:
@@ -880,16 +1022,16 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     obj = classes[argvar.name]
                     if obj:
                         key = argvar.alias.name if argvar.alias else argvar.name
-                        S.classes[-1][key] = { "static": obj.static, 'bound': obj.bound, 'classvars': obj.classvars }
+                        S.classes[-1][key] = {'static': obj.static, 'bound': obj.bound, 'classvars': obj.classvars}
             else:
                 for cname in Object.keys(classes):
                     obj = classes[cname]
                     key = imp.alias.name if imp.alias else imp.key
-                    S.classes[-1][key + '.' + obj.name.name] = { 'static': obj.static, 'bound': obj.bound, 'classvars': obj.classvars }
+                    S.classes[-1][key + '.' + obj.name.name] = {'static': obj.static, 'bound': obj.bound, 'classvars': obj.classvars}
 
         return ans
 
-    def class_():
+    async def class_():
         name = as_symbol(AST_SymbolDefun)
         if not name:
             unexpected()
@@ -898,7 +1040,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         externaldecorator = has_simple_decorator(S.decorators, 'external')
 
         class_details = {
-            "static": {},
+            'static': {},
             'bound': v'[]',
             'classvars': {},
             'processing': name.name,
@@ -908,7 +1050,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         class_parent = None
 
         # read the bases of the class, if any
-        if is_("punc", "("):
+        if is_('punc', '('):
             S.in_parenthesized_expr = True
             next()
             while True:
@@ -916,7 +1058,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     S.in_parenthesized_expr = False
                     next()
                     break
-                a = expr_atom(False)
+                a = await expr_atom(False)
                 if class_parent is None:
                     class_parent = a
                 bases.push(a)
@@ -925,10 +1067,37 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     continue
 
         docstrings = v'[]'
+        cls_decorators = (def ():
+            d = []
+            for decorator in S.decorators:
+                d.push(new AST_Decorator({
+                    'expression': decorator
+                }))
+            S.decorators = v'[]'
+            return d
+        )()
+        cls_body = await (async def (loop, labels):
+            # navigate to correct location in the module tree and append the class
+            S.in_class.push(name.name)
+            S.classes[S.classes.length - 1][name.name] = class_details
+            S.classes.push({})
+            S.scoped_flags.push()
+            S.in_function += 1
+            S.in_loop = 0
+            S.labels = []
+            a = await block_(docstrings)
+            S.in_function -= 1
+            S.scoped_flags.pop()
+            S.classes.pop()
+            S.in_class.pop()
+            S.in_loop = loop
+            S.labels = labels
+            return a
+        )(S.in_loop, S.labels)
         definition = new AST_Class({
             'name': name,
             'docstrings': docstrings,
-            'module_id':module_id,
+            'module_id': module_id,
             'dynamic_properties': Object.create(None),
             'parent': class_parent,
             'bases': bases,
@@ -938,33 +1107,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'external': externaldecorator,
             'bound': class_details.bound,
             'statements': [],
-            'decorators': (def():
-                d = []
-                for decorator in S.decorators:
-                    d.push(new AST_Decorator({
-                        'expression': decorator
-                    }))
-                S.decorators = v'[]'
-                return d
-            )(),
-            'body': (def(loop, labels):
-                # navigate to correct location in the module tree and append the class
-                S.in_class.push(name.name)
-                S.classes[S.classes.length - 1][name.name] = class_details
-                S.classes.push({})
-                S.scoped_flags.push()
-                S.in_function += 1
-                S.in_loop = 0
-                S.labels = []
-                a = block_(docstrings)
-                S.in_function -= 1
-                S.scoped_flags.pop()
-                S.classes.pop()
-                S.in_class.pop()
-                S.in_loop = loop
-                S.labels = labels
-                return a
-            )(S.in_loop, S.labels)
+            'decorators': cls_decorators,
+            'body': cls_body,
         })
         class_details.processing = False
         # find the constructor
@@ -975,10 +1119,11 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     if not descriptor:
                         descriptor = definition.dynamic_properties[stmt.name.name] = {}
                     descriptor['getter' if stmt.is_getter else 'setter'] = stmt
-                elif stmt.name.name is "__init__":
+                elif stmt.name.name is '__init__':
                     definition.init = stmt
         # find the class variables
         class_var_names = {}
+
         # Ensure that if a class variable refers to another class variable in
         # its initialization, the referenced variables' names is correctly
         # mangled.
@@ -996,7 +1141,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     class_var_names[varname] = True
                     definition.classvars[varname] = True
                 elif is_node_type(node, AST_SymbolRef) and has_prop(class_var_names, node.name):
-                    node.thedef = new AST_SymbolDefun({'name':name.name + '.prototype.' + node.name})
+                    node.thedef = new AST_SymbolDefun({'name': name.name + '.prototype.' + node.name})
                 if descend:
                     descend.call(node)
             this._visit = visit_node
@@ -1008,7 +1153,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 definition.statements.push(stmt)
         return definition
 
-    def function_(in_class, is_expression):
+    async def function_(in_class, is_expression, is_async=False):
         name = as_symbol(AST_SymbolDefun if in_class else AST_SymbolLambda) if is_('name') else None
         if in_class and not name:
             croak('Cannot use anonymous function as class methods')
@@ -1024,196 +1169,214 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     croak('A method cannot be both static and a property getter/setter')
                 S.classes[S.classes.length - 2][in_class].static[name.name] = True
                 staticmethod = True
-            elif name.name is not "__init__" and S.scoped_flags.get('bound_methods'):
+            elif name.name is not '__init__' and S.scoped_flags.get('bound_methods'):
                 S.classes[S.classes.length - 2][in_class].bound.push(name.name)
 
-        expect("(")
+        expect('(')
         S.in_parenthesized_expr = True
         ctor = AST_Method if in_class else AST_Function
         return_annotation = None
         is_generator = v'[]'
         docstrings = v'[]'
-        definition = new ctor({
-            'name': name,
-            'is_expression': is_expression,
-            'is_anonymous': is_anonymous,
-            'argnames': (def(a):
-                defaults = {}
-                first = True
-                seen_names = {}
-                def_line = S.input.context().tokline
-                current_arg_name = None
-                name_token = None
-
-                def get_arg():
-                    nonlocal current_arg_name, name_token
-                    current_arg_name = S.token.value
-                    if has_prop(seen_names, current_arg_name):
-                        token_error(prev(), "Can't repeat parameter names")
-                    if current_arg_name is 'arguments':
-                        token_error(prev(), "Can't use the name arguments as a parameter name, it is reserved by JavaScript")
-                    seen_names[current_arg_name] = True
-                    # save these in order to move back if we have an annotation
-                    name_token = S.token
-                    name_ctx = S.input.context()
-                    # check if we have an argument annotation
-                    ntok = peek()
-                    if ntok.type is 'punc' and ntok.value is ':':
-                        next()
-                        expect(':')
-                        annotation = maybe_conditional()
-
-                        # and now, do as_symbol without the next() at the end
-                        # since we are already at the next comma (or end bracket)
-                        if not is_token(name_token, "name"):
-                            # assuming the previous context in case
-                            # the annotation was over the line
-                            croak("Name expected", name_ctx.tokline)
-                            return None
-
-                        sym = new AST_SymbolFunarg({
-                            'name': name_token.value,
-                            'start': S.token,
-                            'end': S.token,
-                            'annotation': annotation
-                        })
-                        return sym
-                    else:
-                        if not is_("name"):
-                            # there is no name, which is an error we should report on the
-                            # same line as the definition, so move to that is we're not already there.
-                            if S.input.context().tokline is not def_line:
-                                croak("Name expected", def_line)
-                            else:
-                                croak("Name expected")
-                            return None
-
-                        sym = new AST_SymbolFunarg({
-                            'name': current_arg_name,
-                            'start': S.token,
-                            'end': S.token,
-                            'annotation': None
-                        })
-                        next()
-                        return sym
 
-                while not is_("punc", ")"):
-                    if first:
-                        first = False
-                    else:
-                        expect(",")
-                        if is_('punc', ')'):
-                            break
-                    if is_('operator', '**'):
-                        # **kwargs
-                        next()
-                        if a.kwargs:
-                            token_error(name_token, "Can't define multiple **kwargs in function definition")
-                        a.kwargs = get_arg()
-                    elif is_('operator', '*'):
-                        # *args
+        fn_argnames = await (async def ():
+            a = v'[]'
+            starargs_sym = undefined
+            kwargs_sym = undefined
+            defaults = {}
+            has_defaults = False
+            first = True
+            seen_names = {}
+            def_line = S.input.context().tokline
+            current_arg_name = None
+            name_token = None
+            async def get_arg():
+                nonlocal current_arg_name, name_token
+                current_arg_name = S.token.value
+                if has_prop(seen_names, current_arg_name):
+                    token_error(prev(), "Can't repeat parameter names")
+                if current_arg_name is 'arguments':
+                    token_error(prev(), "Can't use the name arguments as a parameter name, it is reserved by JavaScript")
+                seen_names[current_arg_name] = True
+                # save these in order to move back if we have an annotation
+                name_token = S.token
+                name_ctx = S.input.context()
+                # check if we have an argument annotation
+                ntok = peek()
+                if ntok.type is 'punc' and ntok.value is ':':
+                    next()
+                    expect(':')
+                    annotation = await maybe_conditional()
+                    # and now, do as_symbol without the next() at the end
+                    # since we are already at the next comma (or end bracket)
+                    if not is_token(name_token, 'name'):
+                        # assuming the previous context in case
+                        # the annotation was over the line
+                        if S.input.context().tokline is not def_line:
+                            croak('Name expected', name_ctx.tokline)
+                        else:
+                            croak('Name expected')
+                        return None
+                    sym = new AST_SymbolFunarg({
+                        'name': name_token.value,
+                        'start': S.token,
+                        'end': S.token,
+                        'annotation': annotation
+                    })
+                    return sym
+                else:
+                    if not is_('name'):
+                        # there is no name, which is an error we should report on the
+                        # same line as the definition, so move to that is we're not already there.
+                        if S.input.context().tokline is not def_line:
+                            croak('Name expected', def_line)
+                        else:
+                            croak('Name expected')
+                        return None
+                    sym = new AST_SymbolFunarg({
+                        'name': current_arg_name,
+                        'start': S.token,
+                        'end': S.token,
+                        'annotation': None
+                    })
+                    next()
+                    return sym
+            while not is_('punc', ')'):
+                if first:
+                    first = False
+                else:
+                    expect(',')
+                    if is_('punc', ')'):
+                        break
+                if is_('operator', '**'):
+                    # **kwargs
+                    next()
+                    if kwargs_sym:
+                        token_error(name_token, "Can't define multiple **kwargs in function definition")
+                    kwargs_sym = await get_arg()
+                elif is_('operator', '*'):
+                    # *args
+                    next()
+                    if starargs_sym:
+                        token_error(name_token, "Can't define multiple *args in function definition")
+                    if kwargs_sym:
+                        token_error(name_token, "Can't define *args after **kwargs in function definition")
+                    starargs_sym = await get_arg()
+                else:
+                    if starargs_sym or kwargs_sym:
+                        token_error(name_token, "Can't define a formal parameter after *args or **kwargs")
+                    a.push(await get_arg())
+                    if is_('operator', '='):
+                        if kwargs_sym:
+                            token_error(name_token, "Can't define an optional formal parameter after **kwargs")
                         next()
-                        if a.starargs:
-                            token_error(name_token, "Can't define multiple *args in function definition")
-                        if a.kwargs:
-                            token_error(name_token, "Can't define *args after **kwargs in function definition")
-                        a.starargs = get_arg()
+                        defaults[current_arg_name] = await expression(False)
+                        has_defaults = True
                     else:
-                        if a.starargs or a.kwargs:
-                            token_error(name_token, "Can't define a formal parameter after *args or **kwargs")
-                        a.push(get_arg())
-                        if is_("operator", "="):
-                            if a.kwargs:
-                                token_error(name_token, "Can't define an optional formal parameter after **kwargs")
-                            next()
-                            defaults[current_arg_name] = expression(False)
-                            a.has_defaults = True
-                        else:
-                            if a.has_defaults:
-                                token_error(name_token, "Can't define required formal parameters after optional formal parameters")
-
+                        if has_defaults:
+                            token_error(name_token, "Can't define required formal parameters after optional formal parameters")
+            next()
+            # check if we have a return type annotation
+            if is_('punc', '->'):
                 next()
-                # check if we have a return type annotation
-                if is_("punc", "->"):
-                    next()
-                    nonlocal return_annotation
-                    return_annotation = maybe_conditional()
-                S.in_parenthesized_expr = False
-                a.defaults = defaults
-                a.is_simple_func = not a.starargs and not a.kwargs and not a.has_defaults
-                return a
-            )(v'[]'),
+                nonlocal return_annotation
+                return_annotation = await maybe_conditional()
+            S.in_parenthesized_expr = False
+            return new AST_ArgsDef({
+                'args': a,
+                'starargs': starargs_sym,
+                'kwargs': kwargs_sym,
+                'defaults': defaults,
+                'has_defaults': has_defaults,
+                'is_simple_func': not starargs_sym and not kwargs_sym and not has_defaults,
+            })
+        )()
+
+        fn_decorators = (def ():
+            d = v'[]'
+            for decorator in S.decorators:
+                d.push(new AST_Decorator({
+                    'expression': decorator
+                }))
+            S.decorators = v'[]'
+            return d
+        )()
+
+        fn_body = await (async def (loop, labels):
+            S.in_class.push(False)
+            S.classes.push({})
+            S.scoped_flags.push()
+            S.in_function += 1
+            S.functions.push({'is_async': is_async})
+            S.in_loop = 0
+            S.labels = []
+            a = await block_(docstrings)
+            S.in_function -= 1
+            S.scoped_flags.pop()
+            is_generator.push(bool(S.functions.pop().is_generator))
+            S.classes.pop()
+            S.in_class.pop()
+            S.in_loop = loop
+            S.labels = labels
+            return a
+        )(S.in_loop, S.labels)
+
+        definition = new ctor({
+            'name': name,
+            'is_expression': is_expression,
+            'is_anonymous': is_anonymous,
+            'argnames': fn_argnames,
             'localvars': [],
-            'decorators': (def():
-                d = v'[]'
-                for decorator in S.decorators:
-                    d.push(new AST_Decorator({
-                        'expression': decorator
-                    }))
-                S.decorators = v'[]'
-                return d
-            )(),
+            'decorators': fn_decorators,
             'docstrings': docstrings,
-            'body': (def(loop, labels):
-                S.in_class.push(False)
-                S.classes.push({})
-                S.scoped_flags.push()
-                S.in_function += 1
-                S.functions.push({})
-                S.in_loop = 0
-                S.labels = []
-                a = block_(docstrings)
-                S.in_function -= 1
-                S.scoped_flags.pop()
-                is_generator.push(bool(S.functions.pop().is_generator))
-                S.classes.pop()
-                S.in_class.pop()
-                S.in_loop = loop
-                S.labels = labels
-                return a
-            )(S.in_loop, S.labels)
+            'body': fn_body,
         })
         definition.return_annotation = return_annotation
         definition.is_generator = is_generator[0]
+        definition.is_async = is_async
         if is_node_type(definition, AST_Method):
             definition.static = staticmethod
             definition.is_getter = property_getter
             definition.is_setter = property_setter
-            if definition.argnames.length < 1 and not definition.static:
+            if definition.argnames.args.length < 1 and not definition.static:
                 croak('Methods of a class must have at least one argument, traditionally named self')
             if definition.name and definition.name.name is '__init__':
                 if definition.is_generator:
                     croak('The __init__ method of a class cannot be a generator (yield not allowed)')
+                if definition.is_async:
+                    croak('The __init__ method of a class cannot be async')
                 if property_getter or property_setter:
                     croak('The __init__ method of a class cannot be a property getter/setter')
+        if definition.is_async and definition.is_generator:
+            croak('A function cannot be both async and a generator')
         if definition.is_generator:
             baselib_items['yield'] = True
 
         # detect local variables, strip function arguments
         assignments = scan_for_local_vars(definition.body)
         for i in range(assignments.length):
-            for j in range(definition.argnames.length+1):
-                if j is definition.argnames.length:
+            for j in range(definition.argnames.args.length + 1):
+                if j is definition.argnames.args.length:
                     definition.localvars.push(new_symbol(AST_SymbolVar, assignments[i]))
-                elif j < definition.argnames.length and assignments[i] is definition.argnames[j].name:
+                elif j < definition.argnames.args.length and assignments[i] is definition.argnames.args[j].name:
                     break
 
         nonlocals = scan_for_nonlocal_defs(definition.body)
         nonlocals = {name for name in nonlocals}
-        definition.localvars = definition.localvars.filter(def(v): return not nonlocals.has(v.name);)
+        definition.localvars = definition.localvars.filter(def (v): return not nonlocals.has(v.name);)
         return definition
 
-    def if_():
-        cond = expression(True)
-        body = statement()
+    async def if_():
+        cond = await expression(True)
+        body = await statement()
         belse = None
-        if is_("keyword", "elif") or is_("keyword", "else"):
-            if is_("keyword", "else"):
+        if is_('keyword', 'elif') or is_('keyword', 'else'):
+            if is_('keyword', 'else'):
                 next()
             else:
-                S.token.value = "if"
+                S.token.value = 'if'
             # effectively converts 'elif' to 'else if'
-            belse = statement()
+            belse = await statement()
 
         return new AST_If({
             'condition': cond,
@@ -1227,15 +1390,27 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 return stmt.body
         return False
 
-    def block_(docstrings):
+    async def block_(docstrings):
         prev_whitespace = S.token.leading_whitespace
-        expect(":")
+        expect(':')
         a = v'[]'
         if not S.token.nlb:
             while not S.token.nlb:
-                if is_("eof"):
+                if is_('eof'):
+                    if options.recover_errors:
+                        break
                     unexpected()
-                stmt = statement()
+                if options.recover_errors:
+                    try:
+                        stmt = await statement()
+                    except as berr:
+                        if is_recoverable_error(berr):
+                            record_recovery(berr)
+                            await skip_to_recovery_point()
+                            continue
+                        raise
+                else:
+                    stmt = await statement()
                 if docstrings:
                     ds = is_docstring(stmt)
                     if ds:
@@ -1246,11 +1421,21 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             current_whitespace = S.token.leading_whitespace
             if current_whitespace.length is 0 or prev_whitespace is current_whitespace:
                 croak('Expected an indented block')
-            while not is_("punc", "}"):
-                if is_("eof"):
+            while not is_('punc', '}'):
+                if is_('eof'):
                     # end of file, terminate block automatically
                     return a
-                stmt = statement()
+                if options.recover_errors:
+                    try:
+                        stmt = await statement()
+                    except as berr:
+                        if is_recoverable_error(berr):
+                            record_recovery(berr)
+                            await skip_to_recovery_point()
+                            continue
+                        raise
+                else:
+                    stmt = await statement()
                 if docstrings:
                     ds = is_docstring(stmt)
                     if ds:
@@ -1260,79 +1445,89 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             next()
         return a
 
-    def try_():
-        body = block_()
+    async def try_():
+        body = await block_()
         bcatch = v'[]'
         bfinally = None
         belse = None
-        while is_("keyword", "except"):
+        while is_('keyword', 'except'):
             start = S.token
             next()
             exceptions = []
-            if not is_("punc", ":") and not is_("keyword", "as"):
+            if not is_('punc', ':') and not is_('keyword', 'as'):
                 exceptions.push(as_symbol(AST_SymbolVar))
-                while is_("punc", ","):
+                while is_('punc', ','):
                     next()
                     exceptions.push(as_symbol(AST_SymbolVar))
 
             name = None
-            if is_("keyword", "as"):
+            if is_('keyword', 'as'):
                 next()
                 name = as_symbol(AST_SymbolCatch)
 
+            except_body = await block_()
             bcatch.push(new AST_Except({
                 'start': start,
                 'argname': name,
                 'errors': exceptions,
-                'body': block_(),
+                'body': except_body,
                 'end': prev()
             }))
 
-        if is_("keyword", "else"):
+        if is_('keyword', 'else'):
             start = S.token
             next()
+            belse_body = await block_()
             belse = new AST_Else({
                 'start': start,
-                'body': block_(),
+                'body': belse_body,
                 'end': prev()
             })
 
-        if is_("keyword", "finally"):
+        if is_('keyword', 'finally'):
             start = S.token
             next()
+            bfinally_body = await block_()
             bfinally = new AST_Finally({
                 'start': start,
-                'body': block_(),
+                'body': bfinally_body,
                 'end': prev()
             })
 
         if not bcatch.length and not bfinally:
-            croak("Missing except/finally blocks")
+            croak('Missing except/finally blocks')
 
         return new AST_Try({
             'body': body,
-            'bcatch': (new AST_Catch({ 'body': bcatch }) if bcatch.length else None),
+            'bcatch': (new AST_Catch({'body': bcatch}) if bcatch.length else None),
             'bfinally': bfinally,
             'belse': belse
         })
 
-    def vardefs(symbol_class):
+    async def vardefs(symbol_class):
         a = []
         while True:
+            vd_start = S.token
+            vd_name = as_symbol(symbol_class)
+            if is_('operator', '='):
+                next()
+                varval = await expression(False)
+            else:
+                varval = None
             a.push(new AST_VarDef({
-                'start': S.token,
-                'name': as_symbol(symbol_class),
-                'value': (next(), expression(False)) if is_('operator', '=') else None,
+                'start': vd_start,
+                'name': vd_name,
+                'value': varval,
                 'end': prev()
             }))
-            if not is_("punc", ","):
+            if not is_('punc', ','):
                 break
             next()
 
         return a
 
-    def nonlocal_(is_global):
-        defs = vardefs(AST_SymbolNonlocal)
+    async def nonlocal_(is_global):
+        defs = await vardefs(AST_SymbolNonlocal)
         if is_global:
             for vardef in defs:
                 S.globals.push(vardef.name.name)
@@ -1342,19 +1537,19 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'end': prev()
         })
 
-    def new_():
+    async def new_():
         start = S.token
-        expect_token("operator", "new")
-        newexp = expr_atom(False)
+        expect_token('operator', 'new')
+        newexp = await expr_atom(False)
 
-        if is_("punc", "("):
+        if is_('punc', '('):
             S.in_parenthesized_expr = True
             next()
-            args = func_call_list()
+            args = await func_call_list()
             S.in_parenthesized_expr = False
         else:
-            args = func_call_list(True)
-        return subscripts(new AST_New({
+            args = await func_call_list(True)
+        return await subscripts(new AST_New({
             'start': start,
             'expression': newexp,
             'args': args,
@@ -1378,40 +1573,40 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
     def token_as_atom_node():
         tok = S.token
         tmp_ = tok.type
-        if tmp_ is "name":
+        if tmp_ is 'name':
             return token_as_symbol(tok, AST_SymbolRef)
-        elif tmp_ is "num":
+        elif tmp_ is 'num':
             return new AST_Number({
                 'start': tok,
                 'end': tok,
                 'value': tok.value
             })
-        elif tmp_ is "string":
+        elif tmp_ is 'string':
             return string_()
-        elif tmp_ is "regexp":
+        elif tmp_ is 'regexp':
             return new AST_RegExp({
                 'start': tok,
                 'end': tok,
                 'value': tok.value
             })
-        elif tmp_ is "atom":
+        elif tmp_ is 'atom':
             tmp__ = tok.value
-            if tmp__ is "False":
+            if tmp__ is 'False':
                 return new AST_False({
                     'start': tok,
                     'end': tok
                 })
-            elif tmp__ is "True":
+            elif tmp__ is 'True':
                 return new AST_True({
                     'start': tok,
                     'end': tok
                 })
-            elif tmp__ is "None":
+            elif tmp__ is 'None':
                 return new AST_Null({
                     'start': tok,
                     'end': tok
                 })
-        elif tmp_ is "js":
+        elif tmp_ is 'js':
             return new AST_Verbatim({
                 'start': tok,
                 'end': tok,
@@ -1424,22 +1619,22 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         next()
         return ret
 
-    def expr_atom(allow_calls):
-        if is_("operator", "new"):
-            return new_()
+    async def expr_atom(allow_calls):
+        if is_('operator', 'new'):
+            return await new_()
 
         start = S.token
-        if is_("punc"):
+        if is_('punc'):
             tmp_ = start.value
-            if tmp_ is "(":
+            if tmp_ is '(':
                 S.in_parenthesized_expr = True
                 next()
                 if is_('punc', ')'):
                     next()
-                    return new AST_Array({'elements':[]})
-                ex = expression(True)
+                    return new AST_Array({'elements': []})
+                ex = await expression(True)
                 if is_('keyword', 'for'):
-                    ret = read_comprehension(new AST_GeneratorComprehension({'statement': ex}), ')')
+                    ret = await read_comprehension(new AST_GeneratorComprehension({'statement': ex}), ')')
                     S.in_parenthesized_expr = False
                     return ret
                 ex.start = start
@@ -1447,153 +1642,168 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 if is_node_type(ex, AST_SymbolRef):
                     ex.parens = True
                 if not is_node_type(ex, AST_GeneratorComprehension):
-                    expect(")")
+                    expect(')')
                 if is_node_type(ex, AST_UnaryPrefix):
                     ex.parenthesized = True
                 S.in_parenthesized_expr = False
-                return subscripts(ex, allow_calls)
-            elif tmp_ is "[":
-                return subscripts(array_(), allow_calls)
-            elif tmp_ is "{":
-                return subscripts(object_(), allow_calls)
+                return await subscripts(ex, allow_calls)
+            elif tmp_ is '[':
+                return await subscripts(await array_(), allow_calls)
+            elif tmp_ is '{':
+                return await subscripts(await object_(), allow_calls)
 
             unexpected()
 
-        if is_("keyword", "class"):
+        if is_('keyword', 'class'):
             next()
-            cls = class_()
+            cls = await class_()
             cls.start = start
             cls.end = prev()
-            return subscripts(cls, allow_calls)
+            return await subscripts(cls, allow_calls)
 
-        if is_("keyword", "def"):
+        if is_('keyword', 'async'):
             next()
-            func = function_(False, True)
+            if not is_('keyword', 'def'):
+                croak("Expected 'def' after 'async'")
+            next()
+            func = await function_(False, True, True)
             func.start = start
             func.end = prev()
-            return subscripts(func, allow_calls)
+            return await subscripts(func, allow_calls)
+
+        if is_('keyword', 'def'):
+            next()
+            func = await function_(False, True)
+            func.start = start
+            func.end = prev()
+            return await subscripts(func, allow_calls)
+
+        if is_('keyword', 'await'):
+            next()
+            if S.in_function is 0 or not S.functions[-1].is_async:
+                croak("'await' outside of async function")
+            value = await expr_atom(True)
+            return new AST_Await({'value': value})
 
         if is_('keyword', 'yield'):
             next()
-            return yield_()
+            return await yield_()
 
         if ATOMIC_START_TOKEN[S.token.type]:
-            return subscripts(as_atom_node(), allow_calls)
+            return await subscripts(as_atom_node(), allow_calls)
 
         unexpected()
 
-    def expr_list(closing, allow_trailing_comma, allow_empty, func_call):
+    async def expr_list(closing, allow_trailing_comma, allow_empty, func_call):
         first = True
         a = []
         saw_starargs = False
-        while not is_("punc", closing):
+        while not is_('punc', closing):
             if saw_starargs:
-                token_error(prev(), "*args must be the last argument in a function call")
+                token_error(prev(), '*args must be the last argument in a function call')
 
             if first:
                 first = False
             else:
-                expect(",")
-            if allow_trailing_comma and is_("punc", closing):
+                expect(',')
+            if allow_trailing_comma and is_('punc', closing):
                 break
 
-            if is_("operator", "*") and func_call:
+            if is_('operator', '*') and func_call:
                 saw_starargs = True
                 next()
 
-            if is_("punc", ",") and allow_empty:
+            if is_('punc', ',') and allow_empty:
                 a.push(new AST_Hole({
                     'start': S.token,
                     'end': S.token
                 }))
             else:
-                a.push(expression(False))
+                a.push(await expression(False))
 
+        next()
         if func_call:
-            tmp = []
-            tmp.kwargs = []
+            pargs = []
+            kwargs = []
             for arg in a:
                 if is_node_type(arg, AST_Assign):
-                    tmp.kwargs.push([arg.left, arg.right])
+                    kwargs.push([arg.left, arg.right])
                 else:
-                    tmp.push(arg)
-            a = tmp
-
-        next()
-        if saw_starargs:
-            a.starargs = True
+                    pargs.push(arg)
+            return new AST_CallArgs({'args': pargs, 'kwargs': kwargs, 'kwarg_items': v'[]', 'starargs': saw_starargs})
         return a
 
-    def func_call_list(empty):
+    async def func_call_list(empty):
         a = v'[]'
         first = True
-        a.kwargs = v'[]'
-        a.kwarg_items = v'[]'
-        a.starargs = False
+        kwargs = v'[]'
+        kwarg_items = v'[]'
+        starargs = False
         if empty:
-            return a
+            return new AST_CallArgs({'args': a, 'kwargs': kwargs, 'kwarg_items': kwarg_items, 'starargs': starargs})
         single_comprehension = False
-        while not is_("punc", ')') and not is_('eof'):
+        while not is_('punc', ')') and not is_('eof'):
             if not first:
-                expect(",")
+                expect(',')
                 if is_('punc', ')'):
                     break
             if is_('operator', '*'):
                 next()
-                arg = expression(False)
-                arg.is_array = True
-                a.push(arg)
-                a.starargs = True
+                arg = await expression(False)
+                a.push(new AST_CallArg({'value': arg, 'is_array': True}))
+                starargs = True
             elif is_('operator', '**'):
                 next()
-                a.kwarg_items.push(as_symbol(AST_SymbolRef, False))
-                a.starargs = True
+                kwarg_items.push(as_symbol(AST_SymbolRef, False))
+                starargs = True
             else:
-                arg = expression(False)
+                arg = await expression(False)
                 if is_node_type(arg, AST_Assign):
-                    a.kwargs.push([arg.left, arg.right])
+                    kwargs.push([arg.left, arg.right])
                 else:
                     if is_('keyword', 'for'):
                         if not first:
                             croak('Generator expression must be parenthesized if not sole argument')
-                        a.push(read_comprehension(new AST_GeneratorComprehension({'statement': arg}), ')'))
+                        comp = await read_comprehension(new AST_GeneratorComprehension({'statement': arg}), ')')
+                        a.push(new AST_CallArg({'value': comp, 'is_array': False}))
                         single_comprehension = True
                         break
-                    a.push(arg)
+                    a.push(new AST_CallArg({'value': arg, 'is_array': False}))
             first = False
         if not single_comprehension:
             next()
-        return a
+        return new AST_CallArgs({'args': a, 'kwargs': kwargs, 'kwarg_items': kwarg_items, 'starargs': starargs})
 
     @embed_tokens
-    def array_():
-        expect("[")
+    async def array_():
+        expect('[')
         expr = []
-        if not is_("punc", "]"):
-            expr.push(expression(False))
-            if is_("keyword", "for"):
+        if not is_('punc', ']'):
+            expr.push(await expression(False))
+            if is_('keyword', 'for'):
                 # list comprehension
-                return read_comprehension(new AST_ListComprehension({'statement': expr[0]}), ']')
+                return await read_comprehension(new AST_ListComprehension({'statement': expr[0]}), ']')
 
-            if not is_("punc", "]"):
-                expect(",")
+            if not is_('punc', ']'):
+                expect(',')
 
+        arr_rest = await expr_list(']', True, True)
         return new AST_Array({
-            'elements': expr.concat(expr_list("]", True, True))
+            'elements': expr.concat(arr_rest)
         })
 
     @embed_tokens
-    def object_():
-        expect("{")
+    async def object_():
+        expect('{')
         first = True
         has_non_const_keys = False
         is_pydict = S.scoped_flags.get('dict_literals', False)
         is_jshash = S.scoped_flags.get('hash_literals', False)
         a = []
-        while not is_("punc", "}"):
+        while not is_('punc', '}'):
             if not first:
-                expect(",")
-            if is_("punc", "}"):
+                expect(',')
+            if is_('punc', '}'):
                 # allow trailing comma
                 break
             first = False
@@ -1603,26 +1813,27 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             orig = ctx.expecting_object_literal_key
             ctx.expecting_object_literal_key = True
             try:
-                left = expression(False)
+                left = await expression(False)
             finally:
                 ctx.expecting_object_literal_key = orig
             if is_('keyword', 'for'):
                 # is_pydict is irrelevant here
-                return read_comprehension(new AST_SetComprehension({'statement':left}), '}')
+                return await read_comprehension(new AST_SetComprehension({'statement': left}), '}')
             if a.length is 0 and (is_('punc', ',') or is_('punc', '}')):
                 end = prev()
-                return set_(start, end, left)
+                return await set_(start, end, left)
             if not is_node_type(left, AST_Constant):
                 has_non_const_keys = True
-            expect(":")
+            expect(':')
+            obj_val = await expression(False)
             a.push(new AST_ObjectKeyVal({
                 'start': start,
                 'key': left,
-                'value': expression(False),
+                'value': obj_val,
                 'end': prev()
             }))
             if a.length is 1 and is_('keyword', 'for'):
-                return dict_comprehension(a, is_pydict, is_jshash)
+                return await dict_comprehension(a, is_pydict, is_jshash)
 
         next()
         return new (AST_ExpressiveObject if has_non_const_keys else AST_Object)({
@@ -1631,50 +1842,58 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'is_jshash': is_jshash,
         })
 
-    def set_(start, end, expr):
+    async def set_(start, end, expr):
         ostart = start
-        a = [new AST_SetItem({'start':start, 'end':end, 'value':expr})]
-        while not is_("punc", "}"):
-            expect(",")
+        a = [new AST_SetItem({'start': start, 'end': end, 'value': expr})]
+        while not is_('punc', '}'):
+            expect(',')
             start = S.token
-            if is_("punc", "}"):
+            if is_('punc', '}'):
                 # allow trailing comma
                 break
-            a.push(new AST_SetItem({'start':start, 'value':expression(False), 'end':prev()}))
+            set_val = await expression(False)
+            a.push(new AST_SetItem({'start': start, 'value': set_val, 'end': prev()}))
         next()
-        return new AST_Set({'items':a, 'start':ostart, 'end':prev()})
+        return new AST_Set({'items': a, 'start': ostart, 'end': prev()})
 
-    def read_comprehension(obj, terminator):
+    async def read_comprehension(obj, terminator):
         if is_node_type(obj, AST_GeneratorComprehension):
             baselib_items['yield'] = True
         S.in_comprehension = True
         S.in_parenthesized_expr = False  # in case we are already in a parenthesized expression
         expect_token('keyword', 'for')
-        forloop = for_(True)
+        forloop = await for_(True)
         obj.init = forloop.init
         obj.name = forloop.name
         obj.object = forloop.object
-        obj.condition = None if is_('punc', terminator) else (expect_token("keyword", "if"), expression(True))
+        if is_('punc', terminator):
+            obj.condition = None
+        else:
+            expect_token('keyword', 'if')
+            obj.condition = await expression(True)
         expect(terminator)
         S.in_comprehension = False
         return obj
 
-    def dict_comprehension(a, is_pydict, is_jshash):
+    async def dict_comprehension(a, is_pydict, is_jshash):
         if a.length:
             left, right = a[0].key, a[0].value
         else:
-            left = expression(False)
+            left = await expression(False)
             if not is_('punc', ':'):
-                return read_comprehension(new AST_SetComprehension({'statement':left}), '}')
+                return await read_comprehension(new AST_SetComprehension({'statement': left}), '}')
             expect(':')
-            right = expression(False)
-        return read_comprehension(new AST_DictComprehension({'statement':left, 'value_statement':right, 'is_pydict':is_pydict, 'is_jshash':is_jshash}), '}')
+            right = await expression(False)
+        return await read_comprehension(
+            new AST_DictComprehension({'statement': left, 'value_statement': right, 'is_pydict': is_pydict, 'is_jshash': is_jshash}),
+            '}'
+        )
 
     def as_name():
         tmp = S.token
         next()
         tmp_ = tmp.type
-        if tmp_ is "name" or tmp_ is "operator" or tmp_ is "keyword" or tmp_ is "atom":
+        if tmp_ is 'name' or tmp_ is 'operator' or tmp_ is 'keyword' or tmp_ is 'atom':
             return tmp.value
         else:
             unexpected()
@@ -1690,9 +1909,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         })
 
     def as_symbol(ttype, noerror):
-        if not is_("name"):
+        if not is_('name'):
             if not noerror:
-                croak("Name expected")
+                croak('Name expected')
             return None
 
         sym = token_as_symbol(S.token, ttype)
@@ -1714,49 +1933,50 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         else:
             return False
 
-    def getitem(expr, allow_calls):
+    async def getitem(expr, allow_calls):
         start = expr.start
         next()
         is_py_sub = S.scoped_flags.get('overload_getitem', False)
         slice_bounds = v'[]'
         is_slice = False
-        if is_("punc", ":"):
+        if is_('punc', ':'):
             # slice [:n]
             slice_bounds.push(None)
         else:
-            slice_bounds.push(expression(False))
+            slice_bounds.push(await expression(False))
 
-        if is_("punc", ":"):
+        if is_('punc', ':'):
             # slice [n:m?]
             is_slice = True
             next()
-            if is_("punc", ":"):
+            if is_('punc', ':'):
                 slice_bounds.push(None)
-            elif not is_("punc", "]"):
-                slice_bounds.push(expression(False))
+            elif not is_('punc', ']'):
+                slice_bounds.push(await expression(False))
 
-        if is_("punc", ":"):
+        if is_('punc', ':'):
             # slice [n:m:o?]
             next()
-            if is_("punc", "]"):
+            if is_('punc', ']'):
                 unexpected()
             else:
-                slice_bounds.push(expression(False))
+                slice_bounds.push(await expression(False))
 
-        expect("]")
+        expect(']')
 
         if is_slice:
-            if is_("operator", '='):
+            if is_('operator', '='):
                 # splice-assignment (arr[start:end] = ...)
                 next()  # swallow the assignment
-                return subscripts(new AST_Splice({
+                splice_assign = await expression(True)
+                return await subscripts(new AST_Splice({
                     'start': start,
                     'expression': expr,
                     'property': slice_bounds[0] or new AST_Number({
                         'value': 0
                     }),
                     'property2': slice_bounds[1],
-                    'assignment': expression(True),
+                    'assignment': splice_assign,
                     'end': prev()
                 }), allow_calls)
             elif slice_bounds.length is 3:
@@ -1768,34 +1988,46 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         slice_bounds.pop()
                 elif not slice_bounds[-2]:
                     slice_bounds[-2] = new AST_Undefined()
-                return subscripts(new AST_Call({
+                _ca = v'[]'
+                _ca.push(new AST_CallArg({'value': expr, 'is_array': False}))
+                for _sb in slice_bounds:
+                    _ca.push(new AST_CallArg({'value': _sb, 'is_array': False}))
+                return await subscripts(new AST_Call({
                     'start': start,
                     'expression': new AST_SymbolRef({
-                        'name': 'ρσ_delslice' if S.in_delete else "ρσ_eslice"
+                        'name': 'ρσ_delslice' if S.in_delete else 'ρσ_eslice'
                     }),
-                    'args': [expr].concat(slice_bounds),
+                    'args': new AST_CallArgs({'args': _ca, 'kwargs': v'[]', 'kwarg_items': v'[]', 'starargs': False}),
                     'end': prev()
                 }), allow_calls)
             else:
                 # regular slice (arr[start:end])
-                slice_bounds = [new AST_Number({'value':0}) if i is None else i for i in slice_bounds]
+                slice_bounds = [new AST_Number({'value': 0}) if i is None else i for i in slice_bounds]
                 if S.in_delete:
-                    return subscripts(new AST_Call({
+                    _ca2 = v'[]'
+                    _ca2.push(new AST_CallArg({'value': expr, 'is_array': False}))
+                    _ca2.push(new AST_CallArg({'value': new AST_Number({'value': 1}), 'is_array': False}))
+                    for _sb in slice_bounds:
+                        _ca2.push(new AST_CallArg({'value': _sb, 'is_array': False}))
+                    return await subscripts(new AST_Call({
                         'start': start,
                         'expression': new AST_SymbolRef({'name': 'ρσ_delslice'}),
-                        'args': [expr, new AST_Number({'value':1})].concat(slice_bounds),
+                        'args': new AST_CallArgs({'args': _ca2, 'kwargs': v'[]', 'kwarg_items': v'[]', 'starargs': False}),
                         'end': prev()
                     }), allow_calls)
 
-                return subscripts(new AST_Call({
+                _ca3 = v'[]'
+                for _sb in slice_bounds:
+                    _ca3.push(new AST_CallArg({'value': _sb, 'is_array': False}))
+                return await subscripts(new AST_Call({
                     'start': start,
                     'expression': new AST_Dot({
                         'start': start,
                         'expression': expr,
-                        'property': "slice",
+                        'property': 'slice',
                         'end': prev()
                     }),
-                    'args': slice_bounds,
+                    'args': new AST_CallArgs({'args': _ca3, 'kwargs': v'[]', 'kwarg_items': v'[]', 'starargs': False}),
                     'end': prev()
                 }), allow_calls)
         else:
@@ -1803,22 +2035,22 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if is_py_sub:
                 assignment = None
                 assign_operator = ''
-                if is_("operator") and ASSIGNMENT[S.token.value]:
+                if is_('operator') and ASSIGNMENT[S.token.value]:
                     assign_operator = S.token.value[:-1]
                     next()
-                    assignment = expression(True)
-                return subscripts(new AST_ItemAccess({
+                    assignment = await expression(True)
+                return await subscripts(new AST_ItemAccess({
                     'start': start,
                     'expression': expr,
                     'property': slice_bounds[0] or new AST_Number({
                         'value': 0
                     }),
-                    'assignment':assignment,
+                    'assignment': assignment,
                     'assign_operator': assign_operator,
                     'end': prev()
                 }), allow_calls)
 
-            return subscripts(new AST_Sub({
+            return await subscripts(new AST_Sub({
                 'start': start,
                 'expression': expr,
                 'property': slice_bounds[0] or new AST_Number({
@@ -1827,16 +2059,17 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'end': prev()
             }), allow_calls)
 
-    def call_(expr):
+    async def call_(expr):
         start = expr.start
         S.in_parenthesized_expr = True
         next()
         if not expr.parens and get_class_in_scope(expr):
             # this is an object being created using a class
-            ret = subscripts(new AST_New({
+            call_args = await func_call_list()
+            ret = await subscripts(new AST_New({
                 'start': start,
                 'expression': expr,
-                'args': func_call_list(),
+                'args': call_args,
                 'end': prev()
             }), True)
             S.in_parenthesized_expr = False
@@ -1849,52 +2082,55 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 # generate class call
                 funcname = expr
 
-                ret = subscripts(new AST_ClassCall({
+                classcall_args = await func_call_list()
+                ret = await subscripts(new AST_ClassCall({
                     'start': start,
-                    "class": expr.expression,
+                    'class': expr.expression,
                     'method': funcname.property,
-                    "static": is_static_method(c, funcname.property),
-                    'args': func_call_list(),
+                    'static': is_static_method(c, funcname.property),
+                    'args': classcall_args,
                     'end': prev()
                 }), True)
                 S.in_parenthesized_expr = False
                 return ret
             elif is_node_type(expr, AST_SymbolRef):
                 tmp_ = expr.name
-                if tmp_ is "jstype":
+                if tmp_ is 'jstype':
+                    jstype_args = await func_call_list()
                     ret = new AST_UnaryPrefix({
                         'start': start,
-                        'operator': "typeof",
-                        'expression': func_call_list()[0],
+                        'operator': 'typeof',
+                        'expression': jstype_args.args[0].value,
                         'end': prev()
                     })
                     S.in_parenthesized_expr = False
                     return ret
-                elif tmp_ is "isinstance":
-                    args = func_call_list()
-                    if args.length is not 2:
+                elif tmp_ is 'isinstance':
+                    args = await func_call_list()
+                    if args.args.length is not 2:
                         croak('isinstance() must be called with exactly two arguments')
                     ret = new AST_Binary({
                         'start': start,
-                        'left': args[0],
+                        'left': args.args[0].value,
                         'operator': 'instanceof',
-                        'right': args[1],
+                        'right': args.args[1].value,
                         'end': prev()
                     })
                     S.in_parenthesized_expr = False
                     return ret
 
             # fall-through to basic function call
-            ret = subscripts(new AST_Call({
+            basic_args = await func_call_list()
+            ret = await subscripts(new AST_Call({
                 'start': start,
                 'expression': expr,
-                'args': func_call_list(),
+                'args': basic_args,
                 'end': prev()
             }), True)
             S.in_parenthesized_expr = False
             return ret
 
-    def get_attr(expr, allow_calls):
+    async def get_attr(expr, allow_calls):
         next()
         prop = as_name()
         c = get_class_in_scope(expr)
@@ -1903,15 +2139,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if classvars and v'classvars[prop]':
                 prop = 'prototype.' + prop
 
-        return subscripts(new AST_Dot({
+        return await subscripts(new AST_Dot({
             'start': expr.start,
             'expression': expr,
             'property': prop,
             'end': prev()
         }), allow_calls)
 
-    def existential(expr, allow_calls):
-        ans = new AST_Existential({'start':expr.start, 'end':S.token, 'expression':expr})
+    async def existential(expr, allow_calls):
+        ans = new AST_Existential({'start': expr.start, 'end': S.token, 'expression': expr})
         next()
         ttype = S.token.type
         val = S.token.value
@@ -1931,49 +2167,49 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             else:
                 ans.after = None
                 return ans
-            return subscripts(ans, allow_calls)
+            return await subscripts(ans, allow_calls)
 
-        ans.after = expression()
+        ans.after = await expression()
         return ans
 
-    def subscripts(expr, allow_calls):
-        if is_("punc", "."):
-            return get_attr(expr, allow_calls)
+    async def subscripts(expr, allow_calls):
+        if is_('punc', '.'):
+            return await get_attr(expr, allow_calls)
 
-        if is_("punc", "[") and not S.token.nlb:
-            return getitem(expr, allow_calls)
+        if is_('punc', '[') and not S.token.nlb:
+            return await getitem(expr, allow_calls)
 
-        if allow_calls and is_("punc", "(") and not S.token.nlb:
-            return call_(expr)
+        if allow_calls and is_('punc', '(') and not S.token.nlb:
+            return await call_(expr)
 
         if is_('punc', '?'):
-            return existential(expr, allow_calls)
+            return await existential(expr, allow_calls)
 
         return expr
 
-    def maybe_unary(allow_calls):
+    async def maybe_unary(allow_calls):
         start = S.token
         if is_('operator', '@'):
             if S.parsing_decorator:
                 croak('Nested decorators are not allowed')
             next()
             S.parsing_decorator = True
-            expr = expression()
+            expr = await expression()
             S.parsing_decorator = False
             S.decorators.push(expr)
-            return new AST_EmptyStatement({'stype':'@', 'start':prev(), 'end':prev()})
-        if is_("operator") and UNARY_PREFIX[start.value]:
+            return new AST_EmptyStatement({'stype': '@', 'start': prev(), 'end': prev()})
+        if is_('operator') and UNARY_PREFIX[start.value]:
             next()
             is_parenthesized = is_('punc', '(')
             S.in_delete = start.value is 'delete'
-            expr = maybe_unary(allow_calls)
+            expr = await maybe_unary(allow_calls)
             S.in_delete = False
             ex = make_unary(AST_UnaryPrefix, start.value, expr, is_parenthesized)
             ex.start = start
             ex.end = prev()
             return ex
 
-        val = expr_atom(allow_calls)
+        val = await expr_atom(allow_calls)
         return val
 
     def make_unary(ctor, op, expr, is_parenthesized):
@@ -1983,19 +2219,19 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'parenthesized': is_parenthesized
         })
 
-    def expr_op(left, min_prec, no_in):
+    async def expr_op(left, min_prec, no_in):
         op = S.token.value if is_('operator') else None
-        if op is "!" and peek().type is "operator" and peek().value is "in":
+        if op is '!' and peek().type is 'operator' and peek().value is 'in':
             next()
             S.token.value = op = 'nin'
 
-        if no_in and (op is "in" or op is 'nin'):
+        if no_in and (op is 'in' or op is 'nin'):
             op = None
 
         prec = PRECEDENCE[op] if op is not None else None
         if prec is not None and prec > min_prec:
             next()
-            right = expr_op(maybe_unary(True), prec, no_in)
+            right = await expr_op(await maybe_unary(True), prec, no_in)
             ret = new AST_Binary({
                 'start': left.start,
                 'left': left,
@@ -2003,24 +2239,25 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'right': right,
                 'end': right.end
             })
-            return expr_op(ret, min_prec, no_in)
+            return await expr_op(ret, min_prec, no_in)
         return left
 
-    def expr_ops(no_in):
-        return expr_op(maybe_unary(True), 0, no_in)
+    async def expr_ops(no_in):
+        return await expr_op(await maybe_unary(True), 0, no_in)
 
-    def maybe_conditional(no_in):
+    async def maybe_conditional(no_in):
         start = S.token
-        expr = expr_ops(no_in)
+        expr = await expr_ops(no_in)
         if (is_('keyword', 'if') and (S.in_parenthesized_expr or (S.statement_starting_token is not S.token and not S.in_comprehension and not S.token.nlb))):
             next()
-            ne = expression(False)
+            ne = await expression(False)
             expect_token('keyword', 'else')
+            cond_alt = await expression(False, no_in)
             conditional = new AST_Conditional({
                 'start': start,
                 'condition': ne,
                 'consequent': expr,
-                'alternative': expression(False, no_in),
+                'alternative': cond_alt,
                 'end': peek()
             })
             return conditional
@@ -2044,28 +2281,30 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         c.provisional_classvars[ans.left.name] = True
         return ans
 
-    def maybe_assign(no_in, only_plain_assignment):
+    async def maybe_assign(no_in, only_plain_assignment):
         start = S.token
-        left = maybe_conditional(no_in)
+        left = await maybe_conditional(no_in)
         val = S.token.value
-        if is_("operator") and ASSIGNMENT[val]:
+        if is_('operator') and ASSIGNMENT[val]:
             if only_plain_assignment and val is not '=':
                 croak('Invalid assignment operator for chained assignment: ' + val)
             next()
+            assign_right = await maybe_assign(no_in, True)
             return create_assign({
                 'start': start,
                 'left': left,
                 'operator': val,
-                'right': maybe_assign(no_in, True),
+                'right': assign_right,
                 'end': prev()
             })
         return left
 
-    def expression(commas, no_in):
+    async def expression(commas, no_in):
         # if there is an assignment, we want the sequences to pivot
         # around it to allow for tuple packing/unpacking
         start = S.token
-        expr = maybe_assign(no_in)
+        expr = await maybe_assign(no_in)
+
         def build_seq(a):
             if a.length is 1:
                 return a[0]
@@ -2078,10 +2317,11 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             })
         if commas:
             left = v'[ expr ]'
-            while is_("punc", ",") and not peek().nlb:
+            while is_('punc', ',') and (not peek().nlb or S.in_parenthesized_expr):
                 next()
                 if is_node_type(expr, AST_Assign):
                     left[-1] = left[-1].left
+                    seq_cdr = await expression(True, no_in)
                     return create_assign({
                         'start': start,
                         'left': (left[0] if left.length is 1 else new AST_Array({
@@ -2090,12 +2330,12 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         'operator': expr.operator,
                         'right': new AST_Seq({
                             'car': expr.right,
-                            'cdr': expression(True, no_in)
+                            'cdr': seq_cdr
                         }),
                         'end': peek()
                     })
 
-                expr = maybe_assign(no_in)
+                expr = await maybe_assign(no_in)
                 left.push(expr)
 
             # if last one was an assignment, fix it
@@ -2114,20 +2354,30 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             return build_seq(left)
         return expr
 
-    def in_loop(cont):
+    async def in_loop(cont):
         S.in_loop += 1
-        ret = cont()
+        ret = await cont()
         S.in_loop -= 1
         return ret
 
-    def run_parser():
+    async def run_parser():
         start = S.token = next()
         body = v'[]'
         docstrings = v'[]'
         first_token = True
         toplevel = options.toplevel
-        while not is_("eof"):
-            element = statement()
+        while not is_('eof'):
+            if options.recover_errors:
+                try:
+                    element = await statement()
+                except as rerr:
+                    if is_recoverable_error(rerr):
+                        record_recovery(rerr)
+                        await skip_to_recovery_point()
+                        continue
+                    raise
+            else:
+                element = await statement()
             if first_token and is_node_type(element, AST_Directive) and element.value.indexOf('#!') is 0:
                 shebang = element.value
             else:
@@ -2152,6 +2402,18 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'docstrings': docstrings,
             })
 
+        if options.recover_errors:
+            # Combine parser-level recovered errors with any collected by the
+            # tokenizer (e.g. skipped unexpected characters) for the LSP to report.
+            recovered = S.recovered_errors
+            try:
+                tok_ctx = S.input.context()
+                if tok_ctx and tok_ctx.recovered_errors and tok_ctx.recovered_errors.length:
+                    recovered = recovered.concat(tok_ctx.recovered_errors)
+            except:
+                pass
+            toplevel.recovered_errors = recovered
+
         toplevel.nonlocalvars = scan_for_nonlocal_defs(toplevel.body).concat(S.globals)
         toplevel.localvars = []
         toplevel.exports = []
@@ -2182,21 +2444,102 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         toplevel.scoped_flags = S.scoped_flags.stack[0]
         importing_modules[module_id] = False
         toplevel.comments_after = S.token.comments_before or v'[]'
+
+        if options._src_hash and options.filename and module_id is not '__main__':
+            try:
+                # Simplified class info (no full AST nodes) for lazy-shell reconstruction.
+                classes_cache = {}
+                for cname in Object.keys(toplevel.classes or {}):
+                    cls = toplevel.classes[cname]
+                    classes_cache[cname] = {'static': cls.static or {}, 'bound': cls.bound or [], 'classvars': cls.classvars or {}}
+                # Single pass: compute has_side_effects + tree-shaking index.
+                # The index lets the tree-shaker resolve the fixpoint iteration
+                # (steps 4-5) without accessing mod.body — no AST deserialization
+                # until step 6 (pruning), which needs actual nodes anyway.
+                has_se = False
+                idx_exec_refs_set = {}
+                idx_import_bindings = {}
+                idx_top_defs = {}
+
+                def _collect_refs_into(node, refs_set):  # noqa
+                    def _rv(n):  # noqa
+                        if is_node_type(n, AST_SymbolRef):
+                            refs_set[n.name] = True
+                    node.walk(new TreeWalker(_rv))
+
+                for idx_stmt in toplevel.body:
+                    if is_node_type(idx_stmt, AST_Function) or is_node_type(idx_stmt, AST_Class):
+                        if idx_stmt.name:
+                            idx_pinned = False
+                            if idx_stmt.decorators:
+                                for idx_dec in idx_stmt.decorators:
+                                    if idx_dec.expression and is_node_type(idx_dec.expression, AST_SymbolRef) and idx_dec.expression.name == 'no_prune':
+                                        idx_pinned = True
+                                        break
+                            idx_def_refs = {}
+                            _collect_refs_into(idx_stmt, idx_def_refs)
+                            idx_top_defs[idx_stmt.name.name] = {'pinned': idx_pinned, 'refs': Object.keys(idx_def_refs)}
+                    elif is_node_type(idx_stmt, AST_Imports):
+                        for idx_imp in idx_stmt.imports:
+                            if idx_imp.argnames and idx_imp.argnames.length:
+                                for idx_arg in idx_imp.argnames:
+                                    idx_local = idx_arg.alias.name if idx_arg.alias else idx_arg.name
+                                    idx_import_bindings[idx_local] = {'source': idx_imp.key, 'imported_name': idx_arg.name, 'is_namespace': False}
+                            else:
+                                idx_local = idx_imp.alias.name if idx_imp.alias else idx_imp.key.split('.')[0]
+                                idx_import_bindings[idx_local] = {'source': idx_imp.key, 'imported_name': None, 'is_namespace': True}
+                    else:
+                        if not has_se:
+                            se_found = v'[false]'
+
+                            def se_check(se_node, se_descend):  # noqa
+                                if is_node_type(se_node, AST_Call):
+                                    se_found[0] = True
+                                    return True
+                            idx_stmt.walk(new TreeWalker(se_check))
+                            if se_found[0]:
+                                has_se = True
+                        _collect_refs_into(idx_stmt, idx_exec_refs_set)
+
+                cache_meta = {
+                    'v': CACHE_VERSION,
+                    'signature': options._src_hash,
+                    'compiler_version': COMPILER_VERSION,
+                    'imported_module_ids': toplevel.imported_module_ids,
+                    'exported_names': [sym.name for sym in toplevel.exports],
+                    'baselib': toplevel.baselib or {},
+                    'classes': classes_cache,
+                    'has_side_effects': has_se,
+                    'index': {
+                        'exec_refs': Object.keys(idx_exec_refs_set),
+                        'import_bindings': idx_import_bindings,
+                        'top_defs': idx_top_defs,
+                    },
+                }
+                if options._file_mtime is not None and options._file_mtime is not undefined:
+                    cache_meta['mtime'] = options._file_mtime
+                # Binary format: metadata JSON followed by newline and binary AST pool.
+                await writefile(cache_file_name(options.filename, options.module_cache_dir), encode_cache(JSON.stringify(cache_meta), toplevel))
+            except as e:
+                console.error('Failed to write AST cache file:', options.filename, 'with error:', e)
+
         return toplevel
 
     return run_parser
 
-def parse(text, options):
+
+async def parse(text, options):
     options = defaults(options, {
-        'filename': None,     # name of the file being parsed
-        'module_id':'__main__', # The id of the module being parsed
+        'filename': None,  # name of the file being parsed
+        'module_id': '__main__',  # The id of the module being parsed
         'toplevel': None,
-        'for_linting': False, # If True certain actions are not performed, such as importing modules
+        'for_linting': False,  # If True certain actions are not performed, such as importing modules
         'import_dirs': v'[]',
-        'classes': undefined, # Map of class names to AST_Class that are available in the global namespace (used by the REPL)
-        'scoped_flags': {},   # Global scoped flags (used by the REPL)
+        'classes': undefined,  # Map of class names to AST_Class that are available in the global namespace (used by the REPL)
+        'scoped_flags': {},  # Global scoped flags (used by the REPL)
         'discard_asserts': False,
         'module_cache_dir': '',
+        'recover_errors': False,  # When True, continue parsing past syntax errors (used by the LSP). See run_parser/block_.
     })
     import_dirs = [x for x in options.import_dirs]
     for location in v'[options.libdir, options.basedir]':
@@ -2207,11 +2550,16 @@ def parse(text, options):
     imported_module_ids = []
     imported_modules = options.imported_modules or {}
     importing_modules = options.importing_modules or {}
+    # loading_promises tracks modules currently being loaded across the entire compilation.
+    # It is shared by passing it through options so parallel sub-parse() calls all see it.
+    if not options.loading_promises:
+        options.loading_promises = {}
     importing_modules[module_id] = True
 
     # The internal state of the parser
     S = {
-        'input': tokenizer(text, options.filename) if jstype(text) is 'string' else text,
+        'input': tokenizer(text, options.filename, options.recover_errors) if jstype(text) is 'string' else text,
+        'recovered_errors': v'[]',  # syntax errors swallowed during recovery (recover_errors mode only)
         'token': None,
         'prev': None,
         'peeked': [],
@@ -2221,9 +2569,9 @@ def parse(text, options):
         'in_parenthesized_expr': False,
         'in_delete': False,
         'in_loop': 0,
-        'in_class': [ False ],
-        'classes': [ {} ],
-        'functions': [ {} ],
+        'in_class': [False],
+        'classes': [{}],
+        'functions': [{}],
         'labels': [],
         'decorators': v'[]',
         'parsing_decorator': False,
@@ -2247,6 +2595,9 @@ def parse(text, options):
     if options.classes:
         for cname in options.classes:
             obj = options.classes[cname]
-            S.classes[0][cname] = { 'static':obj.static, 'bound':obj.bound, 'classvars': obj.classvars }
+            S.classes[0][cname] = {'static': obj.static, 'bound': obj.bound, 'classvars': obj.classvars}
+
+    if jstype(text) is 'string' and not options._src_hash:
+        options._src_hash = sha1sum(text)
 
-    return create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_ids, imported_modules, importing_modules, options)()
+    return await create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_ids, imported_modules, importing_modules, options)()