rapydscript-ng 0.7.23 → 0.8.0

package/src/parse.pyj

@@ -1,13 +1,13 @@
 # vim:fileencoding=utf-8
 # License: BSD Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-# globals: readfile
+# globals: readfile, writefile, stat_file, sha1sum, ast_from_json, ast_to_json, make_lazy_ast_module, decode_cache, encode_cache
 from __python__ import hash_literals
 
 from utils import make_predicate, array_to_hash, defaults, has_prop, cache_file_name
 from errors import SyntaxError, ImportError
 from ast import (
-AST_Array, AST_Assign, AST_Binary, AST_BlockStatement, AST_Break,
-AST_Call, AST_Catch, AST_Class, AST_ClassCall, AST_Conditional,
+AST_ArgsDef, AST_Array, AST_Assign, AST_Binary, AST_BlockStatement, AST_Break,
+AST_Call, AST_CallArg, AST_CallArgs, AST_Catch, AST_Class, AST_ClassCall, AST_Conditional,
 AST_Constant, AST_Continue, AST_DWLoop, AST_Debugger, AST_Decorator,
 AST_Definitions, AST_DictComprehension, AST_Directive, AST_Do, AST_Dot,
 AST_Else, AST_EmptyStatement, AST_Except, AST_ExpressiveObject, AST_False, AST_Finally,
@@ -21,13 +21,14 @@ AST_SymbolCatch, AST_SymbolDefun, AST_SymbolFunarg,
 AST_SymbolLambda, AST_SymbolNonlocal, AST_SymbolRef, AST_SymbolVar, AST_This,
 AST_Throw, AST_Toplevel, AST_True, AST_Try, AST_UnaryPrefix,
 AST_Undefined, AST_Var, AST_VarDef, AST_Verbatim, AST_While, AST_With, AST_WithClause,
-AST_Yield, AST_Assert, AST_Existential, is_node_type
+AST_Yield, AST_Await, AST_Assert, AST_Existential, is_node_type, TreeWalker
 )
 from tokenizer import tokenizer, is_token, RESERVED_WORDS
 
 
 COMPILER_VERSION = '__COMPILER_VERSION__'
 PYTHON_FLAGS = {'dict_literals':True, 'overload_getitem':True, 'bound_methods':True, 'hash_literals':True}
+CACHE_VERSION = 1
 
 
 def get_compiler_version():
@@ -208,6 +209,41 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             token = S.token
         token_error(token, "Unexpected token: " + token.type + " «" + token.value + "»")
 
+    # --- Error recovery (only active when options.recover_errors is True) --------
+    # These helpers let the LSP obtain a usable AST from a file that contains
+    # syntax errors. They are never invoked on the normal compile path, so they
+    # cannot introduce a performance or behavior regression there.
+
+    def is_recoverable_error(e):
+        return v'e instanceof SyntaxError'
+
+    def record_recovery(e):
+        S.recovered_errors.push({
+            'message': e.message, 'line': e.line, 'col': e.col, 'pos': e.pos,
+            'endpos': e.endpos, 'is_eof': e.is_eof,
+        })
+
+    async def skip_to_recovery_point():
+        # Resynchronize after a syntax error: always consume at least one token
+        # (guaranteeing forward progress) then stop at the start of the next
+        # logical line, a block dedent, or eof. next() may itself raise a
+        # tokenizer level SyntaxError (e.g. an unterminated string on the skipped
+        # line); such errors are recorded and skipping continues. The tokenizer
+        # advances its position for every such error (and never raises for an
+        # unexpected character in recover mode), so this loop always terminates.
+        while not is_("eof"):
+            try:
+                next()
+            except as e:
+                if is_recoverable_error(e):
+                    record_recovery(e)
+                    if is_("eof"):
+                        return
+                    continue
+                raise
+            if is_("eof") or is_("punc", "}") or S.token.nlb:
+                return
+
     def expect_token(type, val):
         if is_(type, val):
             return next()
@@ -223,9 +259,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             S.token.nlb = True
 
     def embed_tokens(parser):
-        def with_embedded_tokens():
+        async def with_embedded_tokens():
             start = S.token
-            expr = parser()
+            expr = await parser()
             if expr is undefined:
                 unexpected()
             end = prev()
@@ -404,7 +440,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
         return vars
 
-    def return_():
+    async def return_():
         if is_('punc', ';'):
             semicolon()
             value = None
@@ -413,12 +449,12 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if is_end_of_statement:
                 value = None
             else:
-                value = expression(True)
+                value = await expression(True)
                 semicolon()
         return value
 
     @embed_tokens
-    def statement():
+    async def statement():
         # From Kovid: The next three lines were a hack to try to support statements
         # starting with a regexp literal. However, it did not work, for example:
         # echo 'f=1\n/asd/.test()' | rs -> parse error
@@ -434,7 +470,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         if p and not S.token.nlb and ATOMIC_START_TOKEN[p.type] and not is_('punc', ':'):
             unexpected()
         if tmp_ is "string":
-            return simple_statement()
+            return await simple_statement()
         elif tmp_ is "shebang":
             tmp_ = S.token.value
             next()
@@ -442,17 +478,18 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'value': tmp_
             })
         elif tmp_ is "num" or tmp_ is "regexp" or tmp_ is "operator" or tmp_ is "atom" or tmp_ is "js":
-            return simple_statement()
+            return await simple_statement()
         elif tmp_ is "punc":
             tmp_ = S.token.value
             if tmp_ is ":":
+                bs_body = await block_()
                 return new AST_BlockStatement({
                     'start': S.token,
-                    'body': block_(),
+                    'body': bs_body,
                     'end': prev()
                 })
             elif tmp_ is "{" or tmp_ is "[" or tmp_ is "(":
-                return simple_statement()
+                return await simple_statement()
             elif tmp_ is ";":
                 next()
                 return new AST_EmptyStatement({'stype':';', 'start':prev(), 'end':prev()})
@@ -460,7 +497,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 unexpected()
         elif tmp_ is "name":
             if (is_token(peek(), 'punc', ':')) token_error(peek(), 'invalid syntax, colon not allowed here')
-            return simple_statement()
+            return await simple_statement()
         elif tmp_ is "keyword":
             tmp_ = S.token.value
             next()
@@ -472,44 +509,64 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 semicolon()
                 return new AST_Debugger()
             elif tmp_ is "do":
+                do_body = await in_loop(statement)
+                do_cond = await (async def():
+                    expect(".")
+                    expect_token("keyword", "while")
+                    tmp = await expression(True)
+                    if is_node_type(tmp, AST_Assign):
+                        croak('Assignments in do loop conditions are not allowed')
+                    semicolon()
+                    return tmp
+                )()
                 return new AST_Do({
-                    'body': in_loop(statement),
-                    'condition': (def():
-                        expect(".")
-                        expect_token("keyword", "while")
-                        tmp = expression(True)
-                        if is_node_type(tmp, AST_Assign):
-                            croak('Assignments in do loop conditions are not allowed')
-                        semicolon()
-                        return tmp
-                    )()
+                    'body': do_body,
+                    'condition': do_cond,
                 })
             elif tmp_ is "while":
-                while_cond = expression(True)
+                while_cond = await expression(True)
                 if is_node_type(while_cond, AST_Assign):
                     croak('Assignments in while loop conditions are not allowed')
                 if not is_('punc', ':'):
                     croak('Expected a colon after the while statement')
+                while_body = await in_loop(statement)
                 return new AST_While({
                     'condition': while_cond,
-                    'body': in_loop(statement)
+                    'body': while_body
                 })
             elif tmp_ is "for":
                 if is_('js'):
-                    return for_js()
-                return for_()
+                    return await for_js()
+                return await for_()
             elif tmp_ is "from":
-                return import_(True)
+                return await import_(True)
             elif tmp_ is "import":
-                return import_(False)
+                return await import_(False)
             elif tmp_ is "class":
-                return class_()
+                return await class_()
+            elif tmp_ is "async":
+                if not is_('keyword', 'def'):
+                    croak("Expected 'def' after 'async'")
+                next()
+                start = prev()
+                func = await function_(S.in_class[-1], False, True)
+                func.start = start
+                func.end = prev()
+                chain = await subscripts(func, True)
+                if chain is func:
+                    return func
+                else:
+                    return new AST_SimpleStatement({
+                        'start': start,
+                        'body': chain,
+                        'end': prev()
+                    })
             elif tmp_ is "def":
                 start = prev()
-                func = function_(S.in_class[-1], False)
+                func = await function_(S.in_class[-1], False)
                 func.start = start
                 func.end = prev()
-                chain = subscripts(func, True)
+                chain = await subscripts(func, True)
                 if chain is func:
                     return func
                 else:
@@ -520,14 +577,14 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     })
             elif tmp_ is 'assert':
                 start = prev()
-                cond = expression(False)
+                cond = await expression(False)
                 msg = None
                 if is_('punc', ','):
                     next()
-                    msg = expression(False)
+                    msg = await expression(False)
                 return new AST_Assert({'start': start, 'condition':cond, 'message':msg, 'end':prev()})
             elif tmp_ is "if":
-                return if_()
+                return await if_()
             elif tmp_ is "pass":
                 semicolon()
                 return new AST_EmptyStatement({'stype':'pass', 'start':prev(), 'end':prev()})
@@ -538,9 +595,17 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     croak("'return' not allowed in a function with yield")
                 S.functions[-1].is_generator = False
 
-                return new AST_Return({'value':return_()})
+                ret_val = await return_()
+                return new AST_Return({'value':ret_val})
+            elif tmp_ is "await":
+                if S.in_function is 0 or not S.functions[-1].is_async:
+                    croak("'await' outside of async function")
+                value = await expr_atom(True)
+                node = new AST_Await({'value': value})
+                semicolon()
+                return new AST_SimpleStatement({'body': node})
             elif tmp_ is "yield":
-                return yield_()
+                return await yield_()
             elif tmp_ is "raise":
                 if S.token.nlb:
                     return new AST_Throw({
@@ -549,33 +614,33 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         })
                     })
 
-                tmp = expression(True)
+                tmp = await expression(True)
                 semicolon()
                 return new AST_Throw({
                     'value': tmp
                 })
             elif tmp_ is "try":
-                return try_()
+                return await try_()
             elif tmp_ is "nonlocal":
-                tmp = nonlocal_()
+                tmp = await nonlocal_()
                 semicolon()
                 return tmp
             elif tmp_ is 'global':
-                tmp = nonlocal_(True)
+                tmp = await nonlocal_(True)
                 semicolon()
                 return tmp
             elif tmp_ is "with":
-                return with_()
+                return await with_()
             else:
                 unexpected()
 
-    def with_():
+    async def with_():
         clauses = v'[]'
         start = S.token
         while True:
             if is_('eof'):
                 unexpected()
-            expr = expression()
+            expr = await expression()
             alias = None
             if is_('keyword', 'as'):
                 next()
@@ -590,15 +655,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
         if not clauses.length:
             token_error(start, 'with statement must have at least one clause')
-        body = statement()
+        body = await statement()
 
         return new AST_With({
             'clauses': clauses,
             'body': body
         })
 
-    def simple_statement(tmp):
-        tmp = expression(True)
+    async def simple_statement(tmp):
+        tmp = await expression(True)
         semicolon()
         return new AST_SimpleStatement({
             'body': tmp
@@ -610,22 +675,25 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         semicolon()
         return new t()
 
-    def yield_():
+    async def yield_():
         if S.in_function is 0:
             croak("'yield' outside of function")
+        if S.functions[-1].is_async:
+            croak("'yield' not allowed in an async function")
         if S.functions[-1].is_generator is False:
             croak("'yield' not allowed in a function with return")
         S.functions[-1].is_generator = True
         is_yield_from = is_('keyword', 'from')
         if is_yield_from:
             next()
-        return new AST_Yield({'is_yield_from':is_yield_from, 'value': return_()})
+        yld_val = await return_()
+        return new AST_Yield({'is_yield_from':is_yield_from, 'value': yld_val})
 
-    def for_(list_comp):
+    async def for_(list_comp):
         #        expect("(")
         init = None
         if not is_("punc", ";"):
-            init = expression(True, True)
+            init = await expression(True, True)
             # standardize AST_Seq into array now for consistency
             if is_node_type(init, AST_Seq):
                 if is_node_type(init.car, AST_SymbolRef) and is_node_type(init.cdr, AST_SymbolRef):
@@ -643,13 +711,13 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 if is_node_type(init, AST_Var) and init.definitions.length > 1:
                     croak("Only one variable declaration allowed in for..in loop")
                 next()
-                return for_in(init, list_comp)
+                return await for_in(init, list_comp)
 
         unexpected()
 
-    def for_in(init, list_comp):
+    async def for_in(init, list_comp):
         lhs = init.definitions[0].name if is_node_type(init, AST_Var) else None
-        obj = expression(True)
+        obj = await expression(True)
         #        expect(")")
         if list_comp:
             return {
@@ -658,19 +726,21 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'object': obj
             }
 
+        fi_body = await in_loop(statement)
         return new AST_ForIn({
             'init': init,
             'name': lhs,
             'object': obj,
-            'body': in_loop(statement)
+            'body': fi_body
         })
 
     # A native JavaScript for loop - for v"var i=0; i<5000; i++":
-    def for_js():
+    async def for_js():
         condition = as_atom_node()
+        fj_body = await in_loop(statement)
         return new AST_ForJS({
             'condition': condition,
-            'body': in_loop(statement)
+            'body': fj_body
         })
 
     # scan function/class body for nested class declarations
@@ -712,72 +782,144 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         ctx = S.input.context()
         raise new ImportError(message, ctx.filename, ctx.tokline, ctx.tokcol, ctx.tokpos)
 
-    def do_import(key):
+    async def do_import(key):
         if has_prop(imported_modules, key):
             return
+        # Circular import check must come BEFORE loading_promises so we detect cycles
+        # even when the same module is being loaded by a parallel chain.
         if has_prop(importing_modules, key) and importing_modules[key]:
             import_error('Detected a recursive import of: ' + key + ' while importing: ' + module_id)
+        # If a parallel chain is already loading this module, wait for it instead of
+        # starting a duplicate load.  loading_promises is shared across the entire
+        # compilation via options.loading_promises.
+        loading_promises = options.loading_promises
+        if has_prop(loading_promises, key):
+            await loading_promises[key]
+            return
 
-        # Ensure that the package containing this module is also imported
-        package_module_id = key.split('.')[:-1].join('.')
-        if len(package_module_id) > 0:
-            do_import(package_module_id)
+        # Register this load BEFORE the first await so that any parallel caller
+        # that reaches this point sees the in-progress entry and waits.
+        # Use a single-element list to capture the resolve function from the Promise
+        # executor without needing nonlocal (mutating a container, not rebinding).
+        _rfn = v'[null]'
+        def _capture_resolve(resolve):
+            _rfn[0] = resolve
+        loading_promises[key] = new Promise(_capture_resolve)
 
-        if options.for_linting:
-            imported_modules[key] = {'is_cached':True, 'classes':{}, 'module_id':key, 'exports':[],
-                                     'nonlocalvars':[], 'baselib':{}, 'outputs':{}, 'discard_asserts':options.discard_asserts}
-            return
+        try:
+            # Ensure that the package containing this module is also imported
+            package_module_id = key.split('.')[:-1].join('.')
+            if len(package_module_id) > 0:
+                await do_import(package_module_id)
+
+            if options.for_linting:
+                imported_modules[key] = {'is_cached':True, 'classes':{}, 'module_id':key, 'exports':[],
+                                         'nonlocalvars':[], 'baselib':{}, 'outputs':{}, 'discard_asserts':options.discard_asserts}
+                return
 
-        def safe_read(base_path):
-            for i, path in enumerate([base_path + '.pyj', base_path + '/__init__.pyj']):
-                try:
-                    return [readfile(path, "utf-8"), path]  # noqa:undef
-                except as e:
-                    if e.code is 'ENOENT' or e.code is 'EPERM' or e.code is 'EACCESS':
+            # stat_file returns {mtimeMs, content?}.  For real filesystems mtimeMs is a
+            # number and content is absent; for virtual filesystems content holds the
+            # already-read source to avoid a double read.
+            async def safe_stat(base_path):  # noqa
+                for i, path in enumerate([base_path + '.pyj', base_path + '/__init__.pyj']):
+                    try:
+                        st = await stat_file(path)
+                        return [st.mtimeMs, path, st.content if has_prop(st, 'content') else None]
+                    except as e:
+                        if e.code is 'ENOENT' or e.code is 'EPERM' or e.code is 'EACCESS':
+                            if i is 1:
+                                return None, None, None
                         if i is 1:
-                            return None, None
-                    if i is 1:
-                        raise
+                            raise
 
-        src_code = filename = None
-        modpath = key.replace(/\./g, '/')
+            file_mtime = filename = prefetched_src = None
+            modpath = key.replace(/\./g, '/')
 
-        for location in import_dirs:
-            if location:
-                data, filename = safe_read(location + '/' + modpath)
-                if data is not None:
-                    src_code = data
-                    break
-        if src_code is None:
-            import_error("Failed Import: '" + key + "' module doesn't exist in any of the import directories: " + import_dirs.join(':'))
+            for location in import_dirs:
+                if location:
+                    file_mtime, filename, prefetched_src = await safe_stat(location + '/' + modpath)
+                    if filename is not None:
+                        break
+            if filename is None:
+                import_error("Failed Import: '" + key + "' module doesn't exist in any of the import directories: " + import_dirs.join(':'))
 
-        try:
-            cached = JSON.parse(readfile(cache_file_name(filename, options.module_cache_dir), 'utf-8'))
-        except:
-            cached = None
+            # Read the cache file.  I/O starts here; any CPU work done before the
+            # await can overlap with the disk read.
+            cache_read = readfile(cache_file_name(filename, options.module_cache_dir))
 
-        srchash = sha1sum(src_code)  # noqa:undef
-        if cached and cached.version is COMPILER_VERSION and cached.signature is srchash and cached.discard_asserts is v'!!options.discard_asserts':
-            for ikey in cached.imported_module_ids:
-                do_import(ikey)  # Ensure all modules imported by the cached module are also imported
-            imported_modules[key] = {
-                'is_cached':True, 'classes':cached.classes, 'outputs':cached.outputs, 'module_id':key, 'import_order':Object.keys(imported_modules).length,
-                'nonlocalvars':cached.nonlocalvars, 'baselib':cached.baselib, 'exports':cached.exports, 'discard_asserts':options.discard_asserts,
-                'imported_module_ids':cached.imported_module_ids,
-            }
-        else:
-            parse(src_code, {
-                'filename': filename,
-                'toplevel': None,
-                'basedir': options.basedir,
-                'libdir': options.libdir,
-                'import_dirs': options.import_dirs,
-                'module_id': key,
-                'imported_modules': imported_modules,
-                'importing_modules': importing_modules,
-                'discard_asserts': options.discard_asserts,
-                'module_cache_dir': options.module_cache_dir,
-            })  # This function will add the module to imported_modules itself
+            cached = None
+            ast_json_str = None
+            try:
+                _split = decode_cache(await cache_read)
+                if _split is not None:
+                    cached = JSON.parse(_split.meta)
+                    ast_json_str = _split.ast
+            except:
+                cached = None
+
+            srchash = src_code = None
+            cache_ok = cached and cached.v is CACHE_VERSION and cached.compiler_version is COMPILER_VERSION
+            if cache_ok and file_mtime is not None and cached.mtime is not None and cached.mtime is file_mtime:
+                # Fast path: mtime matches — source file is unchanged, skip reading and hashing it
+                srchash = cached.signature
+            else:
+                # Need source content to validate with sha1sum (or to re-parse on miss)
+                src_code = prefetched_src or await readfile(filename, 'utf-8')
+                srchash = sha1sum(src_code)
+                if not cache_ok or cached.signature is not srchash:
+                    cached = None
+
+            if cached is not None:
+                # Load all deps in parallel — cached module deps have been pre-validated as
+                # non-circular so Promise.all is safe.  The loading_promises dict ensures
+                # that if two parallel chains try to load the same dep, only one actually
+                # does the work; the other waits.
+                pending = v'[]'
+                for ikey in cached.imported_module_ids:
+                    if not has_prop(imported_modules, ikey):
+                        pending.push(do_import(ikey))
+                if pending.length > 0:
+                    await Promise.all(pending)
+                # Reconstruct classes dict from the simplified cache (no full AST nodes).
+                classes = {}
+                for cname in Object.keys(cached.classes or {}):
+                    ci = cached.classes[cname]
+                    classes[cname] = {'name': {'name': cname}, 'static': ci.static or {}, 'bound': ci.bound or [], 'classvars': ci.classvars or {}}
+                meta = {
+                    'module_id': key,
+                    'filename': filename,
+                    'import_order': Object.keys(imported_modules).length,
+                    'imports': imported_modules,
+                    'exports': [{'name': n} for n in cached.exported_names],
+                    'baselib': cached.baselib or {},
+                    'classes': classes,
+                }
+                if cached.has_side_effects is not undefined:
+                    meta['_has_side_effects'] = cached.has_side_effects
+                if cached.index is not undefined:
+                    meta['_cache_index'] = cached.index
+                module = make_lazy_ast_module(ast_json_str, meta)
+                imported_modules[key] = module
+            else:
+                await parse(src_code, {
+                    'filename': filename,
+                    'toplevel': None,
+                    'basedir': options.basedir,
+                    'libdir': options.libdir,
+                    'import_dirs': options.import_dirs,
+                    'module_id': key,
+                    'imported_modules': imported_modules,
+                    'importing_modules': importing_modules,
+                    'discard_asserts': options.discard_asserts,
+                    'module_cache_dir': options.module_cache_dir,
+                    '_src_hash': srchash,
+                    '_file_mtime': file_mtime,
+                    'loading_promises': loading_promises,
+                })  # This function will add the module to imported_modules itself
+        finally:
+            del loading_promises[key]
+            if _rfn[0]:
+                _rfn[0](None)
 
         imported_modules[key].srchash = srchash
 
@@ -811,10 +953,10 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 break
         return new AST_EmptyStatement({'stype':'scoped_flags', 'start':prev(), 'end':prev()})
 
-    def import_(from_import):
+    async def import_(from_import):
         ans = new AST_Imports({'imports':[]})
         while True:
-            tok = tmp = name = last_tok = expression(False)
+            tok = tmp = name = last_tok = await expression(False)
             key = ''
             while is_node_type(tmp, AST_Dot):
                 key = "." + tmp.property + key
@@ -844,7 +986,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 break
 
         for imp in ans['imports']:
-            do_import(imp.key)
+            await do_import(imp.key)
             if imported_module_ids.indexOf(imp.key) is -1:
                 imported_module_ids.push(imp.key)
             classes = imported_modules[key].classes
@@ -889,7 +1031,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
         return ans
 
-    def class_():
+    async def class_():
         name = as_symbol(AST_SymbolDefun)
         if not name:
             unexpected()
@@ -916,7 +1058,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     S.in_parenthesized_expr = False
                     next()
                     break
-                a = expr_atom(False)
+                a = await expr_atom(False)
                 if class_parent is None:
                     class_parent = a
                 bases.push(a)
@@ -925,6 +1067,33 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     continue
 
         docstrings = v'[]'
+        cls_decorators = (def():
+            d = []
+            for decorator in S.decorators:
+                d.push(new AST_Decorator({
+                    'expression': decorator
+                }))
+            S.decorators = v'[]'
+            return d
+        )()
+        cls_body = await (async def(loop, labels):
+            # navigate to correct location in the module tree and append the class
+            S.in_class.push(name.name)
+            S.classes[S.classes.length - 1][name.name] = class_details
+            S.classes.push({})
+            S.scoped_flags.push()
+            S.in_function += 1
+            S.in_loop = 0
+            S.labels = []
+            a = await block_(docstrings)
+            S.in_function -= 1
+            S.scoped_flags.pop()
+            S.classes.pop()
+            S.in_class.pop()
+            S.in_loop = loop
+            S.labels = labels
+            return a
+        )(S.in_loop, S.labels)
         definition = new AST_Class({
             'name': name,
             'docstrings': docstrings,
@@ -938,33 +1107,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'external': externaldecorator,
             'bound': class_details.bound,
             'statements': [],
-            'decorators': (def():
-                d = []
-                for decorator in S.decorators:
-                    d.push(new AST_Decorator({
-                        'expression': decorator
-                    }))
-                S.decorators = v'[]'
-                return d
-            )(),
-            'body': (def(loop, labels):
-                # navigate to correct location in the module tree and append the class
-                S.in_class.push(name.name)
-                S.classes[S.classes.length - 1][name.name] = class_details
-                S.classes.push({})
-                S.scoped_flags.push()
-                S.in_function += 1
-                S.in_loop = 0
-                S.labels = []
-                a = block_(docstrings)
-                S.in_function -= 1
-                S.scoped_flags.pop()
-                S.classes.pop()
-                S.in_class.pop()
-                S.in_loop = loop
-                S.labels = labels
-                return a
-            )(S.in_loop, S.labels)
+            'decorators': cls_decorators,
+            'body': cls_body,
         })
         class_details.processing = False
         # find the constructor
@@ -1008,7 +1152,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 definition.statements.push(stmt)
         return definition
 
-    def function_(in_class, is_expression):
+    async def function_(in_class, is_expression, is_async=False):
         name = as_symbol(AST_SymbolDefun if in_class else AST_SymbolLambda) if is_('name') else None
         if in_class and not name:
             croak('Cannot use anonymous function as class methods')
@@ -1033,169 +1177,193 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         return_annotation = None
         is_generator = v'[]'
         docstrings = v'[]'
-        definition = new ctor({
-            'name': name,
-            'is_expression': is_expression,
-            'is_anonymous': is_anonymous,
-            'argnames': (def(a):
-                defaults = {}
-                first = True
-                seen_names = {}
-                def_line = S.input.context().tokline
-                current_arg_name = None
-                name_token = None
-
-                def get_arg():
-                    nonlocal current_arg_name, name_token
-                    current_arg_name = S.token.value
-                    if has_prop(seen_names, current_arg_name):
-                        token_error(prev(), "Can't repeat parameter names")
-                    if current_arg_name is 'arguments':
-                        token_error(prev(), "Can't use the name arguments as a parameter name, it is reserved by JavaScript")
-                    seen_names[current_arg_name] = True
-                    # save these in order to move back if we have an annotation
-                    name_token = S.token
-                    name_ctx = S.input.context()
-                    # check if we have an argument annotation
-                    ntok = peek()
-                    if ntok.type is 'punc' and ntok.value is ':':
-                        next()
-                        expect(':')
-                        annotation = maybe_conditional()
-
-                        # and now, do as_symbol without the next() at the end
-                        # since we are already at the next comma (or end bracket)
-                        if not is_token(name_token, "name"):
-                            # assuming the previous context in case
-                            # the annotation was over the line
-                            croak("Name expected", name_ctx.tokline)
-                            return None
 
-                        sym = new AST_SymbolFunarg({
-                            'name': name_token.value,
-                            'start': S.token,
-                            'end': S.token,
-                            'annotation': annotation
-                        })
-                        return sym
-                    else:
-                        if not is_("name"):
-                            # there is no name, which is an error we should report on the
-                            # same line as the definition, so move to that is we're not already there.
-                            if S.input.context().tokline is not def_line:
-                                croak("Name expected", def_line)
-                            else:
-                                croak("Name expected")
-                            return None
-
-                        sym = new AST_SymbolFunarg({
-                            'name': current_arg_name,
-                            'start': S.token,
-                            'end': S.token,
-                            'annotation': None
-                        })
-                        next()
-                        return sym
+        fn_argnames = await (async def():
+            a = v'[]'
+            starargs_sym = undefined
+            kwargs_sym = undefined
+            defaults = {}
+            has_defaults = False
+            first = True
+            seen_names = {}
+            def_line = S.input.context().tokline
+            current_arg_name = None
+            name_token = None
+
+            async def get_arg():
+                nonlocal current_arg_name, name_token
+                current_arg_name = S.token.value
+                if has_prop(seen_names, current_arg_name):
+                    token_error(prev(), "Can't repeat parameter names")
+                if current_arg_name is 'arguments':
+                    token_error(prev(), "Can't use the name arguments as a parameter name, it is reserved by JavaScript")
+                seen_names[current_arg_name] = True
+                # save these in order to move back if we have an annotation
+                name_token = S.token
+                name_ctx = S.input.context()
+                # check if we have an argument annotation
+                ntok = peek()
+                if ntok.type is 'punc' and ntok.value is ':':
+                    next()
+                    expect(':')
+                    annotation = await maybe_conditional()
+
+                    # and now, do as_symbol without the next() at the end
+                    # since we are already at the next comma (or end bracket)
+                    if not is_token(name_token, "name"):
+                        # assuming the previous context in case
+                        # the annotation was over the line
+                        if S.input.context().tokline is not def_line:
+                            croak("Name expected", name_ctx.tokline)
+                        else:
+                            croak("Name expected")
+                        return None
+
+                    sym = new AST_SymbolFunarg({
+                        'name': name_token.value,
+                        'start': S.token,
+                        'end': S.token,
+                        'annotation': annotation
+                    })
+                    return sym
+                else:
+                    if not is_("name"):
+                        # there is no name, which is an error we should report on the
+                        # same line as the definition, so move to that is we're not already there.
+                        if S.input.context().tokline is not def_line:
+                            croak("Name expected", def_line)
+                        else:
+                            croak("Name expected")
+                        return None
+
+                    sym = new AST_SymbolFunarg({
+                        'name': current_arg_name,
+                        'start': S.token,
+                        'end': S.token,
+                        'annotation': None
+                    })
+                    next()
+                    return sym
 
-                while not is_("punc", ")"):
-                    if first:
-                        first = False
-                    else:
-                        expect(",")
-                        if is_('punc', ')'):
-                            break
-                    if is_('operator', '**'):
-                        # **kwargs
-                        next()
-                        if a.kwargs:
-                            token_error(name_token, "Can't define multiple **kwargs in function definition")
-                        a.kwargs = get_arg()
-                    elif is_('operator', '*'):
-                        # *args
+            while not is_("punc", ")"):
+                if first:
+                    first = False
+                else:
+                    expect(",")
+                    if is_('punc', ')'):
+                        break
+                if is_('operator', '**'):
+                    # **kwargs
+                    next()
+                    if kwargs_sym:
+                        token_error(name_token, "Can't define multiple **kwargs in function definition")
+                    kwargs_sym = await get_arg()
+                elif is_('operator', '*'):
+                    # *args
+                    next()
+                    if starargs_sym:
+                        token_error(name_token, "Can't define multiple *args in function definition")
+                    if kwargs_sym:
+                        token_error(name_token, "Can't define *args after **kwargs in function definition")
+                    starargs_sym = await get_arg()
+                else:
+                    if starargs_sym or kwargs_sym:
+                        token_error(name_token, "Can't define a formal parameter after *args or **kwargs")
+                    a.push(await get_arg())
+                    if is_("operator", "="):
+                        if kwargs_sym:
+                            token_error(name_token, "Can't define an optional formal parameter after **kwargs")
                         next()
-                        if a.starargs:
-                            token_error(name_token, "Can't define multiple *args in function definition")
-                        if a.kwargs:
-                            token_error(name_token, "Can't define *args after **kwargs in function definition")
-                        a.starargs = get_arg()
+                        defaults[current_arg_name] = await expression(False)
+                        has_defaults = True
                     else:
-                        if a.starargs or a.kwargs:
-                            token_error(name_token, "Can't define a formal parameter after *args or **kwargs")
-                        a.push(get_arg())
-                        if is_("operator", "="):
-                            if a.kwargs:
-                                token_error(name_token, "Can't define an optional formal parameter after **kwargs")
-                            next()
-                            defaults[current_arg_name] = expression(False)
-                            a.has_defaults = True
-                        else:
-                            if a.has_defaults:
-                                token_error(name_token, "Can't define required formal parameters after optional formal parameters")
+                        if has_defaults:
+                            token_error(name_token, "Can't define required formal parameters after optional formal parameters")
 
+            next()
+            # check if we have a return type annotation
+            if is_("punc", "->"):
                 next()
-                # check if we have a return type annotation
-                if is_("punc", "->"):
-                    next()
-                    nonlocal return_annotation
-                    return_annotation = maybe_conditional()
-                S.in_parenthesized_expr = False
-                a.defaults = defaults
-                a.is_simple_func = not a.starargs and not a.kwargs and not a.has_defaults
-                return a
-            )(v'[]'),
+                nonlocal return_annotation
+                return_annotation = await maybe_conditional()
+            S.in_parenthesized_expr = False
+            return new AST_ArgsDef({
+                'args': a,
+                'starargs': starargs_sym,
+                'kwargs': kwargs_sym,
+                'defaults': defaults,
+                'has_defaults': has_defaults,
+                'is_simple_func': not starargs_sym and not kwargs_sym and not has_defaults,
+            })
+        )()
+
+        fn_decorators = (def():
+            d = v'[]'
+            for decorator in S.decorators:
+                d.push(new AST_Decorator({
+                    'expression': decorator
+                }))
+            S.decorators = v'[]'
+            return d
+        )()
+
+        fn_body = await (async def(loop, labels):
+            S.in_class.push(False)
+            S.classes.push({})
+            S.scoped_flags.push()
+            S.in_function += 1
+            S.functions.push({'is_async': is_async})
+            S.in_loop = 0
+            S.labels = []
+            a = await block_(docstrings)
+            S.in_function -= 1
+            S.scoped_flags.pop()
+            is_generator.push(bool(S.functions.pop().is_generator))
+            S.classes.pop()
+            S.in_class.pop()
+            S.in_loop = loop
+            S.labels = labels
+            return a
+        )(S.in_loop, S.labels)
+
+        definition = new ctor({
+            'name': name,
+            'is_expression': is_expression,
+            'is_anonymous': is_anonymous,
+            'argnames': fn_argnames,
             'localvars': [],
-            'decorators': (def():
-                d = v'[]'
-                for decorator in S.decorators:
-                    d.push(new AST_Decorator({
-                        'expression': decorator
-                    }))
-                S.decorators = v'[]'
-                return d
-            )(),
+            'decorators': fn_decorators,
             'docstrings': docstrings,
-            'body': (def(loop, labels):
-                S.in_class.push(False)
-                S.classes.push({})
-                S.scoped_flags.push()
-                S.in_function += 1
-                S.functions.push({})
-                S.in_loop = 0
-                S.labels = []
-                a = block_(docstrings)
-                S.in_function -= 1
-                S.scoped_flags.pop()
-                is_generator.push(bool(S.functions.pop().is_generator))
-                S.classes.pop()
-                S.in_class.pop()
-                S.in_loop = loop
-                S.labels = labels
-                return a
-            )(S.in_loop, S.labels)
+            'body': fn_body,
         })
         definition.return_annotation = return_annotation
         definition.is_generator = is_generator[0]
+        definition.is_async = is_async
         if is_node_type(definition, AST_Method):
             definition.static = staticmethod
             definition.is_getter = property_getter
             definition.is_setter = property_setter
-            if definition.argnames.length < 1 and not definition.static:
+            if definition.argnames.args.length < 1 and not definition.static:
                 croak('Methods of a class must have at least one argument, traditionally named self')
             if definition.name and definition.name.name is '__init__':
                 if definition.is_generator:
                     croak('The __init__ method of a class cannot be a generator (yield not allowed)')
+                if definition.is_async:
+                    croak('The __init__ method of a class cannot be async')
                 if property_getter or property_setter:
                     croak('The __init__ method of a class cannot be a property getter/setter')
+        if definition.is_async and definition.is_generator:
+            croak('A function cannot be both async and a generator')
         if definition.is_generator:
             baselib_items['yield'] = True
 
         # detect local variables, strip function arguments
         assignments = scan_for_local_vars(definition.body)
         for i in range(assignments.length):
-            for j in range(definition.argnames.length+1):
-                if j is definition.argnames.length:
+            for j in range(definition.argnames.args.length+1):
+                if j is definition.argnames.args.length:
                     definition.localvars.push(new_symbol(AST_SymbolVar, assignments[i]))
-                elif j < definition.argnames.length and assignments[i] is definition.argnames[j].name:
+                elif j < definition.argnames.args.length and assignments[i] is definition.argnames.args[j].name:
                     break
 
         nonlocals = scan_for_nonlocal_defs(definition.body)
@@ -1203,9 +1371,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         definition.localvars = definition.localvars.filter(def(v): return not nonlocals.has(v.name);)
         return definition
 
-    def if_():
-        cond = expression(True)
-        body = statement()
+    async def if_():
+        cond = await expression(True)
+        body = await statement()
         belse = None
         if is_("keyword", "elif") or is_("keyword", "else"):
             if is_("keyword", "else"):
@@ -1213,7 +1381,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             else:
                 S.token.value = "if"
             # effectively converts 'elif' to 'else if'
-            belse = statement()
+            belse = await statement()
 
         return new AST_If({
             'condition': cond,
@@ -1227,15 +1395,27 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 return stmt.body
         return False
 
-    def block_(docstrings):
+    async def block_(docstrings):
         prev_whitespace = S.token.leading_whitespace
         expect(":")
         a = v'[]'
         if not S.token.nlb:
             while not S.token.nlb:
                 if is_("eof"):
+                    if options.recover_errors:
+                        break
                     unexpected()
-                stmt = statement()
+                if options.recover_errors:
+                    try:
+                        stmt = await statement()
+                    except as berr:
+                        if is_recoverable_error(berr):
+                            record_recovery(berr)
+                            await skip_to_recovery_point()
+                            continue
+                        raise
+                else:
+                    stmt = await statement()
                 if docstrings:
                     ds = is_docstring(stmt)
                     if ds:
@@ -1250,7 +1430,17 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 if is_("eof"):
                     # end of file, terminate block automatically
                     return a
-                stmt = statement()
+                if options.recover_errors:
+                    try:
+                        stmt = await statement()
+                    except as berr:
+                        if is_recoverable_error(berr):
+                            record_recovery(berr)
+                            await skip_to_recovery_point()
+                            continue
+                        raise
+                else:
+                    stmt = await statement()
                 if docstrings:
                     ds = is_docstring(stmt)
                     if ds:
@@ -1260,8 +1450,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             next()
         return a
 
-    def try_():
-        body = block_()
+    async def try_():
+        body = await block_()
         bcatch = v'[]'
         bfinally = None
         belse = None
@@ -1280,29 +1470,32 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 next()
                 name = as_symbol(AST_SymbolCatch)
 
+            except_body = await block_()
             bcatch.push(new AST_Except({
                 'start': start,
                 'argname': name,
                 'errors': exceptions,
-                'body': block_(),
+                'body': except_body,
                 'end': prev()
             }))
 
         if is_("keyword", "else"):
             start = S.token
             next()
+            belse_body = await block_()
             belse = new AST_Else({
                 'start': start,
-                'body': block_(),
+                'body': belse_body,
                 'end': prev()
             })
 
         if is_("keyword", "finally"):
             start = S.token
             next()
+            bfinally_body = await block_()
             bfinally = new AST_Finally({
                 'start': start,
-                'body': block_(),
+                'body': bfinally_body,
                 'end': prev()
             })
 
@@ -1316,13 +1509,20 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'belse': belse
         })
 
-    def vardefs(symbol_class):
+    async def vardefs(symbol_class):
         a = []
         while True:
+            vd_start = S.token
+            vd_name = as_symbol(symbol_class)
+            if is_('operator', '='):
+                next()
+                varval = await expression(False)
+            else:
+                varval = None
             a.push(new AST_VarDef({
-                'start': S.token,
-                'name': as_symbol(symbol_class),
-                'value': (next(), expression(False)) if is_('operator', '=') else None,
+                'start': vd_start,
+                'name': vd_name,
+                'value': varval,
                 'end': prev()
             }))
             if not is_("punc", ","):
@@ -1331,8 +1531,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
 
         return a
 
-    def nonlocal_(is_global):
-        defs = vardefs(AST_SymbolNonlocal)
+    async def nonlocal_(is_global):
+        defs = await vardefs(AST_SymbolNonlocal)
         if is_global:
             for vardef in defs:
                 S.globals.push(vardef.name.name)
@@ -1342,19 +1542,19 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'end': prev()
         })
 
-    def new_():
+    async def new_():
         start = S.token
         expect_token("operator", "new")
-        newexp = expr_atom(False)
+        newexp = await expr_atom(False)
 
         if is_("punc", "("):
             S.in_parenthesized_expr = True
             next()
-            args = func_call_list()
+            args = await func_call_list()
             S.in_parenthesized_expr = False
         else:
-            args = func_call_list(True)
-        return subscripts(new AST_New({
+            args = await func_call_list(True)
+        return await subscripts(new AST_New({
             'start': start,
             'expression': newexp,
             'args': args,
@@ -1424,9 +1624,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         next()
         return ret
 
-    def expr_atom(allow_calls):
+    async def expr_atom(allow_calls):
         if is_("operator", "new"):
-            return new_()
+            return await new_()
 
         start = S.token
         if is_("punc"):
@@ -1437,9 +1637,9 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 if is_('punc', ')'):
                     next()
                     return new AST_Array({'elements':[]})
-                ex = expression(True)
+                ex = await expression(True)
                 if is_('keyword', 'for'):
-                    ret = read_comprehension(new AST_GeneratorComprehension({'statement': ex}), ')')
+                    ret = await read_comprehension(new AST_GeneratorComprehension({'statement': ex}), ')')
                     S.in_parenthesized_expr = False
                     return ret
                 ex.start = start
@@ -1451,38 +1651,55 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 if is_node_type(ex, AST_UnaryPrefix):
                     ex.parenthesized = True
                 S.in_parenthesized_expr = False
-                return subscripts(ex, allow_calls)
+                return await subscripts(ex, allow_calls)
             elif tmp_ is "[":
-                return subscripts(array_(), allow_calls)
+                return await subscripts(await array_(), allow_calls)
             elif tmp_ is "{":
-                return subscripts(object_(), allow_calls)
+                return await subscripts(await object_(), allow_calls)
 
             unexpected()
 
         if is_("keyword", "class"):
             next()
-            cls = class_()
+            cls = await class_()
             cls.start = start
             cls.end = prev()
-            return subscripts(cls, allow_calls)
+            return await subscripts(cls, allow_calls)
+
+        if is_("keyword", "async"):
+            next()
+            if not is_("keyword", "def"):
+                croak("Expected 'def' after 'async'")
+            next()
+            func = await function_(False, True, True)
+            func.start = start
+            func.end = prev()
+            return await subscripts(func, allow_calls)
 
         if is_("keyword", "def"):
             next()
-            func = function_(False, True)
+            func = await function_(False, True)
             func.start = start
             func.end = prev()
-            return subscripts(func, allow_calls)
+            return await subscripts(func, allow_calls)
+
+        if is_('keyword', 'await'):
+            next()
+            if S.in_function is 0 or not S.functions[-1].is_async:
+                croak("'await' outside of async function")
+            value = await expr_atom(True)
+            return new AST_Await({'value': value})
 
         if is_('keyword', 'yield'):
             next()
-            return yield_()
+            return await yield_()
 
         if ATOMIC_START_TOKEN[S.token.type]:
-            return subscripts(as_atom_node(), allow_calls)
+            return await subscripts(as_atom_node(), allow_calls)
 
         unexpected()
 
-    def expr_list(closing, allow_trailing_comma, allow_empty, func_call):
+    async def expr_list(closing, allow_trailing_comma, allow_empty, func_call):
         first = True
         a = []
         saw_starargs = False
@@ -1507,31 +1724,28 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     'end': S.token
                 }))
             else:
-                a.push(expression(False))
+                a.push(await expression(False))
 
+        next()
         if func_call:
-            tmp = []
-            tmp.kwargs = []
+            pargs = []
+            kwargs = []
             for arg in a:
                 if is_node_type(arg, AST_Assign):
-                    tmp.kwargs.push([arg.left, arg.right])
+                    kwargs.push([arg.left, arg.right])
                 else:
-                    tmp.push(arg)
-            a = tmp
-
-        next()
-        if saw_starargs:
-            a.starargs = True
+                    pargs.push(arg)
+            return new AST_CallArgs({'args': pargs, 'kwargs': kwargs, 'kwarg_items': v'[]', 'starargs': saw_starargs})
         return a
 
-    def func_call_list(empty):
+    async def func_call_list(empty):
         a = v'[]'
         first = True
-        a.kwargs = v'[]'
-        a.kwarg_items = v'[]'
-        a.starargs = False
+        kwargs = v'[]'
+        kwarg_items = v'[]'
+        starargs = False
         if empty:
-            return a
+            return new AST_CallArgs({'args': a, 'kwargs': kwargs, 'kwarg_items': kwarg_items, 'starargs': starargs})
         single_comprehension = False
         while not is_("punc", ')') and not is_('eof'):
             if not first:
@@ -1540,50 +1754,51 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     break
             if is_('operator', '*'):
                 next()
-                arg = expression(False)
-                arg.is_array = True
-                a.push(arg)
-                a.starargs = True
+                arg = await expression(False)
+                a.push(new AST_CallArg({'value': arg, 'is_array': True}))
+                starargs = True
             elif is_('operator', '**'):
                 next()
-                a.kwarg_items.push(as_symbol(AST_SymbolRef, False))
-                a.starargs = True
+                kwarg_items.push(as_symbol(AST_SymbolRef, False))
+                starargs = True
             else:
-                arg = expression(False)
+                arg = await expression(False)
                 if is_node_type(arg, AST_Assign):
-                    a.kwargs.push([arg.left, arg.right])
+                    kwargs.push([arg.left, arg.right])
                 else:
                     if is_('keyword', 'for'):
                         if not first:
                             croak('Generator expression must be parenthesized if not sole argument')
-                        a.push(read_comprehension(new AST_GeneratorComprehension({'statement': arg}), ')'))
+                        comp = await read_comprehension(new AST_GeneratorComprehension({'statement': arg}), ')')
+                        a.push(new AST_CallArg({'value': comp, 'is_array': False}))
                         single_comprehension = True
                         break
-                    a.push(arg)
+                    a.push(new AST_CallArg({'value': arg, 'is_array': False}))
             first = False
         if not single_comprehension:
             next()
-        return a
+        return new AST_CallArgs({'args': a, 'kwargs': kwargs, 'kwarg_items': kwarg_items, 'starargs': starargs})
 
     @embed_tokens
-    def array_():
+    async def array_():
         expect("[")
         expr = []
         if not is_("punc", "]"):
-            expr.push(expression(False))
+            expr.push(await expression(False))
             if is_("keyword", "for"):
                 # list comprehension
-                return read_comprehension(new AST_ListComprehension({'statement': expr[0]}), ']')
+                return await read_comprehension(new AST_ListComprehension({'statement': expr[0]}), ']')
 
             if not is_("punc", "]"):
                 expect(",")
 
+        arr_rest = await expr_list("]", True, True)
         return new AST_Array({
-            'elements': expr.concat(expr_list("]", True, True))
+            'elements': expr.concat(arr_rest)
         })
 
     @embed_tokens
-    def object_():
+    async def object_():
         expect("{")
         first = True
         has_non_const_keys = False
@@ -1603,26 +1818,27 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             orig = ctx.expecting_object_literal_key
             ctx.expecting_object_literal_key = True
             try:
-                left = expression(False)
+                left = await expression(False)
             finally:
                 ctx.expecting_object_literal_key = orig
             if is_('keyword', 'for'):
                 # is_pydict is irrelevant here
-                return read_comprehension(new AST_SetComprehension({'statement':left}), '}')
+                return await read_comprehension(new AST_SetComprehension({'statement':left}), '}')
             if a.length is 0 and (is_('punc', ',') or is_('punc', '}')):
                 end = prev()
-                return set_(start, end, left)
+                return await set_(start, end, left)
             if not is_node_type(left, AST_Constant):
                 has_non_const_keys = True
             expect(":")
+            obj_val = await expression(False)
             a.push(new AST_ObjectKeyVal({
                 'start': start,
                 'key': left,
-                'value': expression(False),
+                'value': obj_val,
                 'end': prev()
             }))
             if a.length is 1 and is_('keyword', 'for'):
-                return dict_comprehension(a, is_pydict, is_jshash)
+                return await dict_comprehension(a, is_pydict, is_jshash)
 
         next()
         return new (AST_ExpressiveObject if has_non_const_keys else AST_Object)({
@@ -1631,7 +1847,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'is_jshash': is_jshash,
         })
 
-    def set_(start, end, expr):
+    async def set_(start, end, expr):
         ostart = start
         a = [new AST_SetItem({'start':start, 'end':end, 'value':expr})]
         while not is_("punc", "}"):
@@ -1640,35 +1856,40 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if is_("punc", "}"):
                 # allow trailing comma
                 break
-            a.push(new AST_SetItem({'start':start, 'value':expression(False), 'end':prev()}))
+            set_val = await expression(False)
+            a.push(new AST_SetItem({'start':start, 'value':set_val, 'end':prev()}))
         next()
         return new AST_Set({'items':a, 'start':ostart, 'end':prev()})
 
-    def read_comprehension(obj, terminator):
+    async def read_comprehension(obj, terminator):
         if is_node_type(obj, AST_GeneratorComprehension):
             baselib_items['yield'] = True
         S.in_comprehension = True
         S.in_parenthesized_expr = False  # in case we are already in a parenthesized expression
         expect_token('keyword', 'for')
-        forloop = for_(True)
+        forloop = await for_(True)
         obj.init = forloop.init
         obj.name = forloop.name
         obj.object = forloop.object
-        obj.condition = None if is_('punc', terminator) else (expect_token("keyword", "if"), expression(True))
+        if is_('punc', terminator):
+            obj.condition = None
+        else:
+            expect_token("keyword", "if")
+            obj.condition = await expression(True)
         expect(terminator)
         S.in_comprehension = False
         return obj
 
-    def dict_comprehension(a, is_pydict, is_jshash):
+    async def dict_comprehension(a, is_pydict, is_jshash):
         if a.length:
             left, right = a[0].key, a[0].value
         else:
-            left = expression(False)
+            left = await expression(False)
             if not is_('punc', ':'):
-                return read_comprehension(new AST_SetComprehension({'statement':left}), '}')
+                return await read_comprehension(new AST_SetComprehension({'statement':left}), '}')
             expect(':')
-            right = expression(False)
-        return read_comprehension(new AST_DictComprehension({'statement':left, 'value_statement':right, 'is_pydict':is_pydict, 'is_jshash':is_jshash}), '}')
+            right = await expression(False)
+        return await read_comprehension(new AST_DictComprehension({'statement':left, 'value_statement':right, 'is_pydict':is_pydict, 'is_jshash':is_jshash}), '}')
 
     def as_name():
         tmp = S.token
@@ -1714,7 +1935,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         else:
             return False
 
-    def getitem(expr, allow_calls):
+    async def getitem(expr, allow_calls):
         start = expr.start
         next()
         is_py_sub = S.scoped_flags.get('overload_getitem', False)
@@ -1724,7 +1945,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             # slice [:n]
             slice_bounds.push(None)
         else:
-            slice_bounds.push(expression(False))
+            slice_bounds.push(await expression(False))
 
         if is_("punc", ":"):
             # slice [n:m?]
@@ -1733,7 +1954,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if is_("punc", ":"):
                 slice_bounds.push(None)
             elif not is_("punc", "]"):
-                slice_bounds.push(expression(False))
+                slice_bounds.push(await expression(False))
 
         if is_("punc", ":"):
             # slice [n:m:o?]
@@ -1741,7 +1962,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if is_("punc", "]"):
                 unexpected()
             else:
-                slice_bounds.push(expression(False))
+                slice_bounds.push(await expression(False))
 
         expect("]")
 
@@ -1749,14 +1970,15 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if is_("operator", '='):
                 # splice-assignment (arr[start:end] = ...)
                 next()  # swallow the assignment
-                return subscripts(new AST_Splice({
+                splice_assign = await expression(True)
+                return await subscripts(new AST_Splice({
                     'start': start,
                     'expression': expr,
                     'property': slice_bounds[0] or new AST_Number({
                         'value': 0
                     }),
                     'property2': slice_bounds[1],
-                    'assignment': expression(True),
+                    'assignment': splice_assign,
                     'end': prev()
                 }), allow_calls)
             elif slice_bounds.length is 3:
@@ -1768,26 +1990,38 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         slice_bounds.pop()
                 elif not slice_bounds[-2]:
                     slice_bounds[-2] = new AST_Undefined()
-                return subscripts(new AST_Call({
+                _ca = v'[]'
+                _ca.push(new AST_CallArg({'value': expr, 'is_array': False}))
+                for _sb in slice_bounds:
+                    _ca.push(new AST_CallArg({'value': _sb, 'is_array': False}))
+                return await subscripts(new AST_Call({
                     'start': start,
                     'expression': new AST_SymbolRef({
                         'name': 'ρσ_delslice' if S.in_delete else "ρσ_eslice"
                     }),
-                    'args': [expr].concat(slice_bounds),
+                    'args': new AST_CallArgs({'args': _ca, 'kwargs': v'[]', 'kwarg_items': v'[]', 'starargs': False}),
                     'end': prev()
                 }), allow_calls)
             else:
                 # regular slice (arr[start:end])
                 slice_bounds = [new AST_Number({'value':0}) if i is None else i for i in slice_bounds]
                 if S.in_delete:
-                    return subscripts(new AST_Call({
+                    _ca2 = v'[]'
+                    _ca2.push(new AST_CallArg({'value': expr, 'is_array': False}))
+                    _ca2.push(new AST_CallArg({'value': new AST_Number({'value': 1}), 'is_array': False}))
+                    for _sb in slice_bounds:
+                        _ca2.push(new AST_CallArg({'value': _sb, 'is_array': False}))
+                    return await subscripts(new AST_Call({
                         'start': start,
                         'expression': new AST_SymbolRef({'name': 'ρσ_delslice'}),
-                        'args': [expr, new AST_Number({'value':1})].concat(slice_bounds),
+                        'args': new AST_CallArgs({'args': _ca2, 'kwargs': v'[]', 'kwarg_items': v'[]', 'starargs': False}),
                         'end': prev()
                     }), allow_calls)
 
-                return subscripts(new AST_Call({
+                _ca3 = v'[]'
+                for _sb in slice_bounds:
+                    _ca3.push(new AST_CallArg({'value': _sb, 'is_array': False}))
+                return await subscripts(new AST_Call({
                     'start': start,
                     'expression': new AST_Dot({
                         'start': start,
@@ -1795,7 +2029,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         'property': "slice",
                         'end': prev()
                     }),
-                    'args': slice_bounds,
+                    'args': new AST_CallArgs({'args': _ca3, 'kwargs': v'[]', 'kwarg_items': v'[]', 'starargs': False}),
                     'end': prev()
                 }), allow_calls)
         else:
@@ -1806,8 +2040,8 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 if is_("operator") and ASSIGNMENT[S.token.value]:
                     assign_operator = S.token.value[:-1]
                     next()
-                    assignment = expression(True)
-                return subscripts(new AST_ItemAccess({
+                    assignment = await expression(True)
+                return await subscripts(new AST_ItemAccess({
                     'start': start,
                     'expression': expr,
                     'property': slice_bounds[0] or new AST_Number({
@@ -1818,7 +2052,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                     'end': prev()
                 }), allow_calls)
 
-            return subscripts(new AST_Sub({
+            return await subscripts(new AST_Sub({
                 'start': start,
                 'expression': expr,
                 'property': slice_bounds[0] or new AST_Number({
@@ -1827,16 +2061,17 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'end': prev()
             }), allow_calls)
 
-    def call_(expr):
+    async def call_(expr):
         start = expr.start
         S.in_parenthesized_expr = True
         next()
         if not expr.parens and get_class_in_scope(expr):
             # this is an object being created using a class
-            ret = subscripts(new AST_New({
+            call_args = await func_call_list()
+            ret = await subscripts(new AST_New({
                 'start': start,
                 'expression': expr,
-                'args': func_call_list(),
+                'args': call_args,
                 'end': prev()
             }), True)
             S.in_parenthesized_expr = False
@@ -1849,12 +2084,13 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 # generate class call
                 funcname = expr
 
-                ret = subscripts(new AST_ClassCall({
+                classcall_args = await func_call_list()
+                ret = await subscripts(new AST_ClassCall({
                     'start': start,
                     "class": expr.expression,
                     'method': funcname.property,
                     "static": is_static_method(c, funcname.property),
-                    'args': func_call_list(),
+                    'args': classcall_args,
                     'end': prev()
                 }), True)
                 S.in_parenthesized_expr = False
@@ -1862,39 +2098,41 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             elif is_node_type(expr, AST_SymbolRef):
                 tmp_ = expr.name
                 if tmp_ is "jstype":
+                    jstype_args = await func_call_list()
                     ret = new AST_UnaryPrefix({
                         'start': start,
                         'operator': "typeof",
-                        'expression': func_call_list()[0],
+                        'expression': jstype_args.args[0].value,
                         'end': prev()
                     })
                     S.in_parenthesized_expr = False
                     return ret
                 elif tmp_ is "isinstance":
-                    args = func_call_list()
-                    if args.length is not 2:
+                    args = await func_call_list()
+                    if args.args.length is not 2:
                         croak('isinstance() must be called with exactly two arguments')
                     ret = new AST_Binary({
                         'start': start,
-                        'left': args[0],
+                        'left': args.args[0].value,
                         'operator': 'instanceof',
-                        'right': args[1],
+                        'right': args.args[1].value,
                         'end': prev()
                     })
                     S.in_parenthesized_expr = False
                     return ret
 
             # fall-through to basic function call
-            ret = subscripts(new AST_Call({
+            basic_args = await func_call_list()
+            ret = await subscripts(new AST_Call({
                 'start': start,
                 'expression': expr,
-                'args': func_call_list(),
+                'args': basic_args,
                 'end': prev()
             }), True)
             S.in_parenthesized_expr = False
             return ret
 
-    def get_attr(expr, allow_calls):
+    async def get_attr(expr, allow_calls):
         next()
         prop = as_name()
         c = get_class_in_scope(expr)
@@ -1903,14 +2141,14 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             if classvars and v'classvars[prop]':
                 prop = 'prototype.' + prop
 
-        return subscripts(new AST_Dot({
+        return await subscripts(new AST_Dot({
             'start': expr.start,
             'expression': expr,
             'property': prop,
             'end': prev()
         }), allow_calls)
 
-    def existential(expr, allow_calls):
+    async def existential(expr, allow_calls):
         ans = new AST_Existential({'start':expr.start, 'end':S.token, 'expression':expr})
         next()
         ttype = S.token.type
@@ -1931,34 +2169,34 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             else:
                 ans.after = None
                 return ans
-            return subscripts(ans, allow_calls)
+            return await subscripts(ans, allow_calls)
 
-        ans.after = expression()
+        ans.after = await expression()
         return ans
 
-    def subscripts(expr, allow_calls):
+    async def subscripts(expr, allow_calls):
         if is_("punc", "."):
-            return get_attr(expr, allow_calls)
+            return await get_attr(expr, allow_calls)
 
         if is_("punc", "[") and not S.token.nlb:
-            return getitem(expr, allow_calls)
+            return await getitem(expr, allow_calls)
 
         if allow_calls and is_("punc", "(") and not S.token.nlb:
-            return call_(expr)
+            return await call_(expr)
 
         if is_('punc', '?'):
-            return existential(expr, allow_calls)
+            return await existential(expr, allow_calls)
 
         return expr
 
-    def maybe_unary(allow_calls):
+    async def maybe_unary(allow_calls):
         start = S.token
         if is_('operator', '@'):
             if S.parsing_decorator:
                 croak('Nested decorators are not allowed')
             next()
             S.parsing_decorator = True
-            expr = expression()
+            expr = await expression()
             S.parsing_decorator = False
             S.decorators.push(expr)
             return new AST_EmptyStatement({'stype':'@', 'start':prev(), 'end':prev()})
@@ -1966,14 +2204,14 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             next()
             is_parenthesized = is_('punc', '(')
             S.in_delete = start.value is 'delete'
-            expr = maybe_unary(allow_calls)
+            expr = await maybe_unary(allow_calls)
             S.in_delete = False
             ex = make_unary(AST_UnaryPrefix, start.value, expr, is_parenthesized)
             ex.start = start
             ex.end = prev()
             return ex
 
-        val = expr_atom(allow_calls)
+        val = await expr_atom(allow_calls)
         return val
 
     def make_unary(ctor, op, expr, is_parenthesized):
@@ -1983,7 +2221,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             'parenthesized': is_parenthesized
         })
 
-    def expr_op(left, min_prec, no_in):
+    async def expr_op(left, min_prec, no_in):
         op = S.token.value if is_('operator') else None
         if op is "!" and peek().type is "operator" and peek().value is "in":
             next()
@@ -1995,7 +2233,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         prec = PRECEDENCE[op] if op is not None else None
         if prec is not None and prec > min_prec:
             next()
-            right = expr_op(maybe_unary(True), prec, no_in)
+            right = await expr_op(await maybe_unary(True), prec, no_in)
             ret = new AST_Binary({
                 'start': left.start,
                 'left': left,
@@ -2003,24 +2241,25 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'right': right,
                 'end': right.end
             })
-            return expr_op(ret, min_prec, no_in)
+            return await expr_op(ret, min_prec, no_in)
         return left
 
-    def expr_ops(no_in):
-        return expr_op(maybe_unary(True), 0, no_in)
+    async def expr_ops(no_in):
+        return await expr_op(await maybe_unary(True), 0, no_in)
 
-    def maybe_conditional(no_in):
+    async def maybe_conditional(no_in):
         start = S.token
-        expr = expr_ops(no_in)
+        expr = await expr_ops(no_in)
         if (is_('keyword', 'if') and (S.in_parenthesized_expr or (S.statement_starting_token is not S.token and not S.in_comprehension and not S.token.nlb))):
             next()
-            ne = expression(False)
+            ne = await expression(False)
             expect_token('keyword', 'else')
+            cond_alt = await expression(False, no_in)
             conditional = new AST_Conditional({
                 'start': start,
                 'condition': ne,
                 'consequent': expr,
-                'alternative': expression(False, no_in),
+                'alternative': cond_alt,
                 'end': peek()
             })
             return conditional
@@ -2044,28 +2283,29 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         c.provisional_classvars[ans.left.name] = True
         return ans
 
-    def maybe_assign(no_in, only_plain_assignment):
+    async def maybe_assign(no_in, only_plain_assignment):
         start = S.token
-        left = maybe_conditional(no_in)
+        left = await maybe_conditional(no_in)
         val = S.token.value
         if is_("operator") and ASSIGNMENT[val]:
             if only_plain_assignment and val is not '=':
                 croak('Invalid assignment operator for chained assignment: ' + val)
             next()
+            assign_right = await maybe_assign(no_in, True)
             return create_assign({
                 'start': start,
                 'left': left,
                 'operator': val,
-                'right': maybe_assign(no_in, True),
+                'right': assign_right,
                 'end': prev()
             })
         return left
 
-    def expression(commas, no_in):
+    async def expression(commas, no_in):
         # if there is an assignment, we want the sequences to pivot
         # around it to allow for tuple packing/unpacking
         start = S.token
-        expr = maybe_assign(no_in)
+        expr = await maybe_assign(no_in)
         def build_seq(a):
             if a.length is 1:
                 return a[0]
@@ -2082,6 +2322,7 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 next()
                 if is_node_type(expr, AST_Assign):
                     left[-1] = left[-1].left
+                    seq_cdr = await expression(True, no_in)
                     return create_assign({
                         'start': start,
                         'left': (left[0] if left.length is 1 else new AST_Array({
@@ -2090,12 +2331,12 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                         'operator': expr.operator,
                         'right': new AST_Seq({
                             'car': expr.right,
-                            'cdr': expression(True, no_in)
+                            'cdr': seq_cdr
                         }),
                         'end': peek()
                     })
 
-                expr = maybe_assign(no_in)
+                expr = await maybe_assign(no_in)
                 left.push(expr)
 
             # if last one was an assignment, fix it
@@ -2114,20 +2355,30 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
             return build_seq(left)
         return expr
 
-    def in_loop(cont):
+    async def in_loop(cont):
         S.in_loop += 1
-        ret = cont()
+        ret = await cont()
         S.in_loop -= 1
         return ret
 
-    def run_parser():
+    async def run_parser():
         start = S.token = next()
         body = v'[]'
         docstrings = v'[]'
         first_token = True
         toplevel = options.toplevel
         while not is_("eof"):
-            element = statement()
+            if options.recover_errors:
+                try:
+                    element = await statement()
+                except as rerr:
+                    if is_recoverable_error(rerr):
+                        record_recovery(rerr)
+                        await skip_to_recovery_point()
+                        continue
+                    raise
+            else:
+                element = await statement()
             if first_token and is_node_type(element, AST_Directive) and element.value.indexOf('#!') is 0:
                 shebang = element.value
             else:
@@ -2152,6 +2403,18 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
                 'docstrings': docstrings,
             })
 
+        if options.recover_errors:
+            # Combine parser-level recovered errors with any collected by the
+            # tokenizer (e.g. skipped unexpected characters) for the LSP to report.
+            recovered = S.recovered_errors
+            try:
+                tok_ctx = S.input.context()
+                if tok_ctx and tok_ctx.recovered_errors and tok_ctx.recovered_errors.length:
+                    recovered = recovered.concat(tok_ctx.recovered_errors)
+            except:
+                pass
+            toplevel.recovered_errors = recovered
+
         toplevel.nonlocalvars = scan_for_nonlocal_defs(toplevel.body).concat(S.globals)
         toplevel.localvars = []
         toplevel.exports = []
@@ -2182,11 +2445,89 @@ def create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_
         toplevel.scoped_flags = S.scoped_flags.stack[0]
         importing_modules[module_id] = False
         toplevel.comments_after = S.token.comments_before or v'[]'
+
+        if options._src_hash and options.filename and module_id is not '__main__':
+            try:
+                # Simplified class info (no full AST nodes) for lazy-shell reconstruction.
+                classes_cache = {}
+                for cname in Object.keys(toplevel.classes or {}):
+                    cls = toplevel.classes[cname]
+                    classes_cache[cname] = {'static': cls.static or {}, 'bound': cls.bound or [], 'classvars': cls.classvars or {}}
+                # Single pass: compute has_side_effects + tree-shaking index.
+                # The index lets the tree-shaker resolve the fixpoint iteration
+                # (steps 4-5) without accessing mod.body — no AST deserialization
+                # until step 6 (pruning), which needs actual nodes anyway.
+                has_se = False
+                idx_exec_refs_set = {}
+                idx_import_bindings = {}
+                idx_top_defs = {}
+
+                def _collect_refs_into(node, refs_set):  # noqa
+                    def _rv(n):  # noqa
+                        if is_node_type(n, AST_SymbolRef):
+                            refs_set[n.name] = True
+                    node.walk(new TreeWalker(_rv))
+
+                for idx_stmt in toplevel.body:
+                    if is_node_type(idx_stmt, AST_Function) or is_node_type(idx_stmt, AST_Class):
+                        if idx_stmt.name:
+                            idx_pinned = False
+                            if idx_stmt.decorators:
+                                for idx_dec in idx_stmt.decorators:
+                                    if idx_dec.expression and is_node_type(idx_dec.expression, AST_SymbolRef) and idx_dec.expression.name == 'no_prune':
+                                        idx_pinned = True
+                                        break
+                            idx_def_refs = {}
+                            _collect_refs_into(idx_stmt, idx_def_refs)
+                            idx_top_defs[idx_stmt.name.name] = {'pinned': idx_pinned, 'refs': Object.keys(idx_def_refs)}
+                    elif is_node_type(idx_stmt, AST_Imports):
+                        for idx_imp in idx_stmt.imports:
+                            if idx_imp.argnames and idx_imp.argnames.length:
+                                for idx_arg in idx_imp.argnames:
+                                    idx_local = idx_arg.alias.name if idx_arg.alias else idx_arg.name
+                                    idx_import_bindings[idx_local] = {'source': idx_imp.key, 'imported_name': idx_arg.name, 'is_namespace': False}
+                            else:
+                                idx_local = idx_imp.alias.name if idx_imp.alias else idx_imp.key.split('.')[0]
+                                idx_import_bindings[idx_local] = {'source': idx_imp.key, 'imported_name': None, 'is_namespace': True}
+                    else:
+                        if not has_se:
+                            se_found = v'[false]'
+                            def se_check(se_node, se_descend):  # noqa
+                                if is_node_type(se_node, AST_Call):
+                                    se_found[0] = True
+                                    return True
+                            idx_stmt.walk(new TreeWalker(se_check))
+                            if se_found[0]:
+                                has_se = True
+                        _collect_refs_into(idx_stmt, idx_exec_refs_set)
+
+                cache_meta = {
+                    'v': CACHE_VERSION,
+                    'signature': options._src_hash,
+                    'compiler_version': COMPILER_VERSION,
+                    'imported_module_ids': toplevel.imported_module_ids,
+                    'exported_names': [sym.name for sym in toplevel.exports],
+                    'baselib': toplevel.baselib or {},
+                    'classes': classes_cache,
+                    'has_side_effects': has_se,
+                    'index': {
+                        'exec_refs': Object.keys(idx_exec_refs_set),
+                        'import_bindings': idx_import_bindings,
+                        'top_defs': idx_top_defs,
+                    },
+                }
+                if options._file_mtime is not None and options._file_mtime is not undefined:
+                    cache_meta['mtime'] = options._file_mtime
+                # Binary format: metadata JSON followed by newline and binary AST pool.
+                await writefile(cache_file_name(options.filename, options.module_cache_dir), encode_cache(JSON.stringify(cache_meta), toplevel))
+            except as e:
+                console.error('Failed to write AST cache file:', options.filename, 'with error:', e)
+
         return toplevel
 
     return run_parser
 
-def parse(text, options):
+async def parse(text, options):
     options = defaults(options, {
         'filename': None,     # name of the file being parsed
         'module_id':'__main__', # The id of the module being parsed
@@ -2197,6 +2538,7 @@ def parse(text, options):
         'scoped_flags': {},   # Global scoped flags (used by the REPL)
         'discard_asserts': False,
         'module_cache_dir': '',
+        'recover_errors': False,  # When True, continue parsing past syntax errors (used by the LSP). See run_parser/block_.
     })
     import_dirs = [x for x in options.import_dirs]
     for location in v'[options.libdir, options.basedir]':
@@ -2207,11 +2549,16 @@ def parse(text, options):
     imported_module_ids = []
     imported_modules = options.imported_modules or {}
     importing_modules = options.importing_modules or {}
+    # loading_promises tracks modules currently being loaded across the entire compilation.
+    # It is shared by passing it through options so parallel sub-parse() calls all see it.
+    if not options.loading_promises:
+        options.loading_promises = {}
     importing_modules[module_id] = True
 
     # The internal state of the parser
     S = {
-        'input': tokenizer(text, options.filename) if jstype(text) is 'string' else text,
+        'input': tokenizer(text, options.filename, options.recover_errors) if jstype(text) is 'string' else text,
+        'recovered_errors': v'[]',  # syntax errors swallowed during recovery (recover_errors mode only)
         'token': None,
         'prev': None,
         'peeked': [],
@@ -2249,4 +2596,7 @@ def parse(text, options):
             obj = options.classes[cname]
             S.classes[0][cname] = { 'static':obj.static, 'bound':obj.bound, 'classvars': obj.classvars }
 
-    return create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_ids, imported_modules, importing_modules, options)()
+    if jstype(text) is 'string' and not options._src_hash:
+        options._src_hash = sha1sum(text)
+
+    return await create_parser_ctx(S, import_dirs, module_id, baselib_items, imported_module_ids, imported_modules, importing_modules, options)()