Typhon-Language 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

Typhon/Grammar/parser.py

@@ -1,53 +1,80 @@
-import ast
-import sys
-import tokenize
-import os
-import io
-
-from typing import (
-    Literal,
-    Union,
-    Optional,
-)
-
-from .tokenizer_custom import TokenizerCustom
-from .token_factory_custom import token_stream_factory
-from ._typhon_parser import parse
-
-
-def parse_file(
-    file_path: str,
-    py_version: Optional[tuple[int, int]] = None,
-    verbose: bool = False,
-) -> ast.Module:
-    """Parse a file."""
-    with open(file_path) as f:
-        tok_stream = token_stream_factory(f.readline)
-        tokenizer = TokenizerCustom(tok_stream, verbose=verbose, path=file_path)
-        parsed = parse(
-            filename=os.path.basename(file_path),
-            tokenizer=tokenizer,
-            mode="file",
-            py_version=py_version,
-            verbose=verbose,
-        )
-    assert isinstance(parsed, ast.Module)
-    return parsed
-
-
-def parse_string(
-    source: str,
-    mode: Union[Literal["eval"], Literal["exec"]] = "exec",
-    py_version: Optional[tuple[int, int]] = None,
-    verbose: bool = False,
-) -> ast.AST | None:
-    """Parse a string."""
-    tok_stream = token_stream_factory(io.StringIO(source).readline)
-    tokenizer = TokenizerCustom(tok_stream, verbose=verbose)
-    return parse(
-        filename="<string>",
-        tokenizer=tokenizer,
-        mode=mode if mode == "eval" else "file",
-        py_version=py_version,
-        verbose=verbose,
-    )
+import ast
+from pathlib import Path
+import sys
+import tokenize
+import os
+import io
+
+from typing import (
+    Literal,
+    Union,
+    Optional,
+)
+
+from ..Driver.debugging import is_debug_verbose
+
+from .tokenizer_custom import TokenizerCustom, show_token
+from .token_factory_custom import token_stream_factory
+from ._typhon_parser import parse
+from .typhon_ast_error import gather_errors
+
+
+def parse_file(
+    file_path: str,
+    py_version: Optional[tuple[int, int]] = None,
+    verbose: bool = False,
+) -> ast.Module:
+    """Parse a file."""
+    with open(file_path) as f:
+        if is_debug_verbose():
+            show_token(Path(file_path).read_text())
+        tok_stream = token_stream_factory(f.readline)
+        tokenizer = TokenizerCustom(tok_stream, verbose=verbose, path=file_path)
+        parsed = parse_tokenizer(
+            tokenizer,
+            py_version=py_version,
+            verbose=verbose,
+        )
+        assert isinstance(parsed, ast.Module), f"Parsing failed: {parsed}"
+        gather_errors(parsed)
+        return parsed
+
+
+def parse_tokenizer(
+    tokenizer: TokenizerCustom,
+    py_version: Optional[tuple[int, int]] = None,
+    verbose: bool = False,
+) -> ast.AST:
+    """Parse using a tokenizer."""
+    parsed = parse(
+        filename="<tokenizer>",
+        tokenizer=tokenizer,
+        mode="file",
+        py_version=py_version,
+        verbose=verbose,
+    )
+    # Must be successful parse
+    assert isinstance(parsed, ast.AST), f"Parsing failed: {parsed}"
+    gather_errors(parsed)
+    return parsed
+
+
+def parse_string(
+    source: str,
+    mode: Union[Literal["eval"], Literal["exec"]] = "exec",
+    py_version: Optional[tuple[int, int]] = None,
+    verbose: bool = False,
+) -> ast.AST | None:
+    """Parse a string."""
+    tok_stream = token_stream_factory(io.StringIO(source).readline)
+    tokenizer = TokenizerCustom(tok_stream, verbose=verbose)
+    parsed = parse(
+        filename="<string>",
+        tokenizer=tokenizer,
+        mode=mode if mode == "eval" else "file",
+        py_version=py_version,
+        verbose=verbose,
+    )
+    if parsed:
+        gather_errors(parsed)
+    return parsed

Typhon/Grammar/parser_helper.py

@@ -6,28 +6,33 @@ import sys
 import tokenize
 
 import enum
-import io
-import itertools
-import os
-import token
 from typing import (
     Any,
     Callable,
     Iterator,
     List,
     Literal,
-    NoReturn,
     Sequence,
+    Unpack,
     Tuple,
     TypeVar,
     Union,
     Optional,
     Protocol,
     cast,
+    Type,
 )
 
 from pegen.tokenizer import Tokenizer
 from pegen.parser import P, memoize, memoize_left_rec, logger, Parser as PegenParser
+from ..Driver.debugging import is_debug_first_error
+from .typhon_ast import (
+    set_anonymous_name_id,
+    PosAttributes,
+    get_pos_attributes,
+    get_anonymous_base_name,
+)
+from .syntax_errors import SkipTokensError, ExpectedTokenError
 
 EXPR_NAME_MAPPING: dict[type, str] = {
     ast.Attribute: "attribute",
@@ -57,6 +62,7 @@ EXPR_NAME_MAPPING: dict[type, str] = {
     ast.NamedExpr: "named expression",
 }
 
+
 # Singleton ast nodes, created once for efficiency
 Load = ast.Load()
 Store = ast.Store()
@@ -83,6 +89,7 @@ class PositionedNode(Protocol):
 class Parser(PegenParser):
     #: Name of the source file, used in error reports
     filename: str
+    _anonymous_id = 0
 
     def __init__(
         self,
@@ -97,8 +104,10 @@ class Parser(PegenParser):
         self.py_version = (
             min(py_version, sys.version_info) if py_version else sys.version_info
         )
+        # Note "invalid_*" rules returns None. Cannot use for error recovery.
+        self.call_invalid_rules = True
 
-    def parse(self, rule: str, call_invalid_rules: bool = False) -> Optional[ast.AST]:
+    def parse(self, rule: str, call_invalid_rules: bool = True) -> Optional[ast.AST]:
         old = self.call_invalid_rules
         self.call_invalid_rules = call_invalid_rules
         res = getattr(self, rule)()
@@ -135,6 +144,15 @@ class Parser(PegenParser):
                 f"{error_msg} is only supported in Python {min_version} and above."
             )
 
+    def make_anonymous_name(
+        self, ctx: ast.expr_context, **kwargs: Unpack[PosAttributes]
+    ) -> tuple[ast.Name, int]:
+        anon_id = self._anonymous_id
+        self._anonymous_id += 1
+        name = ast.Name(f"{get_anonymous_base_name()}_{anon_id}", ctx, **kwargs)
+        set_anonymous_name_id(name, anon_id)
+        return name, anon_id
+
     def raise_indentation_error(self, msg: str) -> None:
         """Raise an indentation error."""
         last_token = self._tokenizer.diagnose()
@@ -359,89 +377,52 @@ class Parser(PegenParser):
             arg.type_comment = type_comment
         return arg
 
-    def make_arguments(
-        self,
-        pos_only: Optional[List[Tuple[ast.arg, None]]],
-        pos_only_with_default: List[Tuple[ast.arg, Any]],
-        param_no_default: Optional[List[ast.arg]],
-        param_default: Optional[List[Tuple[ast.arg, Any]]],
-        after_star: Optional[
-            Tuple[Optional[ast.arg], List[Tuple[ast.arg, Any]], Optional[ast.arg]]
-        ],
-    ) -> ast.arguments:
-        """Build a function definition arguments."""
-        defaults = (
-            [d for _, d in pos_only_with_default if d is not None]
-            if pos_only_with_default
-            else []
-        )
-        defaults += (
-            [d for _, d in param_default if d is not None] if param_default else []
-        )
-
-        pos_only = pos_only or pos_only_with_default
-
-        # Because we need to combine pos only with and without default even
-        # the version with no default is a tuple
-        pos_only_args = [p for p, _ in pos_only]
-        params = (param_no_default or []) + (
-            [p for p, _ in param_default] if param_default else []
-        )
-
-        # If after_star is None, make a default tuple
-        after_star = after_star or (None, [], None)
-
-        return ast.arguments(
-            posonlyargs=pos_only_args,
-            args=params,
-            defaults=defaults,
-            vararg=after_star[0],
-            kwonlyargs=[p for p, _ in after_star[1]],
-            kw_defaults=[d for _, d in after_star[1]],
-            kwarg=after_star[2],
-        )
-
-    def _build_syntax_error(
+    def build_syntax_error[T: SyntaxError](
         self,
         message: str,
-        start: Optional[Tuple[int, int]] = None,
-        end: Optional[Tuple[int, int]] = None,
-    ) -> SyntaxError:
-        line_from_token = start is None and end is None
-        if start is None or end is None:
-            tok = self._tokenizer.diagnose()
-            start = start or tok.start
-            end = end or tok.end
-
-        if line_from_token:
-            line = tok.line
-        else:
-            # End is used only to get the proper text
-            line = "\\n".join(
-                self._tokenizer.get_lines(list(range(start[0], end[0] + 1)))
-            )
+        start: Tuple[int, int],
+        end: Tuple[int, int],
+        error_type: Type[T] = SyntaxError,
+    ) -> T:
+        # End is used only to get the proper text
+        line = "\\n".join(self._tokenizer.get_lines(list(range(start[0], end[0] + 1))))
 
         # tokenize.py index column offset from 0 while Cpython index column
         # offset at 1 when reporting SyntaxError, so we need to increment
         # the column offset when reporting the error.
-        args = (self.filename, start[0], start[1] + 1, line)
+        # But in Typhon, first we take all the ranges as 0-based.
+        args = (self.filename, start[0], start[1], line)
         if sys.version_info >= (3, 10):
-            args += (end[0], end[1] + 1)
+            args += (end[0], end[1])
 
-        return SyntaxError(message, args)
+        result = error_type(message, args)
+        if is_debug_first_error():
+            raise result
+        return result
 
-    def raise_raw_syntax_error(
+    def build_expected_error(
+        self, message: str, start: Tuple[int, int], end: Tuple[int, int]
+    ) -> SyntaxError:
+        error = self.build_syntax_error(
+            f"expected {message}", start, end, ExpectedTokenError
+        )
+        error.expected = message
+        return error
+
+    def build_skip_tokens_error(
         self,
-        message: str,
-        start: Optional[Tuple[int, int]] = None,
-        end: Optional[Tuple[int, int]] = None,
-    ) -> NoReturn:
-        raise self._build_syntax_error(message, start, end)
+        tokens: list[tokenize.TokenInfo],
+        start: Tuple[int, int],
+        end: Tuple[int, int],
+    ) -> SyntaxError:
+        error = self.build_syntax_error("unknown tokens", start, end, SkipTokensError)
+        error.tokens = tokens
+        return error
 
-    def make_syntax_error(
-        self, message: str, filename: str = "<unknown>"
+    def raise_raw_syntax_error(
+        self, message: str, start: Tuple[int, int], end: Tuple[int, int]
     ) -> SyntaxError:
-        return self._build_syntax_error(message)
+        return self.build_syntax_error(message, start, end)
 
     def expect_forced(self, res: Any, expectation: str) -> Optional[tokenize.TokenInfo]:
         if res is None:
@@ -456,10 +437,10 @@ class Parser(PegenParser):
             )
         return res
 
-    def raise_syntax_error(self, message: str) -> NoReturn:
+    def raise_syntax_error(self, message: str) -> SyntaxError:
         """Raise a syntax error."""
         tok = self._tokenizer.diagnose()
-        raise self._build_syntax_error(
+        return self.build_syntax_error(
             message,
             tok.start,
             tok.end if sys.version_info >= (3, 12) or tok.type != 4 else tok.start,
@@ -467,7 +448,7 @@ class Parser(PegenParser):
 
     def raise_syntax_error_known_location(
         self, message: str, node: PositionedNode | tokenize.TokenInfo
-    ) -> NoReturn:
+    ) -> SyntaxError:
         """Raise a syntax error that occured at a given AST node."""
         if isinstance(node, tokenize.TokenInfo):
             start = node.start
@@ -476,14 +457,14 @@ class Parser(PegenParser):
             start = node.lineno, node.col_offset
             end = node.end_lineno, node.end_col_offset
 
-        raise self._build_syntax_error(message, start, end)
+        return self.build_syntax_error(message, start, end)
 
     def raise_syntax_error_known_range(
         self,
         message: str,
         start_node: Union[PositionedNode, tokenize.TokenInfo],
         end_node: Union[PositionedNode, tokenize.TokenInfo],
-    ) -> NoReturn:
+    ) -> SyntaxError:
         if isinstance(start_node, tokenize.TokenInfo):
             start = start_node.start
         else:
@@ -494,11 +475,11 @@ class Parser(PegenParser):
         else:
             end = end_node.end_lineno, end_node.end_col_offset
 
-        raise self._build_syntax_error(message, start, end)
+        return self.build_syntax_error(message, start, end)
 
     def raise_syntax_error_starting_from(
         self, message: str, start_node: Union[PositionedNode, tokenize.TokenInfo]
-    ) -> NoReturn:
+    ) -> SyntaxError:
         if isinstance(start_node, tokenize.TokenInfo):
             start = start_node.start
         else:
@@ -506,7 +487,7 @@ class Parser(PegenParser):
 
         last_token = self._tokenizer.diagnose()
 
-        raise self._build_syntax_error(message, start, last_token.start)
+        return self.build_syntax_error(message, start, last_token.start)
 
     def raise_syntax_error_invalid_target(
         self, target: Target, node: Optional[ast.AST]
@@ -527,6 +508,6 @@ class Parser(PegenParser):
             msg, cast(PositionedNode, invalid_target)
         )
 
-    def raise_syntax_error_on_next_token(self, message: str) -> NoReturn:
+    def raise_syntax_error_on_next_token(self, message: str) -> SyntaxError:
         next_token = self._tokenizer.peek()
-        raise self._build_syntax_error(message, next_token.start, next_token.end)
+        return self.build_syntax_error(message, next_token.start, next_token.end)

Typhon/Grammar/syntax_errors.py

@@ -1,5 +1,6 @@
 import ast
 from pathlib import Path
+from tokenize import TokenInfo
 from .typhon_ast import PosAttributes
 from typing import Unpack, Callable
 from ..Driver.diagnostic import diag_error_file_position, positioned_source_code
@@ -9,7 +10,16 @@ from ..Driver.debugging import debug_print, is_debug_first_error
 _SYNTAX_ERROR_IN_MODULE = "_typh_syntax_error_in_module"
 
 
-class TyphonSyntaxError(Exception):
+# Error recovery skips unrecognized tokens
+class SkipTokensError(SyntaxError):
+    tokens: list[TokenInfo]
+
+
+class ExpectedTokenError(SyntaxError):
+    expected: str
+
+
+class TyphonTransformSyntaxError(SyntaxError):
     message: str
     pos: PosAttributes
 
@@ -22,20 +32,18 @@ class TyphonSyntaxError(Exception):
         return f"{self.message} at {self.pos}"
 
 
-def set_syntax_error_in_module(
-    module: ast.Module, error_details: list[TyphonSyntaxError]
-):
-    setattr(module, _SYNTAX_ERROR_IN_MODULE, error_details)
+def set_syntax_error(node: ast.AST, error_details: list[SyntaxError]):
+    setattr(node, _SYNTAX_ERROR_IN_MODULE, error_details)
 
 
-def add_syntax_error_in_module(module: ast.Module, error_detail: TyphonSyntaxError):
+def add_syntax_error_in_module(module: ast.Module, error_detail: SyntaxError):
     if not hasattr(module, _SYNTAX_ERROR_IN_MODULE):
         setattr(module, _SYNTAX_ERROR_IN_MODULE, [])
     error_list = getattr(module, _SYNTAX_ERROR_IN_MODULE)
     error_list.append(error_detail)
 
 
-def get_syntax_error_in_module(module: ast.Module) -> list[TyphonSyntaxError] | None:
+def get_syntax_error_in_module(module: ast.Module) -> list[SyntaxError] | None:
     return getattr(module, _SYNTAX_ERROR_IN_MODULE, None)
 
 
@@ -49,9 +57,9 @@ def clear_syntax_error_in_module(module: ast.Module) -> None:
 
 
 class TyphonSyntaxErrorList(Exception):
-    errors: list[TyphonSyntaxError]
+    errors: list[SyntaxError]
 
-    def __init__(self, errors: list[TyphonSyntaxError]):
+    def __init__(self, errors: list[SyntaxError]):
         self.errors = errors
         super().__init__(self.errors)
 
@@ -65,7 +73,9 @@ def raise_from_module_syntax_errors(module: ast.Module):
         raise TyphonSyntaxErrorList(errors)
 
 
-def handle_syntax_error(module: ast.Module, syntax_error: TyphonSyntaxError) -> None:
+def handle_syntax_error(
+    module: ast.Module, syntax_error: TyphonTransformSyntaxError
+) -> None:
     if is_debug_first_error():
         debug_print(f"Raising syntax error: {syntax_error}")
         raise syntax_error
@@ -78,12 +88,12 @@ def try_handle_syntax_error_or[T](
 ) -> T:
     try:
         return maybe_syntax_error()
-    except TyphonSyntaxError as syntax_error:
+    except TyphonTransformSyntaxError as syntax_error:
         handle_syntax_error(module, syntax_error)
     return orelse
 
 
-class ScopeError(TyphonSyntaxError):
+class ScopeError(TyphonTransformSyntaxError):
     pass
 
 
@@ -94,7 +104,7 @@ def raise_scope_error(
     raise ScopeError(message, **pos)
 
 
-class ForbiddenStatementError(TyphonSyntaxError):
+class ForbiddenStatementError(TyphonTransformSyntaxError):
     pass
 
 
@@ -105,7 +115,7 @@ def raise_forbidden_statement_error(
     raise ForbiddenStatementError(message, **pos)
 
 
-class TypeAnnotationError(TyphonSyntaxError):
+class TypeAnnotationError(TyphonTransformSyntaxError):
     pass
 
 
@@ -116,7 +126,7 @@ def raise_type_annotation_error(
     raise TypeAnnotationError(message, **pos)
 
 
-class LetMissingElseError(TyphonSyntaxError):
+class LetMissingElseError(TyphonTransformSyntaxError):
     pass
 
 
@@ -128,18 +138,18 @@ def raise_let_missing_else_error(
 
 
 def _get_range_of_error(
-    syntax_error: SyntaxError | TyphonSyntaxError,
+    syntax_error: SyntaxError | TyphonTransformSyntaxError,
 ) -> Range:
-    if isinstance(syntax_error, TyphonSyntaxError):
+    if isinstance(syntax_error, TyphonTransformSyntaxError):
         return Range.from_pos_attr_may_not_end(syntax_error.pos)
     else:
         return Range.from_syntax_error(syntax_error)
 
 
 def _get_message_of_error(
-    syntax_error: SyntaxError | TyphonSyntaxError,
+    syntax_error: SyntaxError | TyphonTransformSyntaxError,
 ) -> str:
-    if isinstance(syntax_error, TyphonSyntaxError):
+    if isinstance(syntax_error, TyphonTransformSyntaxError):
         return syntax_error.message
     else:
         return syntax_error.msg
@@ -152,7 +162,7 @@ def raise_must_have_resolved(msg: str):
 
 
 def diag_error(
-    syntax_error: SyntaxError | TyphonSyntaxError,
+    syntax_error: SyntaxError | TyphonTransformSyntaxError,
     source: Path,
     source_code: str,
 ) -> str:
@@ -173,7 +183,7 @@ def diag_error(
 
 
 def diag_errors(
-    syntax_error: SyntaxError | TyphonSyntaxError | TyphonSyntaxErrorList,
+    syntax_error: SyntaxError | TyphonTransformSyntaxError | TyphonSyntaxErrorList,
     source: Path,
     source_code: str,
 ) -> str: