Typhon-Language 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

Typhon/Grammar/tokenizer_custom.py

@@ -2,10 +2,13 @@ from typing import NamedTuple
 from tokenize import TokenInfo, OP, NAME
 import tokenize
 import token
+import io
 from pegen.tokenizer import Tokenizer as PegenTokenizer
 from typing import override
 from .line_break import line_breakable_after, line_breakable_before
 from .typhon_ast import get_postfix_operator_temp_name
+from ..Driver.debugging import debug_verbose_print
+from .token_factory_custom import token_stream_factory, generate_tokens_ignore_error
 
 
 # Combine sequencial 2 tokens (optionally without space between) into 1
@@ -186,6 +189,7 @@ class TokenizerCustom(PegenTokenizer):
         while True:
             tok = next(self._tokengen)
             if self._is_token_to_skip(tok):
+                self._all_tokens.append(tok)
                 continue
             if tok.type == token.ENDMARKER:
                 self._end_tok = tok
@@ -209,6 +213,7 @@ class TokenizerCustom(PegenTokenizer):
 
     def _commit_token(self, tok: TokenInfo) -> None:
         self._tokens.append(tok)
+        self._all_tokens.append(tok)
         if not self._path and tok.start[0] not in self._lines:
             self._lines[tok.start[0]] = tok.line
 
@@ -295,3 +300,50 @@ class TokenizerCustom(PegenTokenizer):
                 continue
             self._commit_token(tok)
         return self._tokens[self._index]
+
+    def read_all_tokens(self) -> list[TokenInfo]:
+        """Return all tokens including comments."""
+        # Force to consume all tokens
+        debug_verbose_print("Reading all tokens for tokenizer.")
+        while tok := self.getnext():
+            debug_verbose_print(f"  Token: {tok}")
+            if tok.type == token.ENDMARKER:
+                break
+        debug_verbose_print("Finished reading all tokens.")
+        self.reset(0)
+        return sorted(self._all_tokens, key=lambda t: t.start)
+
+
+def tokenizer_for_file(file_path: str) -> TokenizerCustom:
+    """Tokenize the specified file."""
+    with open(file_path) as f:
+        tok_stream = token_stream_factory(f.readline)
+        tokenizer = TokenizerCustom(tok_stream, path=file_path)
+    return tokenizer
+
+
+def tokenizer_for_string(source: str) -> TokenizerCustom:
+    """Tokenize the specified string."""
+    tok_stream = token_stream_factory(io.StringIO(source).readline)
+    tokenizer = TokenizerCustom(tok_stream)
+    return tokenizer
+
+
+def show_token(
+    source: str, show_typhon_token: bool = True, show_python_token: bool = True
+):
+    if show_python_token:
+        print("Tokens of Python tokenizer:")
+        for tok in generate_tokens_ignore_error(io.StringIO(source).readline):
+            print(f"    {tok}")
+    if show_typhon_token:
+        print("Tokens of Typhon Token Factory:")
+        tok_stream = token_stream_factory(io.StringIO(source).readline)
+        for tok in tok_stream:
+            print(f"    {tok}")
+    print("Tokens of Typhon Custom tokenizer:")
+    tok_stream = token_stream_factory(io.StringIO(source).readline)
+    tokenizer = TokenizerCustom(tok_stream, verbose=True)
+    tokens = tokenizer.read_all_tokens()
+    for tok in tokens:
+        print(f"    {tok}")

Typhon/Grammar/typhon_ast.py

@@ -1,11 +1,26 @@
 # Ast Extensions for Typhon
-
+from __future__ import annotations
+from re import A
+from typing import (
+    Union,
+    Unpack,
+    TypedDict,
+    Tuple,
+    cast,
+    TYPE_CHECKING,
+    Optional,
+    List,
+    Any,
+)
 import ast
-from typing import Union, Unpack, TypedDict, Tuple, cast
 from dataclasses import dataclass
 from copy import copy
+from tokenize import TokenInfo
 from ..Driver.debugging import debug_print, debug_verbose_print
 
+if TYPE_CHECKING:
+    from .parser_helper import Parser
+
 
 # Same as ast module's position attributes
 class PosAttributes(TypedDict):
@@ -15,6 +30,25 @@ class PosAttributes(TypedDict):
     end_col_offset: int | None
 
 
+def unpack_pos_default(pos: PosAttributes) -> Tuple[int, int, int, int]:
+    return (
+        pos["lineno"],
+        pos["col_offset"],
+        pos["end_lineno"] or pos["lineno"],
+        pos["end_col_offset"] or pos["col_offset"] + 1,
+    )
+
+
+def unpack_pos_tuple(pos: PosAttributes) -> Tuple[Tuple[int, int], Tuple[int, int]]:
+    return (
+        (pos["lineno"], pos["col_offset"]),
+        (
+            pos["end_lineno"] or pos["lineno"],
+            pos["end_col_offset"] or pos["col_offset"] + 1,
+        ),
+    )
+
+
 class PosRange(TypedDict):
     lineno: int
     col_offset: int
@@ -46,13 +80,21 @@ type PosNode = (
     | ast.excepthandler
     | ast.pattern
     | ast.keyword
+    | ast.match_case
 )
 
 
-def get_pos_attributes(node: PosNode) -> PosAttributes:
+def get_pos_attributes(node: PosNode | TokenInfo) -> PosAttributes:
+    if isinstance(node, TokenInfo):
+        return PosAttributes(
+            lineno=node.start[0],
+            col_offset=node.start[1],
+            end_lineno=node.end[0],
+            end_col_offset=node.end[1],
+        )
     return PosAttributes(
-        lineno=node.lineno,
-        col_offset=node.col_offset,
+        lineno=getattr(node, "lineno", 1),
+        col_offset=getattr(node, "col_offset", 0),
         end_lineno=getattr(node, "end_lineno", None),
         end_col_offset=getattr(node, "end_col_offset", None),
     )
@@ -70,16 +112,16 @@ def get_pos_attributes_if_exists(node: ast.AST) -> PosAttributes | None:
 
 
 def get_empty_pos_attributes() -> PosAttributes:
+    # Python ast position is 1-based for line, 0-based for column
     return PosAttributes(
-        lineno=0,
+        lineno=1,
         col_offset=0,
-        end_lineno=0,
+        end_lineno=1,
         end_col_offset=0,
     )
 
 
 _ANONYMOUS_NAME = "_typh_anonymous"
-_anonymous_global_id = 0
 
 
 def set_anonymous_name_id(node: ast.Name, id: int) -> ast.Name:
@@ -87,6 +129,10 @@ def set_anonymous_name_id(node: ast.Name, id: int) -> ast.Name:
     return node
 
 
+def get_anonymous_base_name() -> str:
+    return _ANONYMOUS_NAME
+
+
 def get_anonymous_name_id(node: ast.Name) -> int | None:
     return getattr(node, _ANONYMOUS_NAME, None)
 
@@ -100,17 +146,6 @@ def is_anonymous_name(node: ast.Name) -> bool:
     return hasattr(node, _ANONYMOUS_NAME)
 
 
-def make_anonymous_name(
-    ctx: ast.expr_context, **kwargs: Unpack[PosAttributes]
-) -> tuple[ast.Name, int]:
-    global _anonymous_global_id
-    anon_id = _anonymous_global_id
-    name = ast.Name(f"{_ANONYMOUS_NAME}_{anon_id}", ctx, **kwargs)
-    set_anonymous_name_id(name, anon_id)
-    _anonymous_global_id += 1
-    return name, anon_id
-
-
 def copy_anonymous_name(src: ast.Name, ctx: ast.expr_context) -> ast.Name:
     result = ast.Name(src.id, ctx, **get_pos_attributes(src))
     anon_id = get_anonymous_name_id(src)
@@ -119,6 +154,13 @@ def copy_anonymous_name(src: ast.Name, ctx: ast.expr_context) -> ast.Name:
     return result
 
 
+_TYPE_INVALID_NAME = "_typh_invalid_name"
+
+
+def get_invalid_name() -> str:
+    return _TYPE_INVALID_NAME
+
+
 _TYPE_IGNORE_NODES = "_typh_type_ignore"
 
 
@@ -147,6 +189,24 @@ def clear_type_ignore_node(node: ast.AST) -> None:
         delattr(node, _TYPE_IGNORE_NODES)
 
 
+# The name is internal when it has no counterpart in input typhon source code.
+_INTERNAL_NAME = "_typh_internal_name"
+
+
+def is_internal_name(name: ast.Name) -> bool:
+    return getattr(name, _INTERNAL_NAME, False)
+
+
+def set_is_internal_name(name: ast.Name, is_internal: bool = True) -> ast.Name:
+    setattr(name, _INTERNAL_NAME, is_internal)
+    return name
+
+
+def clear_internal_name(name: ast.Name) -> None:
+    if hasattr(name, _INTERNAL_NAME):
+        delattr(name, _INTERNAL_NAME)
+
+
 # Normal assignments, let assignments for variable declarations,
 # and constant assignments for constant definitions.
 # They all are Assign/AnnAssign in Python, we distinguish them by
@@ -476,6 +536,7 @@ def declaration_as_withitem(assign: Union[ast.Assign, ast.AnnAssign]) -> ast.wit
 
 
 def _make_with_let_pattern(
+    parser: Parser,
     is_async: bool,
     decl_type: str,
     pattern_subjects: list[tuple[ast.pattern, ast.expr]],
@@ -485,7 +546,9 @@ def _make_with_let_pattern(
     items: list[ast.withitem] = []
     pattern_vars: list[tuple[ast.pattern, ast.expr]] = []
     for pattern, subject in pattern_subjects:
-        var, var_id = make_anonymous_name(ast.Store(), **get_pos_attributes(pattern))
+        var, var_id = parser.make_anonymous_name(
+            ast.Store(), **get_pos_attributes(pattern)
+        )
         item = ast.withitem(context_expr=subject, optional_vars=var)
         _set_is_let_var(item, decl_type)
         items.append(item)
@@ -503,6 +566,7 @@ def _make_with_let_pattern(
 
 
 def make_with_let_pattern(
+    parser: Parser,
     is_async: bool,
     decl_type: str,
     pattern_subjects: list[tuple[ast.pattern, ast.expr]],
@@ -510,6 +574,7 @@ def make_with_let_pattern(
     **kwargs: Unpack[PosAttributes],
 ) -> ast.With | ast.AsyncWith:
     let_pattern_stmt, items = _make_with_let_pattern(
+        parser,
         is_async,
         decl_type,
         pattern_subjects,
@@ -526,6 +591,7 @@ def make_with_let_pattern(
 
 
 def make_inline_with_let_pattern(
+    parser: Parser,
     is_async: bool,
     decl_type: str,
     pattern_subjects: list[tuple[ast.pattern, ast.expr]],
@@ -551,6 +617,7 @@ def make_inline_with_let_pattern(
                 pass
     """
     let_pattern_stmt, items = _make_with_let_pattern(
+        parser,
         is_async,
         decl_type,
         pattern_subjects,
@@ -588,6 +655,7 @@ def set_function_literal_def(
 ):
     setattr(name, _FUNC_DEF, func_def)
     setattr(func_def, _IS_FUNCTION_LITERAL, True)
+    set_is_internal_name(name)
 
 
 def clear_function_literal_def(name: FunctionLiteral):
@@ -798,6 +866,7 @@ def make_for_stmt(
 
 
 def make_for_let_pattern(
+    parser: Parser,
     decl_type: str,
     pattern: ast.pattern,
     iter: ast.expr,
@@ -807,7 +876,9 @@ def make_for_let_pattern(
     is_async: bool,
     **kwargs: Unpack[PosAttributes],
 ):
-    temp_name, anon_id = make_anonymous_name(ast.Load(), **get_pos_attributes(pattern))
+    temp_name, anon_id = parser.make_anonymous_name(
+        ast.Load(), **get_pos_attributes(pattern)
+    )
     let_stmt = make_if_let(
         decl_type,
         pattern_subjects=[(pattern, temp_name)],
@@ -883,8 +954,26 @@ def clear_is_let_else(node: LetElseAnnotatedNode) -> None:
         delattr(node, _IS_LET_ELSE)
 
 
+def _let_pattern_check(
+    parser: Parser,
+    decl_type_str: str,
+    pattern_subjects: list[tuple[ast.pattern, ast.expr]],
+    start_pos: tuple[int, int],
+    end_pos: tuple[int, int],
+) -> bool:
+    if decl_type_str != "let":
+        error = parser.build_syntax_error(
+            "declaration pattern must be 'let' declaration", start_pos, end_pos
+        )
+    if len(pattern_subjects) == 0:
+        parser.build_syntax_error(
+            "declaration pattern must have at least one pattern", start_pos, end_pos
+        )
+    return True
+
+
 def make_if_let(
-    decl_type: str,
+    decl_type: TokenInfo | str,
     pattern_subjects: list[tuple[ast.pattern, ast.expr]],
     cond: ast.expr | None,
     body: list[ast.stmt],
@@ -892,15 +981,13 @@ def make_if_let(
     is_let_else: bool,
     **kwargs: Unpack[PosAttributes],
 ) -> ast.stmt:
-    if len(pattern_subjects) == 0:
-        raise SyntaxError("if let must have at least one pattern")
-    else:
-        return set_is_let_else(
-            _make_if_let_multiple(
-                decl_type, pattern_subjects, cond, body, orelse, is_let_else, **kwargs
-            ),
-            is_let_else,
-        )
+    decl_type_str = decl_type.string if isinstance(decl_type, TokenInfo) else decl_type
+    return set_is_let_else(
+        _make_if_let_multiple(
+            decl_type_str, pattern_subjects, cond, body, orelse, is_let_else, **kwargs
+        ),
+        is_let_else,
+    )
 
 
 def _make_if_let_single_case(
@@ -922,19 +1009,18 @@ def _make_if_let_single_case(
         # Variable capture pattern, e.g. `let x = ...` without condition.
         # In this case, the condition is None check.
         return set_is_let_else(
-            ast.match_case(
+            make_match_case(
                 pattern=pattern,
                 guard=_make_none_check(pattern.name, get_pos_attributes(pattern)),
                 body=body,
+                **get_pos_attributes(pattern),
             ),
             is_let_else,
         )
     else:
         return set_is_let_else(
-            ast.match_case(
-                pattern=pattern,
-                guard=cond,
-                body=body,
+            make_match_case(
+                pattern=pattern, guard=cond, body=body, **get_pos_attributes(pattern)
             ),
             is_let_else,
         )
@@ -1073,10 +1159,7 @@ def make_while_let(
     orelse: list[ast.stmt] | None,
     **kwargs: Unpack[PosAttributes],
 ) -> ast.While:
-    if len(pattern_subjects) == 0:
-        raise SyntaxError("if let must have at least one pattern")
-    else:
-        return _make_while_let(pattern_subjects, cond, body, orelse, **kwargs)
+    return _make_while_let(pattern_subjects, cond, body, orelse, **kwargs)
 
 
 def _make_while_let(
@@ -1143,10 +1226,65 @@ def is_static(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
     return getattr(node, IS_STATIC, False)
 
 
+_DEFINED_NAME = "_typh_defined_name"
+DefinesName = (
+    ast.FunctionDef
+    | ast.AsyncFunctionDef
+    | ast.ClassDef
+    | ast.alias
+    | ast.Attribute
+    | ast.arg
+)
+
+
+def get_defined_name(node: DefinesName) -> ast.Name | None:
+    return getattr(node, _DEFINED_NAME, None)
+
+
+def set_defined_name(
+    node: DefinesName,
+    name: ast.Name,
+):
+    setattr(node, _DEFINED_NAME, name)
+
+
+def maybe_copy_defined_name[T: ast.AST](
+    from_node: T,
+    to_node: T,
+) -> T:
+    if not isinstance(from_node, DefinesName) or not isinstance(to_node, DefinesName):
+        return to_node
+    name = get_defined_name(from_node)
+    if name is not None:
+        set_defined_name(to_node, name)
+    return to_node
+
+
+def set_defined_name_token(
+    node: DefinesName, name: TokenInfo | ast.Name, ctx: ast.expr_context = ast.Store()
+):
+    if isinstance(name, TokenInfo):
+        name = ast.Name(
+            id=name.string,
+            lineno=name.start[0],
+            col_offset=name.start[1],
+            end_lineno=name.end[0],
+            end_col_offset=name.end[1],
+            ctx=ctx,
+        )
+    setattr(node, _DEFINED_NAME, name)
+    return node
+
+
+def clear_defined_name(node: DefinesName):
+    if hasattr(node, _DEFINED_NAME):
+        delattr(node, _DEFINED_NAME)
+
+
 def make_function_def(
     is_async: bool,
     is_static: bool,
-    name: str,
+    name: TokenInfo | str,
     args: ast.arguments,
     returns: ast.expr | None,
     body: list[ast.stmt],
@@ -1156,7 +1294,7 @@ def make_function_def(
 ) -> ast.FunctionDef | ast.AsyncFunctionDef:
     if is_async:
         result = ast.AsyncFunctionDef(
-            name=name,
+            name=name.string if isinstance(name, TokenInfo) else name,
             args=args,
             returns=returns,
             body=body,
@@ -1166,7 +1304,7 @@ def make_function_def(
         )
     else:
         result = ast.FunctionDef(
-            name=name,
+            name=name.string if isinstance(name, TokenInfo) else name,
             args=args,
             returns=returns,
             body=body,
@@ -1175,6 +1313,120 @@ def make_function_def(
             **kwargs,
         )
     set_is_static(result, is_static)
+    if isinstance(name, TokenInfo):
+        set_defined_name_token(result, name)
+    return result
+
+
+def make_class_def(
+    name: TokenInfo | str,
+    bases: list[ast.expr],
+    keywords: list[ast.keyword],
+    body: list[ast.stmt],
+    decorator_list: list[ast.expr],
+    type_params: list[ast.type_param],
+    **kwargs: Unpack[PosAttributes],
+) -> ast.ClassDef:
+    name_str = name.string if isinstance(name, TokenInfo) else name
+    result = ast.ClassDef(
+        name=name_str,
+        bases=bases,
+        keywords=keywords,
+        body=body,
+        decorator_list=decorator_list,
+        type_params=type_params,
+        **kwargs,
+    )
+    set_defined_name_token(
+        result,
+        name
+        if isinstance(name, TokenInfo)
+        else ast.Name(id=name, ctx=ast.Store(), **kwargs),
+    )
+    return result
+
+
+def make_alias(
+    name: list[TokenInfo],
+    asname: TokenInfo | None,
+    **kwargs: Unpack[PosAttributes],
+) -> ast.alias:
+    result = ast.alias(
+        name=".".join(n.string for n in name),
+        asname=asname.string if asname else None,
+        **kwargs,
+    )
+    if asname is not None:
+        set_defined_name_token(result, asname)
+    else:
+        set_defined_name_token(result, name[-1])
+    return result
+
+
+def make_attribute(
+    value: ast.expr,
+    attr: TokenInfo,
+    ctx: ast.expr_context,
+    **kwargs: Unpack[PosAttributes],
+):
+    return set_defined_name_token(
+        ast.Attribute(
+            value=value,
+            attr=attr.string,
+            ctx=ctx,
+            **kwargs,
+        ),
+        attr,
+        ctx,
+    )
+
+
+_IMPORT_FROM_NAMES = "_typh_import_from_names"
+
+
+def get_import_from_names(node: ast.ImportFrom) -> list[ast.Name]:
+    return getattr(node, _IMPORT_FROM_NAMES, [])
+
+
+def set_import_from_names(
+    node: ast.ImportFrom,
+    names: list[ast.Name],
+):
+    setattr(node, _IMPORT_FROM_NAMES, names)
+
+
+def clear_import_from_names(node: ast.ImportFrom):
+    if hasattr(node, _IMPORT_FROM_NAMES):
+        delattr(node, _IMPORT_FROM_NAMES)
+
+
+# Used only the case module part exists.
+def make_import_from(
+    module: list[TokenInfo] | None,
+    names: list[ast.alias],
+    level: int,
+    **kwargs: Unpack[PosAttributes],
+) -> ast.ImportFrom:
+    mod_name = ".".join(n.string for n in module) if module else None
+    result = ast.ImportFrom(
+        module=mod_name,
+        names=names,
+        level=level,
+        **kwargs,
+    )
+    if module:
+        import_names = [
+            ast.Name(
+                id=n.string,
+                lineno=n.start[0],
+                col_offset=n.start[1],
+                end_lineno=n.end[0],
+                end_col_offset=n.end[1],
+                ctx=ast.Load(),
+            )
+            for n in module
+        ]
+        set_import_from_names(result, import_names)
     return result
 
 
@@ -1196,6 +1448,7 @@ def set_control_comprehension_def(
     node: ControlComprehension, func_def: ast.FunctionDef | ast.AsyncFunctionDef
 ):
     setattr(node, _CONTROL_COMPREHENSION, func_def)
+    set_is_internal_name(node)
 
 
 def clear_is_control_comprehension(node: ControlComprehension) -> None:
@@ -1564,7 +1817,7 @@ def make_match_comp(
             ast.Match(
                 subject=subject,
                 cases=[
-                    ast.match_case(
+                    make_match_case(
                         pattern=get_case_comp_case(case)[0],
                         guard=get_case_comp_case(case)[1],
                         body=[
@@ -1573,6 +1826,7 @@ def make_match_comp(
                                 **get_pos_attributes(case),
                             )
                         ],
+                        **get_pos_attributes(case),
                     )
                     for case in cases
                 ],
@@ -2089,3 +2343,77 @@ def is_case_irrefutable(case: ast.match_case) -> bool:
     if case.guard:
         return False
     return is_pattern_irrefutable(case.pattern)
+
+
+def make_match_case(
+    pattern: ast.pattern,
+    guard: ast.expr | None,
+    body: list[ast.stmt],
+    **kwargs: Unpack[PosAttributes],
+) -> ast.match_case:
+    node = ast.match_case(pattern=pattern, guard=guard, body=body)
+    # Append position attributes
+    for key, value in kwargs.items():
+        setattr(node, key, value)
+    return node
+
+
+def make_arg(
+    arg: TokenInfo | None,
+    annotation: ast.expr | None,
+    **kwargs: Unpack[PosAttributes],
+):
+    """Build a function definition argument."""
+    node = ast.arg(
+        arg=arg.string if arg else "",
+        annotation=annotation,
+        **kwargs,
+    )
+    if arg:
+        set_defined_name_token(node, arg)
+    return node
+
+
+def make_arguments(
+    pos_only: Optional[List[Tuple[ast.arg, None]]],
+    pos_only_with_default: List[Tuple[ast.arg, Any]],
+    param_no_default: Optional[List[ast.arg]],
+    param_default: Optional[List[Tuple[ast.arg, Any]]],
+    after_star: Optional[
+        Tuple[Optional[ast.arg], List[Tuple[ast.arg, Any]], Optional[ast.arg]]
+    ],
+    **kwargs: Unpack[PosAttributes],
+) -> ast.arguments:
+    """Build a function definition arguments."""
+    defaults = (
+        [d for _, d in pos_only_with_default if d is not None]
+        if pos_only_with_default
+        else []
+    )
+    defaults += [d for _, d in param_default if d is not None] if param_default else []
+
+    pos_only = pos_only or pos_only_with_default
+
+    # Because we need to combine pos only with and without default even
+    # the version with no default is a tuple
+    pos_only_args = [p for p, _ in pos_only]
+    params = (param_no_default or []) + (
+        [p for p, _ in param_default] if param_default else []
+    )
+
+    # If after_star is None, make a default tuple
+    after_star = after_star or (None, [], None)
+
+    node = ast.arguments(
+        posonlyargs=pos_only_args,
+        args=params,
+        defaults=defaults,
+        vararg=after_star[0],
+        kwonlyargs=[p for p, _ in after_star[1]],
+        kw_defaults=[d for _, d in after_star[1]],
+        kwarg=after_star[2],
+    )
+    # Append position attributes
+    for key, value in kwargs.items():
+        setattr(node, key, value)
+    return node