nadata 0.1.0__py3-none-any.whl

nadata/parser/expressions/tokenizer.py

@@ -0,0 +1,402 @@
+"""
+Tokenizer for Nadata expressions and declarations.
+
+Converts raw source text into a flat list of typed tokens, handling string
+literals with escapes, numeric literals, identifiers, keywords, operators, and
+punctuation. Used by the expression parser and the declaration splitters.
+"""
+
+# Import Modules
+from __future__ import annotations
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class TokenType(Enum):
+    """
+    Enumeration of the token types produced by the Tokenizer.
+    """
+
+    IDENTIFIER = "identifier"
+    KEYWORD = "keyword"
+    INT = "int"
+    FLOAT = "float"
+    STRING = "string"
+    OPERATOR = "operator"
+    PUNCTUATION = "punctuation"
+    EOF = "eof"
+
+
+@dataclass
+class Token:
+    """
+    Single lexical token with its position in the source text.
+
+    Attributes:
+        token_type (TokenType): Category of the token.
+        value (str): Decoded token value (escapes resolved for strings).
+        position (int): Index of the first source character of the token.
+        end_position (int): Index just after the last source character.
+    """
+
+    token_type: TokenType
+    value: str
+    position: int
+    end_position: int
+
+
+# Language keywords (logical operators are keyword-only per the specification)
+KEYWORDS: frozenset[str] = frozenset({
+    "class", "fn", "if", "elif", "else", "return",
+    "true", "false", "and", "or", "xor", "nand", "not",
+    "for", "while", "break", "continue", "in",
+    "try", "catch", "raise",
+})
+
+# Two-character operators, checked before single-character ones
+MULTI_CHAR_OPERATORS: tuple[str, ...] = (
+    "==", "!=", "<=", ">=", "->",
+    "::", "+=", "-=", "*=", "/=", "%=", "++", "--",
+)
+
+# Single-character operator symbols
+SINGLE_CHAR_OPERATORS: str = "+-*/%<>=!"
+
+# Punctuation characters
+PUNCTUATION_CHARS: str = ".,:;()[]{}"
+
+# Escape character used inside string literals
+ESCAPE_CHAR: str = "\\"
+
+# Supported escape sequence decodings inside string literals
+ESCAPE_MAPPINGS: dict[str, str] = {
+    '"': '"',
+    "\\": "\\",
+    "n": "\n",
+    "t": "\t",
+}
+
+
+class Tokenizer:
+    """
+    Converts source text into a list of typed tokens in a single left-to-right
+    scan. Comments are expected to have been removed beforehand.
+    """
+
+    def __init__(self, content: str) -> None:
+        """
+        Initializes the Tokenizer instance.
+
+        Args:
+            content (str): Comment-free source text to tokenize.
+        """
+
+        self.content: str = content
+        self.cursor: int = 0
+        self.token_start: int = 0
+
+    def tokenize(self) -> list[Token]:
+        """
+        Tokenizes the whole content into a token list.
+
+        Returns:
+            list[Token]: Tokens in source order, terminated by an EOF token.
+
+        Raises:
+            SyntaxError: On unterminated strings, invalid escapes, or
+                unexpected characters.
+        """
+
+        tokens: list[Token] = []
+
+        # Scan until the end of the content
+        while self.cursor < len(self.content):
+
+            # Skip insignificant whitespace between tokens
+            if self.content[self.cursor].isspace():
+                self.cursor += 1
+                continue
+
+            previous_cursor: int = self.cursor
+
+            tokens.append(self._scan_next_token())
+
+            # Guard against non-advancing scans (would loop forever)
+            if self.cursor <= previous_cursor:
+                raise RuntimeError(
+                    f"Tokenizer failed to advance at index {previous_cursor}"
+                )
+
+        # Terminate with an EOF token for the parser convenience
+        tokens.append(Token(
+            token_type=TokenType.EOF,
+            value="",
+            position=len(self.content),
+            end_position=len(self.content)
+        ))
+
+        return tokens
+
+    def _scan_next_token(self) -> Token:
+        """
+        Scans a single token starting at the current cursor.
+
+        Returns:
+            Token: The next token.
+
+        Raises:
+            SyntaxError: If no token can start at the cursor.
+        """
+
+        char: str = self.content[self.cursor]
+
+        # Dispatch on the first character
+        if char == "\"":
+            return self._scan_string()
+
+        if char.isdigit():
+            return self._scan_number()
+
+        if char.isalpha() or char == "_":
+            return self._scan_identifier()
+
+        if self.content[self.cursor:self.cursor + 2] in MULTI_CHAR_OPERATORS:
+            return self._scan_multi_char_operator()
+
+        if char in SINGLE_CHAR_OPERATORS:
+            self.token_start = self.cursor
+            self.cursor += 1
+
+            return self._make_token(token_type=TokenType.OPERATOR, value=char)
+
+        if char in PUNCTUATION_CHARS:
+            self.token_start = self.cursor
+            self.cursor += 1
+
+            return self._make_token(token_type=TokenType.PUNCTUATION, value=char)
+
+        raise SyntaxError(
+            f"Unexpected character '{char}' at index {self.cursor}"
+        )
+
+    def _make_token(self, token_type: TokenType, value: Optional[str] = None) -> Token:
+        """
+        Builds a token spanning the characters consumed so far.
+
+        Args:
+            token_type (TokenType): Category of the token.
+            value (Optional[str], optional): Decoded token value. Defaults to
+                None (sliced from the source text).
+
+        Returns:
+            Token: The built token.
+        """
+
+        # Default value is the raw source slice (used by numbers, identifiers...)
+        if value is None:
+            value = self.content[self.token_start:self.cursor]
+
+        token: Token = Token(
+            token_type=token_type,
+            value=value,
+            position=self.token_start,
+            end_position=self.cursor
+        )
+
+        return token
+
+    def _scan_string(self) -> Token:
+        """
+        Scans a double quoted string literal with escape decoding.
+
+        Returns:
+            Token: The STRING token with the decoded value.
+
+        Raises:
+            SyntaxError: On unterminated strings or invalid escape sequences.
+        """
+
+        self.token_start = self.cursor
+        self.cursor += 1
+
+        decoded: list[str] = []
+
+        # Consume characters until the closing quote
+        while self.cursor < len(self.content) and self.content[self.cursor] != "\"":
+
+            # Handle escape sequences
+            if self.content[self.cursor] == ESCAPE_CHAR:
+                decoded.append(self._consume_escape())
+                continue
+
+            # Raw newline inside a string is a syntax error
+            if self.content[self.cursor] == "\n":
+                raise SyntaxError(
+                    f"Unterminated string literal starting at index {self.token_start}"
+                )
+
+            decoded.append(self.content[self.cursor])
+            self.cursor += 1
+
+        # Check the string is terminated
+        if self.cursor >= len(self.content):
+            raise SyntaxError(
+                f"Unterminated string literal starting at index {self.token_start}"
+            )
+
+        # Consume the closing quote
+        self.cursor += 1
+
+        return self._make_token(token_type=TokenType.STRING, value="".join(decoded))
+
+    def _consume_escape(self) -> str:
+        """
+        Consumes a backslash escape sequence at the cursor.
+
+        Returns:
+            str: The decoded character.
+
+        Raises:
+            SyntaxError: If the escape sequence is invalid.
+        """
+
+        # Check the escaped character exists
+        if self.cursor + 1 >= len(self.content):
+            raise SyntaxError(
+                f"Unterminated escape sequence at index {self.cursor}"
+            )
+
+        escaped_char: str = self.content[self.cursor + 1]
+
+        # Check the escape is supported
+        if escaped_char not in ESCAPE_MAPPINGS:
+            raise SyntaxError(
+                f"Invalid escape sequence '\\{escaped_char}' at index {self.cursor}"
+            )
+
+        self.cursor += 2
+
+        return ESCAPE_MAPPINGS[escaped_char]
+
+    def _scan_number(self) -> Token:
+        """
+        Scans an integer or float numeric literal.
+
+        Floats include fractional parts ('1.5') and scientific notation
+        ('1e6', '3.5e-2', '1.5E+3'). The exponent is only consumed when a
+        complete 'e[+|-]<digits>' sequence follows, so '1.5e' keeps the
+        float and leaves 'e' for later postfix handling.
+
+        Returns:
+            Token: The INT or FLOAT token.
+
+        Raises:
+            SyntaxError: On malformed fractional parts.
+        """
+
+        self.token_start = self.cursor
+
+        # Consume integer digits
+        while self.cursor < len(self.content) and self.content[self.cursor].isdigit():
+            self.cursor += 1
+
+        # Check for a fractional part (a '.' followed by a digit)
+        has_fraction: bool = (
+            self.cursor + 1 < len(self.content)
+            and self.content[self.cursor] == "."
+            and self.content[self.cursor + 1].isdigit()
+        )
+
+        if has_fraction:
+            self.cursor += 1
+
+            # Consume fractional digits
+            while self.cursor < len(self.content) and self.content[self.cursor].isdigit():
+                self.cursor += 1
+
+        # Check for a complete exponent ('e'/'E', optional sign, digits)
+        has_exponent: bool = self._scan_exponent()
+
+        if has_fraction or has_exponent:
+            return self._make_token(token_type=TokenType.FLOAT)
+
+        # A '.' not followed by a digit belongs to a later postfix token
+        # (e.g. member access on a float literal like '1.5.method()')
+        return self._make_token(token_type=TokenType.INT)
+
+    def _scan_exponent(self) -> bool:
+        """
+        Scans an optional scientific-notation exponent at the cursor.
+
+        Consumes 'e'/'E', an optional sign, and at least one digit only
+        when the full sequence is present; otherwise leaves the cursor
+        untouched.
+
+        Returns:
+            bool: True if a complete exponent was consumed.
+        """
+
+        cursor: int = self.cursor
+
+        # Exponent marker
+        if cursor >= len(self.content) or self.content[cursor] not in "eE":
+            return False
+
+        cursor += 1
+
+        # Optional sign
+        if cursor < len(self.content) and self.content[cursor] in "+-":
+            cursor += 1
+
+        # At least one digit is required for a valid exponent
+        if cursor >= len(self.content) or not self.content[cursor].isdigit():
+            return False
+
+        while cursor < len(self.content) and self.content[cursor].isdigit():
+            cursor += 1
+
+        self.cursor = cursor
+
+        return True
+
+    def _scan_identifier(self) -> Token:
+        """
+        Scans an identifier or keyword.
+
+        Returns:
+            Token: The IDENTIFIER or KEYWORD token.
+        """
+
+        self.token_start = self.cursor
+
+        # Consume identifier characters
+        while (
+            self.cursor < len(self.content)
+            and (self.content[self.cursor].isalnum() or self.content[self.cursor] == "_")
+        ):
+            self.cursor += 1
+
+        value: str = self.content[self.token_start:self.cursor]
+
+        # Distinguish keywords from plain identifiers
+        token_type: TokenType = (
+            TokenType.KEYWORD if value in KEYWORDS else TokenType.IDENTIFIER
+        )
+
+        return self._make_token(token_type=token_type, value=value)
+
+    def _scan_multi_char_operator(self) -> Token:
+        """
+        Scans a two-character operator at the cursor.
+
+        Returns:
+            Token: The OPERATOR token.
+        """
+
+        self.token_start = self.cursor
+        operator: str = self.content[self.cursor:self.cursor + 2]
+
+        self.cursor += 2
+
+        return self._make_token(token_type=TokenType.OPERATOR, value=operator)

nadata/parser/expressions/type_parser.py

@@ -0,0 +1,187 @@
+"""
+Type annotation parsing for Nadata.
+
+Provides the TmpType intermediate representation (a type name with nested
+children) and the TypeParser converting a raw annotation string into it.
+Shared by the .nastruct and .nadata parsers so types are parsed once into
+structural data instead of being carried as raw strings.
+"""
+
+# Import Modules
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any
+
+from nadata.parser.expressions.tokenizer import Token, TokenType, Tokenizer
+
+
+@dataclass
+class TmpType:
+    """
+    Intermediate representation of a parsed type annotation.
+
+    A simple type (e.g. 'Str') has no children; a generic type (e.g.
+    'Array[Entity]') has its name plus one child per type argument.
+
+    Attributes:
+        name (str): Type constructor name (e.g. 'Str', 'Array', 'Dict').
+        children (list[TmpType]): Type arguments for generic types.
+    """
+
+    name: str
+    children: list[TmpType] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Converts the type to a plain JSON serializable dict.
+
+        Returns:
+            dict[str, Any]: Dict with 'name' and recursive 'children'.
+        """
+
+        return {
+            "name": self.name,
+            "children": [child.to_dict() for child in self.children],
+        }
+
+    def collect_names(self) -> set[str]:
+        """
+        Collects every type constructor name of the tree.
+
+        Returns:
+            set[str]: This type name plus all nested children names.
+        """
+
+        names: set[str] = {self.name}
+
+        for child in self.children:
+            names |= child.collect_names()
+
+        return names
+
+    def __repr__(self) -> str:
+        """
+        Returns string representation of the TmpType.
+
+        Returns:
+            str: Annotation style representation (e.g. 'Array[Entity]').
+        """
+
+        if not self.children:
+            return self.name
+
+        children_text: str = ", ".join(repr(child) for child in self.children)
+
+        return f"{self.name}[{children_text}]"
+
+
+class TypeParser:
+    """
+    Parses a token stream into a TmpType tree.
+    """
+
+    def __init__(self, tokens: list[Token]) -> None:
+        """
+        Initializes the TypeParser instance.
+
+        Args:
+            tokens (list[Token]): Token stream to parse (terminated by EOF).
+        """
+
+        self.tokens: list[Token] = tokens
+        self.cursor: int = 0
+
+    @classmethod
+    def parse_text(cls, text: str) -> TmpType:
+        """
+        Tokenizes and parses a raw type annotation in one step.
+
+        Args:
+            text (str): Raw type annotation (e.g. 'Array[Entity]').
+
+        Returns:
+            TmpType: Parsed type tree.
+
+        Raises:
+            SyntaxError: On malformed or empty annotations.
+        """
+
+        tokens: list[Token] = Tokenizer(content=text).tokenize()
+
+        parsed_type: TmpType
+        cursor: int
+        parsed_type, cursor = cls(tokens=tokens).parse_type_at(cursor=0)
+
+        # Check nothing remains after the type
+        if tokens[cursor].token_type != TokenType.EOF:
+            raise SyntaxError(
+                f"Unexpected token '{tokens[cursor].value}' in type '{text}'"
+            )
+
+        return parsed_type
+
+    def parse_type_at(self, cursor: int) -> tuple[TmpType, int]:
+        """
+        Parses a single type starting at the given token cursor.
+
+        Args:
+            cursor (int): Token index to start parsing from.
+
+        Returns:
+            tuple[TmpType, int]: The parsed type and the new cursor.
+
+        Raises:
+            SyntaxError: On malformed type annotations.
+        """
+
+        token: Token = self.tokens[cursor]
+
+        # A dimension literal (e.g. the 3 in 'Tensor[Float32, 3, 4]') is a
+        # valid type argument and is carried as its digit-string name
+        if token.token_type == TokenType.INT:
+            cursor += 1
+
+            return TmpType(name=token.value), cursor
+
+        # A type starts with its constructor name
+        if token.token_type != TokenType.IDENTIFIER:
+            raise SyntaxError(
+                f"Expected a type name but found '{token.value}' "
+                f"at index {token.position}"
+            )
+
+        name: str = token.value
+        cursor += 1
+
+        children: list[TmpType] = []
+
+        # Parse the optional generic argument list '[...]'
+        if (
+            self.tokens[cursor].token_type == TokenType.PUNCTUATION
+            and self.tokens[cursor].value == "["
+        ):
+            cursor += 1
+
+            # Parse comma separated type arguments
+            while True:
+                child: TmpType
+                child, cursor = self.parse_type_at(cursor=cursor)
+                children.append(child)
+
+                current: Token = self.tokens[cursor]
+
+                # Continue on comma, close on bracket
+                if current.token_type == TokenType.PUNCTUATION and current.value == ",":
+                    cursor += 1
+                    continue
+
+                if current.token_type == TokenType.PUNCTUATION and current.value == "]":
+                    cursor += 1
+                    break
+
+                raise SyntaxError(
+                    f"Expected ',' or ']' in type arguments of '{name}' "
+                    f"but found '{current.value}' at index {current.position}"
+                )
+
+        return TmpType(name=name, children=children), cursor

nadata/parser/nadata_loader/__init__.py

@@ -0,0 +1,8 @@
+"""
+Data loader for Nadata .nadata files.
+
+See loader.py for the implementation of the NaDataLoader and its passes.
+"""
+
+# Import Modules
+from nadata.parser.nadata_loader.loader import NaDataLoader, StructRuntimeBinding