syncraft 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

syncraft/finder.py

@@ -1,16 +1,16 @@
 from __future__ import annotations
 
 from typing import (
-    Any, Tuple, Generator as YieldGen, TypeVar, Generic
+    Any, Tuple, Generator as PyGenerator, TypeVar, Generic
 )
 from dataclasses import dataclass
 from syncraft.algebra import (
-    Algebra, Either, Right, 
+    Algebra, Either, Right, Incomplete
 )
 from syncraft.ast import TokenProtocol, ParseResult, Choice, Many, Then, Marked, Collect
 
 from syncraft.generator import GenState, Generator
-
+from syncraft.cache import Cache
 from syncraft.syntax import Syntax
 
 
@@ -24,7 +24,7 @@ class Finder(Generator[T], Generic[T]):
     such as ``matches`` and ``find``.
     """
     @classmethod
-    def anything(cls)->Algebra[Any, GenState[T]]:
+    def anything(cls, cache: Cache)->Algebra[Any, GenState[T]]:
         """Match any node and return it unchanged.
 
         Succeeds on any input ``GenState`` and returns the current AST node as
@@ -35,15 +35,16 @@ class Finder(Generator[T], Generic[T]):
             Algebra[Any, GenState[T]]: An algebra that always succeeds with the
             tuple ``(input.ast, input)``.
         """
-        def anything_run(input: GenState[T], use_cache:bool) -> Either[Any, Tuple[Any, GenState[T]]]:
+        def anything_run(input: GenState[T], use_cache:bool) -> PyGenerator[Incomplete[GenState[T]] ,GenState[T],Either[Any, Tuple[Any, GenState[T]]]]:
+            yield from ()
             return Right((input.ast, input))
-        return cls(anything_run, name=cls.__name__ + '.anything')
+        return cls(anything_run, name=cls.__name__ + '.anything', cache=cache)
 
 
 
 #: A ``Syntax`` that matches any node and returns it as the result without
 #: consuming or modifying state.
-anything = Syntax(lambda cls: cls.factory('anything')).describe(name="Anything", fixity='infix') 
+anything = Syntax(lambda cls, cache: cls.factory('anything', cache=cache)).describe(name="anything", fixity='infix') 
 
 def _matches(alg: Algebra[Any, GenState[Any]], data: ParseResult[Any])-> bool:
     state = GenState[Any].from_ast(ast = data, restore_pruned=True)
@@ -51,7 +52,7 @@ def _matches(alg: Algebra[Any, GenState[Any]], data: ParseResult[Any])-> bool:
     return isinstance(result, Right)
 
 
-def _find(alg: Algebra[Any, GenState[Any]], data: ParseResult[Any]) -> YieldGen[ParseResult[Any], None, None]:
+def _find(alg: Algebra[Any, GenState[Any]], data: ParseResult[Any]) -> PyGenerator[ParseResult[Any], None, None]:
     if not isinstance(data, (Marked, Collect)):
         if _matches(alg, data):
             yield data
@@ -88,14 +89,14 @@ def matches(syntax: Syntax[Any, Any], data: ParseResult[Any])-> bool:
     Returns:
         bool: ``True`` if the syntax succeeds on ``data``, ``False`` otherwise.
     """
-    gen = syntax(Finder)
+    gen = syntax(Finder, Cache())
     if isinstance(data, (Marked, Collect)):
         return _matches(gen, data.value)
     else:
         return _matches(gen, data)
 
 
-def find(syntax: Syntax[Any, Any], data: ParseResult[Any]) -> YieldGen[ParseResult[Any], None, None]:
+def find(syntax: Syntax[Any, Any], data: ParseResult[Any]) -> PyGenerator[ParseResult[Any], None, None]:
     """Yield all subtrees that match a syntax.
 
     Performs a depth‑first traversal of ``data`` and yields each node where the
@@ -111,5 +112,9 @@ def find(syntax: Syntax[Any, Any], data: ParseResult[Any]) -> YieldGen[ParseResu
         ParseResult[Any]: Each node that satisfies ``syntax`` (pre‑order: the
         current node is tested before visiting its children).
     """
-    gen = syntax(Finder)
-    yield from _find(gen, data)
+    gen = syntax(Finder, Cache())
+    yield from _find(gen, data)
+
+
+
+    

syncraft/generator.py

@@ -2,19 +2,20 @@ from __future__ import annotations
 
 from typing import (
     Any, TypeVar, Tuple, Optional,  Callable, Generic, 
-    List, 
+    List, Generator as PyGenerator
 )
 from functools import cached_property
 from dataclasses import dataclass, replace
 from syncraft.algebra import (
-    Algebra, Either, Left, Right, Error
+    Algebra, Either, Left, Right, Error, Incomplete
 )
+from syncraft.cache import Cache
 
 from syncraft.ast import (
     ParseResult, AST, Token, TokenSpec, 
     Nothing, TokenProtocol,
     Choice, Many, ChoiceKind,
-    Then, ThenKind, Marked
+    Then, ThenKind, SyncraftError
 )
 from syncraft.constraint import FrozenDict
 from syncraft.syntax import Syntax
@@ -142,27 +143,6 @@ class GenState(Bindable, Generic[T]):
         if isinstance(self.ast, Then) and (self.ast.kind != ThenKind.LEFT or self.restore_pruned):
             return replace(self, ast=self.ast.right)
         return replace(self, ast=None)
-
-    def down(self, index: int) -> GenState[T]:
-        """Descend through wrapper nodes to reach the contained value.
-
-        Currently unwraps ``Marked`` nodes. Raises ``TypeError`` for other
-        node types.
-
-        Args:
-            index: Placeholder for a future multi-child descent API.
-
-        Returns:
-            GenState[T]: State focused on the unwrapped child or unchanged when
-            pruned.
-        """
-        if self.ast is None:
-            return self        
-        match self.ast:
-            case Marked(value=value):
-                return replace(self, ast=value)
-            case _:
-                raise TypeError(f"Invalid AST type({self.ast}) for down traversal")
     
     @classmethod
     def from_ast(cls, 
@@ -236,8 +216,9 @@ class TokenGen(TokenSpec):
 
 @dataclass(frozen=True)
 class Generator(Algebra[ParseResult[T], GenState[T]]):  
+    
     @classmethod
-    def state(cls, ast: Optional[ParseResult[T]] = None, seed: int = 0, restore_pruned: bool = False)->GenState[T]:
+    def state(cls, ast: Optional[ParseResult[T]] = None, seed: int = 0, restore_pruned: bool = False)->GenState[T]: # type: ignore
         """Create an initial ``GenState`` for generation or checking.
 
         Args:
@@ -263,32 +244,26 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
         Returns:
             Algebra[B, GenState[T]]: An algebra yielding the final result.
         """
-        def flat_map_run(input: GenState[T], use_cache:bool) -> Either[Any, Tuple[B, GenState[T]]]:
-            try:
-                if not isinstance(input.ast, Then) or isinstance(input.ast, Nothing):
-                    return Left(Error(this=self, 
-                                      message=f"Expect Then got {input.ast}",
-                                      state=input))
-                lft = input.left() 
-                match self.run(lft, use_cache=use_cache):
-                    case Left(error):
-                        return Left(error)
-                    case Right((value, next_input)):
-                        r = input.right() 
-                        match f(value).run(r, use_cache):
-                            case Left(e):
-                                return Left(e)
-                            case Right((result, next_input)):
-                                return Right((result, next_input))
-                raise ValueError("flat_map should always return a value or an error.")
-            except Exception as e:
-                return Left(Error(
-                    message=str(e),
-                    this=self,
-                    state=input,
-                    error=e
-                ))
-        return self.__class__(run_f = flat_map_run, name=self.name) # type: ignore
+        def flat_map_run(input: GenState[T], use_cache:bool) -> PyGenerator[Incomplete[GenState[T]], GenState[T], Either[Any, Tuple[B, GenState[T]]]]:
+            if not isinstance(input.ast, Then) or isinstance(input.ast, Nothing):
+                return Left(Error(this=self, 
+                                    message=f"Expect Then got {input.ast}",
+                                    state=input))
+            lft = input.left() 
+            self_result = yield from self.run(lft, use_cache=use_cache)
+            match self_result:
+                case Left(error):
+                    return Left(error)
+                case Right((value, next_input)):
+                    r = input.right() 
+                    other_result = yield from f(value).run(r, use_cache)
+                    match other_result:
+                        case Left(e):
+                            return Left(e)
+                        case Right((result, next_input)):
+                            return Right((result, next_input))
+            raise SyncraftError("flat_map should always return a value or an error.", offending=self_result, expect=(Left, Right))
+        return self.__class__(flat_map_run, name=self.name, cache=self.cache) # type: ignore
 
 
     def many(self, *, at_least: int, at_most: Optional[int]) -> Algebra[Many[ParseResult[T]], GenState[T]]:
@@ -311,15 +286,16 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
             ValueError: If bounds are invalid.
         """
         if at_least <=0 or (at_most is not None and at_most < at_least):
-            raise ValueError(f"Invalid arguments for many: at_least={at_least}, at_most={at_most}")
-        def many_run(input: GenState[T], use_cache:bool) -> Either[Any, Tuple[Many[ParseResult[T]], GenState[T]]]:
+            raise SyncraftError(f"Invalid arguments for many: at_least={at_least}, at_most={at_most}", offending=(at_least, at_most), expect="at_least>0 and (at_most is None or at_most>=at_least)")
+        def many_run(input: GenState[T], use_cache:bool) -> PyGenerator[Incomplete[GenState[T]], GenState[T], Either[Any, Tuple[Many[ParseResult[T]], GenState[T]]]]:
             if input.pruned:
                 upper = at_most if at_most is not None else at_least + 2
                 count = input.rng("many").randint(at_least, upper)
                 ret: List[Any] = []
                 for i in range(count):
                     forked_input = input.fork(tag=len(ret))
-                    match self.run(forked_input, use_cache):
+                    self_result = yield from self.run(forked_input, use_cache)
+                    match self_result:
                         case Right((value, _)):
                             ret.append(value)
                         case Left(_):
@@ -331,8 +307,9 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
                                       message=f"Expect Many got {input.ast}",
                                       state=input))
                 ret = []
-                for x in input.ast.value: 
-                    match self.run(input.inject(x), use_cache):
+                for x in input.ast.value:
+                    self_result = yield from self.run(input.inject(x), use_cache) 
+                    match self_result:
                         case Right((value, _)):
                             ret.append(value)
                             if at_most is not None and len(ret) > at_most:
@@ -350,7 +327,7 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
                         state=input.inject(x)
                     )) 
                 return Right((Many(value=tuple(ret)), input))
-        return self.__class__(many_run, name=f"many({self.name})")  # type: ignore
+        return self.__class__(many_run, name=f"many({self.name})", cache=self.cache)  # type: ignore
     
  
     def or_else(self, # type: ignore
@@ -368,58 +345,70 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
             Algebra[Choice[ParseResult[T], ParseResult[T]], GenState[T]]: An
             algebra yielding which branch succeeded and its value.
         """
-        def or_else_run(input: GenState[T], use_cache:bool) -> Either[Any, Tuple[Choice[ParseResult[T], ParseResult[T]], GenState[T]]]:
+        def or_else_run(input: GenState[T], use_cache:bool) -> PyGenerator[Incomplete[GenState[T]], GenState[T], Either[Any, Tuple[Choice[ParseResult[T], ParseResult[T]], GenState[T]]]]:
             def exec(kind: ChoiceKind | None, 
                      left: GenState[T], 
-                     right: GenState[T])->Either[Any, Tuple[Choice[ParseResult[T], ParseResult[T]], GenState[T]]]:
+                     right: GenState[T]) -> PyGenerator[Incomplete[GenState[T]], GenState[T], Either[Any, Tuple[Choice[ParseResult[T], ParseResult[T]], GenState[T]]]]:
                 match kind:
                     case ChoiceKind.LEFT:
-                        match self.run(left, use_cache):
+                        self_result = yield from self.run(left, use_cache)
+                        match self_result:
                             case Right((value, next_input)):
                                 return Right((Choice(kind=ChoiceKind.LEFT, value=value), next_input))
                             case Left(error):
                                 return Left(error)
                     case ChoiceKind.RIGHT:
-                        match other.run(right, use_cache):
+                        other_result = yield from other.run(right, use_cache)
+                        match other_result:
                             case Right((value, next_input)):
                                 return Right((Choice(kind=ChoiceKind.RIGHT, value=value), next_input))
                             case Left(error):
                                 return Left(error)
                     case None:
-                        match self.run(left, use_cache):
+                        self_result = yield from self.run(left, use_cache)
+                        match self_result:
                             case Right((value, next_input)):
                                 return Right((Choice(kind=ChoiceKind.LEFT, value=value), next_input))
                             case Left(error):
-                                if isinstance(error, Error) and error.committed:
-                                    return Left(replace(error, committed=False))
-                                match other.run(left, use_cache):
+                                if isinstance(error, Error):
+                                    if error.fatal:
+                                        return Left(error)
+                                    elif error.committed:
+                                        return Left(replace(error, committed=False))
+                                other_result = yield from other.run(right, use_cache)
+                                match other_result:
                                     case Right((value, next_input)):
                                         return Right((Choice(kind=ChoiceKind.RIGHT, value=value), next_input))
                                     case Left(error):
                                         return Left(error)
-                raise ValueError(f"Invalid ChoiceKind: {kind}")
+                raise SyncraftError(f"Invalid ChoiceKind: {kind}", offending=kind, expect=(ChoiceKind.LEFT, ChoiceKind.RIGHT, None))
 
             if input.pruned:
                 forked_input = input.fork(tag="or_else")
                 which = forked_input.rng("or_else").choice((ChoiceKind.LEFT, ChoiceKind.RIGHT))
-                return exec(which, forked_input, forked_input)
+                result = yield from exec(which, forked_input, forked_input)
+                return result
             else:
                 if not isinstance(input.ast, Choice) or isinstance(input.ast, Nothing):
                     return Left(Error(this=self, 
                                       message=f"Expect Choice got {input.ast}",
                                       state=input))
                 else:
-                    return exec(input.ast.kind, 
+                    result = yield from exec(input.ast.kind, 
                                 input.inject(input.ast.value), 
                                 input.inject(input.ast.value))
-        return self.__class__(or_else_run, name=f"or_else({self.name} | {other.name})") # type: ignore
+                    return result
+                    
+        return self.__class__(or_else_run, name=f"or_else({self.name} | {other.name})", cache=self.cache | other.cache) # type: ignore
 
     @classmethod
     def token(cls, 
+              *,
+              cache: Cache,
               token_type: Optional[TokenType] = None, 
               text: Optional[str] = None, 
               case_sensitive: bool = False,
-              regex: Optional[re.Pattern[str]] = None
+              regex: Optional[re.Pattern[str]] = None,
               )-> Algebra[ParseResult[T], GenState[T]]:      
         """Match or synthesize a single token.
 
@@ -439,7 +428,8 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
         """
         gen = TokenGen(token_type=token_type, text=text, case_sensitive=case_sensitive, regex=regex)  
         lazy_self: Algebra[ParseResult[T], GenState[T]]
-        def token_run(input: GenState[T], use_cache:bool) -> Either[Any, Tuple[ParseResult[Token], GenState[T]]]:
+        def token_run(input: GenState[T], use_cache:bool) -> PyGenerator[Incomplete[GenState[T]], GenState[T], Either[Any, Tuple[ParseResult[Token], GenState[T]]]]:
+            yield from ()
             if input.pruned:
                 return Right((gen.gen(), input))
             else:
@@ -449,7 +439,7 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
                                       message=f"Expected a Token({gen.text}), but got {current}.", 
                                       state=input))
                 return Right((current, input))
-        lazy_self = cls(token_run, name=cls.__name__ + f'.token({token_type or text or regex})')  # type: ignore
+        lazy_self = cls(token_run, name=cls.__name__ + f'.token({token_type or text or regex})', cache=cache)  # type: ignore
         return lazy_self
 
 
@@ -459,7 +449,7 @@ def generate_with(
     data: Optional[ParseResult[Any]] = None, 
     seed: int = 0, 
     restore_pruned: bool = False
-) -> Tuple[AST, FrozenDict[str, Tuple[AST, ...]]] | Tuple[Any, None]:
+) -> Tuple[AST, None | FrozenDict[str, Tuple[AST, ...]]]:
     """
     Generate an AST from the given syntax, optionally constrained by a partial parse result.
 
@@ -473,13 +463,17 @@ def generate_with(
         A tuple of (AST, variable bindings) if successful, or (None, None) on failure.
     """
     from syncraft.syntax import run
-    return run(syntax, Generator, not restore_pruned, ast=data, seed=seed, restore_pruned=restore_pruned)
+    v, s = run(syntax=syntax, alg=Generator, use_cache=not restore_pruned, ast=data, seed=seed, restore_pruned=restore_pruned)
+    if s is not None:
+        return v, s.binding.bound()
+    else:
+        return v, None    
 
 
 def validate(
     syntax: Syntax[Any, Any], 
     data: ParseResult[Any]
-) -> Tuple[AST, FrozenDict[str, Tuple[AST, ...]]] | Tuple[Any, None]:
+) -> Tuple[AST, None | FrozenDict[str, Tuple[AST, ...]]]:
     """
     Validate a parse result (AST) against the given syntax.
 
@@ -491,12 +485,16 @@ def validate(
         A tuple of (AST, variable bindings) if valid, or (None, None) if invalid.
     """
     from syncraft.syntax import run
-    return run(syntax, Generator, True, ast=data, seed=0, restore_pruned=True)
+    v, s = run(syntax=syntax, alg=Generator, use_cache=True, ast=data, seed=0, restore_pruned=True)
+    if s is not None:
+        return v, s.binding.bound()
+    else:
+        return v, None    
 
 
 def generate(
     syntax
-) -> Tuple[AST, FrozenDict[str, Tuple[AST, ...]]] | Tuple[Any, None]:
+) -> Tuple[AST, None | FrozenDict[str, Tuple[AST, ...]]]:
     """
     Generate a random AST that conforms to the given syntax.
 
@@ -507,4 +505,8 @@ def generate(
         A tuple of (AST, variable bindings) if successful, or (None, None) on failure.
     """
     from syncraft.syntax import run
-    return run(syntax, Generator, False, ast=None, seed=random.randint(0, 2**32 - 1), restore_pruned=False)
+    v, s = run(syntax=syntax, alg=Generator, use_cache=False, ast=None, seed=random.randint(0, 2**32 - 1), restore_pruned=False)
+    if s is not None:
+        return v, s.binding.bound()
+    else:
+        return v, None

syncraft/lexer.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Optional, Tuple, List, Iterable
+import re
+from enum import Enum, auto
+
+
+import re
+from dataclasses import dataclass, fields, field
+from enum import Enum
+from typing import Optional, Callable, Dict, List, Tuple, Any, Pattern
+
+
+# -----------------------
+# Rule decorator
+# -----------------------
+def rule(state: str = "DEFAULT", next_state: Optional[str] = None):
+    """Decorator to attach lexer metadata to dataclass fields."""
+    def wrapper(f):
+        setattr(f, "_rule_meta", {"state": state, "next_state": next_state})
+        return f
+    return wrapper
+
+
+@dataclass
+class Lexer:
+    TokenType: Enum  # will be generated automatically
+    _states: Dict[str, re.Scanner] = field(default_factory=dict, init=False)
+    @classmethod
+    def build(cls) -> "Lexer":
+        # 1. generate TokenType enum from dataclass fields
+        cls.TokenType = Enum(
+            cls.__name__ + "Type",
+            {f.name: f.name for f in fields(cls)}
+        )
+
+        # 2. collect regex rules per state
+        states: Dict[str, List[Tuple[str, Callable]]] = {}
+
+        for f in fields(cls):
+            regex = getattr(cls, f.name)
+            meta = getattr(f, "_rule_meta", {})
+            state = meta.get("state", "DEFAULT")
+            next_state = meta.get("next_state")
+
+            if state not in states:
+                states[state] = []
+
+            states[state].append(
+                (regex, cls._make_action(f.name, next_state))
+            )
+
+        # 3. build Scanner per state
+        instance = cls.__new__(cls)
+        instance._states = {
+            s: re.Scanner(rules) for s, rules in states.items()
+        }
+        instance._current_state = "DEFAULT"
+
+        # make parent accessible to action functions
+        for scanner in instance._states.values():
+            scanner.parent = instance
+
+        return instance
+
+    @staticmethod
+    def _make_action(name: str, next_state: Optional[str] = None):
+        def action(scanner: Any, token: str):
+            if next_state:
+                scanner.parent.switch(next_state)
+            return (scanner.parent.TokenType[name], token)
+        return action
+
+    def switch(self, state: str) -> None:
+        if state not in self._states:
+            raise ValueError(f"No such state: {state}")
+        self._current_state = state
+
+    def tokenize(self, text: str) -> List[Tuple[Enum, str]]:
+        tokens, remainder = self._states[self._current_state].scan(text)
+        if remainder:
+            raise SyntaxError(f"Unrecognized input: {remainder!r}")
+        return tokens
+
+
+# -----------------------
+# Example SQLite-like lexer
+# -----------------------
+@dataclass(frozen=True)
+class SQLiteLexer(Lexer):
+    # Default state tokens
+    @rule(state="DEFAULT")
+    WS: str = r"[ \t\n]+"
+
+    @rule(state="DEFAULT")
+    NUMBER: str = r"\d+(\.\d+)?([eE][+-]?\d+)?"
+
+    @rule(state="DEFAULT")
+    IDENT: str = r"[a-zA-Z_][a-zA-Z0-9_]*"
+
+    @rule(state="DEFAULT", next_state="STRING")
+    STRING_QUOTE: str = r"'"
+
+    @rule(state="DEFAULT")
+    PLUS: str = r"\+"
+
+    @rule(state="DEFAULT")
+    EQ: str = r"="
+
+    # String state tokens
+    @rule(state="STRING")
+    STRING_TEXT: str = r"[^']+"
+
+    @rule(state="STRING", next_state="DEFAULT")
+    STRING_END: str = r"'"
+
+    # Example comment (line comment)
+    @rule(state="DEFAULT")
+    COMMENT: str = r"--[^\n]*"
+
+
+# -----------------------
+# Usage example
+# -----------------------
+if __name__ == "__main__":
+    lexer = SQLiteLexer.build()
+    sql = "abc + 123 'hello' -- comment"
+
+    tokens = lexer.tokenize(sql)
+    for tok_type, value in tokens:
+        print(f"{tok_type.name:12} : {value!r}")