syncraft 0.2.6__tar.gz → 0.2.8__tar.gz

{syncraft-0.2.6/syncraft.egg-info → syncraft-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: syncraft
-Version: 0.2.6
+Version: 0.2.8
 Summary: Parser combinator library
 Author-email: Michael Afmokt <michael@esacca.com>
 License-Expression: MIT

{syncraft-0.2.6 → syncraft-0.2.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "syncraft"
-version = "0.2.6"
+version = "0.2.8"
 description = "Parser combinator library"
 license = "MIT"
 license-files = ["LICENSE"]

{syncraft-0.2.6 → syncraft-0.2.8}/syncraft/algebra.py

@@ -77,7 +77,7 @@ class Algebra(Generic[A, S]):
 ######################################################## shared among all subclasses ########################################################
     run_f: Callable[[S, bool], Generator[Incomplete[S], S, Either[Any, Tuple[A, S]]]] 
     name: Hashable
-    cache: Cache[ S, Either[Any, Tuple[A, S]] ]
+    cache: Cache[Either[Any, Tuple[A, S]]]
 
     @classmethod
     def state(cls, **kwargs:Any)->Optional[S]: 
@@ -103,10 +103,10 @@ class Algebra(Generic[A, S]):
              cache: Cache) -> Algebra[A, S]:
         def algebra_lazy_run(input: S, use_cache:bool) -> Generator[Incomplete[S], S, Either[Any, Tuple[A, S]]]:
             alg = thunk()
-            print('--' * 20, "Algebra.lazy.algebra_lazy_run", '--' * 20)
-            print('thunk', thunk, id(thunk))
-            print('input', input, id(input))
-            print('alg', alg, id(alg))
+            # print('--' * 20, "Algebra.lazy.algebra_lazy_run", '--' * 20)
+            # print('thunk', thunk, id(thunk))
+            # print('input', input, id(input))
+            # print('alg', alg, id(alg))
             result = yield from alg.run(input, use_cache)
             return result
         return cls(algebra_lazy_run, name=cls.__name__ + '.lazy', cache=cache)

{syncraft-0.2.6 → syncraft-0.2.8}/syncraft/ast.py

@@ -491,6 +491,7 @@ class LazySpec(SyntaxSpec, Generic[A]):
     value: A
 @dataclass(frozen=True)
 class ThenSpec(SyntaxSpec, Generic[A, B]):
+    kind: ThenKind
     left: A
     right: B
 

syncraft-0.2.8/syncraft/cache.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, TypeVar, Hashable, Generic, Callable, Any, Generator, overload, Literal
+from weakref import WeakKeyDictionary
+from syncraft.ast import SyncraftError
+
+
+class RecursionError(SyncraftError):
+    def __init__(self, message: str, offending: Any, expect: Any = None, **kwargs: Any) -> None:
+        super().__init__(message, offending, expect, **kwargs)
+
+
+@dataclass(frozen=True)
+class InProgress:
+    _instance = None
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(InProgress, cls).__new__(cls)
+        return cls._instance
+    def __str__(self)->str:
+        return self.__class__.__name__
+    def __repr__(self)->str:
+        return self.__str__()
+
+
+
+
+Args = TypeVar('Args', bound=Hashable)
+Ret = TypeVar('Ret')
+
+@dataclass
+class Cache(Generic[Ret]):
+    cache: WeakKeyDictionary[Callable[..., Any], Dict[Hashable, Ret | InProgress]] = field(default_factory=WeakKeyDictionary)
+
+    def __contains__(self, f: Callable[..., Any]) -> bool:
+        return f in self.cache
+
+    def __repr__(self) -> str:
+        return f"Cache({({f.__name__: list(c.keys()) for f, c in self.cache.items()})})"
+
+
+    def __or__(self, other: Cache[Any]) -> Cache[Any]:
+        assert self.cache is other.cache, "There should be only one global cache"
+        return self
+
+    @overload
+    def _execute(self, 
+                 f: Callable[..., Ret], 
+                 *args:Any, 
+                 is_gen: Literal[False], 
+                 **kwargs:Any) -> Ret: ...
+    @overload
+    def _execute(self, 
+                 f: Callable[..., Generator[Any, Any, Ret]], 
+                 *args: Any, 
+                 is_gen: Literal[True], 
+                 **kwargs: Any) -> Generator[Any, Any, Ret]: ...
+
+
+    def _execute(self, 
+            f: Callable[..., Any], 
+            *args: Any,
+            is_gen: bool,
+            **kwargs: Any) -> Ret | Generator[Any, Any, Ret]:
+        if f not in self.cache:
+            self.cache.setdefault(f, dict())
+        c: Dict[Hashable, Ret | InProgress] = self.cache[f]
+        key = (args, tuple(sorted(kwargs.items())))
+        if key in c:
+            v = c[key]
+            if isinstance(v, InProgress):
+                raise RecursionError("Left-recursion detected in parser", offending=f, state=args)
+            else:
+                return v        
+        try:
+            c[key] = InProgress()
+            if is_gen:
+                result = yield from f(*args, **kwargs)
+            else:
+                result = f(*args, **kwargs)
+            c[key] = result
+            if kwargs.get('use_cache', True) is False:
+                c.pop(key, None)
+            return result
+        except Exception as e:
+            c.pop(key, None)
+            raise e
+        
+    def gen(self, 
+            f: Callable[..., Generator[Any, Any, Ret]], 
+            *args: Any, 
+            **kwargs: Any) -> Generator[Any, Any, Ret]:
+        if f not in self.cache:
+            self.cache.setdefault(f, dict())
+        c: Dict[Hashable, Ret | InProgress] = self.cache[f]
+        key = (tuple(filter(lambda x: not isinstance(x, Cache), args)), tuple(sorted(filter(lambda item: not isinstance(item[1], Cache), kwargs.items()))))        
+        if key in c:
+            v = c[key]
+            if isinstance(v, InProgress):
+                raise RecursionError("Left-recursion detected in parser", offending=f, state=args)
+            else:
+                return v        
+        try:
+            c[key] = InProgress()
+            result = yield from f(*args, **kwargs)
+            c[key] = result
+            if kwargs.get('use_cache', True) is False:
+                c.pop(key, None)
+            return result
+        except Exception as e:
+            c.pop(key, None)
+            raise e
+        
+
+    def call(self, 
+             f: Callable[..., Ret], 
+             *args:Any, 
+             **kwargs:Any) -> Ret:
+        if f not in self.cache:
+            self.cache.setdefault(f, dict())
+        c: Dict[Hashable, Ret | InProgress] = self.cache[f]
+        key = (tuple(filter(lambda x: not isinstance(x, Cache), args)), tuple(sorted(filter(lambda item: not isinstance(item[1], Cache), kwargs.items()))))
+        if key in c:
+            v = c[key]
+            if isinstance(v, InProgress):
+                raise RecursionError("Left-recursion detected in parser", offending=f, state=args)
+            else:
+                return v        
+        try:
+            c[key] = InProgress()
+            result = f(*args, **kwargs)
+            c[key] = result
+            if kwargs.get('use_cache', True) is False:
+                c.pop(key, None)
+            return result
+        except Exception as e:
+            c.pop(key, None)
+            raise e
+        
+
+

{syncraft-0.2.6 → syncraft-0.2.8}/syncraft/generator.py

@@ -245,7 +245,7 @@ class Generator(Algebra[ParseResult[T], GenState[T]]):
             Algebra[B, GenState[T]]: An algebra yielding the final result.
         """
         def flat_map_run(input: GenState[T], use_cache:bool) -> PyGenerator[Incomplete[GenState[T]], GenState[T], Either[Any, Tuple[B, GenState[T]]]]:
-            if not isinstance(input.ast, Then) or isinstance(input.ast, Nothing):
+            if not input.pruned and (not isinstance(input.ast, Then) or isinstance(input.ast, Nothing)):
                 return Left(Error(this=self, 
                                     message=f"Expect Then got {input.ast}",
                                     state=input))

{syncraft-0.2.6 → syncraft-0.2.8}/syncraft/parser.py

@@ -122,18 +122,6 @@ class ParserState(Bindable, Generic[T]):
 class Parser(Algebra[T, ParserState[T]]):
     @classmethod
     def state(cls, sql: str, dialect: str) -> ParserState[T]: # type: ignore
-        """Tokenize SQL text into an initial ``ParserState``.
-
-        Uses ``sqlglot.tokenize`` for the given dialect and wraps tokens into
-        the project's ``Token`` type.
-
-        Args:
-            sql: The SQL text to tokenize.
-            dialect: The sqlglot dialect name (e.g. "sqlite", "duckdb").
-
-        Returns:
-            ParserState[T]: Initial parser state at index 0.
-        """
         tokens = tuple([Token(token_type=token.token_type, text=token.text) for token in tokenize(sql, dialect=dialect)])
         return ParserState.from_tokens(tokens)  # type: ignore
 
@@ -146,21 +134,6 @@ class Parser(Algebra[T, ParserState[T]]):
               case_sensitive: bool = False,
               regex: Optional[re.Pattern[str]] = None
               )-> Algebra[T, ParserState[T]]:
-        """Match a single token according to a specification.
-
-        Succeeds when the current token satisfies the provided
-        ``TokenSpec`` (by type, exact text, or regex). On failure,
-        an informative ``Error`` is produced with location context.
-
-        Args:
-            token_type: Expected enum type of the token.
-            text: Exact token text to match.
-            case_sensitive: Whether text matching is case sensitive.
-            regex: Regular expression pattern to match token text.
-
-        Returns:
-            Algebra[T, ParserState[T]]: An algebra yielding the matched token.
-        """
         spec = TokenSpec(token_type=token_type, text=text, case_sensitive=case_sensitive, regex=regex)
         def token_run(state: ParserState[T], use_cache:bool) -> Generator[Incomplete[ParserState[T]],ParserState[T], Either[Any, Tuple[T, ParserState[T]]]]:
             while True:

{syncraft-0.2.6 → syncraft-0.2.8}/syncraft/syntax.py

@@ -54,7 +54,7 @@ class Syntax(Generic[A, S]):
     """
     The core signature of Syntax is take an Algebra Class and return an Algebra Instance.
     """
-    alg: Callable[[Type[Algebra[Any, Any]], Cache[Any, Any]], Algebra[A, S]]
+    alg: Callable[[Type[Algebra[Any, Any]], Cache[Any]], Algebra[A, S]]
     meta: Description = field(default_factory=Description, repr=False)
 
     def algebra(self, name: str | MethodType | FunctionType, *args: Any, **kwargs: Any) -> Syntax[A, S]:
@@ -71,7 +71,7 @@ class Syntax(Generic[A, S]):
         Returns:
             A new Syntax reflecting the transformed algebra.
         """
-        def algebra_run(cls: Type[Algebra[Any, S]], cache: Cache[Any, Any]) -> Algebra[Any, S]:
+        def algebra_run(cls: Type[Algebra[Any, S]], cache: Cache[Any]) -> Algebra[Any, S]:
             a = self(cls, cache)
             if isinstance(name, str):
                 attr = getattr(a, name, None) or getattr(cls, name, None)
@@ -97,7 +97,7 @@ class Syntax(Generic[A, S]):
     def as_(self, typ: Type[B]) -> B:
         return cast(typ, self)  # type: ignore
 
-    def __call__(self, alg: Type[Algebra[Any, Any]], cache: Cache[Any, Any]) -> Algebra[A, S]:
+    def __call__(self, alg: Type[Algebra[Any, Any]], cache: Cache[Any]) -> Algebra[A, S]:
         return self.alg(alg, cache)
 
 
@@ -562,37 +562,28 @@ def run(*,
 
 
 def lazy(thunk: Callable[[], Syntax[A, S]]) -> Syntax[A, S]:
-    syntax: Optional[Syntax[A, S]] = None
     algebra: Optional[Algebra[A, S]] = None
+    syntax: Optional[Syntax[A, S]] = None
+    previous_cls: Optional[Type[Algebra[Any, S]]] = None
     def syntax_lazy_run(cls: Type[Algebra[Any, S]], cache: Cache) -> Algebra[A, S]:
-        nonlocal syntax, algebra
-        print('==' * 20, 'Syntax.lazy.syntax_lazy_run', '==' * 20)
-        print('thunk', thunk, id(thunk))
-        print('syntax', syntax, id(syntax))
-        print('algebra', algebra, id(algebra))
+        nonlocal algebra, syntax, previous_cls
+        # print('==' * 20, 'Syntax.lazy.syntax_lazy_run', '==' * 20)
+        # print('thunk', thunk, id(thunk))
+        # print('syntax', syntax, id(syntax))
+        # print('algebra', algebra, id(algebra))
         if syntax is None:
             syntax = thunk()
         def algebra_lazy_f():
             if syntax is None:
                 raise SyncraftError("Lazy thunk did not resolve to a Syntax", offending=thunk, expect="a Syntax")
             return syntax(cls, cache)
-        if algebra is None:
-            algebra = cls.lazy(algebra_lazy_f, cache=cache)  
+        if algebra is None or (previous_cls is not None and previous_cls is not cls):
+            algebra = cls.lazy(algebra_lazy_f, cache=cache)
+            previous_cls = cls
         return algebra
     return Syntax(syntax_lazy_run).describe(name='lazy(?)', fixity='postfix') 
 
 
-# def lazy(thunk: Callable[[], Syntax[A, S]]) -> Syntax[A, S]:
-#     resolved: Optional[Syntax[A, S]] = None
-    
-#     def run_lazy(cls: Type[Algebra[Any, S]], cache: Cache) -> Algebra[A, S]:
-#         nonlocal resolved
-#         if resolved is None:
-#             resolved = thunk()
-#         return resolved(cls, cache)  # reuse the same Algebra instance
-#     return Syntax(run_lazy)
-
-
 
 
 def token(*,

{syncraft-0.2.6 → syncraft-0.2.8}/syncraft/walker.py

@@ -7,13 +7,14 @@ from dataclasses import dataclass, replace, field
 from syncraft.algebra import (
     Algebra, Either, Right, Incomplete, Left, SyncraftError
 )
-from syncraft.ast import TokenSpec, ThenSpec, ManySpec, ChoiceSpec, LazySpec
+from syncraft.ast import TokenSpec, ThenSpec, ManySpec, ChoiceSpec, LazySpec, ThenKind
 from syncraft.parser import TokenType
 from syncraft.constraint import Bindable, FrozenDict
 
 import re
 from syncraft.syntax import Syntax
 from syncraft.cache import Cache
+from rich import print
 
 
 S = TypeVar('S', bound=Bindable)
@@ -31,7 +32,6 @@ SS = TypeVar('SS', bound=Hashable)
 class WalkerState(Bindable, Generic[SS]):
     reducer: Optional[Callable[[Any, SS], SS]] = None
     acc: Optional[SS] = None
-    visited: frozenset = field(default_factory=frozenset)
 
 
     def reduce(self, value: Any) -> WalkerState[SS]:
@@ -41,8 +41,6 @@ class WalkerState(Bindable, Generic[SS]):
         else:
             return replace(self, acc=value)
         
-    def visit(self, key: Hashable) -> WalkerState[SS]:
-        return replace(self, visited=self.visited | {key})
 
 
 
@@ -55,24 +53,40 @@ class Walker(Algebra[SS, WalkerState[SS]]):
         return WalkerState(reducer=reducer, acc=init)
 
 
+
+    @classmethod
+    def token(cls, 
+              *,
+              cache: Cache,
+              token_type: Optional[TokenType] = None, 
+              text: Optional[str] = None, 
+              case_sensitive: bool = False,
+              regex: Optional[re.Pattern[str]] = None
+              )-> Algebra[Any, WalkerState[SS]]:      
+        def token_run(input: WalkerState[SS], use_cache:bool) -> PyGenerator[Incomplete[WalkerState[SS]], WalkerState[SS], Either[Any, Tuple[Any, WalkerState[SS]]]]:
+            yield from ()
+            data = TokenSpec(token_type=token_type, text=text, regex=regex, case_sensitive=case_sensitive)
+            return Right((data, input.reduce(data)))
+        return cls(token_run, name=cls.__name__ + f'.token({token_type or text or regex})', cache=cache)  
+
     @classmethod
-    def lazy(cls, thunk: Callable[[], Algebra[Any, WalkerState[SS]]], cache: Cache) -> Algebra[Any, WalkerState[SS]]:
-        def alazy_run(input: WalkerState[SS], use_cache:bool) -> PyGenerator[Incomplete[WalkerState[SS]], WalkerState[SS], Either[Any, Tuple[Any, WalkerState[SS]]]]:
-            result = yield from thunk().run(input, use_cache)
-            return result
-
-        def lazy_run(input: WalkerState[SS], use_cache:bool) -> PyGenerator[Incomplete[WalkerState[SS]], WalkerState[SS], Either[Any, Tuple[Any, WalkerState[SS]]]]:
-            print('thunk', thunk, input.visited)
-            if thunk in input.visited:
-                return Right((None, input))
-            else:
-                thunk_result = yield from thunk().run(input, use_cache)
-                match thunk_result:
-                    case Right((value, from_thunk)):
-                        data = LazySpec(value=value)
-                        return Right((data, from_thunk.visit(thunk).reduce(data)))
+    def lazy(cls, 
+             thunk: Callable[[], Algebra[Any, WalkerState[SS]]], 
+             cache: Cache) -> Algebra[Any, WalkerState[SS]]:
+        def algebra_lazy_run(input: WalkerState[SS], use_cache:bool) -> PyGenerator[Incomplete[WalkerState[SS]], WalkerState[SS], Either[Any, Tuple[Any, WalkerState[SS]]]]:
+            alg = thunk()
+            print('--' * 20, "Walker.lazy.algebra_lazy_run", '--' * 20)
+            print('thunk', thunk, id(thunk))
+            print('input', input, id(input))
+            print('alg', alg, id(alg))
+            thunk_result = yield from alg.run(input, use_cache)
+            match thunk_result:
+                case Right((value, from_thunk)):
+                    data = LazySpec(value=value)
+                    return Right((data, from_thunk.reduce(data)))
             raise SyncraftError("flat_map should always return a value or an error.", offending=thunk_result, expect=(Left, Right))
-        return cls(lazy_run, name=cls.__name__ + '.lazy', cache=cache)
+        return cls(algebra_lazy_run, name=cls.__name__ + '.lazy', cache=cache)
+
 
 
     def then_both(self, other: Algebra[Any, WalkerState[SS]]) -> Algebra[Any, WalkerState[SS]]:
@@ -83,16 +97,16 @@ class Walker(Algebra[SS, WalkerState[SS]]):
                     other_result = yield from other.run(from_left, use_cache)
                     match other_result:
                         case Right((result, from_right)):
-                            data = ThenSpec(left=value, right=result)
+                            data = ThenSpec(kind=ThenKind.BOTH, left=value, right=result)
                             return Right((data, from_right.reduce(data)))
             raise SyncraftError("flat_map should always return a value or an error.", offending=self_result, expect=(Left, Right))
         return self.__class__(then_run, name=self.name, cache=self.cache | other.cache) 
 
     def then_left(self, other: Algebra[Any, WalkerState[SS]]) -> Algebra[Any, WalkerState[SS]]:
-        return self.then_both(other)  # For simplicity, treat as both
+        return self.then_both(other).map(lambda t: replace(t, kind=ThenKind.LEFT))
 
     def then_right(self, other: Algebra[Any, WalkerState[SS]]) -> Algebra[Any, WalkerState[SS]]:
-        return self.then_both(other)
+        return self.then_both(other).map(lambda t: replace(t, kind=ThenKind.RIGHT))
 
 
     def many(self, *, at_least: int, at_most: Optional[int]) -> Algebra[Any, WalkerState[SS]]:
@@ -121,27 +135,9 @@ class Walker(Algebra[SS, WalkerState[SS]]):
             raise SyncraftError("", offending=self)
         return self.__class__(or_else_run, name=f"or_else({self.name} | {other.name})", cache=self.cache | other.cache) 
 
-    @classmethod
-    def token(cls, 
-              *,
-              cache: Cache,
-              token_type: Optional[TokenType] = None, 
-              text: Optional[str] = None, 
-              case_sensitive: bool = False,
-              regex: Optional[re.Pattern[str]] = None
-              )-> Algebra[Any, WalkerState[SS]]:      
-        def token_run(input: WalkerState[SS], use_cache:bool) -> PyGenerator[Incomplete[WalkerState[SS]], WalkerState[SS], Either[Any, Tuple[Any, WalkerState[SS]]]]:
-            yield from ()
-            data = TokenSpec(token_type=token_type, text=text, regex=regex, case_sensitive=case_sensitive)
-            return Right((data, input.reduce(data)))
-        return cls(token_run, name=cls.__name__ + f'.token({token_type or text or regex})', cache=cache)  
 
 
-def walk(syntax: Syntax[Any, Any], reducer: Callable[[Any, Any], SS], init: SS)-> Optional[SS]:
+def walk(syntax: Syntax[Any, Any], reducer: Callable[[Any, Any], SS], init: SS)-> Tuple[Any, None | SS]:
     from syncraft.syntax import run
-    from rich import print
     v, s = run(syntax=syntax, alg=Walker, use_cache=False, reducer=reducer, init=init)
-    if s is not None:
-        return s.acc
-    else:
-        return None
+    return v, s

{syncraft-0.2.6 → syncraft-0.2.8/syncraft.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: syncraft
-Version: 0.2.6
+Version: 0.2.8
 Summary: Parser combinator library
 Author-email: Michael Afmokt <michael@esacca.com>
 License-Expression: MIT

syncraft-0.2.8/tests/test_lazy.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+from syncraft.parser import token
+import pytest
+from syncraft.walker import walk
+from syncraft.ast import Nothing
+from syncraft.syntax import lazy, literal, regex
+from syncraft.parser import parse
+from syncraft.generator import TokenGen
+from syncraft.cache import RecursionError
+from rich import print
+
+def test_simple_recursion()->None:
+    A = lazy(lambda: literal('a') + ~A | literal('a'))
+    v, s = parse(A, 'a a a', dialect='sqlite')
+    print(v)
+    ast1, inv = v.bimap()
+    print(ast1)
+    assert ast1 == (
+        TokenGen.from_string('a'), 
+        (
+            TokenGen.from_string('a'), 
+            (
+                TokenGen.from_string('a'), 
+                Nothing()
+            )
+        )
+    )
+
+def test_direct_recursion()->None:
+    Expr1 = lazy(lambda: literal('a') + ~Expr1)
+    v, s = parse(Expr1, 'a a a', dialect='sqlite')
+    x, _ = v.bimap()
+    assert x == (
+        TokenGen.from_string('a'), 
+        (
+            TokenGen.from_string('a'), 
+            (
+                TokenGen.from_string('a'), 
+                Nothing()
+            )
+        )
+    )
+
+
+def test_mutual_recursion()->None:
+    A = lazy(lambda: literal('a') + B)
+    B = lazy(lambda: (literal('b') + A) | (literal('c')))
+    v, s = parse(A, 'a b a b a c', dialect='sqlite')
+    print('--' * 20, "test_mutual_recursion", '--' * 20)
+    print(v)
+    ast1, inv = v.bimap()
+    print(ast1)
+    assert ast1 == (
+        TokenGen.from_string('a'), 
+        (
+            TokenGen.from_string('b'), 
+            TokenGen.from_string('a'), 
+            (
+                TokenGen.from_string('b'), 
+                TokenGen.from_string('a'), 
+                TokenGen.from_string('c')
+            )
+        )
+    )
+
+
+
+def test_recursion() -> None:
+    A = literal('a')
+    B = literal('b')
+    L = lazy(lambda: literal("if") >> (A | B) // literal('then'))
+
+    def parens():
+        return A + ~lazy(parens) + B
+    p_code = 'a a b b'
+    LL = parens() | L
+    
+    v, s = parse(LL, p_code, dialect='sqlite')
+    ast1, inv = v.bimap()
+    print(v)
+    print(ast1)
+    assert ast1 == (
+            TokenGen.from_string('a'), 
+            (
+                TokenGen.from_string('a'), 
+                Nothing(), 
+                TokenGen.from_string('b')
+            ), 
+            TokenGen.from_string('b')
+        )
+
+def test_direct_left_recursion()->None:
+    Term = literal('n')
+    Expr = lazy(lambda: Expr + literal('+') + Term | Term)
+    with pytest.raises(RecursionError):
+        v, s = parse(Expr, 'n+n+n', dialect='sqlite')
+
+
+
+def test_indirect_left_recursion()->None:
+    NUMBER = regex(r'\d+').map(int)
+    PLUS = token(text='+')
+    STAR = token(text='*')
+    A = lazy(lambda: (B >> PLUS >> A) | B)
+    B = lazy(lambda: (A >> STAR >> NUMBER) | NUMBER)
+    with pytest.raises(RecursionError):
+        v, s = parse(A, '1 + 2 * 3', dialect='sqlite')

{syncraft-0.2.6 → syncraft-0.2.8}/tests/test_walk.py

@@ -6,17 +6,17 @@ from syncraft.ast import TokenSpec
 
 def test_walk() -> None:
     syntax = literal("test")
-    result = walk(syntax, lambda a, s: s + (a,), ())  
-    assert result == (TokenSpec.create(text='test', case_sensitive=True),)
+    result, s = walk(syntax, lambda a, s: s + (a,), ())  
+    assert s and s.acc == (TokenSpec.create(text='test', case_sensitive=True),)
 
 
 def test_walk_case_insensitive() -> None:
     A = literal('a').many()
     B = literal('b').many()
     syntax = literal("if") >> (A | B) + literal('then')
-    result = walk(syntax, lambda a, s: s + (a,) if isinstance(a, TokenSpec) else s, ())  
+    result, s = walk(syntax, lambda a, s: s + (a,) if isinstance(a, TokenSpec) else s, ())  
     print(result)
-    assert result == (
+    assert s and s.acc == (
         TokenSpec.create(text='if', case_sensitive=True),
         TokenSpec.create(text='a', case_sensitive=True),
         TokenSpec.create(text='b', case_sensitive=True),

syncraft-0.2.6/syncraft/cache.py

@@ -1,113 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Dict, TypeVar, Hashable, Generic, Callable, Any, Generator, overload, Literal
-from weakref import WeakKeyDictionary
-from syncraft.ast import SyncraftError
-
-
-class RecursionError(SyncraftError):
-    def __init__(self, message: str, offending: Any, expect: Any = None, **kwargs: Any) -> None:
-        super().__init__(message, offending, expect, **kwargs)
-
-
-@dataclass(frozen=True)
-class InProgress:
-    _instance = None
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super(InProgress, cls).__new__(cls)
-        return cls._instance
-    def __str__(self)->str:
-        return self.__class__.__name__
-    def __repr__(self)->str:
-        return self.__str__()
-
-
-
-
-Args = TypeVar('Args', bound=Hashable)
-Ret = TypeVar('Ret')
-
-@dataclass
-class Cache(Generic[Args, Ret]):
-    cache: WeakKeyDictionary[Callable[..., Any], Dict[Args, Ret | InProgress]] = field(default_factory=WeakKeyDictionary)
-
-    def __contains__(self, f: Callable[..., Any]) -> bool:
-        return f in self.cache
-
-    def __repr__(self) -> str:
-        return f"Cache({({f.__name__: list(c.keys()) for f, c in self.cache.items()})})"
-
-
-    def __or__(self, other: Cache[Args, Any]) -> Cache[Args, Any]:
-        assert self.cache is other.cache, "There should be only one global cache"
-        if self.cache is other.cache:
-            return self
-        elif len(self.cache) == 0:
-            return other
-        elif len(other.cache) == 0:
-            return self
-        merged = Cache[Args, Ret]()
-        for f, c in self.cache.items():
-            merged.cache[f] = c.copy()
-        for f, c in other.cache.items():
-            merged.cache.setdefault(f, {}).update(c)
-        return merged
-
-    @overload
-    def _execute(self, 
-                 f: Callable[[Args, bool], Ret], 
-                 args: Args, 
-                 use_cache: bool, 
-                 is_gen: Literal[False]) -> Ret: ...
-    @overload
-    def _execute(self, 
-                 f: Callable[[Args, bool], Generator[Any, Any, Ret]], 
-                 args: Args, 
-                 use_cache: bool, 
-                 is_gen: Literal[True]) -> Generator[Any, Any, Ret]: ...
-
-
-    def _execute(self, 
-            f: Callable[[Args, bool], Any], 
-            args: Args, 
-            use_cache:bool,
-            is_gen: bool
-            ) -> Ret | Generator[Any, Any, Ret]:
-        if f not in self.cache:
-            self.cache.setdefault(f, dict())
-        c: Dict[Args, Ret | InProgress] = self.cache[f]
-        if args in c:
-            v = c[args]
-            if isinstance(v, InProgress):
-                raise RecursionError("Left-recursion detected in parser", offending=f, state=args)
-            else:
-                return v        
-        try:
-            c[args] = InProgress()
-            if is_gen:
-                result = yield from f(args, use_cache)
-            else:
-                result = f(args, use_cache)
-            c[args] = result
-            if not use_cache:
-                c.pop(args, None)
-            return result
-        except Exception as e:
-            c.pop(args, None)  
-            raise e
-        
-    def gen(self, 
-            f: Callable[[Args, bool], Generator[Any, Any, Ret]], 
-            args: Args, 
-            use_cache:bool) -> Generator[Any, Any, Ret]:
-        return (yield from self._execute(f, args, use_cache, is_gen=True)) 
-
-    def call(self, 
-            f: Callable[[Args, bool], Ret], 
-            args: Args, 
-            use_cache:bool) -> Ret:
-        return self._execute(f, args, use_cache, is_gen=False) 
-
-

syncraft-0.2.6/tests/test_lazy.py

@@ -1,50 +0,0 @@
-from __future__ import annotations
-from syncraft.parser import token
-
-from syncraft.walker import walk
-from syncraft.ast import Nothing
-from syncraft.syntax import lazy, literal, regex
-from syncraft.parser import parse
-from syncraft.generator import TokenGen
-
-from rich import print
-
-
-def test_recursion() -> None:
-    A = literal('a')
-    B = literal('b')
-    L = lazy(lambda: literal("if") >> (A | B) // literal('then'))
-
-    def parens():
-        return A + ~lazy(parens) + B
-    p_code = 'a a b b'
-    LL = parens() | L
-    
-    v, s = parse(LL, p_code, dialect='sqlite')
-    ast1, inv = v.bimap()
-    print(v)
-    print(ast1)
-    assert ast1 == (
-            TokenGen.from_string('a'), 
-            (
-                TokenGen.from_string('a'), 
-                Nothing(), 
-                TokenGen.from_string('b')
-            ), 
-            TokenGen.from_string('b')
-        )
-
-def test_direct_left_recursion()->None:
-    Term = literal('n')
-    Expr = lazy(lambda: Expr + literal('+') + Term | Term)
-    v, s = parse(Expr, 'n+n+n', dialect='sqlite')
-
-
-
-def test_indirect_left_recursion()->None:
-    NUMBER = regex(r'\d+').map(int)
-    PLUS = token(text='+')
-    STAR = token(text='*')
-    A = lazy(lambda: (B >> PLUS >> A) | B)
-    B = lazy(lambda: (A >> STAR >> NUMBER) | NUMBER)
-    v, s = parse(A, '1 + 2 * 3', dialect='sqlite')