llparse 0.1.1__tar.gz → 0.1.2__tar.gz

{llparse-0.1.1 → llparse-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llparse
-Version: 0.1.1
+Version: 0.1.2
 Summary: A Parody of llparse written for writing C Parsers with Python
 Author-email: Vizonex <VizonexBusiness@gmail.com>
 Requires-Python: >=3.9

{llparse-0.1.1 → llparse-0.1.2}/llparse/C_compiler.py

@@ -29,32 +29,32 @@ class CCompiler:
         out.append("#include <string.h>")
         out.append("")
         # Seems LLParse was updated from /* UNREACHABLE */ abort(); to a Macro, Intresting...
-        out.append('#ifdef __SSE4_2__')
-        out.append(' #ifdef _MSC_VER')
-        out.append('  #include <nmmintrin.h>')
-        out.append(' #else  /* !_MSC_VER */')
-        out.append('  #include <x86intrin.h>')
-        out.append(' #endif  /* _MSC_VER */')
-        out.append('#endif  /* __SSE4_2__ */')
-        out.append('')
-
-        out.append('#ifdef __ARM_NEON__')
-        out.append(' #include <arm_neon.h>')
-        out.append('#endif  /* __ARM_NEON__ */')
-        out.append('')
-
-        out.append('#ifdef __wasm__')
-        out.append(' #include <wasm_simd128.h>')
-        out.append('#endif  /* __wasm__ */')
-        out.append('')
-
-        out.append('#ifdef _MSC_VER')
-        out.append(' #define ALIGN(n) _declspec(align(n))')
-        out.append(' #define UNREACHABLE __assume(0)')
-        out.append('#else  /* !_MSC_VER */')
-        out.append(' #define ALIGN(n) __attribute__((aligned(n)))')
-        out.append(' #define UNREACHABLE __builtin_unreachable()')
-        out.append('#endif  /* _MSC_VER */')
+        out.append("#ifdef __SSE4_2__")
+        out.append(" #ifdef _MSC_VER")
+        out.append("  #include <nmmintrin.h>")
+        out.append(" #else  /* !_MSC_VER */")
+        out.append("  #include <x86intrin.h>")
+        out.append(" #endif  /* _MSC_VER */")
+        out.append("#endif  /* __SSE4_2__ */")
+        out.append("")
+
+        out.append("#ifdef __ARM_NEON__")
+        out.append(" #include <arm_neon.h>")
+        out.append("#endif  /* __ARM_NEON__ */")
+        out.append("")
+
+        out.append("#ifdef __wasm__")
+        out.append(" #include <wasm_simd128.h>")
+        out.append("#endif  /* __wasm__ */")
+        out.append("")
+
+        out.append("#ifdef _MSC_VER")
+        out.append(" #define ALIGN(n) _declspec(align(n))")
+        out.append(" #define UNREACHABLE __assume(0)")
+        out.append("#else  /* !_MSC_VER */")
+        out.append(" #define ALIGN(n) __attribute__((aligned(n)))")
+        out.append(" #define UNREACHABLE __builtin_unreachable()")
+        out.append("#endif  /* _MSC_VER */")
 
         out.append("")
         out.append(
@@ -83,7 +83,7 @@ class CCompiler:
         out.append("}")
         out.append("")
 
-        # TODO (Vizonex) Make llparse_state_t's Name Optional and alterable incase mixed with 
+        # TODO (Vizonex) Make llparse_state_t's Name Optional and alterable incase mixed with
         # llhttp or another parser
         out.append(f"static llparse_state_t {info.prefix}__run(")
         out.append(f"    {info.prefix}_t* {ARG_STATE},")
@@ -182,9 +182,7 @@ class CCompiler:
             else:
                 # TODO (Vizonex) Merge lines 139 & 140 together in a future update
                 callback = (
-                    f"(({info.prefix}__span_cb)"
-                    + ctx.spanCbField(span.index)
-                    + f")"
+                    f"(({info.prefix}__span_cb)" + ctx.spanCbField(span.index) + f")"
                 )
 
             args = [ctx.stateArg(), posField, f"(const char*) {ctx.endPosArg()}"]

{llparse-0.1.1 → llparse-0.1.2}/llparse/compilator.py

@@ -170,7 +170,7 @@ class Load(Field):
     def doBuild(self, ctx: "Compilation", out: list[str]):
         out.append(f"return {self.field(ctx)};")
 
-    
+
 # BIG ONE
 
 
@@ -374,7 +374,6 @@ class Consume(Node):
         index = ctx.stateField(self.ref.field)
         ty = ctx.getFieldType(self.ref.field)
 
-
         if ty == "i64":
             pass
         elif ty == "i32":
@@ -412,7 +411,6 @@ class Empty(Node):
     def __init__(self, ref: _frontend.node.Empty) -> None:
         self.ref = ref
         super().__init__(ref)
-    
 
     def doBuild(self, out: list[str]):
         assert self.ref.otherwise
@@ -884,7 +882,7 @@ class Compilation:
     def buildStateEnum(self, out: list[str]):
         # TODO (Vizonex) Give out other names that you could pass as an enum statename
         # this is incase multiple llparse_state_e states are given to compile
-        # example would be mixing llhttp with some other source... 
+        # example would be mixing llhttp with some other source...
         out.append("enum llparse_state_e {")
         out.append(f"  {STATE_ERROR},")
         for stateName in self.stateDict.keys():
@@ -992,8 +990,7 @@ class Compilation:
             out.append(f"{LABEL_PREFIX}{name} : " + "{")
             for line in lines:
                 out.append(f"  {line}")
-            out.append("  /* UNREACHABLE */;")
-            out.append("  abort();")
+            out.append("  UNREACHABLE;")
             out.append("}")
 
     def buildInternalStates(self, out: list[str]):
@@ -1004,8 +1001,7 @@ class Compilation:
             out.append(f"{LABEL_PREFIX}{name}: " + "{")
             for line in lines:
                 out.append(f"  {line}")
-            out.append("  /* UNREACHABLE */;")
-            out.append("  abort();")
+            out.append("  UNREACHABLE;")
             out.append("}")
 
     def addState(self, state: str, lines: list[str]):

{llparse-0.1.1 → llparse-0.1.2}/llparse/cython_builder.py

@@ -2,6 +2,7 @@
 
 from contextlib import contextmanager
 from typing import Optional
+
 from .frontend import IFrontendResult
 from .pyfront.front import Match
 from .pyfront.nodes import Invoke

{llparse-0.1.1 → llparse-0.1.2}/llparse/dot.py

@@ -208,6 +208,3 @@ class Dot:
 
     def escape(self, value: str):
         return "'" + value.replace("\\", "\\$1").replace('"', "\\$1") + "'"
-
-
-# TODO FIX ALL BUFFERS BACK TO STRINGS!

llparse-0.1.2/llparse/errors.py

@@ -0,0 +1,2 @@
+class Error(Exception):
+    pass

{llparse-0.1.1 → llparse-0.1.2}/llparse/frontend.py

@@ -4,6 +4,7 @@ from typing import Literal, Optional, Union
 from .enumerator import Enumerator
 from .pybuilder import LoopChecker
 from .pybuilder import builder as source
+
 # from pyfront.namespace import code, node , transform
 from .pyfront import namespace as _frontend
 from .pyfront.front import Identifier, IWrap, SpanField
@@ -17,13 +18,10 @@ DEFAULT_MIN_TABLE_SIZE = 32
 DEFAULT_MAX_TABLE_WIDTH = 4
 
 
-
 WrappedNode = IWrap[_frontend.node.Node]
 WrappedCode = IWrap[_frontend.code.Code]
 
 
-
-
 @dataclass
 class ITableLookupTarget:
     trie: TrieEmpty

{llparse-0.1.1 → llparse-0.1.2}/llparse/llparse.py

@@ -4,8 +4,13 @@ from dataclasses import dataclass
 from typing import Optional
 
 from .C_compiler import CCompiler
-from .frontend import (DEFAULT_MAX_TABLE_WIDTH, DEFAULT_MIN_TABLE_SIZE,
-                       Frontend, IImplementation, source)
+from .frontend import (
+    DEFAULT_MAX_TABLE_WIDTH,
+    DEFAULT_MIN_TABLE_SIZE,
+    Frontend,
+    IImplementation,
+    source,
+)
 from .header import HeaderBuilder
 
 

{llparse-0.1.1 → llparse-0.1.2}/llparse/pybuilder/loopchecker.py

@@ -1,7 +1,13 @@
+import logging
 from typing import Any, Literal, Union
 
+from ..errors import Error
 from ..pybuilder.main_code import Node
 
+logger = logging.getLogger("llparse.pybuilder.loopchecker")
+logger.setLevel(logging.INFO)
+
+
 MAX_VALUE = 256
 WORD_SIZE = 32
 SIZE = MAX_VALUE // WORD_SIZE
@@ -19,17 +25,20 @@ assert MAX_VALUE % WORD_SIZE == 0
 
 class Lattice:
     def __init__(
-        self, value: Union[Any, list[int], Literal["empty"], Literal["any"]]
+        self, value: Union[Any, list[int], bytes, Literal["empty"], Literal["any"]]
     ) -> None:
         self.value = value
         self.words: list[int] = []
 
         # allocate space by filling in data with zeros...
-
-        for _ in range(SIZE):
-            self.words.append(0)
-
-        if len(value) > 1:
+        if value != "any":
+            for _ in range(SIZE):
+                self.words.append(0)
+        else:
+            for _ in range(SIZE):
+                self.words.append(WORD_FILL)
+
+        if isinstance(value, (list, bytes)):
             for single in value:
                 self.add(single)
 
@@ -69,6 +78,9 @@ class Lattice:
             result.words[i] = self.words[i] & other.words[i]
         return result
 
+    def __repr__(self):
+        return f"<Lattice {', '.join(f'{k}: {v!r}' for k, v in self.__dict__.items())}>"
+
     def subtract(self, other: "Lattice") -> "Lattice":
         result = Lattice("empty")
         for i in range(SIZE):
@@ -76,7 +88,13 @@ class Lattice:
         return result
 
     def isEqual(self, other: "Lattice"):
-        return True if (self.value == other.value) else False
+        if self.toJSON() == other.toJSON():
+            return True
+        else:
+            for i in range(SIZE):
+                if self.words[i] != other.words[i]:
+                    return False
+        return True
 
     def toJSON(self):
         isEmpty = True
@@ -111,7 +129,9 @@ class Reachability:
             if otherwise:
                 queue.append(otherwise.node)
 
-        return list(res)
+        # Reverse the order so that we always
+        # throw an error on bad configurations...
+        return res
 
 
 EMPTY_VALUE = Lattice("empty")
@@ -124,38 +144,45 @@ class LoopChecker:
         self.terminatedCache: dict[Node, Lattice] = {}
 
     def clear(self, nodes: list[Node]):
-        self.lattice.update({(node, EMPTY_VALUE) for node in nodes})
+        for node in nodes:
+            self.lattice[node] = EMPTY_VALUE
 
     def check(self, root: Node):
         r = Reachability()
         nodes = r.build(root)
+
         for node in nodes:
             self.clear(nodes)
+            logger.debug("checking loops starting from %s" % node.name)
             self.lattice[node] = ANY_VALUE
+            # we must eliminate randomness so that error always throw
             changed: set[Node] = set([root])
-            while len(changed) != 0:
-                next = set()
-                for changedNode in next:
-                    self.propagate(changedNode, next)
-                changed = next
 
-        self.visit(root, list(changed))
+            while changed:
+                logger.debug("changed %s" % [n.name for n in changed])
+                _next = set()
+                for changedNode in changed:
+                    self.propagate(changedNode, _next)
+                changed = _next
+            logger.debug("lattice stabilized")
+            self.visit(root, [])
 
     def propagate(self, node: Node, changed: set[Node]):
         value = self.lattice[node]
-        terminated: "Lattice" = self.terminate(node, value, changed)
-        if terminated.value == EMPTY_VALUE.value:
+        terminated = self.terminate(node)
+        logger.debug("propagate(%r), initial value %r" % (node.name, value.toJSON()))
+        if not terminated.isEqual(EMPTY_VALUE):
+            logger.debug("node %s terminates %r" % (node.name, terminated.toJSON()))
             value = value.subtract(terminated)
             if value.isEqual(EMPTY_VALUE):
                 return
 
-        keysbyTarget: dict[Node, Lattice] = dict()
+        keysbyTarget: dict[Node, Lattice] = {}
 
         for edge in node.getAllEdges():
             if not edge.noAdvance:
                 continue
 
-            targetValue: Lattice
             if keysbyTarget.get(edge.node):
                 targetValue = keysbyTarget[edge.node]
             else:
@@ -170,9 +197,14 @@ class LoopChecker:
                     continue
 
                 targetValue = targetValue.union(edgeValue)
+
             keysbyTarget[edge.node] = targetValue
 
         for child, childValue in keysbyTarget.items():
+            logger.debug(
+                "node %r propagates %r to %r"
+                % (node.name, childValue.toJSON(), child.name)
+            )
             self.update(child, childValue, changed)
         # FINISHED!
 
@@ -182,13 +214,13 @@ class LoopChecker:
             return False
         self.lattice[node] = newValue
         changed.add(node)
+        return True
 
-    def terminate(self, node: Node, value: Lattice, changed: set[Node]):
-        if self.terminatedCache.get(node):
+    def terminate(self, node: Node):
+        if node in self.terminatedCache:
             return self.terminatedCache[node]
 
         terminated: list[int] = []
-
         for edge in node.getAllEdges():
             if edge.noAdvance:
                 continue
@@ -204,43 +236,52 @@ class LoopChecker:
 
     def visit(self, node: Node, path: list[Node]):
         value = self.lattice[node]
+        logger.debug("enter %s, value is %s" % (node.name, value.toJSON()))
+
         terminated = (
-            self.terminatedCache[node]
-            if self.terminatedCache.get(node)
-            else EMPTY_VALUE
+            EMPTY_VALUE
+            if node not in self.terminatedCache
+            else self.terminatedCache[node]
         )
-        if terminated.isEqual(EMPTY_VALUE):
+
+        if not terminated.isEqual(EMPTY_VALUE):
+            logger.debug(f"subtract terminated {terminated}")
             value = value.subtract(terminated)
             if value.isEqual(EMPTY_VALUE):
+                logger.debug("terminated everything")
                 return
 
         for edge in node.getAllEdges():
-            if edge.noAdvance:
+            if not edge.noAdvance:
                 continue
             edgeValue = value
-            if not (not edge.key or isinstance(edge.key, int)):
+            if edge.key is None or isinstance(edge.key, int):
+                pass
+            else:
                 edgeValue = edgeValue.intersect(Lattice([edge.key[0]]))
 
             if edgeValue.isEqual(EMPTY_VALUE):
+                # logger.debug(edge.node.name + " not recursive")
                 continue
 
-            def indexOf(path: list, obj):
-                try:
-                    return path.index(obj)
-                except Exception:
-                    return -1
+            def indexOf(path: list[Node], obj: Node) -> int:
+                for o in path:
+                    if o.name == obj.name:
+                        return 0
+                return -1
 
-            if indexOf(path, node) != -1:
-                if len(path) == 0:
-                    raise Exception(
-                        f'Detected a loop in "{edge.node.name}" though : {edge.node.name}'
+            if indexOf(path, edge.node) != -1:
+                if len(path) == 1:
+                    raise Error(
+                        f'Detected loop in "{edge.node.name}" through "{edge.node.name}"'
                     )
 
-                raise Exception(
+                raise Error(
                     'Detected loop in "'
                     + edge.node.name
                     + '" through chain '
                     + (" -> ").join(['"' + name.name + '"' for name in path])
                 )
 
-            self.visit(edge.node, path.extend([edge.node]))
+            self.visit(edge.node, path + [edge.node])
+        logger.debug("leave %s" % node.name)

{llparse-0.1.1 → llparse-0.1.2}/llparse/pybuilder/main_code.py

@@ -1,6 +1,16 @@
 import re
+import sys
 from dataclasses import dataclass
-from typing import Optional, TypeVar, Union
+from typing import Callable, Literal, Optional, TypeVar, Union
+
+if sys.version_info < (3, 10):
+    from typing_extensions import ParamSpec
+else:
+    from typing import ParamSpec
+
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+
 
 Signature = ["match", "value"]
 
@@ -18,7 +28,7 @@ def toBuffer(value: Union[str, int]):
 # TODO Add text validataion...
 
 
-def validate_text(init):
+def validate_text(init: Callable[_P, _T]) -> Callable[_P, _T]:
     def is_valid(args, kwargs):
         if kwargs.get("field"):
             field = kwargs["field"]
@@ -32,16 +42,20 @@ def validate_text(init):
 
 
 class Code:
-    def __init__(self, signature: str, name: str) -> None:
-        assert signature in Signature
+    def __init__(self, signature: Literal["match", "value"], name: str) -> None:
+        assert signature in Signature, "Invalid signature %s" % signature
 
         self.signature = signature
         self.name = name
+
     def __hash__(self):
         return hash(self.signature + self.name)
 
+
 class Field(Code):
-    def __init__(self, signature: str, name: str, field: str) -> None:
+    def __init__(
+        self, signature: Literal["match", "value"], name: str, field: str
+    ) -> None:
         self.field = field
         # if re.search(r"[//\s\\]+",field):
         #     raise TypeError(f"Can\'t access internal field from user code because the field: {name} conatins invalid characters")
@@ -49,11 +63,10 @@ class Field(Code):
 
 
 class FieldValue(Field):
-    # NOTE I Added for typehinting Here as it refuses to show up on the ide I'm using (Vizonex)
-
-    def __init__(self, signature: str, name: str, field: str, value: int) -> None:
+    def __init__(
+        self, signature: Literal["match", "value"], name: str, field: str, value: int
+    ) -> None:
         self.value = value
-        self.field = field
         super().__init__(signature, name, field)
 
 
@@ -132,6 +145,10 @@ class Node:
         self.otherwiseEdge: Optional["Edge"] = None
         self.privEdges: list["Edge"] = []
 
+    def key(self):
+        """reversed for sorting to prevent python from creating artificial randomness"""
+        return self.name
+
     def __hash__(self) -> int:
         return hash(self.name)
 
@@ -289,7 +306,7 @@ class Edge:
         self.node = node
         self.noAdvance = noAdvance
 
-        self.key = key
+        self.key = key.encode() if isinstance(key, str) else key
         self.value = value
 
         # Validation...

{llparse-0.1.1 → llparse-0.1.2}/llparse/pybuilder/parsemap.py

@@ -1,5 +1,4 @@
-
-from main_code import Edge, Node
+from ..pybuilder.main_code import Edge, Node
 
 
 class ParserMap:
@@ -9,9 +8,7 @@ class ParserMap:
     def Jsonize(self):
         queue = [self.root]
         seen: set[Node] = set()
-
-
-        while len(queue) != 0:
+        while queue:
             node = queue.pop()
 
             if node in seen:
@@ -33,5 +30,4 @@ class ParserMap:
             return {}
         data = edge.__dict__
         data["node"] = edge.node
-        # print(data["node"].name)
         return data

{llparse-0.1.1 → llparse-0.1.2}/llparse/pyfront/front.py

@@ -36,6 +36,7 @@ class Code:
     def __hash__(self):
         return hash(self.cacheKey)
 
+
 class External(Code):
     """Inherits from the `Code` class as a subclass of `Code`"""
 
@@ -46,8 +47,9 @@ class External(Code):
 @dataclass
 class Field(Code):
     """Inherits from `Code`"""
+
     field: str
-    
+
     def __hash__(self):
         return hash(self.cacheKey)
 

{llparse-0.1.1 → llparse-0.1.2}/llparse/pyfront/nodes.py

@@ -34,6 +34,7 @@ class IUniqueName:
     # def __hash__(self):
     #     return hash(self.name)
 
+
 @dataclass
 class IOtherwiseEdge:
     node: IWrap["Node"]
@@ -86,6 +87,7 @@ class Node:
     def __hash__(self):
         return hash(self.id)
 
+
 class Consume(Node):
     def __init__(self, id: IUniqueName, field: str) -> None:
         self.field = field
@@ -179,7 +181,7 @@ class SpanStart(Node):
         self.field = field
         self.callback = callback
         super().__init__(id)
-    
+
 
 class SpanEnd(Node):
     def __init__(

{llparse-0.1.1 → llparse-0.1.2}/llparse/settings.py

@@ -25,7 +25,6 @@ SPAN_CALLBACK_MAYBE = "SPAN_CALLBACK_MAYBE"
 MID_POSTFIX = "_settings"
 
 
-
 @dataclass
 class ApiResult:
     header: str

{llparse-0.1.1 → llparse-0.1.2}/llparse/spanalloc.py

@@ -1,22 +1,10 @@
+from collections import OrderedDict
 from dataclasses import dataclass, field
 from typing import Union
 
+from .errors import Error
 from .pybuilder.main_code import Node, Reachability, Span, SpanEnd, SpanStart
 
-
-class DeadLoop(Exception):
-    """Thrown when this type of loop is detected during complation
-    ```c
-        switch(*p){
-          case dead_loop:
-          dead_loop : {
-            goto dead_loop;
-          } /* Rest in Peace Computer x_x */
-        }
-
-    ```"""
-
-
 SpanSet = set[Span]
 
 
@@ -72,17 +60,15 @@ class SpanAllocator:
                 # print("SPAN:%s  NODE:%s" % (edgeSpans,edge.node.__dict__))
                 for subSpan in edgeSpans:
                     if subSpan not in spans:
-                        raise AssertionError(
-                            f'Unmatched span end for "{subSpan.callback.name}"'
-                            / f'at "{edge.node.name}", coming from "{node.name}"'
+                        raise Error(
+                            f'unmatched span end for "{subSpan.callback.name}"'
+                            f'at "{edge.node.name}", coming from "{node.name}"'
                         )
 
                 if isinstance(edge.node, SpanEnd):
                     span = _id(edge.node)
                     if span not in spans:
-                        raise AssertionError(
-                            f'Unmatched span end for "{span.callback.name}"'
-                        )
+                        raise Error(f'unmatched span end for "{span.callback.name}"')
 
     def computeActive(self, nodes: list[Node]):
         activeMap: dict[Node, SpanSet] = dict()
@@ -111,7 +97,7 @@ class SpanAllocator:
 
                     if isinstance(edgeNode, SpanStart):
                         if _id(edgeNode) == span:
-                            raise DeadLoop(
+                            raise Error(
                                 f'Detected loop in span {span.callback.name} at "{node.name}"'
                             )
 
@@ -136,14 +122,14 @@ class SpanAllocator:
         return overlap
 
     def _allocate(self, span: Span):
-        if self._colors.get(span):
+        if span in self._colors:
             return self._colors[span]
 
         overlap = self._overlapMap[span]
 
         used: set[int] = set()
         for subSpan in overlap:
-            if self._colors.get(subSpan):
+            if subSpan in self._colors:
                 used.add(self._colors.get(subSpan))
         i = 0
         while i in used:
@@ -160,17 +146,16 @@ class SpanAllocator:
 
         self._overlapMap = overlapDict
 
-        Map = {span: self._allocate(span) for span in spans}
+        colors = {span: self._allocate(span) for span in spans}
 
         concurrency = list()
         for _ in range(self._mx + 1):
             # NOTE : concurrency[i] = [] doesn't work but this does :P
             concurrency.append([])
 
-        for s in spans:
+        for s in sorted(spans, key=lambda s: s.callback.name):
             concurrency[self._allocate(s)].append(s)
-
-        return ISpanAllocatorResult(Map, concurrency, self._mx)
+        return ISpanAllocatorResult(colors, concurrency, self._mx)
 
 
 # TODO (Vizonex) Use Indutny's Mini Http parser to help with testing ours to verify that ours is correct...

{llparse-0.1.1 → llparse-0.1.2}/llparse/test.py

@@ -5,6 +5,7 @@ from .pybuilder import Builder
 
 # TODO: Remove and make pytest for it.
 
+
 def smaller_test():
     # from compilator import Compilation, ICompilerOptions
     p = Builder()

{llparse-0.1.1 → llparse-0.1.2}/llparse/trie.py

@@ -18,7 +18,6 @@ class IEdge:
         return self.key < object.key
 
 
-
 @dataclass
 class TrieNode:
     """Mainly Used as an Abstract Object for typing"""
@@ -151,15 +150,3 @@ class Trie:
             children.append(child)
 
         return TrieSingle(children, otherwise)
-
-
-# def test():
-#     t = b"data-to-buffer"
-
-#     for _ in range(2):
-#         t = t[1:4]
-#         print(t)
-
-
-# if __name__ == "__main__":
-#     test()

{llparse-0.1.1 → llparse-0.1.2}/llparse.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llparse
-Version: 0.1.1
+Version: 0.1.2
 Summary: A Parody of llparse written for writing C Parsers with Python
 Author-email: Vizonex <VizonexBusiness@gmail.com>
 Requires-Python: >=3.9

{llparse-0.1.1 → llparse-0.1.2}/llparse.egg-info/SOURCES.txt

@@ -9,13 +9,13 @@ llparse/cython_builder.py
 llparse/debug.py
 llparse/dot.py
 llparse/enumerator.py
+llparse/errors.py
 llparse/frontend.py
 llparse/header.py
 llparse/llparse.py
 llparse/settings.py
 llparse/spanalloc.py
 llparse/test.py
-llparse/tire.py
 llparse/trie.py
 llparse.egg-info/PKG-INFO
 llparse.egg-info/SOURCES.txt
@@ -32,4 +32,6 @@ llparse/pyfront/implementation.py
 llparse/pyfront/namespace.py
 llparse/pyfront/nodes.py
 llparse/pyfront/peephole.py
-llparse/pyfront/transform.py
+llparse/pyfront/transform.py
+tests/test_loop_checker.py
+tests/test_span_allocator.py

{llparse-0.1.1 → llparse-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llparse"
-version = "0.1.1"
+version = "0.1.2"
 description = "A Parody of llparse written for writing C Parsers with Python"
 readme = "README.md"
 authors = [

llparse-0.1.2/tests/test_loop_checker.py

@@ -0,0 +1,160 @@
+from llparse.pybuilder import LoopChecker, Builder
+from llparse.errors import Error
+import pytest
+
+# based off llparse-builder/test/loop-checker.test.ts
+# Time Taken: 7 hours if you count the hard amounts of debugging, I went through.
+
+
+@pytest.fixture()
+def loop_checker() -> tuple[LoopChecker, Builder]:
+    return LoopChecker(), Builder()
+
+
+def test_detect_shallow_loops(loop_checker: tuple[LoopChecker, Builder]) -> None:
+    lc, b = loop_checker
+    start = b.node("start")
+    start.otherwise(start)
+    with pytest.raises(Error, match=r'Detected loop in "start" through "start"'):
+        lc.check(start)
+
+
+def test_detect_loops(loop_checker: tuple[LoopChecker, Builder]) -> None:
+    lc, b = loop_checker
+    start = b.node("start")
+    a = b.node("a")
+    invoke = b.invoke(
+        b.code.match("nop"),
+        {
+            0: start,
+        },
+        b.error(1, "error"),
+    )
+
+    start.peek("a", a).otherwise(b.error(1, "error"))
+
+    a.otherwise(invoke)
+    with pytest.raises(Error, match=r'Detected loop in "a".*"a" -> "invoke_nop"'):
+        lc.check(start)
+
+
+def test_detect_shallow_loops_2(loop_checker: tuple[LoopChecker, Builder]) -> None:
+    lc, b = loop_checker
+    start = b.node("start")
+    loop = b.node("loop")
+
+    start.peek("a", loop).otherwise(b.error(1, "error"))
+    loop.match("a", loop).otherwise(loop)
+    with pytest.raises(Error, match=r'Detected loop in "loop" through "loop"'):
+        lc.check(loop)
+
+
+def test_ignore_loops_through_peek_to_match(
+    loop_checker: tuple[LoopChecker, Builder],
+) -> None:
+    lc, b = loop_checker
+    start = b.node("start")
+    a = b.node("a")
+    invoke = b.invoke(
+        b.code.match("nop"),
+        {
+            0: start,
+        },
+        b.error(1, "error"),
+    )
+
+    start.peek("a", a).otherwise(b.error(1, "error"))
+
+    a.match("abc", invoke).otherwise(start)
+    lc.check(start)
+
+
+def test_ignore_irrelevant_peeks(loop_checker: tuple[LoopChecker, Builder]) -> None:
+    lc, b = loop_checker
+    start = b.node("start")
+    a = b.node("a")
+
+    start.peek("a", a).otherwise(b.error(1, "error"))
+
+    a.peek("b", start).otherwise(b.error(1, "error"))
+    lc.check(start)
+
+
+def test_ignore_loops_with_multi_peek_match(
+    loop_checker: tuple[LoopChecker, Builder],
+) -> None:
+    lc, b = loop_checker
+    start = b.node("start")
+    another = b.node("another")
+
+    NUM: list[str] = [
+        "0",
+        "1",
+        "2",
+        "3",
+        "4",
+        "5",
+        "6",
+        "7",
+        "8",
+        "9",
+    ]
+
+    ALPHA: list[str] = [
+        "a",
+        "b",
+        "c",
+        "d",
+        "e",
+        "f",
+        "g",
+        "h",
+        "i",
+        "j",
+        "k",
+        "l",
+        "m",
+        "n",
+        "o",
+        "p",
+        "q",
+        "r",
+        "s",
+        "t",
+        "u",
+        "v",
+        "w",
+        "x",
+        "y",
+        "z",
+        "A",
+        "B",
+        "C",
+        "D",
+        "E",
+        "F",
+        "G",
+        "H",
+        "I",
+        "J",
+        "K",
+        "L",
+        "M",
+        "N",
+        "O",
+        "P",
+        "Q",
+        "R",
+        "S",
+        "T",
+        "U",
+        "V",
+        "W",
+        "X",
+        "Y",
+        "Z",
+    ]
+    start.match(ALPHA, start).peek(NUM, another).skipTo(start)
+
+    another.match(NUM, another).otherwise(start)
+    lc.check(start)

llparse-0.1.2/tests/test_span_allocator.py

@@ -0,0 +1,121 @@
+from llparse.spanalloc import SpanAllocator
+from llparse.pybuilder import Builder
+from llparse.errors import Error
+
+import pytest
+
+# Brought over and translated from llparse-builder/tests/span-allocator.ts
+
+
+@pytest.fixture()
+def span_alloc() -> tuple[SpanAllocator, Builder]:
+    return SpanAllocator(), Builder()
+
+
+def test_allocate_single_span(span_alloc: tuple[SpanAllocator, Builder]) -> None:
+    sa, b = span_alloc
+    span = b.span(b.code.span("span"))
+    start = b.node("start")
+    body = b.node("body")
+
+    start.otherwise(span.start(body))
+
+    body.skipTo(span.end(start))
+
+    res = sa.allocate(start)
+
+    assert res.max == 0
+
+    assert len(res.concurrency) == 1
+    assert span in res.concurrency[0]
+
+    assert len(res.colors) == 1
+    assert res.colors.get(span) == 0
+
+
+def test_allocate_overlapping_spans(span_alloc: tuple[SpanAllocator, Builder]) -> None:
+    sa, b = span_alloc
+    span1 = b.span(b.code.span("span1"))
+    span2 = b.span(b.code.span("span2"))
+
+    start = b.node("start")
+    body1 = b.node("body1")
+    body2 = b.node("body2")
+
+    start.otherwise(span1.start(body1))
+
+    body1.otherwise(span2.start(body2))
+
+    body2.skipTo(span2.end(span1.end(start)))
+
+    res = sa.allocate(start)
+
+    # TODO: fix it later... it's supposed to be 1
+    assert res.max == 1
+
+    assert len(res.concurrency) == 2
+    
+    # python loves to shuffle things on me :/ but both exist nevertheless
+    assert span2 in res.concurrency[0] or span1 in res.concurrency[0]
+    assert span1 in res.concurrency[1] or span2 in res.concurrency[1]
+
+    assert len(res.colors) == 2
+    assert res.colors.get(span2) in [0, 1]
+    assert res.colors.get(span1) in [0, 1]
+
+
+def test_allocate_non_overlapping_spans(
+    span_alloc: tuple[SpanAllocator, Builder],
+) -> None:
+    sa, b = span_alloc
+    span1 = b.span(b.code.span("span1"))
+    span2 = b.span(b.code.span("span2"))
+
+    start = b.node("start")
+    body1 = b.node("body1")
+    body2 = b.node("body2")
+
+    start.match("a", span1.start(body1)).otherwise(span2.start(body2))
+    body1.skipTo(span1.end(start))
+
+    body2.skipTo(span2.end(start))
+
+    res = sa.allocate(start)
+
+    assert res.max == 0
+
+    assert len(res.concurrency) == 1
+    assert span1 in res.concurrency[0]
+    assert span2 in res.concurrency[0]
+
+    assert len(res.colors) == 2
+    assert res.colors.get(span1) == 0
+    assert res.colors.get(span2) == 0
+
+
+def test_should_throw_on_loops(span_alloc: tuple[SpanAllocator, Builder]) -> None:
+    sa, b = span_alloc
+    start = b.node("start")
+    end = b.node("end")
+    span = b.span(b.code.span("on_data"))
+
+    start.match("a", end).match("b", span.start(end)).otherwise(b.error(1, "error"))
+
+    end.otherwise(span.end(start))
+
+    with pytest.raises(Error, match=r"unmatched.*on_data"):
+        sa.allocate(start)
+
+def test_propagate_through_invoke_map(span_alloc:tuple[SpanAllocator, Builder]):
+    sa, b = span_alloc
+    start = b.node('start')
+    span = b.span(b.code.span('llparse__on_data'))
+
+    b.property('i8', 'custom')
+
+    start.otherwise(b.invoke(b.code.load('custom'), {
+      0: span.end().skipTo(start),
+    }, span.end().skipTo(start)))
+
+    sa.allocate(span.start(start))
+    

llparse-0.1.1/llparse/tire.py

@@ -1,158 +0,0 @@
-from .pybuilder.main_code import Node, Edge
-from typing import Optional
-
-from dataclasses import dataclass
-
-@dataclass
-class IEdge:
-    # NOTE THIS SHOULD BE STRICTLY BYTES !!!
-    key: bytes
-    node: Node 
-    noAdvance:bool
-    value:Optional[int] = None 
-
-    def __lt__(self,object):
-        return self.key < object.key
-
-# TODO RENAME TO TRIE!
-
-class TireNode:
-    """Mainly Used as an Abstract Object for typing"""
-    def __init__(self) -> None:
-        return 
-
-class TireSequence(TireNode):
-    def __init__(self,select:bytes,child:TireNode) -> None:
-        self.select = select
-        self.child = child
-        
-
-class ITireSingleChild:
-    def __init__(self,key:int,noAdvance:bool,node:TireNode) -> None:
-        self.key = key
-        self.noAdvance = noAdvance
-        self.node = node 
-
-class TireEmpty(TireNode):
-    def __init__(self,node:Node,value:int) -> None:
-        self.node = node 
-        self.value = value 
-
-class TireSingle(TireNode):
-    def __init__(self,children:list[ITireSingleChild],otherwise:Optional[TireEmpty] = None) -> None:
-        self.children = children
-        self.otherwise = otherwise 
-        
-
-
-
-
-# TODO Retry making Tire....
-
-class Tire:
-    def __init__(self,name:str) -> None:
-        self.name = name 
-    
-    def build(self,edges:list[Edge]):
-        if len(edges) == 0:
-            return None 
-        
-        internalEdges: list[IEdge] = []
-        
-        for edge in edges:
-            key = str(edge.key) if isinstance(edge.key,int) else edge.key 
-            internalEdges.append(IEdge(key.encode("utf-8") if isinstance(key,str) else key ,edge.node,edge.noAdvance,edge.value))
-        return self.level(internalEdges,[])
-        
-    def level(self,edges:list[IEdge],path:list[bytes]):
-        first = edges[0].key
-        last = edges[-1].key
-
-        if len(edges) == 1 and len(edges[0].key) == 0:
-            return TireEmpty(edges[0].node,edges[0].value)
-        
-        i = 0 
-        for i in range(len(first)):
-            if first[i] != last[i]:
-                break
-
-        if i > 1:
-            # NOTE I think Idutny intended for these sequences 
-            # to advance otherwise not having this would case a recursion error
-            # This is why first[1:i] is used and not first[0:count] like in typescript...
-            return self.sequence(edges,first[: i + 1 ],path)
-
-        return self.single(edges,path) 
-
-    def Slice(self,edges:list[IEdge],off:int):
-        slice = [IEdge(edge.key[off:],edge.node,edge.noAdvance,edge.value) for edge in edges]
-        return sorted(slice, key = lambda k: k.key)
-
-
-    def sequence(self,edges:list[IEdge],prefix:bytes,path:list[bytes]):
-        sliced = self.Slice(edges,len(prefix))
-        assert not any([edge.noAdvance for edge in edges])
-        child = self.level(sliced,path + [prefix])
-        return TireSequence(prefix,child)
-    
-    def single(self,edges:list[IEdge],path:list[bytes]):
-
-        if len(edges[0].key) == 0:
-            if len(path) == 0:
-                AssertionError(f'Empty root entry at "{self.name}"')
-            if not (len(edges) == 1 or len(edges[1].key) != 0):
-                err = f'Duplicate entries in "{self.name}" at: [' + (b", ".join(path).decode("utf-8")) + ']'
-                
-                raise AssertionError(err)
-
-
-        keys : dict[int,list[IEdge]] = {} 
-
-        for edge in edges:
-            if len(edge.key) == 0:
-                otherwise = TireEmpty(edge.node,edge.value)
-                continue
-
-            key = edge.key[0]
-
-            if keys.get(key):
-                keys[key].append(edge)
-            else:
-                keys[key] = [edge]
-        
-   
-        otherwise = None 
-        children : list[ITireSingleChild] = []
-
-        for key, subEdges in keys.items():
-            # TODO LOG FUNCTION's ARGUMENTS TO DETERMINE WEATHER OR NOT IT'S the Problem...
-            # I think this maybe the problem now that I think about it...
-            sliced = self.Slice(subEdges, 1)
-   
-            
-            subPath = path + [chr(key).encode("utf-8")]
-
-            noAdvance = any([e.noAdvance for e in subEdges])
-            allSame = all([e.noAdvance == noAdvance for e in subEdges])
-           
-            if not (allSame or len(subEdges) == 0):
-                err = f'Conflicting `.peek` and `.match` entries in "{self.name}" at: [' + (b", ".join(subPath).decode("utf-8")) + ']'
-                raise TypeError(err)
-            child = ITireSingleChild(key,noAdvance,self.level(sliced,subPath))
-            children.append(child)
-
-       
-        return TireSingle(children,otherwise)
-
-
-
-
-def test():
-    t = b"data-to-buffer"
-
-    for _ in range(2):
-        t = t[1:4]
-        print(t)
-
-if __name__ == "__main__":
-    test()

{llparse-0.1.1 → llparse-0.1.2}/llparse/pyfront/namespace.py

@@ -1,3 +1,3 @@
-from ..pyfront import transform as transform
-from ..pyfront import nodes as node
 from ..pyfront import front as code
+from ..pyfront import nodes as node
+from ..pyfront import transform as transform