llparse 0.1.0__py3-none-any.whl

llparse/llparse.py

@@ -0,0 +1,150 @@
+""""""
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .C_compiler import CCompiler
+from .frontend import (DEFAULT_MAX_TABLE_WIDTH, DEFAULT_MIN_TABLE_SIZE,
+                       Frontend, IImplementation, source)
+from .header import HeaderBuilder
+
+
+@dataclass
+class CompilerResult:
+    c: str
+    """Textual C code"""
+    header: str
+    """Textual C header file"""
+
+    # NOTE Coming soon... I will making a settings compiler which is attached to CompilerResult to Compile your own
+    # settings structures and be able to concate it to the existing header file like in llhttp
+    # I'm also in the works of adding a Cython .pxd import as well...
+
+
+class Compiler:
+    """Used to Compile C code together"""
+
+    def __init__(
+        self,
+        prefix: str,
+        headerGuard: Optional[str] = None,
+        debug: Optional[str] = None,
+        maxTableElemWidth: Optional[int] = None,
+        minTableSize: Optional[int] = None,
+    ):
+        self.prefix = prefix
+        self.headerGuard = headerGuard
+        self.debug = debug
+        self.maxTableElemWidth = maxTableElemWidth
+        self.minTableSize = minTableSize
+
+    def to_frontend(
+        self,
+        root: source.code.Node,
+        properties: list[source.Property],
+        Impl: Optional[IImplementation] = IImplementation(),
+    ):
+        """compiles up the frontend and brings you back the frontend's results.
+        I added documentation to this function so that you can do creative things
+        with the library beyond C..."""
+        return Frontend(
+            self.prefix,
+            Impl,
+            options={
+                "maxTableElemWidth": self.maxTableElemWidth,
+                "minTableSize": self.minTableSize,
+            },
+        ).compile(root, properties)
+
+    def compile(
+        self,
+        root: source.code.Node,
+        properties: list[source.Property],
+        header_name: Optional[str] = None,
+        Impl: Optional[IImplementation] = IImplementation(),
+    ):
+        """Creates the C and header file..."""
+        info = self.to_frontend(root, properties, Impl)
+        hb = HeaderBuilder(self.prefix, self.headerGuard, properties, info.spans)
+        return CompilerResult(
+            CCompiler(header_name, self.debug).compile(info), hb.build()
+        )
+
+
+class LLParse(source.Builder):
+    """
+
+    The prefix controls the names of methods and state struct in generated
+    public C headers:
+
+    ```c
+    // state struct
+    struct PREFIX_t {
+      ...
+    }
+
+    int PREFIX_init(PREFIX_t* state);
+    int PREFIX_execute(PREFIX_t* state, const char p, const char endp);
+    ```
+
+    `prefix`  Prefix to be used when generating public API.
+    """
+
+    def __init__(self, prefix: str) -> None:
+        self.prefix = prefix
+        super().__init__()
+
+    def get_compiler(
+        self,
+        headerGuard: Optional[str] = None,
+        debug: Optional[str] = None,
+        maxTableElemWidth: Optional[int] = None,
+        minTableSize: Optional[int] = None,
+    ):
+        return Compiler(
+            self.prefix,
+            headerGuard,
+            debug,
+            maxTableElemWidth if maxTableElemWidth else DEFAULT_MAX_TABLE_WIDTH,
+            minTableSize if minTableSize else DEFAULT_MIN_TABLE_SIZE,
+        )
+
+    def build(
+        self,
+        root: source.code.Node,
+        headerGuard: Optional[str] = None,
+        debug: Optional[str] = None,
+        maxTableElemWidth: Optional[int] = None,
+        minTableSize: Optional[int] = None,
+        header_name: Optional[str] = None,
+    ):
+        """Builds Graph and then compiles the data into C code , returns with the header and C file inside of a Dataclass"""
+
+        compiler = Compiler(
+            self.prefix,
+            headerGuard,
+            debug,
+            maxTableElemWidth if maxTableElemWidth else DEFAULT_MAX_TABLE_WIDTH,
+            minTableSize if minTableSize else DEFAULT_MIN_TABLE_SIZE,
+        )
+
+        return compiler.compile(root, self.properties(), header_name=header_name)
+
+    def to_frontend(
+        self,
+        root: source.code.Node,
+        headerGuard: Optional[str] = None,
+        debug: Optional[str] = None,
+        maxTableElemWidth: Optional[int] = None,
+        minTableSize: Optional[int] = None,
+    ):
+        """Used as an external hack to get access to the frontend of llparse and extract
+        it's contents to compile the libraries you make other things like cython, This is not in llparse
+        specifically (Yet...)"""
+        return Compiler(
+            self.prefix,
+            headerGuard,
+            debug,
+            maxTableElemWidth if maxTableElemWidth else DEFAULT_MAX_TABLE_WIDTH,
+            minTableSize if minTableSize else DEFAULT_MIN_TABLE_SIZE,
+        ).to_frontend(root, self.properties)

llparse/pybuilder/__init__.py

@@ -0,0 +1,2 @@
+from ..pybuilder.builder import *
+from ..pybuilder.loopchecker import *

llparse/pybuilder/builder.py

@@ -0,0 +1,318 @@
+from typing import Literal, Optional, Union
+
+import pyparse.pybuilder.main_code as code
+
+# from pydot import graph_from_dot_data
+
+
+i8 = "i8"
+i16 = "i16"
+i32 = "i32"
+i64 = "i64"
+ptr = "ptr"
+
+
+PropertyTypes = [i8, i16, i32, i64, ptr]
+
+
+class Property:
+    def __init__(self, ty: str, name: str) -> None:
+        if ty not in PropertyTypes:
+            raise Exception(
+                f"Can't use property : {ty}  Because it is not a valid property type in ['i8' , 'i16' , 'i32' , 'i64' , 'ptr']"
+            )
+        if any([(True if n in name else False) for n in ["\\\\", "/", " "]]):
+            raise Exception(
+                f"Flag/Pointer Name:{name} cannot have spaces or other strange characters in C..."
+            )
+        self.name = name
+        self.ty = ty
+
+
+class Creator:
+    """API for creating external callbacks and intrinsic operations."""
+
+    def __init__(self) -> None:
+        return
+
+    # TODO Make Sure python llparse match api is compatable with 3.10 and above...
+
+    # Thank goodness I can do C example documentation in here :) - Vizonex
+    def match(self, name: str):
+        """Create an external callback that **has no** `value` argument.
+
+        This callback can be used in all `Invoke` nodes except those that are
+        targets of `.select()` method.
+
+        C signature of callback must be:
+
+        ```c
+        int name(llparse_t* state,  char* p,     char* endp)
+        ```
+
+        Where `llparse_t` is parser state's type name.
+
+        Parameters
+        ----------
+        ----------
+        - `name` External function name.
+        """
+        return code._Match(name)
+
+    def value(self, name: str):
+        """
+
+        Create an external callback that **has** `value` argument.
+
+        This callback can be used only in `Invoke` nodes that are targets of
+        `.select()` method.
+
+        C signature of callback must be:
+
+        ```c
+        int name(llparse_t* state, char* p, char* endp, int value)
+        ```
+
+        Where `llparse_t` is parser state's type name.
+
+        Parameters
+        ----------
+        ----------
+        - `name` External function name.
+
+        """
+        return code.Value(name)
+
+    def span(self, name: str):
+        """Create an external span callback.
+
+        This callback can be used only in `Span` constructor.
+
+        The difference is that in typescript it's an Arbitrary Span
+        in python it's called SpanCallback to try not to be as confusing...
+
+        C signature of callback must be:
+
+        ```c
+        int name(llparse_t* state, char* p, char* endp)
+        ```
+
+        NOTE: non-zero return value is treated as resumable error.
+
+        Parameters
+        ----------
+        - `name` External function name.
+        """
+        return code._Span(name)
+
+    def store(self, field: str):
+        """
+        Intrinsic operation. Stores `value` from `.select()` node into the state's
+        property with the name specified by `field`, returns zero.
+
+        ```c
+           state[field] = value;
+           return 0;
+        ```
+
+        Parameters
+        ----------
+
+        - `field`  Property name
+        """
+        return code.Store(field)
+
+    def load(self, field: str):
+        """Intrinsic operation. Loads and returns state's property with the name
+        specified by `field`.
+
+        The value of the property is either truncated or zero-extended to fit into
+        32-bit unsigned integer.
+        ```c
+           return state[field];
+        ```
+        Parameters
+        ----------
+        `field`  Property name.
+        """
+        return code.Load(field)
+
+    def mulAdd(
+        self, field: str, base: int, max: Optional[int] = None, signed: bool = False
+    ):
+        """Intrinsic operation. Takes `value` from `.select()`, state's property
+        with the name `field` and does:
+        ```c
+            field = state[field];
+            field *= options.base;
+            field += value;
+            state[field] = field;
+            return 0;  // or 1 on overflow
+        ```
+        Return values are:
+
+            - 0 - success
+            - 1 - overflow
+
+        Parameters
+        ----------
+        ----------
+        Unlike in Typescript, The values of `IMulAddOptions` have been added here
+        since it's Python , not Typescript
+
+        - `field`    Property name
+
+        - `base` Value to multiply the property with in the first step
+
+        - `max` Maximum value of the property. If at any point of computation the
+           intermediate result exceeds it - `mulAdd` returns 1 (overflow).
+
+        - `signed` If `true` - all arithmetics perfomed by `mulAdd` will be signed.
+           Default value: `false`"""
+        return code.MulAdd(field, base, max, signed)
+
+    def update(self, field: str, value: int):
+        """
+
+        Intrinsic operation. Puts `value` integer into the state's property with
+        the name specified by `field`.
+
+          state[field] = value;
+          return 0;
+
+        Parameters
+        ----------
+        ----------
+        - `field` Property name
+        - `value` Integer value to be stored into the property.
+        """
+        return code.Update(field, value)
+
+    def isEqual(self, field: str, value: str):
+        """Intrinsic operation.
+
+           state[field] &= value
+           return 0;
+
+        Parameters
+        ----------
+        ----------
+        - `field` Property name
+        - `value` Integer value
+        """
+        return code.IsEqual(field, value)
+
+    # NOTE : Unlike in typescript lowercase "and" & "or"
+    # cannot be used this might have to be
+    # throughly addressed - Vizonex
+    def And(self, field: str, value: int):
+        """Intrinsic operation.
+
+          state[field] &= value
+          return 0;
+
+        Parameters
+        ----------
+        ----------
+        - `field` Property name
+        - `value` Integer value"""
+
+        return code.And(field, value)
+
+    def Or(self, field: str, value: int):
+        """
+        Intrinsic operation.
+
+           state[field] |= value
+           return 0;
+
+        Parameters
+        ----------
+        ----------
+        - `field` Property name
+        - `value` Integer value
+
+        This will allow us to set our own flags at will
+        """
+        return code.Or(field, value)
+
+    def test(self, field: str, value: str):
+        """Intrinsic operation.
+
+        return (state[field] & value) == value ? 1 : 0;
+
+        Parameters
+        ----------
+        ----------
+        - `field` Property name
+        - `value` Integer value
+        """
+        return code.Test(field, value)
+
+
+# NOTE: I have Nodes and Codes in the same file called `main_code`
+# as a tiny convienience for the sake a protability of not wanting
+# to Cause Hell for those wishing to move files to other folder
+# quickly this was done during development
+# as a caution of not needing to
+# open too many ides and numerous windows - Vizonex
+# so node.Match in typescript is code.Match in python...
+
+
+# TODO (Vizonex) Add more Documentation later , I got tired of it...
+class Builder:
+    def __init__(self) -> None:
+        self.code = Creator()
+        " API for creating external callbacks and intrinsic operations."
+        self.transform = code.TransfromCreator()
+        self.privProperties: dict[str, Property] = {}
+
+    def node(self, name: str):
+        return code.Match(name)
+
+    def error(self, errorCode: int, reason: str):
+        return code.Error(errorCode, reason)
+
+    def invoke(
+        self,
+        fn: code.Code,
+        Map: Union[dict[int, code.Node], code.Node, None] = None,
+        otherwise: Optional[code.Node] = None,
+    ):
+        if not Map:
+            res = code.Invoke(fn, {})
+
+        elif isinstance(Map, code.Node):
+            res = code.Invoke(fn, {})
+            otherwise = Map
+
+        else:
+            res = code.Invoke(fn, Map)
+
+        if otherwise:
+            res.otherwise(otherwise)
+
+        return res
+
+    def consume(self, field: str):
+        return code.Comsume(field)
+
+    def pause(self, errorCode: int, reason: str):
+        return code.Pause(errorCode, reason)
+
+    # NOTE SpanCallback Can Really be any node, just needed to Calrify that - Vizonex
+    def span(self, callback: code._Span):
+        return code.Span(callback)
+
+    def property(self, ty: Literal["i8", "i16", "i32", "i64", "ptr"], name: str):
+        if ty not in PropertyTypes:
+            raise TypeError(f"ty:{ty} is not an existing Parser Property")
+
+        if self.privProperties.get(name):
+            raise RuntimeError(f"Duplicate property with name:{name}")
+
+        self.privProperties[name] = Property(ty, name)
+
+    def properties(self) -> list[Property]:
+        """Return list of all allocated properties in parser's state."""
+        return list(self.privProperties.values())

llparse/pybuilder/loopchecker.py

@@ -0,0 +1,246 @@
+from typing import Any, Literal, Union
+
+from ..pybuilder.main_code import Node
+
+MAX_VALUE = 256
+WORD_SIZE = 32
+SIZE = MAX_VALUE // WORD_SIZE
+WORD_FILL = -1 | 0
+
+assert MAX_VALUE % WORD_SIZE == 0
+
+
+# NOTE I think it's theroetically Possible to
+# Put Lattice Class into Cython as an optional replacement
+# for the Sake of speed in the near future once I've
+# figured out everything myself from a more deep brain
+# point of view/prespective... - Vizonex
+
+
+class Lattice:
+    def __init__(
+        self, value: Union[Any, list[int], Literal["empty"], Literal["any"]]
+    ) -> None:
+        self.value = value
+        self.words: list[int] = []
+
+        # allocate space by filling in data with zeros...
+
+        for _ in range(SIZE):
+            self.words.append(0)
+
+        if len(value) > 1:
+            for single in value:
+                self.add(single)
+
+    def __iter__(self):
+        for i in range(MAX_VALUE):
+            if self.check(i):
+                yield i
+
+    def check(self, bit: int):
+        if not (0 <= bit and bit < MAX_VALUE):
+            raise AssertionError("Invalid Bit")
+        index = (bit // WORD_SIZE) | 0
+        off = bit % WORD_SIZE
+        return self.words[index] & (1 << off) != 0
+
+    def add(self, bit: int):
+        bit = ord(bit) if isinstance(bit, str) else bit
+        if not (0 <= bit and bit < MAX_VALUE):
+            raise AssertionError("Invalid Bit")
+
+        index = bit // WORD_SIZE
+        off = bit % WORD_SIZE
+
+        self.words[index] |= 1 << off
+
+    def union(self, other: "Lattice") -> "Lattice":
+        result = Lattice("empty")
+
+        for i in range(SIZE):
+            result.words[i] = self.words[i] | other.words[i]
+
+        return result
+
+    def intersect(self, other: "Lattice") -> "Lattice":
+        result = Lattice("empty")
+        for i in range(SIZE):
+            result.words[i] = self.words[i] & other.words[i]
+        return result
+
+    def subtract(self, other: "Lattice") -> "Lattice":
+        result = Lattice("empty")
+        for i in range(SIZE):
+            result.words[i] = self.words[i] & (~other.words[i])
+        return result
+
+    def isEqual(self, other: "Lattice"):
+        return True if (self.value == other.value) else False
+
+    def toJSON(self):
+        isEmpty = True
+        isFull = True
+        for i in range(SIZE):
+            if self.words[i] != 0:
+                isEmpty = False
+            if self.words[i] != WORD_FILL:
+                isFull = False
+        if isEmpty:
+            return "empty"
+        if isFull:
+            return "any"
+        return list(self)
+
+
+class Reachability:
+    def __init__(self) -> None:
+        return
+
+    def build(self, root: Node) -> list[Node]:
+        res: set[Node] = set()
+        queue = [root]
+        while len(queue) != 0:
+            node = queue.pop()
+            if node in res:
+                continue
+            res.add(node)
+            for edge in node:
+                queue.append(edge.node)
+            otherwise = node.getOtherwiseEdge()
+            if otherwise:
+                queue.append(otherwise.node)
+
+        return list(res)
+
+
+EMPTY_VALUE = Lattice("empty")
+ANY_VALUE = Lattice("any")
+
+
+class LoopChecker:
+    def __init__(self) -> None:
+        self.lattice: dict[Node, Lattice] = {}
+        self.terminatedCache: dict[Node, Lattice] = {}
+
+    def clear(self, nodes: list[Node]):
+        self.lattice.update({(node, EMPTY_VALUE) for node in nodes})
+
+    def check(self, root: Node):
+        r = Reachability()
+        nodes = r.build(root)
+        for node in nodes:
+            self.clear(nodes)
+            self.lattice[node] = ANY_VALUE
+            changed: set[Node] = set([root])
+            while len(changed) != 0:
+                next = set()
+                for changedNode in next:
+                    self.propagate(changedNode, next)
+                changed = next
+
+        self.visit(root, list(changed))
+
+    def propagate(self, node: Node, changed: set[Node]):
+        value = self.lattice[node]
+        terminated: "Lattice" = self.terminate(node, value, changed)
+        if terminated.value == EMPTY_VALUE.value:
+            value = value.subtract(terminated)
+            if value.isEqual(EMPTY_VALUE):
+                return
+
+        keysbyTarget: dict[Node, Lattice] = dict()
+
+        for edge in node.getAllEdges():
+            if not edge.noAdvance:
+                continue
+
+            targetValue: Lattice
+            if keysbyTarget.get(edge.node):
+                targetValue = keysbyTarget[edge.node]
+            else:
+                targetValue = self.lattice.get(edge.node)
+
+            if edge.key is None or isinstance(edge.key, int):
+                targetValue = targetValue.union(value)
+            else:
+                # From peek()
+                edgeValue = Lattice([edge.key[0]]).intersect(value)
+                if edgeValue.isEqual(EMPTY_VALUE):
+                    continue
+
+                targetValue = targetValue.union(edgeValue)
+            keysbyTarget[edge.node] = targetValue
+
+        for child, childValue in keysbyTarget.items():
+            self.update(child, childValue, changed)
+        # FINISHED!
+
+    def update(self, node: Node, newValue: Lattice, changed: set[Node]):
+        value = self.lattice[node]
+        if newValue.isEqual(value):
+            return False
+        self.lattice[node] = newValue
+        changed.add(node)
+
+    def terminate(self, node: Node, value: Lattice, changed: set[Node]):
+        if self.terminatedCache.get(node):
+            return self.terminatedCache[node]
+
+        terminated: list[int] = []
+
+        for edge in node.getAllEdges():
+            if edge.noAdvance:
+                continue
+
+            if edge.key is None or isinstance(edge.key, int):
+                continue
+
+            terminated.append(edge.key[0])
+
+        result = Lattice(terminated)
+        self.terminatedCache[node] = result
+        return result
+
+    def visit(self, node: Node, path: list[Node]):
+        value = self.lattice[node]
+        terminated = (
+            self.terminatedCache[node]
+            if self.terminatedCache.get(node)
+            else EMPTY_VALUE
+        )
+        if terminated.isEqual(EMPTY_VALUE):
+            value = value.subtract(terminated)
+            if value.isEqual(EMPTY_VALUE):
+                return
+
+        for edge in node.getAllEdges():
+            if edge.noAdvance:
+                continue
+            edgeValue = value
+            if not (not edge.key or isinstance(edge.key, int)):
+                edgeValue = edgeValue.intersect(Lattice([edge.key[0]]))
+
+            if edgeValue.isEqual(EMPTY_VALUE):
+                continue
+
+            def indexOf(path: list, obj):
+                try:
+                    return path.index(obj)
+                except Exception:
+                    return -1
+
+            if indexOf(path, node) != -1:
+                if len(path) == 0:
+                    raise Exception(
+                        f'Detected a loop in "{edge.node.name}" though : {edge.node.name}'
+                    )
+
+                raise Exception(
+                    'Detected loop in "'
+                    + edge.node.name
+                    + '" through chain '
+                    + (" -> ").join(['"' + name.name + '"' for name in path])
+                )
+
+            self.visit(edge.node, path.extend([edge.node]))