llparse 0.1.0__py3-none-any.whl

llparse/settings.py

@@ -0,0 +1,285 @@
+"""
+Compiles a settings module to use alongside llparse
+
+This is mostly a concept idea for llparse inspired by llhttp to include a settings
+module in the future to make Writing the span callbacks and other important things
+less painful to edit and write entirely.
+
+NOTE: That The objects being passed would come directly from llparse-builder
+
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+from .llparse import LLParse
+from .pybuilder import main_code as source
+
+# MACRO Names , These can be renamed to
+# prevent mixing with llhttp or other parsers
+# allow for mulitple different parsers to be
+# used at the same time...
+
+CALLBACK_MAYBE = "CALLBACK_MAYBE"
+SPAN_CALLBACK_MAYBE = "SPAN_CALLBACK_MAYBE"
+MID_POSTFIX = "_settings"
+
+
+
+@dataclass
+class ApiResult:
+    header: str
+    c: str
+
+
+@dataclass
+class Settings:
+    name: str
+    """The property name to take from the settings module..."""
+    prefix: str
+    """The Prefix of your parser"""
+    postfix: Optional[str] = None
+    """The postfix of your parser"""
+    callbacks: dict[str, bool] = field(default_factory=dict)
+    """These are callbacks to be later edited out into thier appropreate places"""
+    files: dict[str, dict[str, str]] = field(default_factory=dict)
+    """Used to sort out functions if they are to be moved to different places these
+    files will not affect the main api and can be ignored if chosen to be..."""
+
+    external_execute_func: str = field(default_factory=str, init=False)
+    """Allows the use of an external function and it's prefix to be called
+        ```c
+        
+        int POSTFIX_execute(PREFIX_t* parser, const char* data, size_t len) {
+          return PREFIX_execute(parser, data, data + len);
+        }
+        
+        ```
+        WARNIG! THIS SHOULD ONLY BE USED IF A POSTFIX HAS BEEN ADDED!"""
+
+    # It can be annoying to have to recall something over and over again so I added the ability to use
+    # Nodes and Code.Match classes to make up for this annoyance...
+    def move_cb(
+        self,
+        callback: Union[str, source._Match, source.Node, source.Invoke],
+        new_location: str,
+        code: Optional[str] = None,
+    ):
+        """Moves the callback chosen to another file instead of the name of your api
+        this can help you add things like custom C code, to non-api or user-interacted parts...
+        """
+
+        # Grab real name if the callback is a Node or Match otherwise get the string for it...
+
+        name = (
+            callback.name
+            if isinstance(callback, (source.Code, source.Node))
+            else callback
+        )
+        # If not added make a new dictioray / Map if your on typescript...
+        if not self.files.get(new_location):
+            self.files[new_location] = {}
+
+        self.files[new_location][name] = code
+
+        # Remove from default callbacks to be moved to another file in C...
+        # If index fails (IndexError) the callback name doesn't exist...
+        self.callbacks.pop(name)
+        return self
+
+    def add_external_execute(self, postfix_as_struct: bool = False):
+        """Adds an external execute function as long as a postfix variable exists otherwise
+        using this function this is not nessesary!
+        ```c
+        int POSTFIX_execute(PREFIX_t* parser, const char* data, size_t len) {
+          return PREFIX_execute(parser, data, data + len);
+        }
+        ```
+        Parameters
+        ---
+        `postfix_as_struct` use postfix struct instead of inside of POSTFIX_execute's function...
+        """
+        if self.postfix:
+            self.external_execute_func += (
+                f"int {self.postfix}_execute({self.postfix if postfix_as_struct else self.prefix}_t* parser, const char* data, size_t len) "
+                + "{"
+            )
+            self.external_execute_func += (
+                f"\nreturn {self.prefix}_execute(parser, data, data + len);\n" + "}"
+            )
+
+        return self
+
+    def build_functions(self, out: list[str]):
+        # Added newline to this one to reamin consistant with llhttp if the library devs decided to
+        # implement my idea into thier projects
+        out.append("/* CALLBACKS */\n")
+        out.append("")
+        # TODO Vizonex allow custom enum names to be passed as well in the next concept...
+        for cb, is_span in self.callbacks.items():
+            out.append(
+                f"int {cb}({self.prefix}_t * s, const char* p, const char* endp) " + "{"
+            )
+            out.append("  int err;")
+            if is_span:
+                out.append(f"  {SPAN_CALLBACK_MAYBE}(s, {cb} ,p, endp - p);")
+            else:
+                out.append(f"  {CALLBACK_MAYBE}(s, {cb});")
+            out.append("  return err;")
+            # Close function and give some room for the next one to be placed into...
+            out.append("}")
+            out.append("")
+        return
+
+    def buildMacros(
+        self,
+        out: list[str],
+        callback: str = CALLBACK_MAYBE,
+        span_callback: str = SPAN_CALLBACK_MAYBE,
+    ):
+        """Builds Global Macro Callbacks"""
+        # Inspired by llhttp
+        out.append(
+            f"#define {callback}(PARSER, NAME)                                          \\"
+        )
+        out.append(
+            "do {                                                                        \\"
+        )
+        out.append(
+            f"  const {self.prefix}{self.name}_t* settings;                               \\"
+        )
+        out.append(
+            f"  settings = (const {self.prefix}{self.name}_t*) (PARSER)->{self.name};      \\"
+        )
+        out.append(
+            "  if (settings == NULL || settings->NAME == NULL) {                         \\"
+        )
+        out.append(
+            "    err = 0;                                                                \\"
+        )
+        out.append(
+            "    break;                                                                  \\"
+        )
+        out.append(
+            "  }                                                                         \\"
+        )
+        out.append(
+            "  err = settings->NAME((PARSER));                                           \\"
+        )
+        out.append("} while (0)")
+        out.append("")
+
+        out.append(
+            f"#define {span_callback}(PARSER, NAME, START, LEN)                         \\"
+        )
+        out.append(
+            "  do {                                                                        \\"
+        )
+        out.append(
+            f"  const {self.prefix}{self.name}_t* settings;                               \\"
+        )
+        out.append(
+            f"  settings = (const {self.prefix}{self.name}_t*) (PARSER)->{self.name};      \\"
+        )
+        out.append(
+            "    if (settings == NULL || settings->NAME == NULL) {                         \\"
+        )
+        out.append(
+            "      err = 0;                                                                \\"
+        )
+        out.append(
+            "      break;                                                                  \\"
+        )
+        out.append(
+            "    }                                                                         \\"
+        )
+        out.append(
+            "    err = settings->NAME((PARSER), (START), (LEN));                           \\"
+        )
+        # TODO: maybe allow for custom error functions to be raised right here
+        # or from or with both of them optionally? --
+        out.append("  } while (0)")
+        out.append("\n\n\n")
+
+    def build_C(self, cb: str, span_cb: str):
+        out = []
+        out.append("#include <stdlib.h>")
+        out.append("#include <stdio.h>")
+        out.append("#include <string.h>")
+        out.append("")
+        out.append(f'#include "{self.prefix}.h"')
+        out.append("")
+        self.buildMacros(out, cb, span_cb)
+        self.build_functions(out)
+        return "\n".join(out)
+
+    def build_H(self):
+        out = []
+
+        fix = self.postfix if self.postfix else self.prefix
+
+        out.append(f"typedef struct {fix}{self.name}_s {fix}{self.name}_t;")
+        out.append(
+            f"typedef int (*{fix}_data_cb)({fix}_t*, const char *at, size_t length);"
+        )
+        out.append(f"typedef int (*{fix}_cb)({fix}_t*);")
+
+        out.append(f"struct {self.prefix}{self.name}_s " + "{")
+        for cb, is_span in self.callbacks.items():
+            if is_span:
+                out.append(f"  {fix}_data_cb {cb};")
+            else:
+                out.append(f"  {fix}_cb {cb};")
+        out.append("};")
+
+
+class Disassembler:
+    """Takes apart LLParse's calls and sorts the data out to help make
+    different things such as a settings api moudle, Cython pxd file
+    and a Typescript Writer, and a Json model of what's going on...
+    """
+
+    def __init__(self, builder: LLParse) -> None:
+        self.invokes: set[source.Invoke] = set()
+        self.nodes: dict[str, source.Node] = {}
+        """Used to create a small dictionary of all avalible nodes to thier corresponding names.
+        this also includes it's edges used and seen to create a json file if chosen."""
+        self.empty_apis: dict[str, bool] = {}
+        """Carries some of the main api calls to make in a settings module...
+        These carry thier own boolean to see weather or not the callback is a `Span` (True) , 
+        `Match` (False)..."""
+        self.prefix = builder.prefix
+        self.properties = builder.privProperties
+        """Carries important property values"""
+
+    def disassemble(self, root: source.Node):
+        """Takes root apart and enumerates through all the major parts
+        returns itself as a convenience measure..."""
+        reach = source.Reachability()
+        roots = reach.build(root)
+        # Get all span's names and objects themselves...
+        # Turn span_names into a set
+        self.nodes = {n.name: n for n in roots}
+        # print(self.nodes)
+        span_names = list(
+            sorted(
+                {
+                    n.span.callback.name
+                    for n in roots
+                    if isinstance(n, (source.SpanStart, source.SpanEnd))
+                }
+            )
+        )
+        # span_starts = {n.span.callback.name:n for n in roots if isinstance(n,source.SpanStart)}
+
+        self.empty_apis.update({sn: True for sn in span_names})
+
+        invokes = [n for n in roots if isinstance(n, source.Invoke)]
+
+        for i in invokes:
+            # I have Match Underscored because in my version of the library there's two of them in the same file.
+            # The one with the underscore is for use with Invoke...
+            if isinstance(i.code, source._Match):
+                self.empty_apis[i.code.name] = False
+
+        return self

llparse/spanalloc.py

@@ -0,0 +1,176 @@
+from dataclasses import dataclass, field
+from typing import Union
+
+from .pybuilder.main_code import Node, Reachability, Span, SpanEnd, SpanStart
+
+
+class DeadLoop(Exception):
+    """Thrown when this type of loop is detected during complation
+    ```c
+        switch(*p){
+          case dead_loop:
+          dead_loop : {
+            goto dead_loop;
+          } /* Rest in Peace Computer x_x */
+        }
+
+    ```"""
+
+
+SpanSet = set[Span]
+
+
+def _id(node: Union[SpanStart, SpanEnd]):
+    return node.span
+
+
+@dataclass
+class ISpanActiveInfo:
+    active: dict[Node, SpanSet] = field(default_factory=dict)
+    spans: list[Span] = field(default_factory=list)
+
+
+SpanOverlap = dict[Node, SpanSet]
+
+
+@dataclass
+class ISpanAllocatorResult:
+    colors: dict[Span, int] = field(default_factory=dict)
+    concurrency: list[list[Span]] = field(default_factory=list)
+    max: int = field(default_factory=int)
+
+
+class SpanAllocator:
+    def __init__(self) -> None:
+        return
+
+    def allocate(self, root: Node):
+        nodes = Reachability.build(root)
+        info = self.computeActive(nodes)
+
+        self.check(info)
+        overlap = self.computeOverlap(info)
+        return self.color(info.spans, overlap)
+
+    def check(self, info: ISpanActiveInfo):
+        for node, spans in info.active.items():
+            for edge in node.getAllEdges():
+                if isinstance(edge.node, SpanStart):
+                    continue
+
+                # Skip terminal nodes
+                # print(len(edge.node.getAllEdges()))
+                # print(info.active)
+                if len(edge.node.getAllEdges()) == 0:
+                    continue
+
+                # assert node.name != edge.node.name
+                # print("checking edge from %s to %s" % (node.name,edge.node.name))
+                # check edge
+
+                edgeSpans: set[Span] = info.active[edge.node]
+                # print("SPAN:%s  NODE:%s" % (edgeSpans,edge.node.__dict__))
+                for subSpan in edgeSpans:
+                    if subSpan not in spans:
+                        raise AssertionError(
+                            f'Unmatched span end for "{subSpan.callback.name}"'
+                            / f'at "{edge.node.name}", coming from "{node.name}"'
+                        )
+
+                if isinstance(edge.node, SpanEnd):
+                    span = _id(edge.node)
+                    if span not in spans:
+                        raise AssertionError(
+                            f'Unmatched span end for "{span.callback.name}"'
+                        )
+
+    def computeActive(self, nodes: list[Node]):
+        activeMap: dict[Node, SpanSet] = dict()
+        for node in nodes:
+            activeMap[node] = set()
+
+        queue = set(nodes)
+        spans: SpanSet = set()
+        # This fixes an issue when using a for loop which unlike in typescript
+        # we cannot remove items when in a for-loop in python this also ensures
+        # that all spans are visited.
+        while queue:
+            node = queue.pop()
+            active = activeMap[node]
+            if isinstance(node, SpanStart):
+                span = _id(node)
+                spans.add(span)
+                active.add(span)
+
+            for span in active:
+                if isinstance(node, SpanEnd) and span == _id(node):
+                    break
+
+                for edge in node.getAllEdges():
+                    edgeNode = edge.node
+
+                    if isinstance(edgeNode, SpanStart):
+                        if _id(edgeNode) == span:
+                            raise DeadLoop(
+                                f'Detected loop in span {span.callback.name} at "{node.name}"'
+                            )
+
+                    edgeActive = activeMap[edgeNode]
+                    if span in edgeActive:
+                        break
+
+                    edgeActive.add(span)
+                    queue.add(edgeNode)
+
+        return ISpanActiveInfo(active=activeMap, spans=list(spans))
+
+    def computeOverlap(self, info: ISpanActiveInfo):
+        active = info.active
+
+        overlap: dict[Span, set[Span]] = {span: set() for span in info.spans}
+        for _, spans in active.items():
+            for one in spans:
+                for other in spans:
+                    if other != one:
+                        overlap[one].add(other)
+        return overlap
+
+    def _allocate(self, span: Span):
+        if self._colors.get(span):
+            return self._colors[span]
+
+        overlap = self._overlapMap[span]
+
+        used: set[int] = set()
+        for subSpan in overlap:
+            if self._colors.get(subSpan):
+                used.add(self._colors.get(subSpan))
+        i = 0
+        while i in used:
+            i += 1
+
+        self._mx = max(self._mx, i)
+        self._colors[span] = i
+        return i
+
+    def color(self, spans: list[Span], overlapDict: SpanOverlap):
+        # Used _max instead of max because max() is an api called function needed in a bit...
+        self._mx = -1
+        self._colors: dict[Span, int] = {}
+
+        self._overlapMap = overlapDict
+
+        Map = {span: self._allocate(span) for span in spans}
+
+        concurrency = list()
+        for _ in range(self._mx + 1):
+            # NOTE : concurrency[i] = [] doesn't work but this does :P
+            concurrency.append([])
+
+        for s in spans:
+            concurrency[self._allocate(s)].append(s)
+
+        return ISpanAllocatorResult(Map, concurrency, self._mx)
+
+
+# TODO (Vizonex) Use Indutny's Mini Http parser to help with testing ours to verify that ours is correct...

llparse/test.py

@@ -0,0 +1,232 @@
+from .C_compiler import Compilation, ICompilerOptions
+from .frontend import Frontend
+from .llparse import LLParse
+from .pybuilder import Builder
+
+# TODO: Remove and make pytest for it.
+
+def smaller_test():
+    # from compilator import Compilation, ICompilerOptions
+    p = Builder()
+    simple = p.node("simple")
+    complete = p.invoke(
+        p.code.match("On_Complete"), {0: simple}, p.error(1, "On_Complete FAILED!")
+    )
+
+    simple.peek("\n", complete).skipTo(simple)
+
+    front = Frontend("http_parser")
+    f = front.compile(simple, properties=p.properties)
+    comp = Compilation(
+        "http_parser", f.properties, f.resumptionTargets, ICompilerOptions(None, None)
+    )
+    # for r in comp.resumptionTargets:
+    #     print(r)
+    out = []
+    root = comp.unwrapNode(f.root)
+    root.build(comp)
+    comp.reserveSpans(f.spans)
+    comp.buildGlobals(out)
+    comp.buildResumptionStates(out)
+    print(comp.stateDict)
+    # print("\n".join(out))
+
+
+def test1():
+    # I will be using indutny's mini http parser to demonstrate as it's
+    # medium sized and uses all major tricks , bells and whistles...
+
+    p = Builder()
+    method = p.node("method")
+    beforeUrl = p.node("before_url")
+    urlSpan = p.span(p.code.span("on_url"))
+    url = p.node("url")
+    http = p.node("http")
+
+    # print(sliced)(method)
+
+    # // Add custom uint8_t property to the state
+    p.property("i8", "method")
+
+    # Store method inside a custom property
+    onMethod = p.invoke(p.code.store("method"), beforeUrl)
+
+    # Invoke custom C function
+    complete = p.invoke(
+        p.code.match("on_complete"),
+        {
+            #  Restart
+            0: method
+        },
+        p.error(4, "`on_complete` error"),
+    )
+
+    method.select(
+        {
+            "HEAD": 0,
+            "GET": 1,
+            "POST": 2,
+            "PUT": 3,
+            "DELETE": 4,
+            "OPTIONS": 5,
+            "CONNECT": 6,
+            "TRACE": 7,
+            "PATCH": 8,
+        },
+        onMethod,
+    ).otherwise(p.error(5, "Expected method"))
+
+    beforeUrl.match(" ", beforeUrl).otherwise(urlSpan.start(url))
+
+    url.peek(" ", urlSpan.end(http)).skipTo(url)
+
+    http.match(" HTTP/1.1\r\n\r\n", complete).otherwise(
+        p.error(6, "Expected HTTP/1.1 and two newlines")
+    )
+
+    front = Frontend("http_parser")
+    f = front.compile(
+        method,
+        properties=p.properties,
+    )
+    # r = f.root.ref.id.name
+    # NOTE Check that root was addded , this can be a problem laster down the line if it isn't a resumption target...
+    assert f.root in f.resumptionTargets
+    comp = Compilation(
+        "http_parser", p.properties(), f.resumptionTargets, ICompilerOptions()
+    )
+    root = comp.unwrapNode(f.root)
+    root.build(comp)
+
+    assert "s_n_http_parser__n_method" in comp.resumptionTargets
+    out = []
+    comp.reserveSpans(f.spans)
+    comp.buildGlobals(out)
+    out.append("")
+    out.append("/*--RESUMPTION STATES--*/")
+    comp.buildResumptionStates(out)
+    out.append("/*--INTERNAL STATES--*/")
+    comp.buildInternalStates(out)
+    # for k , v in comp.stateDict.items():
+    #     print(k)
+    print("\n".join(out))
+
+
+def test2():
+    from llparse import LLParse
+
+    p = LLParse("http_parser")
+    method = p.node("method")
+    beforeUrl = p.node("before_url")
+    urlSpan = p.span(p.code.span("on_url"))
+    url = p.node("url")
+    http = p.node("http")
+
+    # Add custom uint8_t property to the state
+    p.property("i8", "method")
+
+    # Store method inside a custom property
+    onMethod = p.invoke(p.code.store("method"), beforeUrl)
+
+    # Invoke custom C function
+    complete = p.invoke(
+        p.code.match("on_complete"),
+        {
+            #  Restart
+            0: method
+        },
+        p.error(4, "`on_complete` error"),
+    )
+
+    method.select(
+        {
+            "HEAD": 0,
+            "GET": 1,
+            "POST": 2,
+            "PUT": 3,
+            "DELETE": 4,
+            "OPTIONS": 5,
+            "CONNECT": 6,
+            "TRACE": 7,
+            "PATCH": 8,
+        },
+        onMethod,
+    ).otherwise(p.error(5, "Expected method"))
+
+    beforeUrl.match(" ", beforeUrl).otherwise(urlSpan.start(url))
+
+    url.peek(" ", urlSpan.end(http)).skipTo(url)
+
+    http.match(" HTTP/1.1\r\n\r\n", complete).otherwise(
+        p.error(6, "Expected HTTP/1.1 and two newlines")
+    )
+
+    c = p.build(method)
+    print(c.c)
+    open("http_parser.c", "w").write(c.c)
+    open("http_parser.h", "w").write(c.header)
+    # # sp_alloc = SpanAllocator().allocate(method)
+
+    # TODO Validate Moving parts as they should be compiled correctly...
+
+
+def test3():
+    """Moving Part Validations..."""
+    # A~Sample~Of~Dual Spans...
+    p = LLParse("dual_spans")
+
+    span_a = p.span(p.code.span("on_a"))
+    span_b = p.span(p.code.span("on_b"))
+
+    a = p.node("a_node")
+    b = p.node("b_node")
+
+    start = p.node("start")
+    start.otherwise(span_a.start(a))
+
+    a.peek("~", span_a.end().skipTo(span_b.start(b))).skipTo(a)
+    b.peek("~", span_b.end().skipTo(span_a.start(a))).skipTo(b)
+
+    compiled = p.build(start)
+    print(compiled.c)
+
+
+def test4():
+    """Same Dual Spans but with flags..."""
+    # A~Sample~Of~Dual Spans...
+
+    p = LLParse("dual_spans")
+
+    # Flag
+    p.property("i8", "flag")
+
+    span_a = p.span(p.code.span("on_a"))
+    span_b = p.span(p.code.span("on_b"))
+
+    a = p.node("a_node")
+    b = p.node("b_node")
+
+    # Invoke Updates to Flag
+    # 1 is a , 2 is b
+
+    update_a = p.invoke(p.code.update("flag", 1), span_a.start(a))
+    update_b = p.invoke(p.code.update("flag", 2), span_b.start(b))
+    start = p.node("start")
+    start.otherwise(update_a)
+
+    a.peek("~", span_a.end().skipTo(update_a)).skipTo(a)
+    b.peek("~", span_b.end().skipTo(update_b)).skipTo(b)
+
+    compiled = p.build(start)
+    print(compiled.c)
+
+
+# TODO  Try to Throw more complicated and advanced things at pyllparse to try and solve
+# This will make the code sturdier and more mature as time goes on
+# Make a Test Bag on github for all of these tests to go into incase a single file gets too big
+
+# I encourage anyone to find valid compiled llparse sequences and then translate them over to
+# python to testrun , I will try to make a validator tool whenever I can get to it - Vizonex
+
+if __name__ == "__main__":
+    test2()