llparse 0.1.0__py3-none-any.whl

llparse/constants.py

@@ -0,0 +1,48 @@
+CONTAINER_KEY = "c"
+LABEL_PREFIX = ""
+STATE_PREFIX = "s_n_"
+STATE_ERROR = "s_error"
+BLOB_PREFIX = "llparse_blob"
+ARG_STATE = "state"
+ARG_POS = "p"
+ARG_ENDPOS = "endp"
+VAR_MATCH = "match"
+
+# MatchSequence
+
+SEQUENCE_COMPLETE = "kMatchComplete"
+SEQUENCE_MISMATCH = "kMatchMismatch"
+SEQUENCE_PAUSE = "kMatchPause"
+
+
+# I Thought it might be a little but faster to use tuples instead of lists - Vizonex
+SIGNED_LIMITS: dict[str, tuple[str, str]] = {
+    "i8": ("-0x80", "0x7f"),
+    "i16": ("-0x8000", "0x7fff"),
+    "i32": ("(-0x7fffffff - 1)", "0x7fffffff"),
+    "i64": ("(-0x7fffffffffffffffLL - 1)", "0x7fffffffffffffffLL"),
+}
+
+# TODO (Vizonex) : Propose changes to llparse
+# typescript program which uses two i8's
+# which I belive is an error and a mistake
+UNSIGNED_LIMITS: dict[str, tuple[str, str]] = {
+    "i8": ("0", "0xff"),
+    "i16": ("0", "0xffff"),
+    "i32": ("0", "0xffffffff"),
+    "i64": ("0ULL", "0xffffffffffffffffULL"),
+}
+
+UNSIGNED_TYPES: dict[str, str] = {
+    "i8": "int8_t",
+    "i16": "int16_t",
+    "i32": "int32_t",
+    "i64": "int64_t",
+}
+
+SIGNED_TYPES: dict[str, str] = {
+    "i8": "int8_t",
+    "i16": "int16_t",
+    "i32": "int32_t",
+    "i64": "int64_t",
+}

llparse/cython_builder.py

@@ -0,0 +1,311 @@
+"""Used to build apis and more from llparse WARNING: This may or may not be stable yet!!!"""
+
+from contextlib import contextmanager
+from typing import Optional
+from .frontend import IFrontendResult
+from .pyfront.front import Match
+from .pyfront.nodes import Invoke
+
+# Inspired by Cython's Writer
+
+# You will notice many simillarities
+# because there's was no way to for me to optimize the
+# originals further than what was given by Cython itself.
+# This will be used to help me write my own custom
+# Finite Machine Parts
+
+VA_ARGS_CALLBACK = """#define CALLBACK_MAYBE(PARSER, NAME, ...)                                     \\
+  do {                                                                        \\
+    %s_settings_t* settings;                                              \\
+    settings = (%s_settings_t*) (PARSER)->settings;                       \\
+    if (settings == NULL || settings->NAME == NULL) {                         \\
+      err = 0;                                                                \\
+      break;                                                                  \\
+    }                                                                         \\
+    err = settings->NAME(__VA_ARGS__);                                        \\
+  } while (0)"""
+
+
+class LineWriter(object):
+    def __init__(self) -> None:
+        self.lines: list[str] = []
+        self.s = ""
+
+    def put(self, s: str):
+        self.s += s
+
+    def newline(self):
+        self.lines.append(self.s)
+        self.s = ""
+
+    def putline(self, s: str):
+        self.s += s
+        self.newline()
+
+
+class CodeWriter:
+    """Used as a baseplate for writing code..."""
+
+    line_indent: str = "  "
+
+    def __init__(self) -> None:
+        self._indentures = 0
+        self.lw = LineWriter()
+
+    def __indent(self):
+        self._indentures += 1
+
+    def __dedent(self):
+        self._indentures -= 1
+
+    @contextmanager
+    def indent(self):
+        """Used to mirror/mimic programming with indentures and to make everything cleaner and easier to read"""
+        self.__indent()
+        yield
+        self.__dedent()
+
+    def startline(self, s: str):
+        self.lw.put((self._indentures * self.line_indent) + s)
+
+    def put(self, s: str):
+        self.lw.put(s)
+
+    def putline_with_format(self, s: str, *args):
+        """Makes a cleaner format than what would've been used to workaround formmating with curly brackets `{}`"""
+        self.lw.putline((self._indentures * self.line_indent) + s % args)
+
+    def putline(self, s: str):
+        self.lw.putline((self._indentures * self.line_indent) + s)
+
+    def endline(self, s: str = ""):
+        self.lw.putline(s)
+
+    @property
+    def lines(self) -> list[str]:
+        return self.lw.lines
+
+    @property
+    def code(self) -> str:
+        return "\n".join(self.lines)
+
+
+class CythonWriter(CodeWriter):
+    """Coming soon..."""
+
+    line_indent = "    "
+
+
+# TODO Vizonex Maybe see if Indutny would like to use a special codewriter to
+# help with building llparse's c code in typescript It would be less prone to
+# compile-time errors
+
+
+class MainCompiler:
+    """Used to Create APIS like those seen in llhttp"""
+
+    def __init__(self, info: IFrontendResult) -> None:
+        self.data_cb: set[str] = set()
+        """Used to identify span related Callbacks"""
+        self.cb: set[str] = set()
+        """Used to identify match callbacks that are use-handled"""
+        self.info = info
+
+    def get_user_callbacks(self):
+        for s in self.info.spans:
+            self.data_cb.update(cb.ref.name for cb in s.callbacks)
+        for s in self.info.resumptionTargets:
+            for slot in s.ref.buildSlots():
+                if isinstance(slot.node.ref, Invoke) and isinstance(
+                    slot.node.ref.code.ref, Match
+                ):
+                    self.cb.add(slot.node.ref.code.ref.name)
+                # Some nodes like to hide themselves inside other nodes so this is my only simple solution which is to do it a second time...
+                for _slot in slot.node.ref.getSlots():
+                    if isinstance(_slot.node.ref, Invoke) and isinstance(
+                        _slot.node.ref.code.ref, Match
+                    ):
+                        self.cb.add(_slot.node.ref.code.ref.name)
+
+
+class ApiCompiler(MainCompiler):
+    """Builds external api assuming that our prefix is an internal one"""
+
+    def __init__(
+        self, new_preifx: str, info: IFrontendResult, header_name: Optional[str] = None
+    ) -> None:
+        self.new_prefix = new_preifx
+        super().__init__(info)
+        self.get_user_callbacks()
+        self.header_name = header_name
+
+    def build_C(self):
+        # I find using codewriters to be more elegant so we will be using that instead - Vizonex
+        prefix = self.info.prefix
+        writer = CodeWriter()
+        writer.putline(
+            f'#include <stdlib.h>\n#include <stdio.h>\n#include <string.h>\n#include "{self.header_name or self.new_prefix}.h"'
+        )
+        writer.endline()
+        writer.putline("/* Inspired by llhttp */")
+        writer.endline()
+        writer.putline_with_format(VA_ARGS_CALLBACK, self.new_prefix, self.new_prefix)
+        writer.endline()
+        writer.endline()
+        writer.putline(f"void {self.new_prefix}_init({self.new_prefix}_t* parser,")
+        with writer.indent():
+            writer.putline(f"const {self.new_prefix}_settings_t* settings) " + "{")
+            writer.putline(f"{prefix}_init(parser);")
+            writer.putline("parser->settings = (void*) settings;")
+        writer.endline("}")
+        writer.putline("/* Callbacks */")
+
+        # This is where everything comes together and makes sense
+        for data in sorted(self.data_cb):
+            writer.putline(
+                f"int {data}({self.new_prefix}_t* s, const char* p, const char* endp) "
+                + "{"
+            )
+            with writer.indent():
+                writer.putline("int err;")
+                writer.putline(
+                    f"CALLBACK_MAYBE(s, {data.removeprefix(prefix).strip('_')}, s, p, endp - p);"
+                )
+                writer.putline("return err;")
+            writer.putline("}")
+            writer.endline()
+            writer.endline()
+
+        for data in sorted(self.cb):
+            writer.putline(
+                f"int {data}({self.new_prefix}_t* s, const char* p, const char* endp) "
+                + "{"
+            )
+            with writer.indent():
+                writer.putline("int err;")
+                writer.putline(
+                    f"CALLBACK_MAYBE(s, {data.removeprefix(prefix).strip('_')}, s);"
+                )
+                writer.putline("return err;")
+            writer.putline("}")
+            writer.endline()
+            writer.endline()
+
+        writer.putline(
+            f"int {self.new_prefix}_execute({self.new_prefix}_t* parser, const char* data, size_t len) "
+            + "{"
+        )
+        with writer.indent():
+            writer.putline(f"return {prefix}_execute(parser, data, data + len);")
+        writer.putline("}")
+
+        # Reset Parser
+
+        writer.putline(
+            f"void {self.new_prefix}_settings_init({self.new_prefix}_settings_t* settings)"
+            + " {"
+        )
+        writer.putline("\tmemset(settings, 0, sizeof(*settings));\n}")
+
+        return writer.code
+
+    def build_H(self, headerguard: Optional[str] = None):
+        """Builds Headerfile api extensions..."""
+        writer = CodeWriter()
+        headerguard = self.new_prefix.upper() if not headerguard else headerguard
+        writer.putline(f"\n#ifndef {headerguard}_API_H_")
+        writer.putline(f"#define {headerguard}_API_H_")
+        writer.endline()
+        writer.putline_with_format(
+            "typedef %s_t %s_t;", self.info.prefix, self.new_prefix
+        )
+        writer.putline(
+            f"typedef struct {self.new_prefix}_settings_s {self.new_prefix}_settings_t;"
+        )
+        writer.endline()
+        writer.putline(
+            f"typedef int (*{self.new_prefix}_data_cb)({self.new_prefix}_t*, const char *at, size_t length);"
+        )
+        writer.putline(f"typedef int (*{self.new_prefix}_cb)({self.new_prefix}_t*);")
+        writer.endline()
+        writer.putline(f"struct {self.new_prefix}_settings_s " + "{")
+        with writer.indent():
+            for data in self.data_cb:
+                writer.putline(
+                    f"{self.new_prefix}_data_cb {data.removeprefix(self.info.prefix).strip('_')};"
+                )
+            for data in self.cb:
+                writer.putline(
+                    f"{self.new_prefix}_cb {data.removeprefix(self.info.prefix).strip('_')};"
+                )
+        writer.putline("};")
+        writer.endline()
+        writer.putline(
+            f"int {self.new_prefix}_execute({self.new_prefix}_t* parser, const char* data, size_t len);"
+        )
+        writer.putline(f"void {self.new_prefix}_init({self.new_prefix}_t* parser,")
+        with writer.indent():
+            writer.putline(f"const {self.new_prefix}_settings_t* settings);")
+        writer.putline(
+            f"void {self.new_prefix}_settings_init({self.new_prefix}_settings_t* settings);"
+        )
+        writer.putline(f"#endif /* {headerguard}_API_H_ */")
+        return writer.code
+
+    def build_pxd(self):
+        writer = CythonWriter()
+
+        writer.putline("#cython: language_level = 3")
+        writer.endline()
+        writer.putline("from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t")
+        writer.endline()
+        writer.putline("# Automatically generated in pyparse a parody of llparse")
+        writer.putline(f'cdef extern from "{self.header_name or self.new_prefix}.h":')
+        with writer.indent():
+            writer.putline(f"struct {self.new_prefix}_t:")
+            with writer.indent():
+                ty = ""
+
+                for prop in self.info.properties():
+                    if prop.ty == "i8":
+                        ty = "uint8_t"
+                    elif prop.ty == "i16":
+                        ty = "uint16_t"
+                    elif prop.ty == "i32":
+                        ty = "uint32_t"
+                    elif prop.ty == "i64":
+                        ty = "uint64_t"
+                    elif prop.ty == "ptr":
+                        ty = "void*"
+                    else:
+                        raise Exception(f'Unknown state property type: "{prop.ty}"')
+                    writer.putline("%s %s" % (ty, prop.name))
+                writer.putline("void* data")
+
+            writer.endline()
+            writer.putline(
+                f"ctypedef int (*{self.new_prefix}_data_cb)({self.new_prefix}_t*, const char *at, size_t length)"
+            )
+            writer.putline(
+                f"ctypedef int (*{self.new_prefix}_cb)({self.new_prefix}_t*)"
+            )
+            writer.endline()
+            writer.putline(f"struct {self.new_prefix}_settings_t:")
+            with writer.indent():
+                for data in sorted(self.data_cb):
+                    writer.putline(
+                        f"{self.new_prefix}_data_cb {data.removeprefix(self.info.prefix).strip('_')}"
+                    )
+                for data in sorted(self.cb):
+                    writer.putline(
+                        f"{self.new_prefix}_cb {data.removeprefix(self.info.prefix).strip('_')}"
+                    )
+            writer.endline()
+            writer.putline(
+                f"int {self.new_prefix}_execute({self.new_prefix}_t* parser, const char* data, size_t len)"
+            )
+            writer.putline(
+                f"void {self.new_prefix}_init({self.new_prefix}_t* parser, const {self.new_prefix}_settings_t* settings)"
+            )
+        writer.endline()
+        return writer.code

llparse/debug.py

@@ -0,0 +1,23 @@
+from .pybuilder import Node
+
+
+class Debugger:
+    @staticmethod
+    def getAllNodes(root: Node):
+        nodes: set[Node] = set()
+        queue: list[Node] = [root]
+
+        while queue:
+            node = queue.pop()
+            print(node.name)
+            if node.name == "nmethods":
+                print(node.getEdges())
+            if edges := node.getEdges():
+                for slot in edges:
+                    if slot.node in nodes:
+                        continue
+
+                    nodes.add(slot.node)
+                    queue.append(slot.node)
+
+        return list(nodes)

llparse/dot.py

@@ -0,0 +1,213 @@
+from pathlib import Path
+from typing import Optional, Union
+
+from .pybuilder.main_code import Edge, Node
+
+# TODO: Fix all graphs and more It's currently broken...
+
+COLOR_ADVANCE = "black"
+COLOR_NO_ADVANCE = "blue"
+COLOR_INVOKE = "green"
+COLOR_OTHERWISE = "red"
+
+
+class Dot:
+    """Used to create a graphviz of your parser"""
+
+    def __init__(self) -> None:
+        self.idCache: dict[Node, str] = {}
+        self.ns: set[str] = set()
+
+    def dump_to_file(self, filename: Union[str, Path], root: Node):
+        open(filename, "w").write(self.build(root))
+
+    def build(self, root: Node):
+        res = ""
+        res += "digraph {\n"
+        res += '  concentrate="true"\n'
+
+        for node in self.enumerateNodes(root):
+            res += self.buildNode(node)
+
+        res += "}\n"
+        return res
+
+    def enumerateNodes(self, root: Node):
+        queue = [root]
+        seen: set[Node] = set()
+
+        while queue:
+            node = queue.pop()
+            if node in seen:
+                continue
+
+            seen.add(node)
+
+            for edge in node:
+                queue.append(edge.node)
+
+            otherwise = node.getOtherwiseEdge()
+            if otherwise:
+                queue.append(otherwise.node)
+
+        return seen
+
+    def buildNode(self, node: Node):
+        res: str = ""
+        edges = list(node)
+        otherwise = node.getOtherwiseEdge()
+        if otherwise:
+            edges.append(otherwise)
+
+        advance: dict[Node, list[Edge]] = {}
+        noAdvance: dict[Node, list[Edge]] = {}
+
+        for edge in edges:
+            targets = noAdvance if edge.noAdvance else advance
+
+            if targets.get(edge.node):
+                targets[edge.node].append(edge)
+            else:
+                targets[edge.node] = [edge]
+
+        res += self.buildEdgeMap(node, advance, "advance")
+        res += self.buildEdgeMap(node, noAdvance, "noAdvance")
+
+        return res
+
+    def buildEdgeMap(self, node: Node, Map: dict[Node, list[Edge]], kind: str):
+        res = ""
+        for target, edges in Map.items():
+            otherwise: list[Edge] = []
+            single: list[Edge] = []
+            sequence: list[Edge] = []
+            code: list[Edge] = []
+
+            for edge in edges:
+                if not edge.key:
+                    otherwise.append(edge)
+                elif isinstance(edge.key, int):
+                    code.append(edge)
+                elif len(edge.key) == 1:
+                    single.append(edge)
+                else:
+                    sequence.append(edge)
+            labels: list[str] = []
+            # print(target.name,otherwise,code,single,sequence)
+
+            # end:int node:Node start:int
+            ranges: list[dict[str, Union[int, Node]]] = []
+
+            firstKey: Optional[int] = None
+            lastKey: Optional[int] = None
+
+            for edge in single:
+                key = (
+                    edge.key[0]
+                    if isinstance(edge.key, (bytes, list))
+                    else (
+                        edge.key
+                        if not isinstance(edge.key, str)
+                        else edge.key.encode()[0]
+                    )
+                )
+
+                if lastKey and lastKey == key - 1:
+                    lastKey = key
+                    continue
+
+                if lastKey is not None:
+                    ranges.append({"start": firstKey, "end": lastKey, "node": target})
+
+                firstKey = key
+                lastKey = key
+
+            if lastKey:
+                assert firstKey
+                ranges.append({"start": firstKey, "end": lastKey, "node": target})
+
+            for _range in ranges:
+                labels.append(self.buildRangeLabel(node, _range))
+
+            for edge in sequence:
+                labels.append(self.buildEdgeLabel(node, edge))
+
+            for edge in code:
+                labels.append(self.buildInvokeLabel(node, edge))
+
+            for edge in otherwise:
+                labels.append(self.buildOtherwiseLabel(node, edge))
+
+            color = COLOR_NO_ADVANCE if kind == "noAdvance" else COLOR_ADVANCE
+
+            res += (
+                f'  "{self.id(node)}" -> "{self.id(target)}"'
+                f'[label="{"|".join(labels)}" color="{color}" decorate=true];\n'
+            )
+
+        return res
+
+    def buildRangeLabel(self, node: Node, _range: dict[str, Union[int, Node]]):
+        start = self.buildChar(_range["start"])
+        end = self.buildChar(_range["end"])
+        # return range.start === range.end ? start : `${start}:${end}`;
+        return start if _range["start"] == _range["end"] else f"{start}:{end}"
+
+    def buildEdgeLabel(self, node: Node, edge: Edge):
+        return f"{self.buildBuffer(edge.key)}"
+
+    def buildInvokeLabel(self, node: Node, edge: Edge):
+        return f"code={int(edge.key)}"
+
+    def buildOtherwiseLabel(self, node: Node, edge: Edge):
+        return "otherwise" if edge.noAdvance else "skipTo"
+
+    def buildChar(self, code: int):
+        if not isinstance(code, int):
+            code = ord(code)
+        if code == 0x0A:
+            return self.escape("'\\n'")
+        if code == 0x0D:
+            return self.escape("'\\r'")
+        if code == 0x09:
+            return self.escape("'\\t'")
+
+        if 0x20 <= code and code <= 0x7E:
+            return self.escape(chr(code))
+        # I Don't know how accurate this is but it was worth a shot
+        res = hex(code)
+        return res
+
+    def buildBuffer(self, buffer: bytes):
+        s = buffer.decode() if isinstance(buffer, bytes) else buffer
+        return (
+            "'"
+            + s.replace("\n", "\\n")
+            .replace("\t", "\\t")
+            .replace("\r", "\\r")
+            .replace("\\", "\\$1")
+            + "'"
+        )
+
+    def id(self, node: Node):
+        if self.idCache.get(node):
+            return self.idCache[node]
+
+        res = node.name
+        if res in self.ns:
+            for i in range(len(self.ns)):
+                if (res + "_%i" % i) in self.ns:
+                    break
+
+            res += "_%i" % i
+
+        self.ns.add(res)
+        res = self.escape(res)
+        self.idCache[node] = res
+        return res
+
+    def escape(self, value: str):
+        return "'" + value.replace("\\", "\\$1").replace('"', "\\$1") + "'"
+
+
+# TODO FIX ALL BUFFERS BACK TO STRINGS!

llparse/enumerator.py

@@ -0,0 +1,20 @@
+from .pyfront.front import IWrap
+from .pyfront.nodes import Node
+
+
+class Enumerator:
+    @staticmethod
+    def getAllNodes(root: IWrap[Node]):
+        nodes: set[IWrap[Node]] = set()
+        queue: list[IWrap[Node]] = [root]
+
+        while queue:
+            node = queue.pop()
+            for slot in node.ref.getSlots():
+                if slot.node in nodes:
+                    continue
+
+                nodes.add(slot.node)
+                queue.append(slot.node)
+
+        return list(nodes)