edison-parser 0.1__py3-none-any.whl

edison/__init__.py

@@ -0,0 +1,3 @@
+from .tokenizer import *  # ruff:ignore[undefined-local-with-import-star]
+from .envelope import *  # ruff:ignore[undefined-local-with-import-star]
+from .parser import *  # ruff:ignore[undefined-local-with-import-star]

edison/envelope.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import logging
+from typing import NamedTuple
+
+from .tokenizer import tokenize_segments_iter
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+    "EnvelopeFooters",
+    "EnvelopeHeaders",
+    "extract_envelope_footers",
+    "extract_envelope_headers",
+]
+
+
+class EnvelopeHeaders(NamedTuple):
+    unb: str | None = None
+    unh: str | None = None
+
+
+class EnvelopeFooters(NamedTuple):
+    unt: str | None = None
+    unz: str | None = None
+
+
+def extract_envelope_headers(edi: str) -> EnvelopeHeaders:
+    r"""Extract EDIFACT envelope headers from the beginning of a message.
+
+    This uses the iterative tokenizer from the start and stops as soon as the
+    message header boundary is found.
+
+    >>> extract_envelope_headers("UNB+1'UNH+2+ORDERS:D:96A:UN'BGM'")
+    EnvelopeHeaders(unb='UNB+1', unh='UNH+2+ORDERS:D:96A:UN')
+    """
+    unb = unh = None
+
+    for segment in tokenize_segments_iter(edi):
+        tag = segment[:3]
+        if tag == "UNB" and unb is None:
+            unb = segment
+        elif tag == "UNH":
+            unh = segment
+            break
+
+    return EnvelopeHeaders(unb, unh)
+
+
+def extract_envelope_footers(edi: str) -> EnvelopeFooters:
+    r"""Extract EDIFACT envelope footers from the end of a message.
+
+    This uses the iterative tokenizer from the end and stops as soon as the
+    message footer boundary is found.
+
+    >>> extract_envelope_footers("BGM'UNT+3'UNZ+4'")
+    EnvelopeFooters(unt='UNT+3', unz='UNZ+4')
+    """
+    unt = unz = None
+
+    for segment in tokenize_segments_iter(edi, reverse=True):
+        tag = segment[:3]
+        if tag == "UNZ" and unz is None:
+            unz = segment
+        elif tag == "UNT":
+            unt = segment
+            break
+
+    return EnvelopeFooters(unt, unz)

edison/parser.py

@@ -0,0 +1,471 @@
+#!/usr/bin/env python
+
+import logging
+from bisect import bisect_right
+from operator import itemgetter
+from collections.abc import Iterator, Callable, Sequence
+from typing import Any, NamedTuple, Literal, TypedDict, TYPE_CHECKING
+from typing_extensions import TypeIs
+import platform
+
+from .tokenizer import tokenize_segments, tokenize_segments_with_blb
+
+PYPY = platform.python_implementation() == "PyPy"
+
+log = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    import pydot
+
+
+__all__ = ["BaseParser"]
+
+
+type VersionTuple = tuple[int | None, int | None]
+type SchemaVersion = tuple[int, int]
+
+type EDITag = str
+type EDIPath = tuple[EDITag, ...]
+type EDIAction = str | None  # method to call when this segment is found
+type EDIRepetition = int  # number of times you can pass on a node
+type EDIFlag = Literal["M", "C"]  # mandatory or conditional
+
+# NodeId is the index of a node in the schema, starting at 0 for the root node.
+type NodeId = int
+
+# RawEDISegment is the input schema, which is a recursive lists of lists.
+# EDISegment is the normalized view used while building transitions.
+type RawEDISegment = list[Any]
+
+
+class EDISegment(NamedTuple):
+    id_: NodeId
+    tag: EDITag
+    children_raw: Sequence[RawEDISegment] = ()
+    action: EDIAction = None
+    repetition: EDIRepetition = 99999
+    flag: EDIFlag = "C"
+
+
+class EDISegmentFactory:
+    def __init__(self, start_id: NodeId) -> None:
+        self._next_id = start_id
+
+    def create_from_raw(self, raw_segment: RawEDISegment) -> EDISegment:
+        node = EDISegment(self._next_id, *raw_segment)
+        if not isinstance(node.children_raw, (list, tuple)):
+            raise ValueError(f"{raw_segment!r} should have a list in second position")
+        self._next_id += 1
+        return node
+
+
+class Transition(NamedTuple):
+    id_: NodeId
+    path: EDIPath
+    action: str | None
+
+
+type NodeTransitions = dict[EDITag, Transition]
+type NodeTransitionsList = list[tuple[EDITag, Transition]]
+type SegmentTokenizer = Callable[[str], Iterator[str]]
+
+
+class CompiledSchema(NamedTuple):
+    version: SchemaVersion
+    tokenize: SegmentTokenizer
+    transitions: dict[NodeId, NodeTransitions]
+
+
+def format_path(path: EDIPath, sep: str = "/") -> str:
+    """Format a path in transition object.
+
+    >>> format_path(())
+    '/'
+    >>> format_path(("PTK",))
+    '/PTK'
+    >>> format_path(("PTK", "COR"))
+    '/PTK/COR'
+    """
+    return f"{sep}{sep.join(path)}"
+
+
+def build_transitions(schema: list[RawEDISegment]) -> dict[NodeId, NodeTransitions]:
+    transitions: dict[NodeId, NodeTransitions] = {}
+    transitions_cache: dict[NodeId, Transition] = {}
+
+    def get_transition(parent_path: EDIPath, node: EDISegment) -> Transition:
+        # We cache the transition objects to avoid creating multiple instances for
+        # the same node. Indeed the transitions object we return is a mapping of
+        # tag -> Transition(node_id, path, action), so this object can be reused
+        # for all incoming transitions to the same node.
+        # We also delay the creation of the path until we actually need it.
+        try:
+            return transitions_cache[node.id_]
+        except KeyError:
+            transition = Transition(node.id_, (*parent_path, node.tag), node.action)
+            transitions_cache[node.id_] = transition
+            return transition
+
+    factory = EDISegmentFactory(start_id=0)
+    root = factory.create_from_raw(["", schema])
+    stack: list[tuple[EDISegment, EDIPath, NodeTransitionsList]] = [(root, (), [])]
+
+    while stack:
+        node, node_path, parent_transitions = stack.pop()
+        child_nodes = [factory.create_from_raw(raw) for raw in node.children_raw]
+        transitions[node.id_] = node_trans = {}
+
+        # We store the index of the next mandatory segment for each child.
+        next_mandatory_indexes: list[int | None] = [None] * len(child_nodes)
+        next_mandatory_index = None
+
+        for i in range(len(child_nodes) - 1, 0, -1):
+            child = child_nodes[i]
+            if child.flag == "M":
+                next_mandatory_index = i
+            next_mandatory_indexes[i - 1] = next_mandatory_index
+
+        # We store the index of the first mandatory child.
+        for i, child in enumerate(child_nodes):
+            if child.flag == "M":
+                first_mandatory_index = i
+                break
+        else:  # no break
+            first_mandatory_index = None
+
+        # Now we iterate over the children to push their children in the stack
+        for i, child in enumerate(child_nodes):
+            child_path = (*node_path, child.tag)
+
+            # In cases where you have several segments with the same name in a group
+            # we only keep the transition to the first in the parent
+            if child.tag not in node_trans:
+                # We do not transition after the first mandatory child.
+                if first_mandatory_index is None or i <= first_mandatory_index:
+                    node_trans[child.tag] = get_transition(node_path, child)
+
+            # repetition is the number of times you can pass on a node, today we only
+            # consider 1 or more-than-one options, to put a transition to self or not
+            if child.repetition > 1:
+                i_start = i
+            else:
+                i_start = i + 1
+
+            # Sibling transitions stop at the first mandatory sibling.
+            next_mandatory_index = next_mandatory_indexes[i]
+
+            if next_mandatory_index is None:
+                i_end = None
+            else:
+                i_end = next_mandatory_index + 1
+
+            sibling_transitions = [
+                (sibling.tag, get_transition(node_path, sibling))
+                for sibling in child_nodes[i_start:i_end]
+            ]
+
+            # Siblings are before parents because local transitions inside group
+            # have priority over parents
+            # Also note that siblings may contain the child if repetition > 1
+            # in that case child transitions have priority over self if same tag name
+            if i_end is None:
+                child_parent_transitions = sibling_transitions + parent_transitions
+            else:
+                child_parent_transitions = sibling_transitions
+
+            stack.append((child, child_path, child_parent_transitions))
+
+        # We do not add links to parents if there is a mandatory child.
+        if first_mandatory_index is None:
+            for parent_tag, transition in parent_transitions:
+                if parent_tag not in node_trans:
+                    node_trans[parent_tag] = transition
+
+    return transitions
+
+
+def _has_blb_segment(transitions: dict[NodeId, NodeTransitions]) -> bool:
+    """Scan transitions to find out if a BLB segment is there."""
+    return any("BLB" in node_trans for node_trans in transitions.values())
+
+
+# PYDOT GLOBAL OPTIONS
+#
+class NodeOptions(TypedDict):
+    fontname: str
+    shape: str
+    style: str
+
+
+class EdgeOptions(TypedDict):
+    fontname: str
+
+
+N_OPTIONS: NodeOptions = {
+    "fontname": "Monospace",
+    "shape": "box",
+    "style": "filled",
+}
+E_OPTIONS: EdgeOptions = {
+    "fontname": "Monospace",
+}
+
+
+# SHOW MISC.
+#
+COLORIZE = {
+    "bold": "\033[1m{0}\033[0m",
+    "grey": "\033[01;90m{0}\033[0m",
+    "white": "\033[01;97m{0}\033[0m",
+    "red": "\033[01;31m{0}\033[0m",
+    "green": "\033[01;32m{0}\033[0m",
+    "yellow": "\033[01;33m{0}\033[0m",
+    "blue": "\033[01;34m{0}\033[0m",
+    "magenta": "\033[01;35m{0}\033[0m",
+    "cyan": "\033[01;36m{0}\033[0m",
+}
+
+STYLE = {
+    "bold": "\033[1m{0}\033[0m",
+    "dim": "\033[2m{0}\033[0m",
+    "italic": "\033[3m{0}\033[0m",
+    "underline": "\033[4m{0}\033[0m",
+    "blink": "\033[5m{0}\033[0m",
+    "reverse": "\033[7m{0}\033[0m",
+    "hidden": "\033[8m{0}\033[0m",
+    "strikethrough": "\033[9m{0}\033[0m",
+}
+
+blink = STYLE["blink"].format
+
+
+def is_full_version(version: VersionTuple | None) -> TypeIs[tuple[int, int]]:
+    if version is None:
+        return False
+    if version[0] is None or version[1] is None:
+        log.warning("Message version %r contains None", version)
+        return False
+    return True
+
+
+class BaseParser:
+    """This is a generic parser that should be subclassed wih a schemas."""
+
+    __slots__ = []
+
+    # Those can/should be overriden when subclassing
+    #
+    header: list[RawEDISegment] = []
+    footer: list[RawEDISegment] = []
+    schemas: dict[SchemaVersion, list[RawEDISegment]] = {}
+
+    # Assigned in compile() during class definition
+    #
+    _compiled_schemas: list[CompiledSchema] = []
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        cls.compile()
+
+    @classmethod
+    def compile(cls) -> None:
+        """This is automatically called when a BaseParser subclass is defined."""
+        cls._compiled_schemas = []
+        for version in sorted(cls.schemas):
+            schema = [*cls.header, *cls.schemas[version], *cls.footer]
+            transitions = build_transitions(schema)
+            tokenize: SegmentTokenizer
+            if _has_blb_segment(transitions):
+                tokenize = tokenize_segments_with_blb
+            else:
+                tokenize = tokenize_segments
+            cls._compiled_schemas.append(CompiledSchema(version, tokenize, transitions))
+
+    def set_up(self) -> None:
+        """To be subclassed to trigger something at the beginning of parsing."""
+
+    def tear_down(self) -> None:
+        """To be subclassed to trigger something at the end of parsing."""
+
+    @classmethod
+    def find_transitions(
+        cls,
+        version: VersionTuple | None = None,
+    ) -> CompiledSchema:
+        # Each (min_version, tokenize, transitions) in cls._compiled_schemas is sorted
+        # by min_version. We want to find the largest min_version that is less than
+        # or equal to the given version. We can use bisect_right to find the insertion
+        # point for the given version in the sorted list of min_versions.
+        # We have a fast path before doing the bisect because in practice most input
+        # version are higher than the latest defined schema version.
+        if not cls._compiled_schemas:
+            raise ValueError("No transitions found, did you forget to set a schema?")
+
+        latest = cls._compiled_schemas[-1]
+        if not is_full_version(version):
+            return latest
+        if version >= latest.version:  # fast path, common
+            return latest
+
+        index = bisect_right(cls._compiled_schemas, version, key=itemgetter(0))
+        if index > 0:
+            return cls._compiled_schemas[index - 1]
+
+        raise ValueError(
+            f"Schemas do not cover all versions for {cls.__name__}:"
+            f" {[t.version for t in cls._compiled_schemas]}"
+        )
+
+    def parse(self, edi: str, version: VersionTuple | None = None) -> None:
+        _, tokenize, transitions = self.find_transitions(version)
+        id_ = 0  # starting at root
+        self.set_up()
+        for segment in tokenize(edi):
+            tag = segment[:3]
+            if (tr := transitions[id_].get(tag)) is not None:
+                id_, _, action = tr
+                if action is not None:
+                    getattr(self, action)(segment)
+        self.tear_down()
+
+    @classmethod
+    def validate(cls, edi: str, version: VersionTuple | None = None) -> bool:
+        _, tokenize, transitions = cls.find_transitions(version)
+        id_: NodeId = 0  # starting at root
+        path: EDIPath = ()
+        for segment in tokenize(edi):
+            tag = segment[:3]
+            if (tr := transitions[id_].get(tag)) is not None:
+                id_, path, _ = tr
+                continue
+            # If we reach this point, it means the segment is not valid.
+            if not transitions[id_]:
+                log.warning(
+                    "[%s] Cannot continue from %s, but still got %r",
+                    cls.__name__,
+                    format_path(path),
+                    tag,
+                )
+            else:  # only other case is 'tag is not in transitions[id_]'
+                log.warning(
+                    "[%s] Cannot continue from %s with %r, should be one of:\n%s",
+                    cls.__name__,
+                    format_path(path),
+                    tag,
+                    "\n".join(
+                        f"{k!r} -> {format_path(v.path)}"
+                        for k, v in transitions[id_].items()
+                    ),
+                )
+            return False
+        return True
+
+    # Color related stuff
+    #
+    _depth_colors = [
+        "bold",
+        "blue",
+        "green",
+        "yellow",
+        "magenta",
+        "blue",
+        "green",
+        "yellow",
+        "magenta",
+    ]
+    _hidden: set[str] = set()
+
+    @classmethod
+    def show(
+        cls,
+        edi: str,
+        version: VersionTuple | None = None,
+        formatting: bool = True,
+    ) -> str:
+        _, tokenize, transitions = cls.find_transitions(version)
+        res = []
+        id_, depth = 0, 0  # starting at root
+
+        for segment in tokenize(edi):
+            tag = segment[:3]
+            segment = f"{segment}'"
+
+            if (tr := transitions[id_].get(tag)) is not None:
+                id_, path, _ = tr
+                depth = len(path)
+                known = True
+            else:
+                known = False
+
+            if not formatting:
+                color = None
+            elif not known:
+                color = "red"
+            elif tag in cls._hidden:
+                color = "grey"
+            else:
+                color = cls._depth_colors[(depth - 1) % len(cls._depth_colors)]
+
+            if not known:
+                filler = "-" * 2 * (depth - 1)
+            else:
+                filler = " " * 2 * (depth - 1)
+
+            if color is not None:
+                # "blink" actually adds an almost-invisible property to the filler,
+                # so we can filter on it in CSS and set the user-select to none.
+                filler = COLORIZE[color].format(blink(filler))
+                segment = COLORIZE[color].format(segment)
+
+            res.append(filler + segment)
+
+        return "\n".join(res)
+
+    @classmethod
+    def build_dot_object(
+        cls,
+        version: VersionTuple | None = None,
+        *,
+        include_up_links: bool = True,
+    ) -> "pydot.Dot":
+        import pydot
+
+        _, _, transitions = cls.find_transitions(version)
+        colors = cls._depth_colors
+        dot = pydot.Dot(graph_type="digraph")
+
+        nodes = {id_: pydot.Node(str(id_), **N_OPTIONS) for id_ in transitions}
+
+        for source_id in transitions:
+            for tag in transitions[source_id]:
+                target_id, path, _ = transitions[source_id][tag]
+                fillcolor = colors[(len(path) - 1) % len(colors)]
+                node = nodes[target_id]
+                node.set("label", "\\n".join(path))
+                node.set("fillcolor", fillcolor)
+                node.set(
+                    "fontcolor",
+                    "white" if fillcolor in {"bold", "blue"} else "black",
+                )
+
+        for source_id in transitions:
+            source_label = nodes[source_id].get("label")
+            if source_label in cls._hidden:
+                continue
+            dot.add_node(nodes[source_id])
+            for tag in transitions[source_id]:
+                target_id, path, _ = transitions[source_id][tag]
+                target_label = nodes[target_id].get("label")
+                if target_label in cls._hidden:
+                    continue
+                if (
+                    not include_up_links
+                    and source_label
+                    and len(source_label) > len(target_label)
+                ):
+                    continue
+                dot.add_edge(
+                    pydot.Edge(nodes[source_id], nodes[target_id], **E_OPTIONS)
+                )
+
+        return dot

edison/tokenizer.py

@@ -0,0 +1,373 @@
+#!/usr/bin/env python
+
+import re
+import logging
+from collections.abc import Iterator
+import platform
+
+PYPY = platform.python_implementation() == "PyPy"
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+    "translate_ebcdic_separators",
+    "normalize_ebcdic_separators",
+    "escape",
+    "unescape",
+    "tokenize_segments",
+    "tokenize_segments_iter",
+    "tokenize_segments_with_blb",
+]
+
+
+# EBCDIC/ESCAPING CORNER
+#
+# Note that the backslash itself is not escaped, as in practice
+# this character never appears in messages. If it ever does, it
+# will only be an issue if it is just before an EBCDIC symbol, in
+# that case the symbol will look like it is escaped, and this can
+# mess up the parsing. This is most likely never going to happen.
+
+ASCII_SYMBOLS = "+:*'"
+EBCDIC_SYMBOLS = "\x1d\x1f\x19\x1c"
+
+TABLE_FORMAT = str.maketrans(EBCDIC_SYMBOLS, ASCII_SYMBOLS)
+TABLE_ESC = [(a, f"\\{a}") for a in ASCII_SYMBOLS]
+
+_RE_FIND_ESCAPED = re.compile(rb"\\[+:*']")
+_RE_SUB_ESCAPED = re.compile(r"\\([+:*'])")
+
+# This will be used on a memoryview, where find() is not possible
+_RE_QUOTE_B = re.compile(rb"'")
+
+
+def translate_ebcdic_separators(edi: str) -> str:
+    r"""Converts EBCDIC symbols to ASCII, with no escaping.
+
+    >>> translate_ebcdic_separators("ORG\x1D00\x1C")
+    "ORG+00'"
+    """
+    return edi.translate(TABLE_FORMAT)
+
+
+def normalize_ebcdic_separators(edi: str, scan_limit: int = 100) -> str:
+    r"""Converts EBCDIC symbols to ASCII, with escaping if necessary.
+
+    We only look for EBCDIC symbols in the first hundred characters
+    for performance reasons, as on very large strings it adds up.
+
+    If already converted, this function is no-op.
+
+    >>> normalize_ebcdic_separators("UNB\x1DSENDER\x1FREC")
+    'UNB+SENDER:REC'
+    >>> normalize_ebcdic_separators("UNB\x1DSENDER\x1FREC+\x1C")  # escaping
+    "UNB+SENDER:REC\\+'"
+    >>> normalize_ebcdic_separators(r"UNB+SENDER:REC\+'")  # no-op
+    "UNB+SENDER:REC\\+'"
+    """
+    if edi.find("\x1d", 0, scan_limit) == -1:  # not found
+        return edi
+    return translate_ebcdic_separators(escape(edi))
+
+
+def escape(data: str) -> str:
+    r"""Escaping ASCII symbols that are used as EBCDIC control characters.
+
+    This function is typically used right before generating EDIFACT or XML,
+    to make sure there is a difference between control characters and true
+    content that happens to be +:*'.
+
+    Escaping is basically a re.sub(r"([+:*'])", r"\\\1", data), but it is
+    actually faster to do repeated replace().
+
+    >>> escape("REC+")
+    'REC\\+'
+    """
+    for c, ec in TABLE_ESC:
+        data = data.replace(c, ec)
+    return data
+
+
+def unescape(
+    data: str,
+    _re_escaped: re.Pattern[str] = _RE_SUB_ESCAPED,
+) -> str:
+    r"""Unescaping is the opposite of escaping: removing backslash where relevant.
+
+    This function is typically used right after parsing the EDIFACT content,
+    where we want to get rid of the escaping backslashes that were useful
+    when tokenizing the segments.
+
+    Unescaping is basically a re.sub(r"\\([+:*'])", r"\1", data), but on PyPy
+    it is faster to do repeated replace().
+    And if Unicode characters are involved, then PyPy has a bug (#4028) where
+    the performance is completely broken using regular expressions.
+
+    >>> unescape(r"REC\+")
+    'REC+'
+    """
+    if not PYPY:
+        return _re_escaped.sub(r"\1", data)
+
+    for c, ec in TABLE_ESC:
+        data = data.replace(ec, c)
+    return data
+
+
+# SEGMENTS TOKENIZING
+#
+_EOL = "& \t\r\n"
+_EOL_ORD = frozenset(_EOL.encode())
+
+
+def tokenize_segments(edi: str) -> Iterator[str]:
+    r"""Tokenizer of EDIFACT segments.
+
+    The only trick here is to split on quotes, but not if they were escaped
+    with a backslash. For this we first replace escaped quotes but a special
+    character, then we split normally on quotes.
+
+    This is basically a re.split(r'(?<!\\)"', edi), but it is actually faster.
+
+    >>> list(tokenize_segments(r"ERC'IFT+1+USER\'DATA'"))
+    ['ERC', "IFT+1+USER\\'DATA"]
+    >>> list(tokenize_segments("SDI'SPI+'UNZ'TRUNC"))
+    ['SDI', 'SPI+', 'UNZ']
+    """
+    edi = normalize_ebcdic_separators(edi)
+
+    segments = edi.replace(r"\'", "\x1a").split("'")
+    segments.pop()  # remove the last empty segment after the last quote
+    for segment in segments:
+        yield segment.replace("\x1a", r"\'").lstrip(_EOL)
+
+
+def tokenize_segments_iter(
+    edi: str,
+    reverse: bool = False,
+) -> Iterator[str]:
+    r"""Tokenizer of EDIFACT segments, the iterator version.
+
+    This is the iterator version of the normal tokenizer, in the sense that it
+    lazily yields segments without processing the whole string. It is used to
+    parse EDIFACT headers, where you only care about the first segments.
+    This is not the default tokenizer as it is 2-3x slower than the normal version,
+    if you use it to tokenize the full message. But it is 2-15x faster to use this
+    version if you just want the first segments, depending on message size.
+
+    Note that we exclude manually the escaped quotes, we could do it using a
+    lookbehind clause with a regular expression, but it is a bit slower.
+
+    We support a reverse mode to tokenize from the end of the EDI, this is useful
+    when testing for truncation.
+
+    >>> # Same tests as tokenize_segments
+    >>> list(tokenize_segments_iter(r"ERC'IFT+1+USER\'DATA'"))
+    ['ERC', "IFT+1+USER\\'DATA"]
+    >>> list(tokenize_segments_iter("SDI'SPI+'UNZ'TRUNC"))
+    ['SDI', 'SPI+', 'UNZ']
+    >>>
+    >>> # The reverse mode that starts from the end
+    >>> list(tokenize_segments_iter(r"ERC'IFT+1+USER\'DATA'", reverse=True))
+    ["IFT+1+USER\\'DATA", 'ERC']
+    >>> list(tokenize_segments_iter("SDI'SPI+'UNZ'TRUNC", reverse=True))
+    ['UNZ', 'SPI+', 'SDI']
+    """
+    edi = normalize_ebcdic_separators(edi)
+
+    if not reverse:
+        # * find_start_index is the index for the beginning of the find()
+        # * start_index is the index for the beginning of the segment
+        find_start_index = start_index = 0
+        while True:
+            index = edi.find("'", find_start_index)
+            if index == -1:  # between the last quote and the end
+                return
+            if index == 0 or edi[index - 1] != "\\":
+                yield edi[start_index:index].lstrip(_EOL)
+                start_index = index + 1
+            find_start_index = index + 1
+    else:
+        # * rfind_end_index is the index for the end of the rfind()
+        # * end_index is the index for the end of the segment
+        # We use those end indices "plus one" as we use them for slicing.
+        rfind_end_index_plus_one = end_index_plus_one = edi_len = len(edi)
+        while True:
+            index = edi.rfind("'", 0, rfind_end_index_plus_one)
+            if index == -1:
+                if end_index_plus_one < edi_len:  # before last quote
+                    yield edi[:end_index_plus_one].lstrip(_EOL)
+                return
+            if index == 0 or edi[index - 1] != "\\":
+                if end_index_plus_one < edi_len:  # before last quote
+                    yield edi[index + 1 : end_index_plus_one].lstrip(_EOL)
+                end_index_plus_one = index
+            rfind_end_index_plus_one = index
+
+
+def tokenize_segments_with_blb(
+    edi: str,
+    _re_quote: re.Pattern[bytes] = _RE_QUOTE_B,
+) -> Iterator[str]:
+    r"""Tokenizer of EDIFACT segments, handling BLB segments as well.
+
+    Note that this tokenizer could be used everywhere instead of tokenize_segments,
+    but it 10x slower. This version understands that BLB segments may contain
+    quotes, so we cannot just tokenize on them. When finding a BLB segment, it will
+    read the number of bytes specified, and basically "jump over" it.
+
+    >>> list(tokenize_segments_with_blb("BLB+8++EQN'PTK''"))
+    ["BLB+8++EQN'PTK'"]
+    >>> list(tokenize_segments_with_blb("BLB+8++EQN'PTK''UNT'"))
+    ["BLB+8++EQN'PTK'", 'UNT']
+    >>> list(tokenize_segments_with_blb("BLB'AAA'"))
+    ['BLB', 'AAA']
+    >>>
+    >>> # Even though the BLB count is 0, the BLB extends until a quote is found
+    >>> list(tokenize_segments_with_blb("UNH'BLB+0++<xml>A</xml>'UNT'"))
+    ['UNH', 'BLB+0++<xml>A</xml>', 'UNT']
+    >>>
+    >>> # Quotes are jumped over, because of the BLB count
+    >>> list(tokenize_segments_with_blb("UNH'BLB+26++<xml>Quotes''inside'</xml>'UNT'"))
+    ['UNH', "BLB+26++<xml>Quotes''inside'</xml>", 'UNT']
+    >>>
+    >>> # Escaped characters (not normal backslashes) are excluded from the BLB count
+    >>> for s in tokenize_segments_with_blb(r"UNH'BLB+2++1\''BLB+2++\\:'UNT'"):
+    ...     print(s)
+    UNH
+    BLB+2++1\'
+    BLB+2++\\:
+    UNT
+    >>>
+    >>> # Unicode characters are counted as their bytes length
+    >>> list(tokenize_segments_with_blb("UNH'BLB+11++TRADEMARK®'UNT'"))
+    ['UNH', 'BLB+11++TRADEMARK®', 'UNT']
+    """
+    edi = normalize_ebcdic_separators(edi)
+
+    # The BLB length is given in bytes, so to perform the BLB extraction properly,
+    # we need to convert the EDI to bytes(). In this context, every character is
+    # actually an int, so comparisons are done against their ASCII numbers via ord().
+    edi_as_bytes = edi.encode()
+
+    # The additional memoryview() is to avoid copying the data when slicing, before
+    # converting to str(). It is a bit faster and reduces memory consumption for large
+    # messages, but then we cannot use view.find(b"'"), and rely on a regex instead.
+    view = memoryview(edi_as_bytes)
+
+    index, edi_len = 0, len(view)
+
+    while index < edi_len:
+        while index < edi_len and view[index] in _EOL_ORD:  # skip over EOL, if any
+            index += 1
+
+        if view[index : index + 3].tobytes() == b"BLB":  # BLB+270++BLOB
+            minimum_end_index = _get_blb_end_index(view, index, edi_len)
+        else:
+            minimum_end_index = index + 3
+
+        # We now look for a true quote, one that is not coming after a backslash:
+        # * if we find one, we yield the segment and move the index further
+        # * if we never find one, it means the segment is truncated and we exit
+        for match in _re_quote.finditer(view, pos=minimum_end_index):
+            quote_index = match.start()
+            if quote_index == 0 or view[quote_index - 1] != 92:  # ord("\\")
+                yield str(view[index:quote_index], "utf8")
+                index = quote_index + 1
+                break
+        else:
+            break
+
+    # We do not return the last truncated segment unless it is a BLB, in that case
+    # the parser of the BLB should exclude its own last truncated segment
+    if view[index : index + 3].tobytes() == b"BLB":
+        yield str(view[index:], "utf8")
+
+
+def _get_blb_end_index(
+    view: memoryview,
+    index: int,
+    edi_len: int,
+    _re_escaped: re.Pattern[bytes] = _RE_FIND_ESCAPED,
+) -> int:
+    r"""Compute the minimum possible index of the end of BLB.
+
+    From a beginning "index", this function returns an integer representing the
+    index of the end of BLB. We read the BLB length and compute that index.
+    Note that sometimes the BLB will extend further, so we need to find the next
+    true quote in the caller.
+
+    >>> _get_blb_end_index(memoryview(b"BLB+3++ABC'UNT'"), 0, 15)
+    10
+    >>> _get_blb_end_index(memoryview(b"BLB+3++AB\\+'UNT'"), 0, 16)
+    11
+    >>> _get_blb_end_index(memoryview(b"BLB'"), 0, 4)
+    3
+    >>> _get_blb_end_index(memoryview(b"BLB+30"), 0, 6)
+    6
+    >>> _get_blb_end_index(memoryview(b"BLB+30++AAA"), 0, 11)
+    38
+    >>> _get_blb_end_index(memoryview(b"BLB+COMPUTED++AAA'"), 0, 18)
+    16
+    >>> _get_blb_end_index(memoryview(b"BLB+ERROR++AAA"), 0, 14)
+    11
+    """
+    # We begin by jumping over the segment name
+    index += 3
+
+    # We move the index forward, until the beginning of the actual BLB content
+    field_number = 0
+    while field_number != 3 and index < edi_len:
+        if view[index] == 39:  # ord("'")
+            return index
+        if view[index] == 43:  # ord("+")
+            field_number += 1
+            if field_number == 1:
+                blb_len_start = index + 1
+            elif field_number == 2:
+                blb_len_end = index - 1
+        index += 1
+
+    # This means the message was probably truncated, because we reached the
+    # end of message, despite only trying to move until the beginning of the
+    # actual BLB content.
+    if index == edi_len:
+        return index
+
+    blb_len_view = view[blb_len_start : blb_len_end + 1]
+
+    # This can happen as some test messages have a 'COMPUTED' in place of the
+    # length. In that case we return the end of message, as it seems this works
+    # best in most cases, meaning the BLB is effectively "running until the end
+    # of message".
+    # Note that in the general case this is a bad idea: some messages have
+    # multiple BLB segments, or other segments after the first BLB.
+    # We could also return the current index, and leave it to the caller to
+    # extend it until the next true quote, but this is going to cause issues
+    # if the BLB itself contains quotes.
+    if blb_len_view.tobytes() == b"COMPUTED":
+        log.info("BLB length was set to 'COMPUTED', assuming the rest is BLB")
+        return edi_len - 2  # penultimate character (last would be the quote)
+
+    try:
+        blb_len = int(blb_len_view)
+    except ValueError:
+        # In that case we return the current index, as we do not want to
+        # "go too far" in estimating the BLB end index. The caller will then
+        # extend it until the next true quote. Obviously, this is going to
+        # cause issues if the BLB itself contains quotes ¯\_(ツ)_/¯
+        log.error("Could not parse BLB length from %r", blb_len_view.tobytes())
+        return index
+
+    blb_end_index = index + blb_len
+
+    # We have to scan the BLB for potential escaped delimiters:
+    # if we find any, we have to increment the BLB end index
+    for match in _re_escaped.finditer(view, pos=index):
+        # match spans 2 characters and we want match.end() before/at blb_end_index
+        if match.start() < blb_end_index:
+            blb_end_index += 1
+        else:
+            break
+
+    return blb_end_index

edison_parser-0.1.dist-info/METADATA

@@ -0,0 +1,426 @@
+Metadata-Version: 2.4
+Name: edison-parser
+Version: 0.1
+Summary: An EDIFACT parser with a real syntax tree.
+Author-email: Alex Prengère <alex.prengere@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/AmadeusITGroup/edison
+Project-URL: Changelog, https://github.com/AmadeusITGroup/edison/blob/main/CHANGELOG.md
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydot>=3.0.0
+Requires-Dist: typing-extensions>=4.10
+Dynamic: license-file
+
+<div align="center">
+    <img src="docs/edison.png" width="360" height="360" alt="Edison">
+    <h1>Edison</h1>
+    <p><strong>An EDIFACT parser with a real syntax tree.</strong></p>
+</div>
+
+Edison is a Python library for parsing EDIFACT messages. Instead of treating a
+message as a flat stream of segments, it models valid messages as a tree of
+transitions. You define the schema once, then parse, validate, or pretty-print
+messages against that structure.
+
+## Highlights
+
+- Define EDIFACT syntax trees directly in Python.
+- Attach parsing callbacks to the segments you care about.
+- Select schemas by message version.
+- Model conditional, mandatory, and repeatable segments.
+- Handle escaped EDIFACT separators and EBCDIC control characters.
+- Properly tokenize segments with specified lengths like `BLB`
+- Export transition graphs with `pydot`.
+
+## Installation
+
+Edison requires Python 3.12 or newer.
+
+```bash
+pip install edison-parser
+```
+
+## Quick start
+
+Subclass `BaseParser`, define one or more schemas, and attach callback method
+names to the schema nodes you want to process.
+
+```python
+from edison import BaseParser
+
+
+class PIZZAQParser(BaseParser):
+    schemas = {
+        (0, 0): [
+            ["UNH"],
+            ["CVR"],
+            ["REF"],
+            [
+                "PIZ",
+                [
+                    ["TYP", [], "parse_typ"],
+                    ["REF", [], "parse_ref"],
+                ],
+            ],
+            ["UNT"],
+        ],
+    }
+
+    def __init__(self):
+        self.summary = {}
+        self.pizza = None
+
+    def parse_typ(self, segment: str):
+        self.pizza = segment[4:]
+
+    def parse_ref(self, segment: str):
+        refs = segment[4:].split("*")
+        self.summary[self.pizza] = len(refs)
+
+
+edi = """
+UNH+1+PIZZAQ:1:1:1A+00012480825035'
+CVR+USD'
+REF+LUIS'
+PIZ+1'
+TYP+MARGARITA'
+REF+1*2'
+PIZ+2'
+TYP+PESTO'
+REF+3'
+UNT+19+1'
+"""
+
+parser = PIZZAQParser()
+parser.parse(edi)
+
+assert parser.summary == {"MARGARITA": 2, "PESTO": 1}
+assert PIZZAQParser.validate(edi)
+print(PIZZAQParser.show(edi))
+```
+
+<div align="center">
+    <img src="docs/Image.png" width="360" height="360" alt="Edison">
+    <p><strong>Tree-based structure.</strong></p>
+</div>
+
+## Schema format
+
+A schema is a dictionary keyed by the minimum supported version:
+
+```python
+schemas = {
+    (0, 0): [
+        ["UNH"],
+        ["BGM"],
+        ["UNT"],
+    ],
+}
+```
+
+Each segment node accepts these positional fields:
+
+```python
+["TAG", children, action, repetition, flag]
+```
+
+| Field | Default | Meaning |
+| --- | --- | --- |
+| `TAG` | required | Three-character EDIFACT segment tag. |
+| `children` | `[]` | Nested segment group. |
+| `action` | `None` | Parser method name called with the raw segment. |
+| `repetition` | `99999` | `1` means single-use; any higher value means repeatable. |
+| `flag` | `"C"` | `"C"` for conditional, `"M"` for mandatory. |
+
+## Main API
+
+| API | Use |
+| --- | --- |
+| `parser.parse(edi, version=None)` | Walk an EDI message and call matching segment actions. |
+| `Parser.validate(edi, version=None)` | Return `True` when all segments follow the schema transitions. |
+| `Parser.show(edi, version=None, colorize=True)` | Return a formatted, indented representation of an EDI message. |
+| `Parser.build_dot_object(version=None, include_up_links=True)` | Build a `pydot.Dot` transition graph. |
+| `extract_envelope_headers(edi)` | Quickly return the `UNB` and `UNH` envelope segments. |
+| `extract_envelope_footers(edi)` | Quickly return the `UNT` and `UNZ` envelope segments. |
+| `escape(data)` / `unescape(data)` | Escape or unescape EDIFACT separators: `+`, `:`, `*`, `'`. |
+| `normalize_ebcdic_separators(edi)` | Convert EBCDIC EDIFACT separators to ASCII safely. |
+
+## Advanced usage
+
+<details>
+<summary>Versioned syntax trees</summary>
+
+Define multiple syntax trees when the same message evolves across versions. The
+parser selects the highest schema version less than or equal to the `version`
+argument, while parsing callbacks can stay the same.
+
+```python
+from edison import BaseParser
+
+
+class RouteParser(BaseParser):
+    schemas = {
+        (0, 0): [
+            ["UNH"],
+            ["ORG", [], "parse_origin"],
+            ["DST", [], "parse_destination"],
+            ["UNT"],
+        ],
+        (2, 0): [
+            ["UNH"],
+            [
+                "LEG",
+                [
+                    ["ORG", [], "parse_origin"],
+                    ["DST", [], "parse_destination"],
+                ],
+            ],
+            ["UNT"],
+        ],
+    }
+
+    def __init__(self):
+        self.route = {}
+
+    def parse_origin(self, segment: str):
+        self.route["origin"] = segment[4:]
+
+    def parse_destination(self, segment: str):
+        self.route["destination"] = segment[4:]
+
+
+legacy = RouteParser()
+legacy.parse("UNH'ORG+PAR'DST+NCE'UNT'", version=(1, 0))
+
+modern = RouteParser()
+modern.parse("UNH'LEG+1'ORG+PAR'DST+NCE'UNT'", version=(2, 0))
+
+assert legacy.route == modern.route == {
+    "origin": "PAR",
+    "destination": "NCE",
+}
+assert RouteParser.validate("UNH'ORG+PAR'DST+NCE'UNT'", version=(1, 0))
+assert RouteParser.validate("UNH'LEG+1'ORG+PAR'DST+NCE'UNT'", version=(2, 0))
+```
+
+When `version` is omitted, incomplete, or newer than all known schemas, Edison
+uses the latest schema. If a requested version is older than the first schema,
+`find_transitions()` raises `ValueError`.
+
+</details>
+
+<details>
+<summary>Mandatory and repeatable segments</summary>
+
+`repetition` controls whether the transition graph can come back to the same
+node:
+
+- `1` means the node can be used once in a given sequence.
+- Any value greater than `1` means the node is repeatable. The transition graph
+  only distinguishes `1` from "repeatable"; it does not count exact occurrence
+  limits such as `2` versus `99999`.
+
+Mandatory nodes constrain valid paths through the tree. Once a mandatory node is
+next in a sequence, the parser cannot skip over it to reach a later sibling, a
+parent, or a child that would bypass it. A mandatory child does not make its
+parent mandatory by itself: if the whole group is required, mark the group node
+as `"M"` too.
+
+```python
+from edison import BaseParser
+
+
+class OrderParser(BaseParser):
+    schemas = {
+        (0, 0): [
+            ["UNH"],
+            [
+                "ITM",
+                [
+                    ["DSC"],
+                    ["QTY", [], "parse_quantity", 1, "M"],
+                    ["PRI"],
+                ],
+                None,
+                99999,
+                "M",
+            ],
+            ["UNT"],
+        ],
+    }
+
+    def __init__(self):
+        self.quantities = []
+
+    def parse_quantity(self, segment: str):
+        self.quantities.append(segment[4:])
+
+
+assert OrderParser.validate("UNH'ITM'DSC'QTY+2'PRI'ITM'QTY+1'UNT'")
+assert not OrderParser.validate("UNH'UNT'")  # ITM is mandatory.
+assert not OrderParser.validate("UNH'ITM'PRI'UNT'")  # QTY is mandatory.
+assert not OrderParser.validate("UNH'ITM'QTY+2'QTY+3'UNT'")  # QTY repeats once.
+```
+
+</details>
+
+<details>
+<summary>Shared headers, footers, and parser hooks</summary>
+
+Use `header` and `footer` when several schemas share envelope segments. They are
+automatically prepended and appended to every schema at class compilation time.
+
+```python
+from edison import BaseParser
+
+
+class EnvelopeParser(BaseParser):
+    header = [["UNH", [], "on_segment"]]
+    footer = [["UNT", [], "on_segment"]]
+    schemas = {(0, 0): [["BGM", [], "on_segment"]]}
+
+    def __init__(self):
+        self.segments = []
+
+    def set_up(self):
+        self.segments.clear()
+
+    def on_segment(self, segment: str):
+        self.segments.append(segment)
+
+    def tear_down(self):
+        self.count = len(self.segments)
+```
+
+`set_up()` runs before parsing starts, and `tear_down()` runs after parsing
+finishes.
+
+</details>
+
+<details>
+<summary>Transition graphs</summary>
+
+`pydot` is installed with Edison. To write PNG files, make sure Graphviz is
+available on your system, then call `build_dot_object()` on a `BaseParser`
+subclass.
+
+```python
+from edison import BaseParser
+
+
+class Parser(BaseParser):
+    schemas = {
+        (0, 0): [
+            ["UNH"],
+            [
+                "BGM",
+                [
+                    ["DTM"],
+                ],
+            ],
+            ["UNT"],
+        ],
+    }
+
+
+dot = Parser.build_dot_object()
+dot.write_png("transitions.png")
+```
+
+You can pass the same version selector used by parsing, and hide upward links
+for a simpler tree-like graph:
+
+```python
+dot = Parser.build_dot_object(version=(13, 2), include_up_links=False)
+dot.write_png("transitions-v13.png")
+```
+
+Because `build_dot_object()` returns the underlying `pydot.Dot` object, you can
+customize it before writing.
+
+</details>
+
+<details>
+<summary>Escaping and EBCDIC EDIFACT separators</summary>
+
+Edison understands escaped EDIFACT separators and can normalize EBCDIC separator
+characters often found in binary payloads.
+
+```python
+from edison import normalize_ebcdic_separators, escape, unescape
+
+
+assert escape("REC+") == r"REC\+"
+assert unescape(r"REC\+") == "REC+"
+assert normalize_ebcdic_separators("UNB\x1dSENDER\x1fREC") == "UNB+SENDER:REC"
+```
+
+The segment tokenizer also handles escaped quotes, so content such as
+`IFT+1+AIRLINE\'SERVER'` stays in a single EDIFACT segment.
+
+</details>
+
+<details>
+<summary>EDIFACT envelope extraction</summary>
+
+For the common case of reading EDIFACT envelope boundaries, use
+`extract_envelope_headers()` to lazily return the first `UNB`/`UNH` segments or
+`extract_envelope_footers()` to lazily return the last `UNT`/`UNZ` segments:
+
+```python
+from edison import extract_envelope_footers, extract_envelope_headers
+
+edi = "UNB+SENDER+RECEIVER'UNH+1+PIZZAQ:1:1:1A'BGM+1'UNT+3+1'UNZ+1+1'"
+headers = extract_envelope_headers(edi)
+footers = extract_envelope_footers(edi)
+
+assert headers.unb == "UNB+SENDER+RECEIVER"
+assert headers.unh == "UNH+1+PIZZAQ:1:1:1A"
+assert footers.unt == "UNT+3+1"
+assert footers.unz == "UNZ+1+1"
+```
+
+</details>
+
+## Development
+
+<details>
+<summary>Local commands</summary>
+
+This repository uses `uv`, `tox`, `pytest`, `ruff`, `black`, `ty`, and
+pre-commit hooks.
+
+```bash
+uv tool install tox --with tox-uv
+tox -e py
+```
+
+Build distributions with:
+
+```bash
+uv build --sdist --wheel -o dist/
+```
+
+Run the full pre-commit suite with:
+
+```bash
+uv tool install prek==0.4.10
+prek run --all-files
+```
+
+CI runs tests through tox on CPython 3.12 and 3.14 across Linux, Windows, and
+macOS, then builds source and wheel distributions.
+
+</details>

edison_parser-0.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+edison/__init__.py,sha256=g1IVyG0eemMdRQInqjMSwDCZfNjOx6epv-C9c9_fvfE,218
+edison/envelope.py,sha256=o2Y58wth0Jy58_AG-PpIqbvJ5G65B5jn06CTdq1iKBI,1759
+edison/parser.py,sha256=yYx-hOT0P3U7YpYK7RtUs7xko16mjJuzs1UZWlKgt9I,15890
+edison/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+edison/tokenizer.py,sha256=Q2NQirhJsX0pnlNJVri7oFqcySvqJsT4MKrjhtm3Qqw,14079
+edison_parser-0.1.dist-info/licenses/LICENSE,sha256=qpBUePzboadefuykg6yqvOFSNtWCl0Q86qlnBG0Xod4,1065
+edison_parser-0.1.dist-info/METADATA,sha256=JSw3GD6QmLmE7czLsSzepqZjEHoAYzPRNBmxvLvITz4,11534
+edison_parser-0.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+edison_parser-0.1.dist-info/top_level.txt,sha256=pimY8xrIwHa0a7JsZ6Ikp6bCyLQK90iA8a28SyyStc0,7
+edison_parser-0.1.dist-info/RECORD,,

edison_parser-0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

edison_parser-0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Amadeus s.a.s
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

edison_parser-0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+edison