omextra 0.0.0.dev495__tar.gz → 0.0.0.dev497__tar.gz

{omextra-0.0.0.dev495/omextra.egg-info → omextra-0.0.0.dev497}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omextra
-Version: 0.0.0.dev495
+Version: 0.0.0.dev497
 Summary: omextra
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev495
+Requires-Dist: omlish==0.0.0.dev497
 Dynamic: license-file
 
 # Overview

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra/text/abnf/__init__.py

@@ -5,20 +5,18 @@ Originally based on library by Charles Yeomans (see LICENSE file):
 
   https://github.com/declaresub/abnf/tree/561ced67c0a8afc869ad0de5b39dbe4f6e71b0d8/src/abnf
 
-It has however been nearly entirely rewritten.
+It has however been entirely rewritten.
 
 ====
 
 TODO:
- - cache lol
- - get greedier
- - match-powered optimizer
-  - greedily compile regexes
+ - opto
  - error reporting
- - codegen
+ - codegen?
  - fix_ws problem
   - auto? no, need to keep lines / offsets accurate for errors
   - relax CRLF rule by default?
+  - grammar transform? helper kwarg?
 """
 from omlish import dataclasses as _dc  # noqa
 
@@ -82,6 +80,10 @@ from .ops import (  # noqa
     rule,
 )
 
+from .opto import (  # noqa
+    optimize_op,
+)
+
 from .utils import (  # noqa
     strip_insignificant_match_rules,
     only_match_rules,

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra/text/abnf/base.py

@@ -1,3 +1,4 @@
+import abc
 import io
 import itertools
 import typing as ta
@@ -10,6 +11,7 @@ from .errors import AbnfIncompleteParseError
 
 
 with lang.auto_proxy_import(globals()):
+    from . import internal
     from . import ops
     from . import parsing
 
@@ -53,6 +55,8 @@ class Match(ta.NamedTuple):
             write(f'literal<{self.start}-{self.end}>({o.value!r})')
         elif isinstance(o, ops.RangeLiteral):
             write(f'literal<{self.start}-{self.end}>({o.value.lo!r}-{o.value.hi!r})')
+        elif isinstance(o, internal.Regex):
+            write(f'regex<{self.start}-{self.end}>({o.pat.pattern!r})')
         else:
             write(f'{o.__class__.__name__.lower()}<{self.start}-{self.end}>')
             if isinstance(o, ops.RuleRef):
@@ -110,8 +114,28 @@ class Op(lang.Abstract, lang.PackageSealed):
         return f'{self.__class__.__name__}@{id(self):x}'
 
 
+class CompositeOp(Op, lang.Abstract):
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        try:
+            leaf_op_cls = LeafOp
+        except NameError:
+            pass
+        else:
+            check.not_issubclass(cls, leaf_op_cls)
+
+    @property
+    @abc.abstractmethod
+    def children(self) -> ta.Sequence[Op]:
+        raise NotImplementedError
+
+
 class LeafOp(Op, lang.Abstract):
-    pass
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        check.not_issubclass(cls, CompositeOp)
 
 
 ##
@@ -128,13 +152,21 @@ class Rule(lang.Final):
         super().__init__()
 
         self._name = check.non_empty_str(name)
-        self._name_f = name.casefold()
         self._op = check.isinstance(op, Op)
         self._insignificant = insignificant
 
+        self._name_f = name.casefold()
+
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}({self._name!r})'
 
+    def replace_op(self, op: Op) -> 'Rule':
+        return Rule(
+            self._name,
+            op,
+            insignificant=self._insignificant,
+        )
+
     @property
     def name(self) -> str:
         return self._name
@@ -165,6 +197,7 @@ class Grammar(lang.Final):
         rules_by_name_f: dict[str, Rule] = {}
         rules_by_op: dict[Op, Rule] = {}
         for gr in rules:
+            check.isinstance(gr, Rule)
             check.not_in(gr, rules_set)
             check.not_in(gr._name, rules_by_name)  # noqa
             check.not_in(gr._name_f, rules_by_name_f)  # noqa
@@ -196,6 +229,7 @@ class Grammar(lang.Final):
             *,
             start: int = 0,
             debug: int = 0,
+            **kwargs: ta.Any,
     ) -> ta.Iterator[Match]:
         if root is None:
             if (root := self._root) is None:
@@ -212,6 +246,7 @@ class Grammar(lang.Final):
             root._op,  # noqa
             start,
             debug=debug,
+            **kwargs,
         )
 
     def parse(
@@ -222,12 +257,14 @@ class Grammar(lang.Final):
             start: int = 0,
             complete: bool = False,
             debug: int = 0,
+            **kwargs: ta.Any,
     ) -> Match | None:
         if (match := longest_match(self.iter_parse(
             source,
             root,
             start=start,
             debug=debug,
+            **kwargs,
         ))) is None:
             return None
 

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra/text/abnf/internal.py

@@ -29,4 +29,4 @@ class Regex(InternalOp, LeafOp, lang.Final):
         return self._pat
 
     def __repr__(self) -> str:
-        return f'{self.__class__.__name__}@{id(self):x}({self._pat!r})'
+        return f'{self.__class__.__name__}@{id(self):x}({self._pat.pattern!r})'

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra/text/abnf/meta.py

@@ -20,6 +20,7 @@ from .ops import literal
 from .ops import option
 from .ops import repeat
 from .ops import rule
+from .opto import optimize_op
 from .utils import fix_ws
 from .utils import parse_rules
 from .visitors import RuleMatchVisitor
@@ -558,8 +559,9 @@ class MetaGrammarRuleMatchVisitor(RuleMatchVisitor[ta.Any]):
 def parse_grammar(
         source: str,
         *,
-        no_core_rules: bool = False,
         root: str | None = None,
+        no_core_rules: bool = False,
+        no_optimize: bool = False,
         **kwargs: ta.Any,
 ) -> Grammar:
     source = fix_ws(source)
@@ -575,7 +577,16 @@ def parse_grammar(
     check.isinstance(mg_m.op, Repeat)
 
     mg_rmv = MetaGrammarRuleMatchVisitor(source)
-    rules = [mg_rmv.visit_match(gg_cm) for gg_cm in mg_m.children]
+    rules = [
+        check.isinstance(mg_rmv.visit_match(gg_cm), Rule)
+        for gg_cm in mg_m.children
+    ]
+
+    if not no_optimize:
+        rules = [
+            r.replace_op(optimize_op(r.op))
+            for r in rules
+        ]
 
     return Grammar(
         *rules,

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra/text/abnf/ops.py

@@ -4,6 +4,7 @@ from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
 
+from .base import CompositeOp
 from .base import LeafOp
 from .base import Op
 
@@ -54,8 +55,8 @@ class RangeLiteral(Literal, lang.Final):
         hi: str
 
         def __post_init__(self) -> None:
-            check.non_empty_str(self.lo)
-            check.non_empty_str(self.hi)
+            for c in (self.lo, self.hi):
+                check.equal(len(check.non_empty_str(c)), 1)
             check.state(self.hi >= self.lo)
 
     def __init__(self, value: Range) -> None:
@@ -101,7 +102,7 @@ def literal(*args, case_sensitive=None):
 
 
 @ta.final
-class Concat(Op, lang.Final):
+class Concat(CompositeOp, lang.Final):
     def __init__(self, *children: Op) -> None:
         super().__init__()
 
@@ -124,7 +125,7 @@ concat = Concat
 
 
 @ta.final
-class Repeat(Op, lang.Final):
+class Repeat(CompositeOp, lang.Final):
     @dc.dataclass(frozen=True)
     class Times:
         min: int = 0
@@ -159,6 +160,10 @@ class Repeat(Op, lang.Final):
     def child(self) -> Op:
         return self._child
 
+    @property
+    def children(self) -> ta.Sequence[Op]:
+        return (self._child,)
+
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}@{id(self):x}({self._times}, {self._child!r})'
 
@@ -223,7 +228,7 @@ def option(child: Op) -> Repeat:
 
 
 @ta.final
-class Either(Op, lang.Final):
+class Either(CompositeOp, lang.Final):
     def __init__(self, *children: Op, first_match: bool = False) -> None:
         super().__init__()
 

omextra-0.0.0.dev497/omextra/text/abnf/opto.py

@@ -0,0 +1,154 @@
+"""
+TODO:
+ - Merge concat
+ - Merge concatted literals
+ - Regex
+"""
+import re
+import typing as ta
+
+from omlish import check
+
+from .base import Op
+from .internal import Regex
+from .ops import CaseInsensitiveStringLiteral
+from .ops import CompositeOp
+from .ops import Concat
+from .ops import Either
+from .ops import RangeLiteral
+from .ops import Repeat
+from .ops import RuleRef
+from .ops import StringLiteral
+
+
+##
+
+
+def _build_op_regex_pat(op: Op, pats_by_op: ta.Mapping[Op, str | None]) -> str | None:
+    if isinstance(op, StringLiteral):
+        return re.escape(op.value)
+
+    elif isinstance(op, CaseInsensitiveStringLiteral):
+        return f'(?i:{re.escape(op.value)})'
+
+    elif isinstance(op, RangeLiteral):
+        lo = re.escape(op.value.lo)
+        hi = re.escape(op.value.hi)
+        return f'[{lo}-{hi}]'
+
+    elif isinstance(op, RuleRef):
+        return None
+
+    elif isinstance(op, Regex):
+        return op.pat.pattern
+
+    elif isinstance(op, Concat):
+        child_pats = [pats_by_op[child] for child in op.children]
+        if not all(ca is not None for ca in child_pats):
+            return None
+        return ''.join(ta.cast(str, ca) for ca in child_pats)
+
+    elif isinstance(op, Repeat):
+        if (child_pat := pats_by_op[op.child]) is None:
+            return None
+
+        # Wrap the child pattern in a non-capturing group if needed to ensure correct quantification. A pattern needs
+        # wrapping if it contains multiple elements or operators (e.g., 'ab', 'a|b'). Single character classes [a-z] and
+        # single escaped chars don't need wrapping.
+        needs_group = (
+            len(child_pat) > 1 and
+            not (child_pat.startswith('[') and child_pat.endswith(']'))
+        )
+        if needs_group:
+            child_pat = f'(?:{child_pat})'
+
+        times = op.times
+        if times.min == 0 and times.max is None:
+            quantifier = '*'
+        elif times.min == 1 and times.max is None:
+            quantifier = '+'
+        elif times.min == 0 and times.max == 1:
+            quantifier = '?'
+        elif times.max is None:
+            quantifier = f'{{{times.min},}}'
+        elif times.min == times.max:
+            quantifier = f'{{{times.min}}}'
+        else:
+            quantifier = f'{{{times.min},{times.max}}}'
+
+        return child_pat + quantifier
+
+    elif isinstance(op, Either):
+        # Only convert Either if first_match is True, as regex alternation uses first-match semantics. ABNF Either with
+        # first_match=False uses longest-match semantics, which differs from regex.
+        if not op.first_match:
+            return None
+
+        child_pats = [pats_by_op[child] for child in op.children]
+        if not all(ca is not None for ca in child_pats):
+            return None
+
+        # Build regex alternation. Use a capturing group for the alternation
+        return f'({"|".join(ta.cast("ta.Sequence[str]", child_pats))})'
+
+    else:
+        raise TypeError(op)
+
+
+def _regex_transform_single_op(op: Op, pats_by_op: ta.Mapping[Op, str | None]) -> Op:
+    pat = pats_by_op[op]
+
+    if pat is not None:
+        if isinstance(op, Regex):
+            return op
+
+        return Regex(re.compile(pat))
+
+    if isinstance(op, Concat):
+        new_children = tuple(_regex_transform_single_op(child, pats_by_op) for child in op.children)
+        if new_children == op.children:
+            return op
+
+        return Concat(*new_children)
+
+    elif isinstance(op, Repeat):
+        new_child = _regex_transform_single_op(op.child, pats_by_op)
+        if new_child == op.child:
+            return op
+
+        return Repeat(op.times, new_child)
+
+    elif isinstance(op, Either):
+        new_children = tuple(_regex_transform_single_op(child, pats_by_op) for child in op.children)
+        if new_children == op.children:
+            return op
+
+        return Either(*new_children, first_match=op.first_match)
+
+    return op
+
+
+def regex_transform_op(op: Op) -> Op:
+    pats_by_op: dict[Op, str | None] = {}
+
+    def analyze_tree(o: Op) -> None:
+        check.not_in(o, pats_by_op)
+
+        if isinstance(o, CompositeOp):
+            for child in o.children:
+                analyze_tree(child)
+
+        pats_by_op[o] = _build_op_regex_pat(o, pats_by_op)
+
+    analyze_tree(op)
+
+    return _regex_transform_single_op(op, pats_by_op)
+
+
+##
+
+
+def optimize_op(op: Op) -> Op:
+    op = regex_transform_op(op)
+
+    return op

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra/text/abnf/parsing.py

@@ -19,15 +19,21 @@ from .ops import StringLiteral
 
 
 class _Parser:
+    class MaxStepsExceededError(Exception):
+        pass
+
     def __init__(
             self,
             grammar: Grammar,
             source: str,
+            *,
+            max_steps: int | None = None,
     ) -> None:
         super().__init__()
 
         self._grammar = grammar
         self._source = source
+        self._max_steps = max_steps
 
         self._dispatch: dict[type[Op], ta.Any] = {
             StringLiteral: self._iter_parse_string_literal,
@@ -40,6 +46,10 @@ class _Parser:
             Regex: self._iter_parse_regex,
         }
 
+        self._memo: dict[tuple[Op, int], tuple[Match, ...]] = {}
+
+        self._cur_step = 0
+
     def _iter_parse_string_literal(self, op: StringLiteral, start: int) -> ta.Iterator[Match]:
         if start < len(self._source):  # noqa
             source = self._source[start : start + len(op._value)]  # noqa
@@ -62,49 +72,90 @@ class _Parser:
             yield Match(op, start, start + 1, ())
 
     def _iter_parse_concat(self, op: Concat, start: int) -> ta.Iterator[Match]:
-        i = 0
         match_tups: list[tuple[Match, ...]] = [()]
+
+        i = 0
         for cp in op._children:  # noqa
             next_match_tups: list[tuple[Match, ...]] = []
+
             for mt in match_tups:
                 for cm in self.iter_parse(cp, mt[-1].end if mt else start):
                     next_match_tups.append((*mt, cm))
                     i += 1
+
             if not next_match_tups:
                 return
+
             match_tups = next_match_tups
+
         if not i:
             return
+
         for mt in sorted(match_tups, key=len, reverse=True):
             yield Match(op, start, mt[-1].end if mt else start, mt)
 
     def _iter_parse_repeat(self, op: Repeat, start: int) -> ta.Iterator[Match]:
-        match_tup_set: set[tuple[Match, ...]] = set()
-        last_match_tup_set: set[tuple[Match, ...]] = {()}
+        # Map from (repetition_count, end_position) to longest match tuple
+        matches_by_count_pos: dict[tuple[int, int], tuple[Match, ...]] = {(0, start): ()}
+        max_end_by_count: dict[int, int] = {0: start}
+
         i = 0
         while True:
             if op._times.max is not None and i == op._times.max:  # noqa
                 break
-            next_match_tup_set: set[tuple[Match, ...]] = set()
-            for mt in last_match_tup_set:
-                for cm in self.iter_parse(op._child, mt[-1].end if mt else start):  # noqa
-                    next_match_tup_set.add((*mt, cm))
-            if not next_match_tup_set or next_match_tup_set < match_tup_set:
+
+            if self._max_steps is not None and self._cur_step > self._max_steps:
+                raise _Parser.MaxStepsExceededError(self._cur_step)
+            self._cur_step += 1
+
+            next_matches: dict[tuple[int, int], tuple[Match, ...]] = {}
+            next_max_end = max_end_by_count.get(i, -1)
+
+            for (count, end_pos), mt in matches_by_count_pos.items():
+                if count != i:
+                    continue
+
+                for cm in self.iter_parse(op._child, end_pos):  # noqa
+                    next_mt = (*mt, cm)
+                    next_key = (i + 1, cm.end)
+
+                    # Keep only the longest match tuple for each (count, position)
+                    if next_key not in next_matches or len(next_mt) > len(next_matches[next_key]):
+                        next_matches[next_key] = next_mt
+                        if cm.end > next_max_end:
+                            next_max_end = cm.end
+
+            if not next_matches:
+                break
+
+            # Check if we made progress (reached new positions)
+            if next_max_end <= max_end_by_count.get(i, -1):
                 break
+
             i += 1
-            match_tup_set |= next_match_tup_set
-            last_match_tup_set = next_match_tup_set
+            matches_by_count_pos.update(next_matches)
+            max_end_by_count[i] = next_max_end
+
         if i < op._times.min:  # noqa
             return
-        for mt in sorted(match_tup_set or [()], key=len, reverse=True):
-            yield Match(op, start, mt[-1].end if mt else start, mt)  # noqa
+
+        # Collect valid matches and sort by (end_position, repetition_count) descending
+        valid_matches: list[tuple[int, int, tuple[Match, ...]]] = []
+        for (count, end_pos), mt in matches_by_count_pos.items():
+            if op._times.min <= count <= (op._times.max if op._times.max is not None else i):  # noqa
+                valid_matches.append((end_pos, count, mt))
+
+        for end_pos, _, mt in sorted(valid_matches, key=lambda x: (x[0], x[1]), reverse=True):
+            yield Match(op, start, end_pos, mt)
 
     def _iter_parse_either(self, op: Either, start: int) -> ta.Iterator[Match]:
         for cp in op._children:  # noqa
             found = False
+
             for cm in self.iter_parse(cp, start):
                 found = True
                 yield Match(op, start, cm.end, (cm,))
+
             if found and op._first_match:  # noqa
                 return
 
@@ -114,10 +165,21 @@ class _Parser:
             yield Match(op, cm.start, cm.end, (cm,))
 
     def _iter_parse_regex(self, op: Regex, start: int) -> ta.Iterator[Match]:
-        raise NotImplementedError
+        if (m := op._pat.match(self._source, start)) is not None:  # noqa
+            yield Match(op, start, m.end(), ())
 
     def iter_parse(self, op: Op, start: int) -> ta.Iterator[Match]:
-        return self._dispatch[op.__class__](op, start)
+        if (key := (op, start)) in self._memo:
+            yield from self._memo[key]
+            return
+
+        if self._max_steps is not None and self._cur_step >= self._max_steps:
+            raise _Parser.MaxStepsExceededError(self._cur_step)
+        self._cur_step += 1
+
+        matches = tuple(self._dispatch[op.__class__](op, start))
+        self._memo[key] = matches
+        yield from matches
 
 
 ##
@@ -131,8 +193,9 @@ class _DebugParser(_Parser):
             level: int = 1,
             *,
             write: ta.Callable[[str], None] | None = None,
+            **kwargs: ta.Any,
     ) -> None:
-        super().__init__(grammar, source)
+        super().__init__(grammar, source, **kwargs)
 
         self._level = level
         if write is None:
@@ -195,18 +258,21 @@ def _iter_parse(
         start: int,
         *,
         debug: int = 0,
+        max_steps: int | None = None,
 ) -> ta.Iterator[Match]:
     parser: _Parser
     if debug:
         parser = _DebugParser(
             grammar,
             source,
+            max_steps=max_steps,
             level=debug,
         )
     else:
         parser = _Parser(
             grammar,
             source,
+            max_steps=max_steps,
         )
 
     return parser.iter_parse(op, start)

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497/omextra.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omextra
-Version: 0.0.0.dev495
+Version: 0.0.0.dev497
 Summary: omextra
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev495
+Requires-Dist: omlish==0.0.0.dev497
 Dynamic: license-file
 
 # Overview

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/omextra.egg-info/SOURCES.txt

@@ -83,6 +83,7 @@ omextra/text/abnf/errors.py
 omextra/text/abnf/internal.py
 omextra/text/abnf/meta.py
 omextra/text/abnf/ops.py
+omextra/text/abnf/opto.py
 omextra/text/abnf/parsing.py
 omextra/text/abnf/utils.py
 omextra/text/abnf/visitors.py

omextra-0.0.0.dev497/omextra.egg-info/requires.txt

@@ -0,0 +1 @@
+omlish==0.0.0.dev497

{omextra-0.0.0.dev495 → omextra-0.0.0.dev497}/pyproject.toml

@@ -13,7 +13,7 @@ urls = {source = 'https://github.com/wrmsr/omlish'}
 license = 'BSD-3-Clause'
 readme = 'README.md'
 requires-python = '>=3.13'
-version = '0.0.0.dev495'
+version = '0.0.0.dev497'
 classifiers = [
     'Development Status :: 2 - Pre-Alpha',
     'Intended Audience :: Developers',
@@ -24,7 +24,7 @@ classifiers = [
 ]
 description = 'omextra'
 dependencies = [
-    'omlish == 0.0.0.dev495',
+    'omlish == 0.0.0.dev497',
 ]
 
 [project.optional-dependencies]

omextra-0.0.0.dev495/omextra.egg-info/requires.txt

@@ -1 +0,0 @@
-omlish==0.0.0.dev495