omlish 0.0.0.dev306__py3-none-any.whl → 0.0.0.dev308__py3-none-any.whl

omlish/formats/edn/parsing.py

@@ -1,28 +1,88 @@
+# ruff: noqa: PYI055 UP007
 """
 TODO:
+ - \u0123 in strings
+ - https://clojure.org/reference/reader
  - reader meta - ^:foo
+ - read table
 """
-# https://github.com/jorinvo/edn-data/blob/1e5824f63803eb58f35e98839352000053d47115/test/parse.test.ts
+import dataclasses as dc
 import datetime
 import enum
+import io
 import re
 import typing as ta
 
 from ... import check
+from ...funcs.genmachine import GenMachine
+from .lexing import Position
+from .lexing import StreamLexer
+from .lexing import Token
 from .values import Char
+from .values import Collection
 from .values import Keyword
 from .values import List
 from .values import Map
 from .values import Set
 from .values import Symbol
-from .values import TaggedVal
+from .values import Tagged
 from .values import Vector
 
 
 ##
 
 
-class ListParser:
+WORD_CONST_VALUES: ta.Mapping[str, ta.Any] = {
+    'true': True,
+    'false': False,
+    'nil': None,
+
+    '##Inf': float('inf'),
+    '##-Inf': float('-inf'),
+    '##NaN': float('nan'),
+}
+
+
+STRING_ESCAPE_MAP: ta.Mapping[str, str] = {
+    '"': '"',
+    '\\': '\\',
+    'b': '\b',
+    'f': '\f',
+    'n': '\n',
+    'r': '\r',
+    't': '\t',
+}
+
+
+CHAR_ESCAPE_MAP: ta.Mapping[str, str] = {
+    'backspace': '\b',
+    'formfeed': '\f',
+    'newline': '\n',
+    'return': '\r',
+    'space': ' ',
+    'tab': '\t',
+}
+
+
+##
+
+
+@dc.dataclass()
+class StreamParseError(Exception):
+    message: str
+
+    pos: Position | None = None
+
+
+@dc.dataclass(frozen=True)
+class MetaMaker:
+    fn: ta.Callable[..., ta.Any]
+
+    def __call__(self, *args: ta.Any, meta: ta.Any | None = None) -> ta.Any:
+        return self.fn(*args, meta=meta)
+
+
+class StreamParser(GenMachine[Token, ta.Any]):
     DEFAULT_TAG_HANDLERS: ta.ClassVar[ta.Mapping[str, ta.Callable[..., ta.Any]]] = {
         'inst': lambda val: datetime.datetime.fromisoformat(val) if isinstance(val, str) else None,
     }
@@ -30,19 +90,17 @@ class ListParser:
     def __init__(
             self,
             *,
-            keyword_maker: ta.Callable[..., ta.Any] = Keyword,
-            char_maker: ta.Callable[..., ta.Any] = Char,
-            symbol_maker: ta.Callable[..., ta.Any] = Symbol,
+            keyword_maker: ta.Callable[..., ta.Any] = MetaMaker(Keyword),
+            char_maker: ta.Callable[..., ta.Any] = MetaMaker(Char),
+            symbol_maker: ta.Callable[..., ta.Any] = MetaMaker(Symbol),
 
-            list_maker: ta.Callable[..., ta.Any] = List.new,
-            vector_maker: ta.Callable[..., ta.Any] = Vector.new,
-            set_maker: ta.Callable[..., ta.Any] = Set.new,
-            map_maker: ta.Callable[..., ta.Any] = Map.new,
+            list_maker: ta.Callable[..., ta.Any] = MetaMaker(List.new),
+            vector_maker: ta.Callable[..., ta.Any] = MetaMaker(Vector.new),
+            set_maker: ta.Callable[..., ta.Any] = MetaMaker(Set.new),
+            map_maker: ta.Callable[..., ta.Any] = MetaMaker(Map.new),
 
             tag_handlers: ta.Mapping[str, ta.Callable[..., ta.Any]] | None = None,
     ) -> None:
-        super().__init__()
-
         self._keyword_maker = keyword_maker
         self._char_maker = char_maker
         self._symbol_maker = symbol_maker
@@ -57,299 +115,262 @@ class ListParser:
             **(tag_handlers or {}),
         }
 
-        self._stack: list[tuple[ListParser._ParseMode | ListParser._StackItem, ta.Any]] = []
-        self._mode: ListParser._ParseMode = ListParser._ParseMode.IDLE
-        self._state = ''
-        self._result: ta.Any = self._UNDEFINED
+        self._stack: list[
+            tuple[
+                ta.Union[
+                    type[Collection],
+                    StreamParser._StackSpecial,
+                ],
+                list[ta.Any],
+            ],
+        ] = []
 
-    #
+        super().__init__(self._do_main())
 
-    class _UNDEFINED:  # noqa
-        def __new__(cls, *args, **kwargs):  # noqa
-            raise TypeError
+    class _StackSpecial(enum.Enum):  # noqa
+        DISCARD = enum.auto()
+        TAG = enum.auto()
 
-    class _ParseMode(enum.Enum):
-        IDLE = 0
-        STRING = 1
-        ESCAPE = 2
-        COMMENT = 3
+    def _emit_value(self, value: ta.Any) -> tuple[ta.Any, ...]:
+        while self._stack and self._stack[-1][0] is StreamParser._StackSpecial.TAG:
+            cc, cl = self._stack.pop()
+            ts = check.non_empty_str(check.single(cl))
+            value = Tagged(ts, value)
 
-    class _StackItem(enum.Enum):
-        VECTOR = 0
-        LIST = 1
-        MAP = 2
-        SET = 3
-        TAG = 4
+        if not self._stack:
+            return (value,)
 
-    #
+        cc, cl = self._stack[-1]
+
+        if cc is StreamParser._StackSpecial.DISCARD:
+            check.empty(cl)
+            self._stack.pop()
+            return ()
 
-    def _update_stack(self) -> None:
-        if not self._stack or self._result is self._UNDEFINED:
-            return
+        elif cc is StreamParser._StackSpecial.TAG:
+            ts = check.non_empty_str(check.single(cl))
+            self._stack.pop()
+            tv = Tagged(ts, value)
+            return (tv,)
+
+        elif cc is Map:
+            if cl and len(cl[-1]) < 2:
+                cl[-1] = (*cl[-1], value)
+            else:
+                cl.append((value,))
+            return ()
+
+        elif isinstance(cc, type) and issubclass(cc, Collection):
+            cl.append(value)
+            return ()
+
+        else:
+            raise RuntimeError(cc)
+
+    def _do_main(self):
+        while True:
+            tok: Token
+            try:
+                tok = yield None  # noqa
+            except GeneratorExit:
+                if self._stack:
+                    raise StreamParseError('Expected value') from None
+                else:
+                    raise
 
-        stack_item, prev_state = self._stack[-1]
+            # ignored
 
-        if stack_item == ListParser._StackItem.VECTOR:
-            prev_state.append(self._result)
+            if tok.kind in ('SPACE', 'COMMENT'):
+                continue
 
-        elif stack_item == ListParser._StackItem.LIST:
-            prev_state.append(self._result)
+            # scalars
 
-        elif stack_item == ListParser._StackItem.SET:
-            prev_state.append(self._result)
+            value: ta.Any
 
-        elif stack_item == ListParser._StackItem.MAP:
-            if len(prev_state[1]) > 0:
-                prev_state[0].append([prev_state[1].pop(), self._result])
-            else:
-                prev_state[1].append(self._result)
+            if tok.kind == 'STRING':
+                value = self._parse_string(tok)
 
-        elif stack_item == ListParser._StackItem.TAG:
-            self._stack.pop()
+            elif tok.kind == 'CHAR':
+                value = self._parse_char(tok)
 
-            if prev_state == '_':
-                self._result = self._UNDEFINED
+            elif tok.kind == 'WORD':
+                if tok.src.startswith('#'):
+                    # FIXME: more dispatching
+                    self._stack.append((StreamParser._StackSpecial.TAG, [tok.src[1:]]))
+                    continue
 
-            else:
-                tag_handler = self._tag_handlers.get(prev_state)
-                if tag_handler:
-                    self._result = tag_handler(self._result)
                 else:
-                    self._result = TaggedVal(prev_state, self._result)
-
-            self._update_stack()
-            return
-
-        # TODO: else error
-        # Reset result after updating stack
-        self._result = self._UNDEFINED
-
-    #
-
-    _INT_PAT = re.compile(r'^[-+]?(0|[1-9][0-9]*)$')
-    _BIGINT_PAT = re.compile(r'^[-+]?(0|[1-9][0-9]*)N$')
-    _FLOAT_PAT = re.compile(r'^[-+]?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?(0|[1-9][0-9]*))?M?$')
-
-    def _match(self) -> None:
-        if self._state == 'nil':
-            self._result = None
-
-        elif self._state == 'true':
-            self._result = True
-
-        elif self._state == 'false':
-            self._result = False
-
-        elif self._state.startswith(':'):
-            # Keyword
-            self._result = self._keyword_maker(self._state[1:])
-
-        elif self._state.startswith('#'):
-            # Tag
-            self._stack.append((ListParser._StackItem.TAG, self._state[1:]))
-            self._result = self._UNDEFINED
-
-        elif self._INT_PAT.match(self._state):
-            # Int
-            self._result = int(self._state)
-
-        elif self._FLOAT_PAT.match(self._state):
-            # Float
-            self._result = float(self._state)
-
-        elif self._BIGINT_PAT.match(self._state):
-            # BigInt
-            self._result = int(self._state[:-1])  # In Python we don't need special handling for bigint
-
-        elif self._state.startswith('\\'):
-            # Char
-            check.state(len(self._state) > 1)
-            if self._state == '\\space':
-                c = ' '
-            elif self._state == '\\newline':
-                c = '\n'
-            elif self._state == '\\return':
-                c = '\r'
-            elif self._state == '\\tab':
-                c = '\t'
-            elif self._state == '\\\\':
-                c = '\\'
-            elif self._state.startswith('\\u'):
-                check.state(len(self._state) == 6)
-                c = chr(int(self._state[2:], 16))
-            else:
-                check.state(len(self._state) == 2)
-                c = self._state[1:]
+                    value = self._parse_word(tok)
 
-            self._result = self._char_maker(c)
+            # open
 
-        elif self._state:
-            # Symbol
-            self._result = self._symbol_maker(self._state)
+            elif tok.kind == 'LPAREN':
+                self._stack.append((List, []))
+                continue
 
-        self._state = ''
+            elif tok.kind == 'LBRACKET':
+                self._stack.append((Vector, []))
+                continue
 
-    #
+            elif tok.kind == 'HASH_LBRACE':
+                self._stack.append((Set, []))
+                continue
 
-    _SPACE_CHARS: ta.ClassVar[ta.AbstractSet[str]] = frozenset([',', ' ', '\t', '\n', '\r'])
+            elif tok.kind == 'LBRACE':
+                self._stack.append((Map, []))
+                continue
 
-    _STRING_ESCAPE_MAP: ta.ClassVar[ta.Mapping[str, str]] = {
-        't': '\t',
-        'r': '\r',
-        'n': '\n',
-        '\\': '\\',
-        '"': '"',
-    }
+            elif tok.kind == 'HASH_UNDERSCORE':
+                self._stack.append((StreamParser._StackSpecial.DISCARD, []))
+                continue
 
-    def parse(self, src: str) -> list[ta.Any]:
-        values = []
+            # close
 
-        i = -1
-        for i in range(len(src)):
-            if not self._stack and self._result is not self._UNDEFINED:
-                values.append(self._result)
-                self._result = self._UNDEFINED
+            elif tok.kind == 'RPAREN':
+                cc, cl = self._stack.pop()
+                check.state(cc is List)
+                value = self._list_maker(cl)
 
-            char = src[i]
+            elif tok.kind == 'RBRACKET':
+                cc, cl = self._stack.pop()
+                check.state(cc is Vector)
+                value = self._vector_maker(cl)
 
-            if self._mode == ListParser._ParseMode.IDLE:
-                if char == '"':
-                    self._match()
-                    self._update_stack()
-                    self._mode = ListParser._ParseMode.STRING
-                    self._state = ''
-                    continue
+            elif tok.kind == 'RBRACE':
+                cc, cl = self._stack.pop()
 
-                if char == ';':
-                    self._mode = ListParser._ParseMode.COMMENT
-                    continue
+                if cc is Set:
+                    value = self._set_maker(cl)
 
-                if char in self._SPACE_CHARS:
-                    self._match()
-                    self._update_stack()
-                    continue
+                elif cc is Map:
+                    if cl and len(cl[-1]) != 2:
+                        raise RuntimeError('Mismatched map entries')
+                    value = self._map_maker(cl)
 
-                if char == '}':
-                    self._match()
-                    self._update_stack()
+                else:
+                    raise RuntimeError(cc)
 
-                    if self._stack:
-                        stack_item, prev_state = self._stack.pop()
+            # nyi
 
-                        if stack_item == ListParser._StackItem.MAP:
-                            check.empty(prev_state[1])
-                            self._result = self._map_maker(prev_state[0])
+            elif tok.kind == 'META':
+                raise NotImplementedError
 
-                        else:  # Set
-                            # FIXME:
-                            # check.state(stack_item == ListParser._StackItem.SET)
-                            self._result = self._set_maker(prev_state)
+            elif tok.kind == 'QUOTE':
+                raise NotImplementedError
 
-                    self._update_stack()
-                    continue
+            # failure
 
-                if char == ']':
-                    self._match()
-                    self._update_stack()
-                    stack_item, prev_state = self._stack.pop()
-                    self._result = self._vector_maker(tuple(prev_state))
-                    self._update_stack()
-                    continue
+            else:
+                raise ValueError(tok.kind)
+
+            # emit
+
+            if (ev := self._emit_value(value)):
+                yield ev
+
+    def _parse_string(self, tok: Token) -> str:
+        check.state(tok.kind == 'STRING')
+        src = tok.src
+        check.state(src[0] == '"')
+        check.state(src[-1] == '"')
+        check.state(len(src) > 1)
+
+        p = 1
+        end = len(src) - 1
+        if src.find('\\', p, end) < 0:
+            return src[1:-1]
+
+        sb = io.StringIO()
+        while True:
+            if (n := src.find('\\', p, end)) < 0:
+                sb.write(src[p:end])
+                break
+
+            sb.write(src[p:n])
+            p = n + 1
+            check.state(p < end)
+            x = src[p]
+            p += 1
+
+            if x == 'u':
+                check.state(p < end - 4)
+                r = chr(int(src[p:p + 4], 16))
+                p += 4
 
-                if char == ')':
-                    self._match()
-                    self._update_stack()
-                    stack_item, prev_state = self._stack.pop()
-                    self._result = self._list_maker(prev_state)
-                    self._update_stack()
-                    continue
+            else:
+                r = STRING_ESCAPE_MAP[x]
 
-                if char == '[':
-                    self._match()
-                    self._update_stack()
-                    self._stack.append((ListParser._StackItem.VECTOR, []))
-                    continue
+            sb.write(r)
 
-                if char == '(':
-                    self._match()
-                    self._update_stack()
-                    self._stack.append((ListParser._StackItem.LIST, []))
-                    continue
+        return sb.getvalue()
 
-                state_plus_char = self._state + char
-                if state_plus_char == '#_':
-                    self._stack.append((ListParser._StackItem.TAG, char))
-                    self._result = self._UNDEFINED
-                    self._state = ''
-                    continue
+    def _parse_char(self, tok: Token) -> ta.Any:
+        check.state(tok.kind == 'CHAR')
+        src = tok.src
+        check.state(len(src) > 1)
+        check.state(src.startswith('\\'))
 
-                if state_plus_char.endswith('#{'):
-                    self._state = self._state[:-1]  # Remove the '#'
-                    self._match()
-                    self._update_stack()
-                    self._stack.append((ListParser._StackItem.SET, []))
-                    self._state = ''
-                    continue
+        if len(src) == 2:
+            c = src[1]
 
-                if char == '{':
-                    self._match()
-                    self._update_stack()
-                    self._stack.append((ListParser._StackItem.MAP, [[], []]))
-                    self._state = ''
-                    continue
+        elif src[1] == 'u':
+            check.state(len(src) == 6)
+            c = chr(int(src[2:], 16))
 
-                self._state += char
-                continue
+        elif src[1] == 'o':
+            # \oXXX -> octal
+            raise NotImplementedError
 
-            elif self._mode == ListParser._ParseMode.STRING:  # noqa
-                if char == '\\':
-                    self._stack.append((self._mode, self._state))
-                    self._mode = ListParser._ParseMode.ESCAPE
-                    self._state = ''
-                    continue
+        else:
+            c = CHAR_ESCAPE_MAP[src[1:]]
 
-                if char == '"':
-                    self._mode = ListParser._ParseMode.IDLE
-                    self._result = self._state
-                    self._update_stack()
-                    self._state = ''
-                    continue
+        return self._char_maker(c)
 
-                self._state += char
+    _INT_PAT = re.compile(r'[-+]?(0|[1-9][0-9]*)N?')
+    _FLOAT_PAT = re.compile(r'[-+]?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?(0|[1-9][0-9]*))?M?')
 
-            elif self._mode == ListParser._ParseMode.ESCAPE:
-                # TODO what should happen when escaping other char
-                escaped_char = self._STRING_ESCAPE_MAP.get(char, char)
-                stack_item, prev_state = self._stack.pop()
-                self._mode = check.isinstance(stack_item, ListParser._ParseMode)
-                self._state = prev_state + escaped_char
+    def _parse_word(self, tok: Token) -> ta.Any:
+        check.state(tok.kind == 'WORD')
+        src = tok.src
+        check.non_empty_str(src)
+        check.state(not src.startswith('#'))
 
-            elif self._mode == ListParser._ParseMode.COMMENT:
-                if char == '\n':
-                    self._mode = ListParser._ParseMode.IDLE
+        if src in WORD_CONST_VALUES:
+            return WORD_CONST_VALUES[src]
 
-            else:
-                raise RuntimeError(self._mode)
+        elif src.startswith(':'):
+            return self._keyword_maker(src[1:])
 
-        if i >= 0:
-            self._match()
-            self._update_stack()
+        elif self._INT_PAT.fullmatch(src):
+            # FIXME: numbers lol
+            # 2r101010, 052, 8r52, 0x2a, 36r16, and 42 are all the same Long.
+            # Floating point numbers are read as Doubles; with M suffix they are read as BigDecimals.
+            # Ratios are supported, e.g. 22/7.
+            if src.endswith('N'):
+                return int(src[:-1])
 
-        check.state(not self._stack)
+            else:
+                return int(src)
 
-        if self._result is not self._UNDEFINED:
-            values.append(self._result)
-        return values
+        elif self._FLOAT_PAT.fullmatch(src):
+            return float(src)
 
+        else:
+            return self._symbol_maker(src)
 
-#
 
+##
 
-def parse_list(src: str, **kwargs: ta.Any) -> list[ta.Any]:
-    """Parse an edn string and return the corresponding Python object."""
 
-    parser = ListParser(**kwargs)
-    return parser.parse(src)
+def parse_list(src: str, **kwargs: ta.Any) -> list[ta.Any]:
+    r: list[ta.Any] = []
+    with StreamLexer() as l:
+        with StreamParser(**kwargs) as p:
+            for c in [*src, '']:
+                for t in l(c):
+                    for o in p(t):
+                        r.append(o)  # noqa
+    return r
 
 
 def parse(src: str, **kwargs: ta.Any) -> ta.Any | None:

omlish/formats/edn/values.py

@@ -15,7 +15,7 @@ _DEBUG = __debug__
 
 @dc.dataclass(frozen=True)
 class Value(lang.Abstract, lang.Sealed):
-    pass
+    meta: ta.Any | None = dc.field(default=None, kw_only=True)
 
 
 #
@@ -87,8 +87,8 @@ class List(Collection, lang.Final):
             check.isinstance(self.items, tuple)
 
     @classmethod
-    def new(cls, items: ta.Iterable[ta.Any]) -> 'List':
-        return cls(tuple(items))
+    def new(cls, items: ta.Iterable[ta.Any], *, meta: ta.Any | None = None) -> 'List':
+        return cls(tuple(items), meta=meta)
 
 
 @dataclass_cache_hash()
@@ -104,8 +104,8 @@ class Vector(Collection, lang.Final):
             check.isinstance(self.items, tuple)
 
     @classmethod
-    def new(cls, items: ta.Iterable[ta.Any]) -> 'Vector':
-        return cls(tuple(items))
+    def new(cls, items: ta.Iterable[ta.Any], *, meta: ta.Any | None = None) -> 'Vector':
+        return cls(tuple(items), meta=meta)
 
 
 @dataclass_cache_hash()
@@ -121,8 +121,8 @@ class Set(Collection, lang.Final):
             check.isinstance(self.items, tuple)
 
     @classmethod
-    def new(cls, items: ta.Iterable[ta.Any]) -> 'Set':
-        return cls(tuple(items))
+    def new(cls, items: ta.Iterable[ta.Any], *, meta: ta.Any | None = None) -> 'Set':
+        return cls(tuple(items), meta=meta)
 
 
 @dataclass_cache_hash()
@@ -141,8 +141,8 @@ class Map(Collection, lang.Final):
                 check.equal(len(t), 2)
 
     @classmethod
-    def new(cls, items: ta.Iterable[ta.Iterable[ta.Any]]) -> 'Map':
-        return cls(tuple((k, v) for k, v in items))
+    def new(cls, items: ta.Iterable[ta.Iterable[ta.Any]], *, meta: ta.Any | None = None) -> 'Map':
+        return cls(tuple((k, v) for k, v in items), meta=meta)
 
 
 #
@@ -150,7 +150,7 @@ class Map(Collection, lang.Final):
 
 @dataclass_cache_hash()
 @dc.dataclass(frozen=True)
-class TaggedVal(Value, lang.Final):
+class Tagged(Value, lang.Final):
     t: str
     v: ta.Any