omlish 0.0.0.dev104__py3-none-any.whl → 0.0.0.dev105__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev104'
-__revision__ = '1ffe2eb6d42217c0ee8b049c6c3b04fd970f9e66'
+__version__ = '0.0.0.dev105'
+__revision__ = '933f83fc2d561338d613829ed857297f339dcccd'
 
 
 #

omlish/fnpipes.py

@@ -6,6 +6,9 @@ T = ta.TypeVar('T')
 U = ta.TypeVar('U')
 
 
+##
+
+
 class Fn(abc.ABC, ta.Generic[T]):
     @abc.abstractmethod
     def __call__(self, *args: ta.Any, **kwargs: ta.Any) -> T:
@@ -24,6 +27,9 @@ class Fn(abc.ABC, ta.Generic[T]):
         return self.apply(fn)
 
 
+##
+
+
 class Bind(Fn[T]):
     def __init__(self, fn: ta.Callable[..., T], *args: ta.Any, **kwargs: ta.Any) -> None:
         super().__init__()
@@ -54,6 +60,12 @@ class Bind(Fn[T]):
         return self._fn(*fa, **fkw)
 
 
+bind = Bind
+
+
+##
+
+
 class Pipe(Fn[T]):
     def __init__(self, lfns: ta.Sequence[ta.Callable], rfn: ta.Callable[..., T]) -> None:
         super().__init__()
@@ -66,6 +78,14 @@ class Pipe(Fn[T]):
         return o
 
 
+def pipe(*fns: ta.Callable) -> Pipe:
+    *lfns, rfn = fns
+    return Pipe(lfns, rfn)
+
+
+##
+
+
 class Apply(Fn[T]):
     def __init__(self, *fns: ta.Callable[[T], ta.Any]) -> None:
         super().__init__()
@@ -77,6 +97,4 @@ class Apply(Fn[T]):
         return o
 
 
-bind = Bind
-pipe = Pipe
 apply = Apply

omlish/formats/json/__init__.py

@@ -3,6 +3,8 @@ from .consts import (  # noqa
     COMPACT_SEPARATORS,
     PRETTY_INDENT,
     PRETTY_KWARGS,
+    PRETTY_SEPARATORS,
+    Separators,
 )
 
 from .encoding import (  # noqa

omlish/formats/json/cli/cli.py

@@ -5,7 +5,7 @@ TODO:
 
 ==
 
-jq Command options:
+jq Command options)
   -n, --null-input          use `null` as the single input value;
   -R, --raw-input           read each line as string instead of JSON;
   -s, --slurp               read all inputs into an array and use it as the single input value;
@@ -38,47 +38,36 @@ jq Command options:
   --                        terminates argument processing;
 """
 import argparse
-import codecs
 import contextlib
+import dataclasses as dc
 import io
-import json
 import os
 import subprocess
 import sys
 import typing as ta
 
 from .... import check
+from .... import fnpipes as fp
 from .... import lang
-from .... import term
-from ....lite.io import DelimitingBuffer
-from ..render import JsonRenderer
-from ..stream.build import JsonObjectBuilder
-from ..stream.lex import JsonStreamLexer
-from ..stream.parse import JsonStreamParser
-from ..stream.render import StreamJsonRenderer
 from .formats import FORMATS_BY_NAME
+from .formats import Format
 from .formats import Formats
+from .parsing import DelimitingParser
+from .parsing import EagerParser
+from .parsing import StreamBuilder
+from .parsing import StreamParser
+from .processing import ProcessingOptions
+from .processing import Processor
+from .rendering import EagerRenderer
+from .rendering import RenderingOptions
+from .rendering import StreamRenderer
 
 
-if ta.TYPE_CHECKING:
-    from ....specs import jmespath
-else:
-    jmespath = lang.proxy_import('....specs.jmespath', __package__)
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
 
 
-##
-
-
-def term_color(o: ta.Any, state: JsonRenderer.State) -> tuple[str, str]:
-    if state is JsonRenderer.State.KEY:
-        return term.SGR(term.SGRs.FG.BRIGHT_BLUE), term.SGR(term.SGRs.RESET)
-    elif isinstance(o, str):
-        return term.SGR(term.SGRs.FG.GREEN), term.SGR(term.SGRs.RESET)
-    else:
-        return '', ''
-
-
-def _main() -> None:
+def _build_args_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
 
     parser.add_argument('file', nargs='?')
@@ -93,17 +82,52 @@ def _main() -> None:
     parser.add_argument('-f', '--format')
 
     parser.add_argument('-x', '--jmespath-expr')
+    parser.add_argument('-F', '--flat', action='store_true')
 
     parser.add_argument('-z', '--compact', action='store_true')
     parser.add_argument('-p', '--pretty', action='store_true')
     parser.add_argument('-i', '--indent')
     parser.add_argument('-s', '--sort-keys', action='store_true')
-
+    parser.add_argument('-R', '--raw', action='store_true')
+    parser.add_argument('-U', '--unicode', action='store_true')
     parser.add_argument('-c', '--color', action='store_true')
 
     parser.add_argument('-L', '--less', action='store_true')
 
-    args = parser.parse_args()
+    return parser
+
+
+def _parse_args(args: ta.Any = None) -> ta.Any:
+    return _build_args_parser().parse_args(args)
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class RunConfiguration:
+    format: Format
+    processing: ProcessingOptions
+    rendering: RenderingOptions
+
+
+def _process_args(args: ta.Any) -> RunConfiguration:
+    fmt_name = args.format
+    if fmt_name is None:
+        if args.file is not None:
+            ext = args.file.rpartition('.')[2]
+            if ext in FORMATS_BY_NAME:
+                fmt_name = ext
+    if fmt_name is None:
+        fmt_name = 'json'
+    format = FORMATS_BY_NAME[fmt_name]  # noqa
+
+    if args.stream:
+        check.arg(format is Formats.JSON.value)
+
+    #
+
+    processing = ProcessingOptions(
+        jmespath_expr=args.jmespath_expr,
+        flat=args.flat,
+    )
 
     #
 
@@ -120,48 +144,30 @@ def _main() -> None:
         except ValueError:
             indent = args.indent
 
-    kw: dict[str, ta.Any] = dict(
+    rendering = RenderingOptions(
         indent=indent,
         separators=separators,
         sort_keys=args.sort_keys,
+        raw=args.raw,
+        unicode=args.unicode,
+        color=args.color,
     )
 
-    if args.jmespath_expr is not None:
-        jp_expr = jmespath.compile(args.jmespath_expr)
-    else:
-        jp_expr = None
+    #
 
-    def render_one(v: ta.Any) -> str:
-        if jp_expr is not None:
-            v = jp_expr.search(v)
+    return RunConfiguration(
+        format=format,
+        processing=processing,
+        rendering=rendering,
+    )
 
-        if args.color:
-            return JsonRenderer.render_str(
-                v,
-                **kw,
-                style=term_color,
-            )
 
-        else:
-            return json.dumps(
-                v,
-                **kw,
-            )
+def _main() -> None:
+    args = _parse_args()
 
     #
 
-    fmt_name = args.format
-    if fmt_name is None:
-        if args.file is not None:
-            ext = args.file.rpartition('.')[2]
-            if ext in FORMATS_BY_NAME:
-                fmt_name = ext
-    if fmt_name is None:
-        fmt_name = 'json'
-    fmt = FORMATS_BY_NAME[fmt_name]
-
-    if args.stream:
-        check.arg(fmt is Formats.JSON.value)
+    cfg = _process_args(args)
 
     #
 
@@ -172,20 +178,31 @@ def _main() -> None:
         else:
             in_file = es.enter_context(open(args.file, 'rb'))
 
+        def yield_input() -> ta.Generator[bytes, None, None]:
+            fd = check.isinstance(in_file.fileno(), int)
+
+            while True:
+                buf = os.read(fd, args.read_buffer_size)
+
+                yield buf
+
+                if not buf:
+                    break
+
         #
 
         if args.less:
             less = subprocess.Popen(
                 [
                     'less',
-                    *(['-R'] if args.color else []),
+                    *(['-R'] if cfg.rendering.color else []),
                 ],
                 stdin=subprocess.PIPE,
                 encoding='utf-8',
             )
             out = check.not_none(less.stdin)
 
-            def close_less():
+            def close_less() -> None:
                 out.close()
                 less.wait()
 
@@ -196,67 +213,86 @@ def _main() -> None:
 
         #
 
-        if args.stream:
-            fd = in_file.fileno()
-            decoder = codecs.getincrementaldecoder('utf-8')()
+        parser: ta.Any
+        renderer: ta.Any
 
+        if args.stream:
             with contextlib.ExitStack() as es2:
-                lex = es2.enter_context(JsonStreamLexer())
-                parse = es2.enter_context(JsonStreamParser())
+                parser = es2.enter_context(StreamParser())
 
-                if args.stream_build:
-                    build = es2.enter_context(JsonObjectBuilder())
-                    renderer = None
+                def flush_output(
+                        fn: ta.Callable[[T], ta.Iterable[U]],
+                        i: T,
+                ) -> ta.Generator[U, None, None]:
+                    n = 0
+                    for o in fn(i):
+                        yield o
+                        n += 1
+                    if n:
+                        out.flush()
 
-                else:
-                    renderer = StreamJsonRenderer(
-                        out,
-                        style=term_color if args.color else None,
-                        delimiter='\n',
-                        **kw,
-                    )
-                    build = None
-
-                while True:
-                    buf = os.read(fd, args.read_buffer_size)
+                pipeline: ta.Any
 
-                    for s in decoder.decode(buf, not buf):
-                        n = 0
-                        for c in s:
-                            for t in lex(c):
-                                for e in parse(t):
-                                    if renderer is not None:
-                                        renderer.render((e,))
+                if args.stream_build:
+                    builder: StreamBuilder = es2.enter_context(StreamBuilder())
+                    processor = Processor(cfg.processing)
+                    renderer = EagerRenderer(cfg.rendering)
+                    trailing_newline = False
+
+                    def append_newlines(
+                            fn: ta.Callable[[T], ta.Iterable[str]],
+                            i: T,
+                    ) -> ta.Generator[str, None, None]:
+                        yield from fn(i)
+                        yield '\n'
+
+                    pipeline = lambda v: (renderer.render(v),)  # Any -> [str]  # noqa
+                    pipeline = fp.bind(append_newlines, pipeline)  # Any -> [str]
+                    pipeline = fp.bind(lang.flatmap, pipeline)  # [Any] -> [str]
+                    pipeline = fp.pipe(fp.bind(lang.flatmap, processor.process), pipeline)  # [Any] -> [str]
+                    pipeline = fp.pipe(fp.bind(lang.flatmap, builder.build), pipeline)  # [JsonStreamParserEvent] -> [str]  # noqa
+                    pipeline = fp.pipe(parser.parse, pipeline)  # bytes -> [str]
 
-                                    if build is not None:
-                                        for v in build(e):
-                                            print(render_one(v), file=out)
+                else:
+                    renderer = StreamRenderer(cfg.rendering)
+                    trailing_newline = True
 
-                                    n += 1
+                    pipeline = renderer.render  # JsonStreamParserEvent -> [str]
+                    pipeline = fp.bind(lang.flatmap, pipeline)  # [JsonStreamParserEvent] -> [str]
+                    pipeline = fp.pipe(parser.parse, pipeline)  # bytes -> [str]
 
-                        if n:
-                            out.flush()
+                pipeline = fp.bind(flush_output, pipeline)  # bytes -> [str]
 
-                    if not buf:
-                        break
+                for buf in yield_input():
+                    for s in pipeline(buf):
+                        print(s, file=out, end='')
 
-                if renderer is not None:
-                    out.write('\n')
+                if trailing_newline:
+                    print(file=out)
 
         elif args.lines:
-            fd = in_file.fileno()
-            db = DelimitingBuffer()
+            parser = DelimitingParser(cfg.format)
+            processor = Processor(cfg.processing)
+            renderer = EagerRenderer(cfg.rendering)
 
-            while buf := os.read(fd, args.read_buffer_size):
-                for chunk in db.feed(buf):
-                    s = check.isinstance(chunk, bytes).decode('utf-8')
-                    v = fmt.load(io.StringIO(s))
-                    print(render_one(v), file=out)
+            for buf in yield_input():
+                if buf:
+                    for v in parser.parse(buf):
+                        for e in processor.process(v):
+                            s = renderer.render(e)
+                            print(s, file=out)
 
         else:
-            with io.TextIOWrapper(in_file) as tw:
-                v = fmt.load(tw)
-            print(render_one(v), file=out)
+            parser = EagerParser(cfg.format)
+            processor = Processor(cfg.processing)
+            renderer = EagerRenderer(cfg.rendering)
+
+            with io.TextIOWrapper(in_file) as tf:
+                v = parser.parse(tf)
+
+            for e in processor.process(v):
+                s = renderer.render(e)
+                print(s, file=out)
 
 
 if __name__ == '__main__':

omlish/formats/json/cli/parsing.py

@@ -0,0 +1,82 @@
+import codecs
+import io
+import typing as ta
+
+from .... import check
+from .... import lang
+from ....lite.io import DelimitingBuffer
+from ..stream.build import JsonObjectBuilder
+from ..stream.lex import JsonStreamLexer
+from ..stream.parse import JsonStreamParser
+from ..stream.parse import JsonStreamParserEvent
+from .formats import Format
+
+
+##
+
+
+class EagerParser:
+    def __init__(self, fmt: Format) -> None:
+        super().__init__()
+
+        self._fmt = fmt
+
+    def parse(self, f: ta.TextIO) -> ta.Generator[ta.Any, None, None]:
+        return self._fmt.load(f)
+
+
+##
+
+
+class DelimitingParser:
+    def __init__(
+            self,
+            fmt: Format,
+            *,
+            delimiters: ta.Iterable[int] = b'\n',
+    ) -> None:
+        super().__init__()
+
+        self._fmt = fmt
+
+        self._db = DelimitingBuffer(delimiters)
+
+    def parse(self, b: bytes) -> ta.Generator[ta.Any, None, None]:
+        for chunk in self._db.feed(b):
+            s = check.isinstance(chunk, bytes).decode('utf-8')
+            v = self._fmt.load(io.StringIO(s))
+            yield v
+
+
+##
+
+
+class StreamBuilder(lang.ExitStacked):
+    _builder: JsonObjectBuilder | None = None
+
+    def __enter__(self) -> ta.Self:
+        super().__enter__()
+        self._builder = self._enter_context(JsonObjectBuilder())
+        return self
+
+    def build(self, e: JsonStreamParserEvent) -> ta.Generator[ta.Any, None, None]:
+        yield from check.not_none(self._builder)(e)
+
+
+class StreamParser(lang.ExitStacked):
+    _decoder: codecs.IncrementalDecoder
+    _lex: JsonStreamLexer
+    _parse: JsonStreamParser
+
+    def __enter__(self) -> ta.Self:
+        super().__enter__()
+        self._decoder = codecs.getincrementaldecoder('utf-8')()
+        self._lex = self._enter_context(JsonStreamLexer())
+        self._parse = self._enter_context(JsonStreamParser())
+        return self
+
+    def parse(self, b: bytes) -> ta.Generator[JsonStreamParserEvent, None, None]:
+        for s in self._decoder.decode(b, not b):
+            for c in s:
+                for t in self._lex(c):
+                    yield from self._parse(t)

omlish/formats/json/cli/processing.py

@@ -0,0 +1,44 @@
+import dataclasses as dc
+import typing as ta
+
+from .... import lang
+
+
+if ta.TYPE_CHECKING:
+    from ....specs import jmespath
+else:
+    jmespath = lang.proxy_import('....specs.jmespath', __package__)
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class ProcessingOptions:
+    jmespath_expr: ta.Any | None = None
+    flat: bool = False
+
+
+class Processor:
+    def __init__(self, opts: ProcessingOptions) -> None:
+        super().__init__()
+
+        self._opts = opts
+
+        jmespath_expr = opts.jmespath_expr
+        if isinstance(jmespath_expr, str):
+            jmespath_expr = jmespath.compile(jmespath_expr)
+        self._jmespath_expr: ta.Any | None = jmespath_expr
+
+    def process(self, v: ta.Any) -> ta.Iterable[ta.Any]:
+        if self._jmespath_expr is not None:
+            v = self._jmespath_expr.search(v)
+
+        if self._opts.flat:
+            if isinstance(v, str):
+                raise TypeError(f'Flat output must be arrays, got {type(v)}', v)
+
+            yield from v
+
+        else:
+            yield v

omlish/formats/json/cli/rendering.py

@@ -0,0 +1,92 @@
+import dataclasses as dc
+import json
+import typing as ta
+
+from .... import lang
+from .... import term
+from ..render import JsonRenderer
+from ..stream.parse import JsonStreamParserEvent
+from ..stream.render import StreamJsonRenderer
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class RenderingOptions:
+    indent: int | str | None = None
+    separators: tuple[str, str] | None = None
+    sort_keys: bool = False
+    raw: bool = False
+    unicode: bool = False
+    color: bool = False
+
+
+def make_render_kwargs(opts: RenderingOptions) -> ta.Mapping[str, ta.Any]:
+    return dict(
+        indent=opts.indent,
+        separators=opts.separators,
+        sort_keys=opts.sort_keys,
+        ensure_ascii=not opts.unicode,
+    )
+
+
+class Renderer(lang.Abstract):
+    def __init__(self, opts: RenderingOptions) -> None:
+        super().__init__()
+        self._opts = opts
+        self._kw = make_render_kwargs(opts)
+
+
+##
+
+
+def term_color(o: ta.Any, state: JsonRenderer.State) -> tuple[str, str]:
+    if state is JsonRenderer.State.KEY:
+        return term.SGR(term.SGRs.FG.BRIGHT_BLUE), term.SGR(term.SGRs.RESET)
+    elif isinstance(o, str):
+        return term.SGR(term.SGRs.FG.GREEN), term.SGR(term.SGRs.RESET)
+    else:
+        return '', ''
+
+
+##
+
+
+class EagerRenderer(Renderer):
+    def render(self, v: ta.Any) -> str:
+        if self._opts.raw:
+            if not isinstance(v, str):
+                raise TypeError(f'Raw output must be strings, got {type(v)}', v)
+
+            return v
+
+        elif self._opts.color:
+            return JsonRenderer.render_str(
+                v,
+                **self._kw,
+                style=term_color,
+            )
+
+        else:
+            return json.dumps(
+                v,
+                **self._kw,
+            )
+
+
+##
+
+
+class StreamRenderer(Renderer):
+    def __init__(self, opts: RenderingOptions) -> None:
+        super().__init__(opts)
+
+        self._renderer = StreamJsonRenderer(
+            style=term_color if self._opts.color else None,
+            delimiter='\n',
+            **self._kw,
+        )
+
+    def render(self, e: JsonStreamParserEvent) -> ta.Generator[str, None, None]:
+        return self._renderer.render((e,))

omlish/formats/json/consts.py

@@ -4,8 +4,18 @@ import typing as ta
 ##
 
 
+class Separators(ta.NamedTuple):
+    comma: str
+    colon: str
+
+
+##
+
+
 PRETTY_INDENT = 2
 
+PRETTY_SEPARATORS = Separators(', ', ': ')
+
 PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
     indent=PRETTY_INDENT,
 )
@@ -14,7 +24,7 @@ PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
 ##
 
 
-COMPACT_SEPARATORS = (',', ':')
+COMPACT_SEPARATORS = Separators(',', ':')
 
 COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
     indent=None,