omlish 0.0.0.dev101__tar.gz → 0.0.0.dev112__tar.gz

{omlish-0.0.0.dev101/omlish.egg-info → omlish-0.0.0.dev112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev101
+Version: 0.0.0.dev112
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/.manifests.json

@@ -23,18 +23,6 @@
       }
     }
   },
-  {
-    "module": ".diag.pycharm.cli",
-    "attr": "_CLI_MODULE",
-    "file": "omlish/diag/pycharm/cli.py",
-    "line": 38,
-    "value": {
-      "$omdev.cli.types.CliModule": {
-        "cmd_name": "pycharm",
-        "mod_name": "omlish.diag.pycharm.cli"
-      }
-    }
-  },
   {
     "module": ".formats.json.cli.__main__",
     "attr": "_CLI_MODULE",

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev101'
-__revision__ = '44c6ff624e1a09a5024ea5cbc8a6cdcb1f54f013'
+__version__ = '0.0.0.dev112'
+__revision__ = 'b12f9a554e6b2fb475bd820af89e0860c0b46215'
 
 
 #

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/concurrent/threadlets.py

@@ -94,3 +94,6 @@ class GreenletThreadlets(Threadlets):
 
     def get_current(self) -> Threadlet:
         return GreenletThreadlet(greenlet.getcurrent())
+
+
+GREENLET_THREADLETS = GreenletThreadlets()

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/dataclasses/impl/reflect.py

@@ -3,7 +3,6 @@ TODO:
  - more cache-recursive reuse - fields, mro, etc
 """
 import dataclasses as dc
-import inspect
 import sys
 import typing as ta
 import weakref
@@ -28,22 +27,9 @@ from .params import get_params_extras
 from .utils import Namespace
 
 
-try:
-    import annotationlib  # noqa
-except ImportError:
-    annotationlib = None
-
-
 MISSING = dc.MISSING
 
 
-def _get_annotations(obj):
-    if annotationlib is not None:
-        return annotationlib.get_annotations(obj, format=annotationlib.Format.FORWARDREF)  # noqa
-    else:
-        return inspect.get_annotations(obj)
-
-
 class ClassInfo:
 
     def __init__(self, cls: type, *, _constructing: bool = False) -> None:
@@ -67,7 +53,10 @@ class ClassInfo:
 
     @cached.property
     def cls_annotations(self) -> ta.Mapping[str, ta.Any]:
-        return _get_annotations(self._cls)
+        # Does not use ta.get_type_hints because that's what std dataclasses do [1]. Might be worth revisiting? A part
+        # of why they don't is to not import typing for efficiency but we don't care about that degree of startup speed.
+        # [1]: https://github.com/python/cpython/blob/54c63a32d06cb5f07a66245c375eac7d7efb964a/Lib/dataclasses.py#L985-L986  # noqa
+        return rfl.get_annotations(self._cls)
 
     ##
 

{omlish-0.0.0.dev101/omlish/diag/pycharm → omlish-0.0.0.dev112/omlish/diag}/pycharm.py

@@ -1,19 +1,21 @@
 import dataclasses as dc
+import json
 import os.path
 import plistlib
 import re
+import shutil
 import subprocess
 import sys
 import typing as ta
 
-from ... import check
-from ... import lang
+from .. import check
+from .. import lang
 
 
 if ta.TYPE_CHECKING:
-    docker = lang.proxy_import('...docker')
+    docker = lang.proxy_import('..docker')
 else:
-    from ... import docker
+    from .. import docker
 
 
 ##
@@ -43,6 +45,26 @@ def read_darwin_pycharm_info_plist() -> ta.Mapping[str, ta.Any] | None:
     return root
 
 
+#
+
+
+UBUNTU_PYCHARM_HOME = '/snap/pycharm-professional/current'
+
+
+def read_ubuntu_pycharm_product_info() -> ta.Mapping[str, ta.Any] | None:
+    json_file = os.path.join(UBUNTU_PYCHARM_HOME, 'product-info.json')
+    if not os.path.isfile(json_file):
+        return None
+
+    with open(json_file) as f:
+        root = json.load(f)
+
+    return root
+
+
+#
+
+
 @lang.cached_function
 def get_pycharm_version() -> str | None:
     if sys.platform == 'darwin':
@@ -54,6 +76,17 @@ def get_pycharm_version() -> str | None:
         check.state(ver.startswith('PY-'))
         return ver[3:]
 
+    elif sys.platform == 'linux':
+        if shutil.which('lsb_release') is not None:
+            lsb_id = subprocess.check_output(['lsb_release', '-is']).decode().strip()
+            if lsb_id == 'Ubuntu':
+                pi = read_ubuntu_pycharm_product_info()
+                if pi is not None:
+                    ver = check.non_empty_str(pi['buildNumber'])
+                    return ver
+
+        return None
+
     else:
         return None
 

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/docker/helpers.py

@@ -26,7 +26,7 @@ def timebomb_payload(delay_s: float, name: str = _DEFAULT_TIMEBOMB_NAME) -> str:
 DOCKER_FOR_MAC_HOSTNAME = 'docker.for.mac.localhost'
 
 
-_LIKELY_IN_DOCKER_PATTERN = re.compile(r'^overlay / .*/docker/')
+_LIKELY_IN_DOCKER_PATTERN = re.compile(r'^overlay / .*/(docker|desktop-containerd)/')
 
 
 def is_likely_in_docker() -> bool:

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/fnpairs.py

@@ -26,7 +26,6 @@ from . import lang
 if ta.TYPE_CHECKING:
     import bz2 as _bz2
     import gzip as _gzip
-    import json as _json
     import lzma as _lzma
     import pickle as _pickle
     import struct as _struct
@@ -40,10 +39,11 @@ if ta.TYPE_CHECKING:
     import yaml as _yaml
     import zstandard as _zstandard
 
+    from .formats import json as _json
+
 else:
     _bz2 = lang.proxy_import('bz2')
     _gzip = lang.proxy_import('gzip')
-    _json = lang.proxy_import('json')
     _lzma = lang.proxy_import('lzma')
     _pickle = lang.proxy_import('pickle')
     _struct = lang.proxy_import('struct')
@@ -57,6 +57,8 @@ else:
     _yaml = lang.proxy_import('yaml')
     _zstandard = lang.proxy_import('zstandard')
 
+    _json = lang.proxy_import('.formats.json', __package__)
+
 
 ##
 
@@ -277,7 +279,7 @@ def _register_extension(*ss):
     def inner(cls):
         for s in ss:
             if s in _EXTENSION_REGISTRY:
-                raise Exception(s)
+                raise KeyError(s)
             _EXTENSION_REGISTRY[s] = cls
         return cls
     return inner
@@ -409,22 +411,30 @@ class Pickle(ObjectBytes_):
         return _pickle.loads(t)
 
 
+class _Json(ObjectStr_, lang.Abstract):  # noqa
+    def backward(self, t: str) -> ta.Any:
+        return _json.loads(t)
+
+
 @_register_extension('json')
-@dc.dataclass(frozen=True)
-class Json(ObjectStr_):
-    indent: int | str | None = dc.field(default=None, kw_only=True)
-    separators: tuple[str, str] | None = dc.field(default=None, kw_only=True)
+class Json(_Json):
+    def forward(self, f: ta.Any) -> str:
+        return _json.dumps(f)
+
 
+class JsonPretty(_Json):
     def forward(self, f: ta.Any) -> str:
-        return _json.dumps(f, indent=self.indent, separators=self.separators)
+        return _json.dumps_pretty(f)
 
-    def backward(self, t: str) -> ta.Any:
-        return _json.loads(t)
+
+class JsonCompact(_Json):
+    def forward(self, f: ta.Any) -> str:
+        return _json.dumps_compact(f)
 
 
 JSON = Json()
-PRETTY_JSON = Json(indent=2)
-COMPACT_JSON = Json(separators=(',', ':'))
+PRETTY_JSON = JsonPretty()
+COMPACT_JSON = JsonCompact()
 
 
 @_register_extension('jsonl')

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/fnpipes.py

@@ -6,22 +6,34 @@ T = ta.TypeVar('T')
 U = ta.TypeVar('U')
 
 
+##
+
+
 class Fn(abc.ABC, ta.Generic[T]):
     @abc.abstractmethod
     def __call__(self, *args: ta.Any, **kwargs: ta.Any) -> T:
         raise NotImplementedError
 
     def pipe(self, fn: ta.Callable[..., U], *args: ta.Any, **kwargs: ta.Any) -> 'Fn[U]':
-        return Pipe([self], Bind(fn, *args, **kwargs))
+        return pipe(self, bind(fn, *args, **kwargs))
 
     def __or__(self, fn: ta.Callable[..., U]) -> 'Fn[U]':
-        return self.pipe(fn)
+        return pipe(self, fn)
+
+    def __ror__(self, fn: ta.Callable[..., U]) -> 'Fn[U]':
+        return pipe(fn, self)
 
     def apply(self, fn: ta.Callable[[T], ta.Any], *args: ta.Any, **kwargs: ta.Any) -> 'Fn[T]':
-        return Pipe([self], Apply(Bind(fn, *args, **kwargs)))
+        return pipe(self, apply(bind(fn, *args, **kwargs)))
 
     def __and__(self, fn: ta.Callable[[T], ta.Any]) -> 'Fn[T]':
-        return self.apply(fn)
+        return pipe(self, apply(fn))
+
+    def __rand__(self, fn: ta.Callable[[T], ta.Any]) -> 'Fn[T]':
+        return pipe(fn, apply(self))
+
+
+##
 
 
 class Bind(Fn[T]):
@@ -54,6 +66,12 @@ class Bind(Fn[T]):
         return self._fn(*fa, **fkw)
 
 
+bind = Bind
+
+
+##
+
+
 class Pipe(Fn[T]):
     def __init__(self, lfns: ta.Sequence[ta.Callable], rfn: ta.Callable[..., T]) -> None:
         super().__init__()
@@ -66,6 +84,14 @@ class Pipe(Fn[T]):
         return o
 
 
+def pipe(*fns: ta.Callable) -> Pipe:
+    *lfns, rfn = fns
+    return Pipe(lfns, rfn)
+
+
+##
+
+
 class Apply(Fn[T]):
     def __init__(self, *fns: ta.Callable[[T], ta.Any]) -> None:
         super().__init__()
@@ -77,6 +103,4 @@ class Apply(Fn[T]):
         return o
 
 
-bind = Bind
-pipe = Pipe
 apply = Apply

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/formats/dotenv.py

@@ -324,10 +324,13 @@ def _with_warn_for_invalid_lines(mappings: ta.Iterator[Binding]) -> ta.Iterator[
         yield mapping
 
 
+StrMutableMappingT = ta.TypeVar('StrMutableMappingT', bound=ta.MutableMapping[str, str])
+
+
 class DotEnv:
     def __init__(
         self,
-        path: StrPath | None,
+        path: StrPath | None = None,
         stream: ta.IO[str] | None = None,
         verbose: bool = False,
         encoding: str | None = None,
@@ -373,6 +376,14 @@ class DotEnv:
 
         return self._dict
 
+    def apply_to(self, dst: StrMutableMappingT) -> StrMutableMappingT:
+        for k, v in self.dict().items():
+            if v is not None:
+                dst[k] = v
+            elif k in dst:
+                del dst[k]
+        return dst
+
     def parse(self) -> ta.Iterator[tuple[str, str | None]]:
         with self._get_stream() as stream:
             for mapping in _with_warn_for_invalid_lines(parse_stream(stream)):

{omlish-0.0.0.dev101 → omlish-0.0.0.dev112}/omlish/formats/json/__init__.py

@@ -1,8 +1,16 @@
+# ruff: noqa: I001
+import typing as _ta
+
+from ... import lang as _lang
+
+
 from .consts import (  # noqa
     COMPACT_KWARGS,
     COMPACT_SEPARATORS,
     PRETTY_INDENT,
     PRETTY_KWARGS,
+    PRETTY_SEPARATORS,
+    Separators,
 )
 
 from .encoding import (  # noqa
@@ -21,6 +29,11 @@ from .json import (  # noqa
     loads,
 )
 
-from .render import (  # noqa
-    JsonRenderer,
-)
+if _ta.TYPE_CHECKING:
+    from .render import (  # noqa
+        JsonRenderer,
+    )
+else:
+    _lang.proxy_init(globals(), '.render', [
+        'JsonRenderer',
+    ])

omlish-0.0.0.dev112/omlish/formats/json/cli/cli.py

@@ -0,0 +1,298 @@
+"""
+TODO:
+ - read from http
+ - jmespath output flat, unquoted strs like jq '.[]'
+
+==
+
+jq Command options:
+  -n, --null-input          use `null` as the single input value;
+  -R, --raw-input           read each line as string instead of JSON;
+  -s, --slurp               read all inputs into an array and use it as the single input value;
+  -c, --compact-output      compact instead of pretty-printed output;
+  -r, --raw-output          output strings without escapes and quotes;
+      --raw-output0         implies -r and output NUL after each output;
+  -j, --join-output         implies -r and output without newline after each output;
+  -a, --ascii-output        output strings by only ASCII characters using escape sequences;
+  -S, --sort-keys           sort keys of each object on output;
+  -C, --color-output        colorize JSON output;
+  -M, --monochrome-output   disable colored output;
+      --tab                 use tabs for indentation;
+      --indent n            use n spaces for indentation (max 7 spaces);
+      --unbuffered          flush output stream after each output;
+      --stream              parse the input value in streaming fashion;
+      --stream-errors       implies --stream and report parse error as an array;
+      --seq                 parse input/output as application/json-seq;
+  -f, --from-file file      load filter from the file;
+  -L directory              search modules from the directory;
+      --arg name value      set $name to the string value;
+      --argjson name value  set $name to the JSON value;
+      --slurpfile name file set $name to an array of JSON values read from the file;
+      --rawfile name file   set $name to string contents of file;
+      --args                consume remaining arguments as positional string values;
+      --jsonargs            consume remaining arguments as positional JSON values;
+  -e, --exit-status         set exit status code based on the output;
+  -V, --version             show the version;
+  --build-configuration     show jq's build configuration;
+  -h, --help                show the help;
+  --                        terminates argument processing;
+"""
+import argparse
+import contextlib
+import dataclasses as dc
+import io
+import os
+import subprocess
+import sys
+import typing as ta
+
+from .... import check
+from .... import fnpipes as fp
+from .... import lang
+from .formats import FORMATS_BY_NAME
+from .formats import Format
+from .formats import Formats
+from .parsing import DelimitingParser
+from .parsing import EagerParser
+from .parsing import StreamBuilder
+from .parsing import StreamParser
+from .processing import ProcessingOptions
+from .processing import Processor
+from .rendering import EagerRenderer
+from .rendering import RenderingOptions
+from .rendering import StreamRenderer
+
+
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
+
+
+def _build_args_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('file', nargs='?')
+
+    parser.add_argument('--stream', action='store_true')
+    parser.add_argument('--stream-build', action='store_true')
+
+    parser.add_argument('-l', '--lines', action='store_true')
+
+    parser.add_argument('--read-buffer-size', type=int, default=0x4000)
+
+    parser.add_argument('-f', '--format')
+
+    parser.add_argument('-x', '--jmespath-expr')
+    parser.add_argument('-F', '--flat', action='store_true')
+
+    parser.add_argument('-z', '--compact', action='store_true')
+    parser.add_argument('-p', '--pretty', action='store_true')
+    parser.add_argument('-i', '--indent')
+    parser.add_argument('-s', '--sort-keys', action='store_true')
+    parser.add_argument('-R', '--raw', action='store_true')
+    parser.add_argument('-U', '--unicode', action='store_true')
+    parser.add_argument('-c', '--color', action='store_true')
+
+    parser.add_argument('-L', '--less', action='store_true')
+
+    return parser
+
+
+def _parse_args(args: ta.Any = None) -> ta.Any:
+    return _build_args_parser().parse_args(args)
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class RunConfiguration:
+    format: Format
+    processing: ProcessingOptions
+    rendering: RenderingOptions
+
+
+def _process_args(args: ta.Any) -> RunConfiguration:
+    fmt_name = args.format
+    if fmt_name is None:
+        if args.file is not None:
+            ext = args.file.rpartition('.')[2]
+            if ext in FORMATS_BY_NAME:
+                fmt_name = ext
+    if fmt_name is None:
+        fmt_name = 'json'
+    format = FORMATS_BY_NAME[fmt_name]  # noqa
+
+    if args.stream:
+        check.arg(format is Formats.JSON.value)
+
+    #
+
+    processing = ProcessingOptions(
+        jmespath_expr=args.jmespath_expr,
+        flat=args.flat,
+    )
+
+    #
+
+    separators = None
+    if args.compact:
+        separators = (',', ':')
+
+    indent = None
+    if args.pretty:
+        indent = 2
+    if args.indent:
+        try:
+            indent = int(args.indent)
+        except ValueError:
+            indent = args.indent
+
+    rendering = RenderingOptions(
+        indent=indent,
+        separators=separators,
+        sort_keys=args.sort_keys,
+        raw=args.raw,
+        unicode=args.unicode,
+        color=args.color,
+    )
+
+    #
+
+    return RunConfiguration(
+        format=format,
+        processing=processing,
+        rendering=rendering,
+    )
+
+
+def _main() -> None:
+    args = _parse_args()
+
+    #
+
+    cfg = _process_args(args)
+
+    #
+
+    with contextlib.ExitStack() as es:
+        if args.file is None:
+            in_file = sys.stdin.buffer
+
+        else:
+            in_file = es.enter_context(open(args.file, 'rb'))
+
+        def yield_input() -> ta.Generator[bytes, None, None]:
+            fd = check.isinstance(in_file.fileno(), int)
+
+            while True:
+                buf = os.read(fd, args.read_buffer_size)
+
+                yield buf
+
+                if not buf:
+                    break
+
+        #
+
+        if args.less:
+            less = subprocess.Popen(
+                [
+                    'less',
+                    *(['-R'] if cfg.rendering.color else []),
+                ],
+                stdin=subprocess.PIPE,
+                encoding='utf-8',
+            )
+            out = check.not_none(less.stdin)
+
+            def close_less() -> None:
+                out.close()
+                less.wait()
+
+            es.enter_context(lang.defer(close_less))  # noqa
+
+        else:
+            out = sys.stdout
+
+        #
+
+        parser: ta.Any
+        renderer: ta.Any
+
+        if args.stream:
+            with contextlib.ExitStack() as es2:
+                parser = es2.enter_context(StreamParser())
+
+                def flush_output(
+                        fn: ta.Callable[[T], ta.Iterable[U]],
+                        i: T,
+                ) -> ta.Generator[U, None, None]:
+                    n = 0
+                    for o in fn(i):
+                        yield o
+                        n += 1
+                    if n:
+                        out.flush()
+
+                pipeline: ta.Any
+
+                if args.stream_build:
+                    builder: StreamBuilder = es2.enter_context(StreamBuilder())
+                    processor = Processor(cfg.processing)
+                    renderer = EagerRenderer(cfg.rendering)
+                    trailing_newline = False
+
+                    def append_newlines(
+                            fn: ta.Callable[[T], ta.Iterable[str]],
+                            i: T,
+                    ) -> ta.Generator[str, None, None]:
+                        yield from fn(i)
+                        yield '\n'
+
+                    pipeline = lambda v: (renderer.render(v),)  # Any -> [str]  # noqa
+                    pipeline = fp.bind(append_newlines, pipeline)  # Any -> [str]
+                    pipeline = fp.bind(lang.flatmap, pipeline)  # [Any] -> [str]
+                    pipeline = fp.pipe(fp.bind(lang.flatmap, processor.process), pipeline)  # [Any] -> [str]
+                    pipeline = fp.pipe(fp.bind(lang.flatmap, builder.build), pipeline)  # [JsonStreamParserEvent] -> [str]  # noqa
+                    pipeline = fp.pipe(parser.parse, pipeline)  # bytes -> [str]
+
+                else:
+                    renderer = StreamRenderer(cfg.rendering)
+                    trailing_newline = True
+
+                    pipeline = renderer.render  # JsonStreamParserEvent -> [str]
+                    pipeline = fp.bind(lang.flatmap, pipeline)  # [JsonStreamParserEvent] -> [str]
+                    pipeline = fp.pipe(parser.parse, pipeline)  # bytes -> [str]
+
+                pipeline = fp.bind(flush_output, pipeline)  # bytes -> [str]
+
+                for buf in yield_input():
+                    for s in pipeline(buf):
+                        print(s, file=out, end='')
+
+                if trailing_newline:
+                    print(file=out)
+
+        elif args.lines:
+            parser = DelimitingParser(cfg.format)
+            processor = Processor(cfg.processing)
+            renderer = EagerRenderer(cfg.rendering)
+
+            for buf in yield_input():
+                for v in parser.parse(buf):
+                    for e in processor.process(v):
+                        s = renderer.render(e)
+                        print(s, file=out)
+
+        else:
+            parser = EagerParser(cfg.format)
+            processor = Processor(cfg.processing)
+            renderer = EagerRenderer(cfg.rendering)
+
+            with io.TextIOWrapper(in_file) as tf:
+                v = parser.parse(tf)
+
+            for e in processor.process(v):
+                s = renderer.render(e)
+                print(s, file=out)
+
+
+if __name__ == '__main__':
+    _main()