omlish 0.0.0.dev2__py3-none-any.whl → 0.0.0.dev4__py3-none-any.whl

omlish/{replserver → diag/replserver}/server.py

@@ -23,8 +23,8 @@ import threading
 import typing as ta
 import weakref
 
-from .. import check
-from .. import dataclasses as dc
+from ... import check
+from ... import dataclasses as dc
 from .console import InteractiveSocketConsole
 
 
@@ -62,12 +62,12 @@ class ReplServer:
     def path(self) -> str:
         return self._config.path
 
-    def __enter__(self):
+    def __enter__(self) -> ta.Self:
         check.state(not self._is_running)
         check.state(not self._is_shutdown.is_set())
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
         if not self._is_shutdown.is_set():
             self.shutdown(True, self._config.exit_timeout)
 
@@ -91,7 +91,7 @@ class ReplServer:
                 while not self._should_shutdown:
                     try:
                         conn, _ = self._socket.accept()
-                    except sock.timeout:
+                    except TimeoutError:
                         continue
 
                     log.info(f'Got repl server connection on file {self._config.path}')
@@ -117,13 +117,13 @@ class ReplServer:
                         name=self.CONNECTION_THREAD_NAME)
                     thread.start()
 
-                for thread, console in self._consoles_by_threads.items():
+                for console in self._consoles_by_threads.values():
                     try:
                         console.conn.close()
                     except Exception:
                         log.exception('Error shutting down')
 
-                for thread in self._consoles_by_threads.keys():
+                for thread in self._consoles_by_threads:
                     try:
                         thread.join(self._config.exit_timeout)
                     except Exception:
@@ -141,6 +141,6 @@ class ReplServer:
             self._is_shutdown.wait(timeout=timeout)
 
 
-def run():
+def run() -> None:
     with ReplServer(ReplServer.Config('repl.sock')) as repl_server:
         repl_server.run()

omlish/dispatch/dispatch.py

@@ -1,4 +1,5 @@
 import abc
+import contextlib
 import typing as ta
 import weakref
 
@@ -13,14 +14,12 @@ T = ta.TypeVar('T')
 ##
 
 
-_IMPL_FUNC_CLS_SET_CACHE: ta.MutableMapping[ta.Callable, ta.FrozenSet[type]] = weakref.WeakKeyDictionary()
+_IMPL_FUNC_CLS_SET_CACHE: ta.MutableMapping[ta.Callable, frozenset[type]] = weakref.WeakKeyDictionary()
 
 
-def get_impl_func_cls_set(func: ta.Callable) -> ta.FrozenSet[type]:
-    try:
+def get_impl_func_cls_set(func: ta.Callable) -> frozenset[type]:
+    with contextlib.suppress(KeyError):
         return _IMPL_FUNC_CLS_SET_CACHE[func]
-    except KeyError:
-        pass
 
     ann = getattr(func, '__annotations__', {})
     if not ann:
@@ -78,10 +77,8 @@ class Dispatcher(ta.Generic[T]):
         def cache_remove(k, self_ref=weakref.ref(self)):
             if (ref_self := self_ref()) is not None:
                 cache = ref_self._get_dispatch_cache()  # noqa
-                try:
+                with contextlib.suppress(KeyError):
                     del cache[k]
-                except KeyError:
-                    pass
 
         cache_token: ta.Any = None
         self._get_cache_token = lambda: cache_token

omlish/dispatch/functions.py

@@ -10,7 +10,7 @@ from .dispatch import get_impl_func_cls_set
 USE_EXTENSION = False
 
 
-def function(func):
+def function(func):  # noqa
     disp = Dispatcher()  # type: ignore
     disp.register(func, [object])
 

omlish/dispatch/methods.py

@@ -4,6 +4,7 @@ TODO:
  - ALT: A.f(super(), ... ? :/
  - classmethod/staticmethod
 """
+import contextlib
 import functools
 import typing as ta
 import weakref
@@ -23,7 +24,7 @@ def build_mro_dct(instance_cls: type, owner_cls: type | None = None) -> ta.Mappi
     try:
         pos = mro.index(owner_cls)
     except ValueError:
-        raise TypeError(f'Owner class {owner_cls} not in mro of instance class {instance_cls}')
+        raise TypeError(f'Owner class {owner_cls} not in mro of instance class {instance_cls}') from None
     dct: dict[str, ta.Any] = {}
     for cur_cls in mro[:pos + 1]:
         dct.update(cur_cls.__dict__)
@@ -55,10 +56,8 @@ class Method:
         def dispatch_func_cache_remove(k, self_ref=weakref.ref(self)):
             if (ref_self := self_ref()) is not None:
                 cache = ref_self._dispatch_func_cache  # noqa
-                try:
+                with contextlib.suppress(KeyError):
                     del cache[k]
-                except KeyError:
-                    pass
 
         self._dispatch_func_cache_remove = dispatch_func_cache_remove
 
@@ -158,5 +157,5 @@ class Method:
         return func.__get__(instance)(*args, **kwargs)  # noqa
 
 
-def method(func):
+def method(func):  # noqa
     return Method(func)

omlish/docker.py

@@ -127,7 +127,7 @@ class ComposeConfig:
 
     @lang.cached_function
     def get_config(self) -> ta.Mapping[str, ta.Any]:
-        with open(check.not_none(self._file_path), 'r') as f:
+        with open(check.not_none(self._file_path)) as f:
             buf = f.read()
         return yaml.safe_load(buf)
 

omlish/dynamic.py

@@ -25,7 +25,7 @@ _HOISTED_CODE_DEPTH: ta.MutableMapping[types.CodeType, int] = weakref.WeakKeyDic
 _MAX_HOIST_DEPTH = 0
 
 
-def hoist(depth=0):
+def hoist(depth=0):  # noqa
     def inner(fn):
         _HOISTED_CODE_DEPTH[fn.__code__] = depth
         global _MAX_HOIST_DEPTH
@@ -49,7 +49,7 @@ class Var(ta.Generic[T]):
 
     def __init__(
             self,
-            default: type[MISSING] | T = MISSING,  # type: ignore
+            default: type[MISSING] | T = MISSING,
             *,
             new: ta.Callable[[], T] | type[MISSING] = MISSING,
             validate: ta.Callable[[T], None] | None = None,
@@ -87,7 +87,7 @@ class Var(ta.Generic[T]):
             self._validate(self.value)
         return Binding(self, value, offset=offset)
 
-    def with_binding(self, value):
+    def with_binding(self, value):  # noqa
         def outer(fn):
             @functools.wraps(fn)
             def inner(*args, **kwargs):
@@ -96,7 +96,7 @@ class Var(ta.Generic[T]):
             return inner
         return outer
 
-    def with_binding_fn(self, binding_fn):
+    def with_binding_fn(self, binding_fn):  # noqa
         this = self
 
         def outer(fn):
@@ -104,7 +104,7 @@ class Var(ta.Generic[T]):
 
                 @staticmethod
                 @functools.wraps(fn)
-                def __call__(*args, **kwargs):
+                def __call__(*args, **kwargs):  # noqa
                     with this.binding(binding_fn(*args, **kwargs)):
                         return fn(*args, **kwargs)
 
@@ -119,7 +119,7 @@ class Var(ta.Generic[T]):
 
                     return inner
 
-            dct = dict((k, getattr(fn, k)) for k in functools.WRAPPER_ASSIGNMENTS)
+            dct: dict[str, ta.Any] = {k: getattr(fn, k) for k in functools.WRAPPER_ASSIGNMENTS}
             return lang.new_type(fn.__name__, (Descriptor,), dct)()
 
         return outer
@@ -133,12 +133,12 @@ class Var(ta.Generic[T]):
             except KeyError:
                 pass
             else:
-                for level, frame_binding in sorted(frame_bindings.items()):
+                for level, frame_binding in sorted(frame_bindings.items()):  # noqa
                     yield frame_binding._value  # noqa
             frame = frame.f_back
 
         if self._new is not MISSING:
-            yield self._new()
+            yield self._new()  # type: ignore
 
     def __iter__(self) -> ta.Iterator[T]:
         return self.values
@@ -148,7 +148,7 @@ class Var(ta.Generic[T]):
         try:
             return next(self.values)
         except StopIteration:
-            raise UnboundVarError
+            raise UnboundVarError from None
 
 
 class Binding(ta.Generic[T]):
@@ -213,7 +213,7 @@ class _GeneratorContextManager(contextlib._GeneratorContextManager):  # noqa
         return super().__enter__()
 
 
-def contextmanager(fn):
+def contextmanager(fn):  # noqa
     @functools.wraps(fn)
     def helper(*args, **kwds):
         return _GeneratorContextManager(fn, args, kwds)

omlish/fnpairs.py

@@ -0,0 +1,400 @@
+"""
+TODO:
+ - objects
+  - csv
+  - csvloader
+  - cbor
+  - cloudpickle
+  - alt json backends
+ - compression
+  - snappy
+  - lz4
+ - wrapped (wait for usecase)
+"""
+import abc
+import codecs
+import dataclasses as dc
+import typing as ta
+
+from . import lang
+
+if ta.TYPE_CHECKING:
+    import bz2 as _bz2
+    import cloudpickle as _cloudpickle
+    import gzip as _gzip
+    import json as _json
+    import lz4.frame as _lz4_frame
+    import lzma as _lzma
+    import pickle as _pickle
+    import snappy as _snappy
+    import struct as _struct
+    import tomllib as _tomllib
+    import yaml as _yaml
+    import zstd as _zstd
+
+else:
+    _bz2 = lang.proxy_import('bz2')
+    _cloudpickle = lang.proxy_import('cloudpickle')
+    _gzip = lang.proxy_import('gzip')
+    _json = lang.proxy_import('json')
+    _lz4_frame = lang.proxy_import('lz4.frame')
+    _lzma = lang.proxy_import('lzma')
+    _pickle = lang.proxy_import('pickle')
+    _snappy = lang.proxy_import('snappy')
+    _struct = lang.proxy_import('struct')
+    _tomllib = lang.proxy_import('tomllib')
+    _yaml = lang.proxy_import('yaml')
+    _zstd = lang.proxy_import('zstd')
+
+
+##
+
+
+F = ta.TypeVar('F')
+F2 = ta.TypeVar('F2')
+T = ta.TypeVar('T')
+T2 = ta.TypeVar('T2')
+U = ta.TypeVar('U')
+
+
+class FnPair(ta.Generic[F, T], abc.ABC):
+    @abc.abstractmethod
+    def forward(self, f: F) -> T:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def backward(self, t: T) -> F:
+        raise NotImplementedError
+
+    ##
+
+    def __call__(self, f: F) -> T:
+        return self.forward(f)
+
+    def invert(self) -> 'FnPair[T, F]':
+        if isinstance(self, Inverted):
+            return self.fp
+        return Inverted(self)
+
+    def compose(self, nxt: 'FnPair[T, U]') -> 'FnPair[F, U]':
+        return Composite((self, nxt))
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Simple(FnPair[F, T]):
+    forward: ta.Callable[[F], T]  # type: ignore
+    backward: ta.Callable[[T], F]  # type: ignore
+
+    def _forward(self, f: F) -> T:
+        return self.forward(f)
+
+    def _backward(self, t: T) -> F:
+        return self.backward(t)
+
+
+# HACK: ABC workaround. Our dataclasses handle this with `override=True` but we don't want to dep that in here.
+Simple.forward = Simple._forward  # type: ignore  # noqa
+Simple.backward = Simple._backward  # type: ignore  # noqa
+Simple.__abstractmethods__ = frozenset()  # noqa
+
+
+of = Simple
+
+NOP: FnPair[ta.Any, ta.Any] = of(lang.identity, lang.identity)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Inverted(FnPair[F, T]):
+    fp: FnPair[T, F]
+
+    def forward(self, f: F) -> T:
+        return self.fp.backward(f)
+
+    def backward(self, t: T) -> F:
+        return self.fp.forward(t)
+
+
+@dc.dataclass(frozen=True)
+class Composite(FnPair[F, T]):
+    children: ta.Sequence[FnPair]
+
+    def forward(self, f: F) -> T:
+        for c in self.children:
+            f = c.forward(f)
+        return ta.cast(T, f)
+
+    def backward(self, t: T) -> F:
+        for c in reversed(self.children):
+            t = c.backward(t)
+        return ta.cast(F, t)
+
+
+def compose(*ps: FnPair) -> FnPair:
+    if not ps:
+        return NOP
+    if len(ps) == 1:
+        return ps[0]
+    return Composite(ps)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Text(FnPair[str, bytes]):
+    ci: codecs.CodecInfo
+    encode_errors: str = dc.field(default='strict', kw_only=True)
+    decode_errors: str = dc.field(default='strict', kw_only=True)
+
+    def forward(self, f: str) -> bytes:
+        # Python ignores the returned length:
+        #   https://github.com/python/cpython/blob/7431c3799efbd06ed03ee70b64420f45e83b3667/Python/codecs.c#L424
+        t, _ = self.ci.encode(f, self.encode_errors)
+        return t
+
+    def backward(self, t: bytes) -> str:
+        f, _ = self.ci.decode(t, self.decode_errors)
+        return f
+
+
+def text(name: str, *, encode_errors: str = 'strict', decode_errors: str = 'strict') -> Text:
+    ci = codecs.lookup(name)
+    if not ci._is_text_encoding:  # noqa
+        raise TypeError(f'must be text codec: {name}')
+    return Text(ci, encode_errors=encode_errors, decode_errors=decode_errors)
+
+
+UTF8 = text('utf-8')
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class Optional(FnPair[ta.Optional[F], ta.Optional[T]]):
+    fp: FnPair[F, T]
+
+    def forward(self, f: ta.Optional[F]) -> ta.Optional[T]:
+        return None if f is None else self.fp.forward(f)
+
+    def backward(self, t: ta.Optional[T]) -> ta.Optional[F]:
+        return None if t is None else self.fp.backward(t)
+
+
+class Lines(FnPair[ta.Sequence[str], str]):
+    def forward(self, f: ta.Sequence[str]) -> str:
+        return '\n'.join(f)
+
+    def backward(self, t: str) -> ta.Sequence[str]:
+        return t.splitlines()
+
+
+##
+
+
+_EXTENSION_REGISTRY: dict[str, type[FnPair]] = {}
+
+
+def _register_extension(*ss):
+    def inner(cls):
+        for s in ss:
+            if s in _EXTENSION_REGISTRY:
+                raise Exception(s)
+            _EXTENSION_REGISTRY[s] = cls
+        return cls
+    return inner
+
+
+def get_for_extension(ext: str) -> FnPair:
+    return compose(*[_EXTENSION_REGISTRY[p]() for p in ext.split('.')])
+
+
+##
+
+
+class Compression(FnPair[bytes, bytes], abc.ABC):
+    pass
+
+
+@_register_extension('bz2')
+@dc.dataclass(frozen=True)
+class Bz2(Compression):
+    compresslevel: int = 9
+
+    def forward(self, f: bytes) -> bytes:
+        return _bz2.compress(f, compresslevel=self.compresslevel)
+
+    def backward(self, t: bytes) -> bytes:
+        return _bz2.decompress(t)
+
+
+@_register_extension('gz')
+@dc.dataclass(frozen=True)
+class Gzip(Compression):
+    compresslevel: int = 9
+
+    def forward(self, f: bytes) -> bytes:
+        return _gzip.compress(f, compresslevel=self.compresslevel)
+
+    def backward(self, t: bytes) -> bytes:
+        return _gzip.decompress(t)
+
+
+@_register_extension('lzma')
+class Lzma(Compression):
+    def forward(self, f: bytes) -> bytes:
+        return _lzma.compress(f)
+
+    def backward(self, t: bytes) -> bytes:
+        return _lzma.decompress(t)
+
+
+#
+
+
+@_register_extension('lz4')
+@dc.dataclass(frozen=True)
+class Lz4(Compression):
+    compression_level: int = 0
+
+    def forward(self, f: bytes) -> bytes:
+        return _lz4_frame.compress(f, compression_level=self.compression_level)
+
+    def backward(self, t: bytes) -> bytes:
+        return _lz4_frame.decompress(t)
+
+
+@_register_extension('snappy')
+class Snappy(Compression):
+    def forward(self, f: bytes) -> bytes:
+        return _snappy.compress(f)
+
+    def backward(self, t: bytes) -> bytes:
+        return _snappy.decompress(t)
+
+
+@_register_extension('zstd')
+class Zstd(Compression):
+    def forward(self, f: bytes) -> bytes:
+        return _zstd.compress(f)
+
+    def backward(self, t: bytes) -> bytes:
+        return _zstd.decompress(t)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Struct(FnPair[tuple, bytes]):
+    fmt: str
+
+    def forward(self, f: tuple) -> bytes:
+        return _struct.pack(self.fmt, *f)
+
+    def backward(self, t: bytes) -> tuple:
+        return _struct.unpack(self.fmt, t)
+
+
+##
+
+
+class Object(FnPair[ta.Any, T], lang.Abstract):  # noqa
+    pass
+
+
+class ObjectStr(Object[str], lang.Abstract):  # noqa
+    pass
+
+
+class ObjectBytes(Object[bytes], lang.Abstract):  # noqa
+    pass
+
+
+#
+
+
+@_register_extension('pkl')
+@dc.dataclass(frozen=True)
+class Pickle(ObjectBytes):
+    protocol: int | None = None
+
+    def forward(self, f: ta.Any) -> bytes:
+        return _pickle.dumps(f, protocol=self.protocol)
+
+    def backward(self, t: bytes) -> ta.Any:
+        return _pickle.loads(t)
+
+
+@_register_extension('json')
+@dc.dataclass(frozen=True)
+class Json(ObjectStr):
+    indent: int | str | None = dc.field(default=None, kw_only=True)
+    separators: tuple[str, str] | None = dc.field(default=None, kw_only=True)
+
+    def forward(self, f: ta.Any) -> str:
+        return _json.dumps(f, indent=self.indent, separators=self.separators)
+
+    def backward(self, t: str) -> ta.Any:
+        return _json.loads(t)
+
+
+JSON = Json()
+PRETTY_JSON = Json(indent=2)
+COMPACT_JSON = Json(separators=(',', ':'))
+
+
+@_register_extension('jsonl')
+class JsonLines(FnPair[ta.Sequence[ta.Any], str]):
+    def forward(self, f: ta.Sequence[ta.Any]) -> str:
+        return '\n'.join(_json.dumps(e) for e in f)
+
+    def backward(self, t: str) -> ta.Sequence[ta.Any]:
+        return [_json.loads(l) for l in t.splitlines()]
+
+
+@_register_extension('toml')
+class Toml(ObjectStr):
+    def forward(self, f: ta.Any) -> str:
+        raise NotImplementedError
+
+    def backward(self, t: str) -> ta.Any:
+        return _tomllib.loads(t)
+
+
+#
+
+
+@_register_extension('cpkl')
+@dc.dataclass(frozen=True)
+class Cloudpickle(ObjectBytes):
+    protocol: int | None = None
+
+    def forward(self, f: ta.Any) -> bytes:
+        return _cloudpickle.dumps(f, protocol=self.protocol)
+
+    def backward(self, t: bytes) -> ta.Any:
+        return _cloudpickle.loads(t)
+
+
+@_register_extension('yml', 'yaml')
+class Yaml(ObjectStr):
+    def forward(self, f: ta.Any) -> str:
+        return _yaml.dump(f)
+
+    def backward(self, t: str) -> ta.Any:
+        return _yaml.safe_load(t)
+
+
+class YamlUnsafe(ObjectStr):
+    def forward(self, f: ta.Any) -> str:
+        return _yaml.dump(f)
+
+    def backward(self, t: str) -> ta.Any:
+        return _yaml.load(t, _yaml.FullLoader)

omlish/graphs/trees.py

@@ -18,7 +18,7 @@ NodeWalker = ta.Callable[[NodeT], ta.Iterable[NodeT]]
 NodeGenerator = ta.Generator[NodeT, None, None]
 
 
-class NodeException(ta.Generic[NodeT], Exception):
+class NodeError(ta.Generic[NodeT], Exception):
     def __init__(self, node: NodeT, msg: str, *args, **kwargs) -> None:
         super().__init__(msg, *args, **kwargs)  # noqa
         self._node = node
@@ -28,12 +28,12 @@ class NodeException(ta.Generic[NodeT], Exception):
         return self._node
 
 
-class DuplicateNodeException(NodeException[NodeT]):
+class DuplicateNodeError(NodeError[NodeT]):
     def __init__(self, node: NodeT, *args, **kwargs) -> None:
         super().__init__(node, f'Duplicate node: {node!r}', *args, **kwargs)
 
 
-class UnknownNodeException(NodeException[NodeT]):
+class UnknownNodeError(NodeError[NodeT]):
     def __init__(self, node: NodeT, *args, **kwargs) -> None:
         super().__init__(node, f'Unknown node: {node!r}', *args, **kwargs)
 
@@ -55,19 +55,19 @@ class BasicTreeAnalysis(ta.Generic[NodeT]):
 
         self._set_fac: ta.Callable[..., ta.MutableSet[NodeT]] = col.IdentitySet if identity else set
         self._dict_fac: ta.Callable[..., ta.MutableMapping[NodeT, ta.Any]] = col.IdentityKeyDict if identity else dict
-        self._idx_seq_fac: ta.Callable[..., col.IndexedSeq[NodeT]] = functools.partial(col.IndexedSeq, identity=identity)  # noqa
+        self._idx_seq_fac: ta.Callable[..., col.IndexedSeq[NodeT]] = functools.partial(col.IndexedSeq, identity=identity)  # type: ignore  # noqa
 
         def walk(cur: NodeT, parent: NodeT | None) -> None:
             check.not_none(cur)
             if cur in node_set:
-                raise DuplicateNodeException(cur)
+                raise DuplicateNodeError(cur)
 
             nodes.append(cur)
             node_set.add(cur)
             if parent is None:
                 check.state(cur is root)
             elif parent not in node_set:
-                raise UnknownNodeException(parent)
+                raise UnknownNodeError(parent)
 
             parents_by_node[cur] = parent
 
@@ -77,19 +77,19 @@ class BasicTreeAnalysis(ta.Generic[NodeT]):
                 walk(child, cur)
 
         nodes: list[NodeT] = []
-        node_set: ta.MutableSet[NodeT] = self._set_fac()  # type: ignore
-        children_by_node: ta.MutableMapping[NodeT | None, ta.Sequence[NodeT]] = self._dict_fac()  # type: ignore
-        child_sets_by_node: ta.MutableMapping[ta.Optional[NodeT], ta.AbstractSet[NodeT]] = self._dict_fac()  # type: ignore  # noqa
-        parents_by_node: ta.MutableMapping[NodeT, NodeT | None] = self._dict_fac()  # type: ignore
+        node_set: ta.MutableSet[NodeT] = self._set_fac()
+        children_by_node: ta.MutableMapping[NodeT | None, ta.Sequence[NodeT]] = self._dict_fac()
+        child_sets_by_node: ta.MutableMapping[ta.Optional[NodeT], ta.AbstractSet[NodeT]] = self._dict_fac()
+        parents_by_node: ta.MutableMapping[NodeT, NodeT | None] = self._dict_fac()
 
         children_by_node[None] = [root]
-        child_sets_by_node[None] = self._set_fac([root])  # type: ignore
+        child_sets_by_node[None] = self._set_fac([root])
 
         walk(root, None)
 
-        self._nodes = self._idx_seq_fac(nodes)  # type: ignore
+        self._nodes = self._idx_seq_fac(nodes)
         self._node_set: ta.AbstractSet[NodeT] = node_set
-        self._children_by_node: ta.Mapping[NodeT | None, col.IndexedSeq[NodeT]] = self._dict_fac(  # type: ignore
+        self._children_by_node: ta.Mapping[NodeT | None, col.IndexedSeq[NodeT]] = self._dict_fac(
             [(n, self._idx_seq_fac(cs)) for n, cs in children_by_node.items()])
         self._child_sets_by_node: ta.Mapping[NodeT | None, ta.AbstractSet[NodeT]] = child_sets_by_node
         self._parents_by_node: ta.Mapping[NodeT, NodeT | None] = parents_by_node