omlish 0.0.0.dev3__py3-none-any.whl → 0.0.0.dev5__py3-none-any.whl

omlish/dataclasses/impl/main.py

@@ -24,6 +24,7 @@ from .simple import MatchArgsProcessor
 from .simple import OverridesProcessor
 from .slots import add_slots
 
+
 if ta.TYPE_CHECKING:
     from . import metaclass
 else:

omlish/dataclasses/impl/metaclass.py

@@ -39,7 +39,12 @@ def confer_kwargs(
         for ck in bmp.confer:
             if ck in kwargs:
                 continue
-            if ck in ('frozen', 'generic_init', 'kw_only'):
+            if ck in (
+                    'frozen',
+                    'generic_init',
+                    'kw_only',
+                    'reorder',
+            ):
                 confer_kwarg(out, ck, get_params(base).frozen)
             elif ck == 'confer':
                 confer_kwarg(out, 'confer', bmp.confer)

omlish/dataclasses/impl/order.py

@@ -1,7 +1,7 @@
 import typing as ta
 
-from .utils import Namespace
 from .processing import Processor
+from .utils import Namespace
 from .utils import create_fn
 from .utils import set_new_attribute
 from .utils import tuple_str

omlish/dataclasses/impl/reflect.py

@@ -28,9 +28,22 @@ from .params import get_params_extras
 from .utils import Namespace
 
 
+try:
+    import annotationlib  # noqa
+except ImportError:
+    annotationlib = None
+
+
 MISSING = dc.MISSING
 
 
+def _get_annotations(obj):
+    if annotationlib is not None:
+        return annotationlib.get_annotations(obj, format=annotationlib.Format.FORWARDREF)  # noqa
+    else:
+        return inspect.get_annotations(obj)
+
+
 class ClassInfo:
 
     def __init__(self, cls: type, *, _constructing: bool = False) -> None:
@@ -54,7 +67,7 @@ class ClassInfo:
 
     @cached.property
     def cls_annotations(self) -> ta.Mapping[str, ta.Any]:
-        return inspect.get_annotations(self._cls)
+        return _get_annotations(self._cls)
 
     ##
 
@@ -136,7 +149,7 @@ class ClassInfo:
 
     @cached.property
     def generic_mro_lookup(self) -> ta.Mapping[type, rfl.Type]:
-        return col.unique_dict((check.not_none(rfl.get_concrete_type(g)), g) for g in self.generic_mro)
+        return col.unique_map((check.not_none(rfl.get_concrete_type(g)), g) for g in self.generic_mro)
 
     @cached.property
     def generic_replaced_field_types(self) -> ta.Mapping[str, rfl.Type]:

omlish/defs.py

@@ -141,7 +141,7 @@ def hash_eq(cls_dct, *attrs):
     def __eq__(self, other):  # noqa
         if type(other) is not type(self):
             return False
-        for attr in attrs:
+        for attr in attrs:  # noqa
             if getattr(self, attr) != getattr(other, attr):
                 return False
         return True

omlish/diag/procfs.py

@@ -15,6 +15,7 @@ from .. import iterators as it
 from .. import json
 from .. import lang
 from .. import os as oos
+from .procstats import ProcStats
 
 
 log = logging.getLogger(__name__)
@@ -23,6 +24,9 @@ log = logging.getLogger(__name__)
 PidLike = int | str
 
 
+##
+
+
 RLIMIT_RESOURCES = {
     getattr(resource, k): k
     for k in dir(resource)
@@ -98,6 +102,9 @@ def _check_linux() -> None:
         raise OSError
 
 
+##
+
+
 def get_process_stats(pid: PidLike = 'self') -> list[str]:
     """http://man7.org/linux/man-pages/man5/proc.5.html -> /proc/[pid]/stat"""
 
@@ -109,6 +116,18 @@ def get_process_stats(pid: PidLike = 'self') -> list[str]:
     return [pid.strip(), comm, *r.strip().split(' ')]
 
 
+def get_process_procstats(pid: int | None = None) -> ProcStats:
+    st = get_process_stats('self' if pid is None else pid)
+    return ProcStats(
+        pid=int(st[ProcStat.PID]),
+
+        rss=int(st[ProcStat.RSS]),
+    )
+
+
+##
+
+
 def get_process_chain(pid: PidLike = 'self') -> list[tuple[int, str]]:
     _check_linux()
     lst = []
@@ -148,6 +167,9 @@ def set_process_oom_score_adj(score: str, pid: PidLike = 'self') -> None:
         f.write(str(score))
 
 
+##
+
+
 MAP_LINE_RX = re.compile(
     r'^'
     r'(?P<address>[A-Fa-f0-9]+)-(?P<end_address>[A-Fa-f0-9]+)\s+'
@@ -156,7 +178,7 @@ MAP_LINE_RX = re.compile(
     r'(?P<device>\S+)\s+'
     r'(?P<inode>\d+)\s+'
     r'(?P<path>.*)'
-    r'$'
+    r'$',
 )
 
 
@@ -198,6 +220,9 @@ def get_process_maps(pid: PidLike = 'self', sharing: bool = False) -> ta.Iterato
             yield d
 
 
+##
+
+
 PAGEMAP_KEYS = (
     'address',
     'pfn',
@@ -243,6 +268,9 @@ def get_process_pagemaps(pid: PidLike = 'self') -> ta.Iterable[dict[str, int]]:
         yield from get_process_range_pagemaps(m['address'], m['end_address'], pid)
 
 
+##
+
+
 def _dump_cmd(args: ta.Any) -> None:
     total = 0
     dirty_total = 0

omlish/diag/procstats.py

@@ -0,0 +1,32 @@
+import dataclasses as dc
+import os
+import typing as ta
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import psutil as _psutil
+else:
+    _psutil = lang.proxy_import('psutil')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class ProcStats:
+    pid: int
+
+    rss: int
+
+
+def get_psutil_procstats(pid: int | None = None) -> ProcStats:
+    if pid is None:
+        pid = os.getpid()
+
+    proc = _psutil.Process(pid)
+    mi = proc.memory_info()
+
+    return ProcStats(
+        pid=pid,
+
+        rss=mi.rss,
+    )

omlish/diag/replserver/console.py

@@ -74,7 +74,7 @@ class InteractiveSocketConsole:
     CPRT = 'Type "help", "copyright", "credits" or "license" for more information.'
 
     def interact(self, banner: str | None = None, exitmsg: str | None = None) -> None:
-        log.info(f'Console {id(self)} on thread {threading.current_thread().ident} interacting')
+        log.info('Console %x on thread %r interacting', id(self), threading.current_thread().ident)
 
         try:
             ps1 = getattr(sys, 'ps1', '>>> ')
@@ -115,7 +115,7 @@ class InteractiveSocketConsole:
                 pass
 
         finally:
-            log.info(f'Console {id(self)} on thread {threading.current_thread().ident} finished')
+            log.info('Console %x on thread %r finished', id(self), threading.current_thread().ident)
 
     def push_line(self, line: str) -> bool:
         self._buffer.append(line)
@@ -208,7 +208,7 @@ class InteractiveSocketConsole:
             exec(code, self._locals)
         except SystemExit:
             raise
-        except Exception:
+        except Exception:  # noqa
             self.show_traceback()
         else:
             if self._count == self._write_count:

omlish/diag/replserver/server.py

@@ -84,7 +84,7 @@ class ReplServer:
         with contextlib.closing(self._socket):
             self._socket.listen(1)
 
-            log.info(f'Repl server listening on file {self._config.path}')
+            log.info('Repl server listening on file %s', self._config.path)
 
             self._is_running = True
             try:
@@ -94,7 +94,7 @@ class ReplServer:
                     except TimeoutError:
                         continue
 
-                    log.info(f'Got repl server connection on file {self._config.path}')
+                    log.info('Got repl server connection on file %s', self._config.path)
 
                     def run(conn):
                         with contextlib.closing(conn):
@@ -104,9 +104,10 @@ class ReplServer:
                             variables['__console__'] = console
 
                             log.info(
-                                f'Starting console {id(console)} repl server connection '
-                                f'on file {self._config.path} '
-                                f'on thread {threading.current_thread().ident}',
+                                'Starting console %x repl server connection on file %s on thread %r',
+                                id(console),
+                                self._config.path,
+                                threading.current_thread().ident,
                             )
                             self._consoles_by_threads[threading.current_thread()] = console
                             console.interact()

omlish/diag/threads.py

@@ -0,0 +1,86 @@
+import io
+import itertools
+import os
+import signal
+import sys
+import threading
+import time
+import traceback
+import typing as ta
+
+
+_DEBUG_THREAD_COUNTER = itertools.count()
+
+
+def create_thread_dump_thread(
+        *,
+        interval_s: float = 5.,
+        out: ta.TextIO = sys.stderr,
+        start: bool = False,
+        nodaemon: bool = False,
+) -> threading.Thread:
+    def dump():
+        cthr = threading.current_thread()
+        thrs_by_tid = {t.ident: t for t in threading.enumerate()}
+
+        buf = io.StringIO()
+        for tid, fr in sys._current_frames().items():  # noqa
+            if tid == cthr.ident:
+                continue
+
+            try:
+                thr = thrs_by_tid[tid]
+            except KeyError:
+                thr_rpr = repr(tid)
+            else:
+                thr_rpr = repr(thr)
+
+            tb = traceback.format_stack(fr)
+
+            buf.write(f'{thr_rpr}\n')
+            buf.write('\n'.join(l.strip() for l in tb))
+            buf.write('\n\n')
+
+        out.write(buf.getvalue())
+
+    def proc():
+        while True:
+            time.sleep(interval_s)
+            try:
+                dump()
+            except Exception as e:  # noqa
+                out.write(repr(e) + '\n\n')
+
+    dthr = threading.Thread(
+        target=proc,
+        daemon=not nodaemon,
+        name=f'thread-dump-thread-{next(_DEBUG_THREAD_COUNTER)}',
+    )
+    if start:
+        dthr.start()
+    return dthr
+
+
+def create_suicide_thread(
+        *,
+        sig: int = signal.SIGKILL,
+        interval_s: float = 1.,
+        parent_thread: threading.Thread | None = None,
+        start: bool = False,
+) -> threading.Thread:
+    if parent_thread is None:
+        parent_thread = threading.current_thread()
+
+    def proc():
+        while True:
+            parent_thread.join(interval_s)
+            if not parent_thread.is_alive():
+                os.kill(os.getpid(), sig)
+
+    dthr = threading.Thread(
+        target=proc,
+        name=f'suicide-thread-{next(_DEBUG_THREAD_COUNTER)}',
+    )
+    if start:
+        dthr.start()
+    return dthr

omlish/docker.py

@@ -26,12 +26,16 @@ from . import json
 from . import lang
 from . import marshal as msh
 
+
 if ta.TYPE_CHECKING:
     import yaml
 else:
     yaml = lang.proxy_import('yaml')
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class PsItem(lang.Final):
     dc.metadata(msh.ObjectMetadata(
@@ -113,6 +117,9 @@ def cli_inspect(ids: list[str]) -> list[Inspect]:
     return msh.unmarshal(json.loads(o.decode()), list[Inspect])
 
 
+##
+
+
 class ComposeConfig:
     def __init__(
             self,
@@ -141,6 +148,18 @@ class ComposeConfig:
         return ret
 
 
+def get_compose_port(cfg: ta.Mapping[str, ta.Any], default: int) -> int:
+    return check.single(
+        int(l)
+        for p in cfg['ports']
+        for l, r in [p.split(':')]
+        if int(r) == default
+    )
+
+
+##
+
+
 def timebomb_payload(delay_s: float, name: str = 'omlish-docker-timebomb') -> str:
     return (
         '('

omlish/dynamic.py

@@ -49,7 +49,7 @@ class Var(ta.Generic[T]):
 
     def __init__(
             self,
-            default: type[MISSING] | T = MISSING,  # type: ignore
+            default: type[MISSING] | T = MISSING,
             *,
             new: ta.Callable[[], T] | type[MISSING] = MISSING,
             validate: ta.Callable[[T], None] | None = None,
@@ -138,7 +138,7 @@ class Var(ta.Generic[T]):
             frame = frame.f_back
 
         if self._new is not MISSING:
-            yield self._new()
+            yield self._new()  # type: ignore
 
     def __iter__(self) -> ta.Iterator[T]:
         return self.values

omlish/fnpairs.py

@@ -1,3 +1,16 @@
+"""
+TODO:
+ - objects
+  - csv
+  - csvloader
+  - cbor
+  - cloudpickle
+  - alt json backends
+ - compression
+  - snappy
+  - lz4
+ - wrapped (wait for usecase)
+"""
 import abc
 import codecs
 import dataclasses as dc
@@ -5,8 +18,9 @@ import typing as ta
 
 from . import lang
 
+
 if ta.TYPE_CHECKING:
-    import bzip2 as _bzip2
+    import bz2 as _bz2
     import gzip as _gzip
     import json as _json
     import lzma as _lzma
@@ -14,8 +28,14 @@ if ta.TYPE_CHECKING:
     import struct as _struct
     import tomllib as _tomllib
 
+    import cloudpickle as _cloudpickle
+    import lz4.frame as _lz4_frame
+    import snappy as _snappy
+    import yaml as _yaml
+    import zstd as _zstd
+
 else:
-    _bzip2 = lang.proxy_import('bzip2')
+    _bz2 = lang.proxy_import('bz2')
     _gzip = lang.proxy_import('gzip')
     _json = lang.proxy_import('json')
     _lzma = lang.proxy_import('lzma')
@@ -23,15 +43,20 @@ else:
     _struct = lang.proxy_import('struct')
     _tomllib = lang.proxy_import('tomllib')
 
-_zstd = lang.proxy_import('zstd')
-_yaml = lang.proxy_import('yaml')
+    _cloudpickle = lang.proxy_import('cloudpickle')
+    _lz4_frame = lang.proxy_import('lz4.frame')
+    _snappy = lang.proxy_import('snappy')
+    _yaml = lang.proxy_import('yaml')
+    _zstd = lang.proxy_import('zstd')
 
 
 ##
 
 
 F = ta.TypeVar('F')
+F2 = ta.TypeVar('F2')
 T = ta.TypeVar('T')
+T2 = ta.TypeVar('T2')
 U = ta.TypeVar('U')
 
 
@@ -44,6 +69,11 @@ class FnPair(ta.Generic[F, T], abc.ABC):
     def backward(self, t: T) -> F:
         raise NotImplementedError
 
+    ##
+
+    def __call__(self, f: F) -> T:
+        return self.forward(f)
+
     def invert(self) -> 'FnPair[T, F]':
         if isinstance(self, Inverted):
             return self.fp
@@ -76,6 +106,8 @@ Simple.__abstractmethods__ = frozenset()  # noqa
 
 of = Simple
 
+NOP: FnPair[ta.Any, ta.Any] = of(lang.identity, lang.identity)
+
 
 ##
 
@@ -91,9 +123,6 @@ class Inverted(FnPair[F, T]):
         return self.fp.forward(t)
 
 
-##
-
-
 @dc.dataclass(frozen=True)
 class Composite(FnPair[F, T]):
     children: ta.Sequence[FnPair]
@@ -109,6 +138,14 @@ class Composite(FnPair[F, T]):
         return ta.cast(F, t)
 
 
+def compose(*ps: FnPair) -> FnPair:
+    if not ps:
+        return NOP
+    if len(ps) == 1:
+        return ps[0]
+    return Composite(ps)
+
+
 ##
 
 
@@ -143,13 +180,13 @@ UTF8 = text('utf-8')
 
 
 @dc.dataclass(frozen=True)
-class Optional(FnPair[ta.Optional[F], ta.Optional[T]]):
+class Optional(FnPair[F | None, T | None]):
     fp: FnPair[F, T]
 
-    def forward(self, f: ta.Optional[F]) -> ta.Optional[T]:
+    def forward(self, f: F | None) -> T | None:
         return None if f is None else self.fp.forward(f)
 
-    def backward(self, t: ta.Optional[T]) -> ta.Optional[F]:
+    def backward(self, t: T | None) -> F | None:
         return None if t is None else self.fp.backward(t)
 
 
@@ -177,6 +214,10 @@ def _register_extension(*ss):
     return inner
 
 
+def get_for_extension(ext: str) -> FnPair:
+    return compose(*[_EXTENSION_REGISTRY[p]() for p in ext.split('.')])
+
+
 ##
 
 
@@ -184,28 +225,28 @@ class Compression(FnPair[bytes, bytes], abc.ABC):
     pass
 
 
-@_register_extension('gz')
+@_register_extension('bz2')
 @dc.dataclass(frozen=True)
-class Gzip(Compression):
+class Bz2(Compression):
     compresslevel: int = 9
 
     def forward(self, f: bytes) -> bytes:
-        return _gzip.compress(f, compresslevel=self.compresslevel)
+        return _bz2.compress(f, compresslevel=self.compresslevel)
 
     def backward(self, t: bytes) -> bytes:
-        return _gzip.decompress(t)
+        return _bz2.decompress(t)
 
 
-@_register_extension('bz2')
+@_register_extension('gz')
 @dc.dataclass(frozen=True)
-class Bzip2(Compression):
+class Gzip(Compression):
     compresslevel: int = 9
 
     def forward(self, f: bytes) -> bytes:
-        return _bzip2.compress(f, compresslevel=self.compresslevel)
+        return _gzip.compress(f, compresslevel=self.compresslevel)
 
     def backward(self, t: bytes) -> bytes:
-        return _bzip2.decompress(t)
+        return _gzip.decompress(t)
 
 
 @_register_extension('lzma')
@@ -220,6 +261,27 @@ class Lzma(Compression):
 #
 
 
+@_register_extension('lz4')
+@dc.dataclass(frozen=True)
+class Lz4(Compression):
+    compression_level: int = 0
+
+    def forward(self, f: bytes) -> bytes:
+        return _lz4_frame.compress(f, compression_level=self.compression_level)
+
+    def backward(self, t: bytes) -> bytes:
+        return _lz4_frame.decompress(t)
+
+
+@_register_extension('snappy')
+class Snappy(Compression):
+    def forward(self, f: bytes) -> bytes:
+        return _snappy.compress(f)
+
+    def backward(self, t: bytes) -> bytes:
+        return _snappy.decompress(t)
+
+
 @_register_extension('zstd')
 class Zstd(Compression):
     def forward(self, f: bytes) -> bytes:
@@ -243,9 +305,32 @@ class Struct(FnPair[tuple, bytes]):
         return _struct.unpack(self.fmt, t)
 
 
+##
+
+
+Object: ta.TypeAlias = FnPair[ta.Any, T]
+ObjectStr: ta.TypeAlias = Object[str]
+ObjectBytes: ta.TypeAlias = Object[bytes]
+
+
+class Object_(FnPair[ta.Any, T], lang.Abstract):  # noqa
+    pass
+
+
+class ObjectStr_(Object_[str], lang.Abstract):  # noqa
+    pass
+
+
+class ObjectBytes_(Object_[bytes], lang.Abstract):  # noqa
+    pass
+
+
+#
+
+
 @_register_extension('pkl')
 @dc.dataclass(frozen=True)
-class Pickle(FnPair[ta.Any, bytes]):
+class Pickle(ObjectBytes_):
     protocol: int | None = None
 
     def forward(self, f: ta.Any) -> bytes:
@@ -257,7 +342,7 @@ class Pickle(FnPair[ta.Any, bytes]):
 
 @_register_extension('json')
 @dc.dataclass(frozen=True)
-class Json(FnPair[ta.Any, str]):
+class Json(ObjectStr_):
     indent: int | str | None = dc.field(default=None, kw_only=True)
     separators: tuple[str, str] | None = dc.field(default=None, kw_only=True)
 
@@ -283,7 +368,7 @@ class JsonLines(FnPair[ta.Sequence[ta.Any], str]):
 
 
 @_register_extension('toml')
-class Toml(FnPair[ta.Any, str]):
+class Toml(ObjectStr_):
     def forward(self, f: ta.Any) -> str:
         raise NotImplementedError
 
@@ -294,8 +379,20 @@ class Toml(FnPair[ta.Any, str]):
 #
 
 
+@_register_extension('cpkl')
+@dc.dataclass(frozen=True)
+class Cloudpickle(ObjectBytes_):
+    protocol: int | None = None
+
+    def forward(self, f: ta.Any) -> bytes:
+        return _cloudpickle.dumps(f, protocol=self.protocol)
+
+    def backward(self, t: bytes) -> ta.Any:
+        return _cloudpickle.loads(t)
+
+
 @_register_extension('yml', 'yaml')
-class Yaml(FnPair[ta.Any, str]):
+class Yaml(ObjectStr_):
     def forward(self, f: ta.Any) -> str:
         return _yaml.dump(f)
 
@@ -303,9 +400,9 @@ class Yaml(FnPair[ta.Any, str]):
         return _yaml.safe_load(t)
 
 
-class UnsafeYaml(FnPair[ta.Any, str]):
+class YamlUnsafe(ObjectStr_):
     def forward(self, f: ta.Any) -> str:
         return _yaml.dump(f)
 
     def backward(self, t: str) -> ta.Any:
-        return _yaml.safe_load(t, loader=_yaml.FullLoader)
+        return _yaml.load(t, _yaml.FullLoader)