omlish 0.0.0.dev4__py3-none-any.whl → 0.0.0.dev6__py3-none-any.whl

omlish/dataclasses/utils.py

@@ -0,0 +1,44 @@
+import collections
+import dataclasses as dc
+import types
+import typing as ta
+
+from .. import check
+
+
+T = ta.TypeVar('T')
+
+
+def maybe_post_init(sup: ta.Any) -> bool:
+    try:
+        fn = sup.__post_init__
+    except AttributeError:
+        return False
+    fn()
+    return True
+
+
+def opt_repr(o: ta.Any) -> str | None:
+    return repr(o) if o is not None else None
+
+
+class field_modifier:  # noqa
+    def __init__(self, fn: ta.Callable[[dc.Field], dc.Field]) -> None:
+        super().__init__()
+        self.fn = fn
+
+    def __ror__(self, other: T) -> T:
+        return self(other)
+
+    def __call__(self, f: T) -> T:
+        return check.isinstance(self.fn(check.isinstance(f, dc.Field)), dc.Field)  # type: ignore
+
+
+def chain_metadata(*mds: ta.Mapping) -> types.MappingProxyType:
+    return types.MappingProxyType(collections.ChainMap(*mds))  # type: ignore  # noqa
+
+
+def update_field_metadata(f: dc.Field, nmd: ta.Mapping) -> dc.Field:
+    check.isinstance(f, dc.Field)
+    f.metadata = chain_metadata(nmd, f.metadata)
+    return f

omlish/defs.py

@@ -141,7 +141,7 @@ def hash_eq(cls_dct, *attrs):
     def __eq__(self, other):  # noqa
         if type(other) is not type(self):
             return False
-        for attr in attrs:
+        for attr in attrs:  # noqa
             if getattr(self, attr) != getattr(other, attr):
                 return False
         return True

omlish/diag/__init__.py

@@ -0,0 +1,4 @@
+"""
+TODO:
+ - https://github.com/aristocratos/btop/blob/00c90884c328eb3e44a0ab094e833161092aae9c/src/linux/btop_collect.cpp#L443
+"""

omlish/diag/procfs.py

@@ -12,9 +12,10 @@ import sys
 import typing as ta
 
 from .. import iterators as it
-from .. import json
 from .. import lang
 from .. import os as oos
+from ..serde import json
+from .procstats import ProcStats
 
 
 log = logging.getLogger(__name__)
@@ -23,6 +24,9 @@ log = logging.getLogger(__name__)
 PidLike = int | str
 
 
+##
+
+
 RLIMIT_RESOURCES = {
     getattr(resource, k): k
     for k in dir(resource)
@@ -38,7 +42,7 @@ def parse_size(s: str) -> int:
     return int(v) * us[u]
 
 
-class ProcStat(lang.Namespace):
+class ProcStat(lang.Namespace, lang.Final):
     PID = 0
     COMM = 1
     STATE = 2
@@ -98,6 +102,9 @@ def _check_linux() -> None:
         raise OSError
 
 
+##
+
+
 def get_process_stats(pid: PidLike = 'self') -> list[str]:
     """http://man7.org/linux/man-pages/man5/proc.5.html -> /proc/[pid]/stat"""
 
@@ -109,6 +116,18 @@ def get_process_stats(pid: PidLike = 'self') -> list[str]:
     return [pid.strip(), comm, *r.strip().split(' ')]
 
 
+def get_process_procstats(pid: int | None = None) -> ProcStats:
+    st = get_process_stats('self' if pid is None else pid)
+    return ProcStats(
+        pid=int(st[ProcStat.PID]),
+
+        rss=int(st[ProcStat.RSS]),
+    )
+
+
+##
+
+
 def get_process_chain(pid: PidLike = 'self') -> list[tuple[int, str]]:
     _check_linux()
     lst = []
@@ -148,6 +167,9 @@ def set_process_oom_score_adj(score: str, pid: PidLike = 'self') -> None:
         f.write(str(score))
 
 
+##
+
+
 MAP_LINE_RX = re.compile(
     r'^'
     r'(?P<address>[A-Fa-f0-9]+)-(?P<end_address>[A-Fa-f0-9]+)\s+'
@@ -156,7 +178,7 @@ MAP_LINE_RX = re.compile(
     r'(?P<device>\S+)\s+'
     r'(?P<inode>\d+)\s+'
     r'(?P<path>.*)'
-    r'$'
+    r'$',
 )
 
 
@@ -198,6 +220,9 @@ def get_process_maps(pid: PidLike = 'self', sharing: bool = False) -> ta.Iterato
             yield d
 
 
+##
+
+
 PAGEMAP_KEYS = (
     'address',
     'pfn',
@@ -243,6 +268,9 @@ def get_process_pagemaps(pid: PidLike = 'self') -> ta.Iterable[dict[str, int]]:
         yield from get_process_range_pagemaps(m['address'], m['end_address'], pid)
 
 
+##
+
+
 def _dump_cmd(args: ta.Any) -> None:
     total = 0
     dirty_total = 0

omlish/diag/procstats.py

@@ -0,0 +1,32 @@
+import dataclasses as dc
+import os
+import typing as ta
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import psutil as _psutil
+else:
+    _psutil = lang.proxy_import('psutil')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class ProcStats:
+    pid: int
+
+    rss: int
+
+
+def get_psutil_procstats(pid: int | None = None) -> ProcStats:
+    if pid is None:
+        pid = os.getpid()
+
+    proc = _psutil.Process(pid)
+    mi = proc.memory_info()
+
+    return ProcStats(
+        pid=pid,
+
+        rss=mi.rss,
+    )

omlish/{testing → diag}/pydevd.py

@@ -32,6 +32,7 @@ import os
 import sys
 import tempfile
 import textwrap
+import threading
 import types
 import typing as ta
 
@@ -243,3 +244,37 @@ def maybe_reexec(
         args = [args[0], bootstrap_path]
 
     os.execvp(sys.executable, args)
+
+
+def debug_unhandled_exception(exc_info: ta.Any = None) -> None:
+    if exc_info is None:
+        exc_info = sys.exc_info()
+
+    try:
+        import pydevd
+        from pydevd import pydevd_tracing
+
+    except ImportError:
+        return
+
+    exctype, value, traceback = exc_info
+    frames = []
+    while traceback:
+        frames.append(traceback.tb_frame)
+        traceback = traceback.tb_next
+
+    thread = threading.current_thread()
+    frames_by_id = {id(frame): frame for frame in frames}
+    frame = frames[-1]
+    exception = (exctype, value, traceback)
+
+    if hasattr(thread, 'additional_info'):
+        thread.additional_info.pydev_message = 'server exception'
+    try:
+        debugger = pydevd.debugger  # noqa
+    except AttributeError:
+        debugger = pydevd.get_global_debugger()  # noqa
+
+    pydevd_tracing.SetTrace(None)
+
+    debugger.stop_on_unhandled_exception(thread, frame, frames_by_id, exception)

omlish/diag/replserver/console.py

@@ -74,7 +74,7 @@ class InteractiveSocketConsole:
     CPRT = 'Type "help", "copyright", "credits" or "license" for more information.'
 
     def interact(self, banner: str | None = None, exitmsg: str | None = None) -> None:
-        log.info(f'Console {id(self)} on thread {threading.current_thread().ident} interacting')
+        log.info('Console %x on thread %r interacting', id(self), threading.current_thread().ident)
 
         try:
             ps1 = getattr(sys, 'ps1', '>>> ')
@@ -115,7 +115,7 @@ class InteractiveSocketConsole:
                 pass
 
         finally:
-            log.info(f'Console {id(self)} on thread {threading.current_thread().ident} finished')
+            log.info('Console %x on thread %r finished', id(self), threading.current_thread().ident)
 
     def push_line(self, line: str) -> bool:
         self._buffer.append(line)
@@ -208,7 +208,7 @@ class InteractiveSocketConsole:
             exec(code, self._locals)
         except SystemExit:
             raise
-        except Exception:
+        except Exception:  # noqa
             self.show_traceback()
         else:
             if self._count == self._write_count:

omlish/diag/replserver/server.py

@@ -84,7 +84,7 @@ class ReplServer:
         with contextlib.closing(self._socket):
             self._socket.listen(1)
 
-            log.info(f'Repl server listening on file {self._config.path}')
+            log.info('Repl server listening on file %s', self._config.path)
 
             self._is_running = True
             try:
@@ -94,7 +94,7 @@ class ReplServer:
                     except TimeoutError:
                         continue
 
-                    log.info(f'Got repl server connection on file {self._config.path}')
+                    log.info('Got repl server connection on file %s', self._config.path)
 
                     def run(conn):
                         with contextlib.closing(conn):
@@ -104,9 +104,10 @@ class ReplServer:
                             variables['__console__'] = console
 
                             log.info(
-                                f'Starting console {id(console)} repl server connection '
-                                f'on file {self._config.path} '
-                                f'on thread {threading.current_thread().ident}',
+                                'Starting console %x repl server connection on file %s on thread %r',
+                                id(console),
+                                self._config.path,
+                                threading.current_thread().ident,
                             )
                             self._consoles_by_threads[threading.current_thread()] = console
                             console.interact()

omlish/diag/threads.py

@@ -0,0 +1,86 @@
+import io
+import itertools
+import os
+import signal
+import sys
+import threading
+import time
+import traceback
+import typing as ta
+
+
+_DEBUG_THREAD_COUNTER = itertools.count()
+
+
+def create_thread_dump_thread(
+        *,
+        interval_s: float = 5.,
+        out: ta.TextIO = sys.stderr,
+        start: bool = False,
+        nodaemon: bool = False,
+) -> threading.Thread:
+    def dump():
+        cthr = threading.current_thread()
+        thrs_by_tid = {t.ident: t for t in threading.enumerate()}
+
+        buf = io.StringIO()
+        for tid, fr in sys._current_frames().items():  # noqa
+            if tid == cthr.ident:
+                continue
+
+            try:
+                thr = thrs_by_tid[tid]
+            except KeyError:
+                thr_rpr = repr(tid)
+            else:
+                thr_rpr = repr(thr)
+
+            tb = traceback.format_stack(fr)
+
+            buf.write(f'{thr_rpr}\n')
+            buf.write('\n'.join(l.strip() for l in tb))
+            buf.write('\n\n')
+
+        out.write(buf.getvalue())
+
+    def proc():
+        while True:
+            time.sleep(interval_s)
+            try:
+                dump()
+            except Exception as e:  # noqa
+                out.write(repr(e) + '\n\n')
+
+    dthr = threading.Thread(
+        target=proc,
+        daemon=not nodaemon,
+        name=f'thread-dump-thread-{next(_DEBUG_THREAD_COUNTER)}',
+    )
+    if start:
+        dthr.start()
+    return dthr
+
+
+def create_suicide_thread(
+        *,
+        sig: int = signal.SIGKILL,
+        interval_s: float = 1.,
+        parent_thread: threading.Thread | None = None,
+        start: bool = False,
+) -> threading.Thread:
+    if parent_thread is None:
+        parent_thread = threading.current_thread()
+
+    def proc():
+        while True:
+            parent_thread.join(interval_s)
+            if not parent_thread.is_alive():
+                os.kill(os.getpid(), sig)
+
+    dthr = threading.Thread(
+        target=proc,
+        name=f'suicide-thread-{next(_DEBUG_THREAD_COUNTER)}',
+    )
+    if start:
+        dthr.start()
+    return dthr

omlish/dispatch/_dispatch2.py

@@ -0,0 +1,65 @@
+import abc
+import typing as ta
+import weakref
+
+from .dispatch import find_impl
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class Dispatcher(ta.Generic[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._impls_by_arg_cls: dict[type, T] = {}
+        self._dispatch_cache: dict[ta.Any, T | None] = {}
+
+        def cache_remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                cache = ref_self._dispatch_cache  # noqa
+                try:
+                    del cache[k]
+                except KeyError:
+                    pass
+
+        self._cache_remove = cache_remove
+
+        self._cache_token: ta.Any = None
+
+    def cache_size(self) -> int:
+        return len(self._dispatch_cache)
+
+    def register(self, impl: T, cls_col: ta.Iterable[type]) -> T:
+        for cls in cls_col:
+            self._impls_by_arg_cls[cls] = impl
+
+            if self._cache_token is None and hasattr(cls, '__abstractmethods__'):
+                self._cache_token = abc.get_cache_token()
+
+        self._dispatch_cache.clear()
+        return impl
+
+    def dispatch(self, cls: type) -> T | None:
+        if self._cache_token is not None and (current_token := abc.get_cache_token()) != self._cache_token:
+            self._dispatch_cache.clear()
+            self._cache_token = current_token
+
+        cls_ref = weakref.ref(cls)
+        try:
+            return self._dispatch_cache[cls_ref]
+        except KeyError:
+            pass
+        del cls_ref
+
+        impl: T | None
+        try:
+            impl = self._impls_by_arg_cls[cls]
+        except KeyError:
+            impl = find_impl(cls, self._impls_by_arg_cls)
+
+        self._dispatch_cache[weakref.ref(cls, self._cache_remove)] = impl
+        return impl

omlish/dispatch/_dispatch3.py

@@ -0,0 +1,104 @@
+import abc
+import typing as ta
+import weakref
+
+from .dispatch import find_impl
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class DispatchCacheProtocol(ta.Protocol[T]):
+    def size(self) -> int: ...
+    def prepare(self, cls: type) -> None: ...
+    def clear(self) -> None: ...
+    def put(self, cls: type, impl: T) -> None: ...
+    def get(self, cls: type) -> T: ...  # Raises[KeyError]
+
+
+class DispatcherProtocol(ta.Protocol[T]):
+    def cache_size(self) -> int: ...
+    def register(self, impl: T, cls_col: ta.Iterable[type]) -> T: ...
+    def dispatch(self, cls: type) -> T | None: ...
+
+
+##
+
+
+class DispatchCache(DispatchCacheProtocol[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._dct: dict[ta.Any, T] = {}
+
+        def remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                dct = ref_self._dct  # noqa
+                try:
+                    del dct[k]
+                except KeyError:
+                    pass
+
+        self._remove = remove
+
+        self._token: ta.Any = None
+
+    def size(self) -> int:
+        return len(self._dct)
+
+    def prepare(self, cls: type) -> None:
+        if self._token is None and hasattr(cls, '__abstractmethods__'):
+            self._token = abc.get_cache_token()
+
+        self.clear()
+
+    def clear(self) -> None:
+        if self._dct:
+            self._dct.clear()
+
+    def put(self, cls: type, impl: T) -> None:
+        self._dct[weakref.ref(cls, self._remove)] = impl
+
+    def get(self, cls: type) -> T:
+        if self._token is not None and (current_token := abc.get_cache_token()) != self._token:
+            self._dct.clear()
+            self._token = current_token
+
+        cls_ref = weakref.ref(cls)
+        return self._dct[cls_ref]
+
+
+class Dispatcher(DispatcherProtocol[T]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._impls_by_arg_cls: dict[type, T] = {}
+        self._cache: DispatchCache[T | None] = DispatchCache()
+
+    def cache_size(self) -> int:
+        return self._cache.size()
+
+    def register(self, impl: T, cls_col: ta.Iterable[type]) -> T:
+        for cls in cls_col:
+            self._impls_by_arg_cls[cls] = impl
+            self._cache.prepare(cls)
+
+        return impl
+
+    def dispatch(self, cls: type) -> T | None:
+        try:
+            return self._cache.get(cls)
+        except KeyError:
+            pass
+
+        impl: T | None
+        try:
+            impl = self._impls_by_arg_cls[cls]
+        except KeyError:
+            impl = find_impl(cls, self._impls_by_arg_cls)
+
+        self._cache.put(cls, impl)
+        return impl

omlish/docker.py

@@ -22,9 +22,10 @@ import typing as ta
 
 from . import check
 from . import dataclasses as dc
-from . import json
 from . import lang
 from . import marshal as msh
+from .serde import json
+
 
 if ta.TYPE_CHECKING:
     import yaml
@@ -32,6 +33,9 @@ else:
     yaml = lang.proxy_import('yaml')
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class PsItem(lang.Final):
     dc.metadata(msh.ObjectMetadata(
@@ -113,6 +117,9 @@ def cli_inspect(ids: list[str]) -> list[Inspect]:
     return msh.unmarshal(json.loads(o.decode()), list[Inspect])
 
 
+##
+
+
 class ComposeConfig:
     def __init__(
             self,
@@ -141,6 +148,18 @@ class ComposeConfig:
         return ret
 
 
+def get_compose_port(cfg: ta.Mapping[str, ta.Any], default: int) -> int:
+    return check.single(
+        int(l)
+        for p in cfg['ports']
+        for l, r in [p.split(':')]
+        if int(r) == default
+    )
+
+
+##
+
+
 def timebomb_payload(delay_s: float, name: str = 'omlish-docker-timebomb') -> str:
     return (
         '('