omlish 0.0.0.dev447__py3-none-any.whl → 0.0.0.dev493__py3-none-any.whl

omlish/bootstrap/diag.py

@@ -13,23 +13,14 @@ from .base import ContextBootstrap
 from .base import SimpleBootstrap
 
 
-if ta.TYPE_CHECKING:
+with lang.auto_proxy_import(globals()):
     import cProfile  # noqa
     import pstats
 
-    from ..diag import debug as ddbg
-    from ..diag import pycharm as diagpc
-    from ..diag import replserver as diagrs
-    from ..diag import threads as diagt
-
-else:
-    cProfile = lang.proxy_import('cProfile')  # noqa
-    pstats = lang.proxy_import('pstats')
-
-    ddbg = lang.proxy_import('..diag.debug', __package__)
-    diagpc = lang.proxy_import('..diag.pycharm', __package__)
-    diagrs = lang.proxy_import('..diag.replserver', __package__)
-    diagt = lang.proxy_import('..diag.threads', __package__)
+    from ..diag import debug as d_debug
+    from ..diag import pycharm as d_pycharm
+    from ..diag import replserver
+    from ..diag import threads as d_threads
 
 
 ##
@@ -110,7 +101,7 @@ class ThreadDumpBootstrap(ContextBootstrap['ThreadDumpBootstrap.Config']):
     @contextlib.contextmanager
     def enter(self) -> ta.Iterator[None]:
         if self._config.interval_s:
-            tdt = diagt.create_thread_dump_thread(
+            tdt = d_threads.create_thread_dump_thread(
                 interval_s=self._config.interval_s,
                 start=True,
                 nodaemon=self._config.nodaemon,
@@ -119,7 +110,7 @@ class ThreadDumpBootstrap(ContextBootstrap['ThreadDumpBootstrap.Config']):
             tdt = None
 
         if self._config.on_sigquit:
-            dump_threads_str = diagt.dump_threads_str
+            dump_threads_str = d_threads.dump_threads_str
 
             def handler(signum, frame):
                 print(dump_threads_str(), file=sys.stderr)
@@ -153,7 +144,7 @@ class TimebombBootstrap(ContextBootstrap['TimebombBootstrap.Config']):
             yield
             return
 
-        tbt = diagt.create_timebomb_thread(
+        tbt = d_threads.create_timebomb_thread(
             self._config.delay_s,
             start=True,
         )
@@ -173,8 +164,8 @@ class PycharmBootstrap(SimpleBootstrap['PycharmBootstrap.Config']):
 
     def run(self) -> None:
         if self._config.debug is not None:
-            prd = diagpc.PycharmRemoteDebugger.parse(self._config.debug)
-            diagpc.pycharm_remote_debugger_attach(prd)
+            prd = d_pycharm.PycharmRemoteDebugger.parse(self._config.debug)
+            d_pycharm.pycharm_remote_debugger_attach(prd)
 
 
 ##
@@ -190,7 +181,7 @@ class ReplServerBootstrap(ContextBootstrap['ReplServerBootstrap.Config']):
         if self._config.path is None:
             return
 
-        with diagrs.ReplServer(diagrs.ReplServer.Config(
+        with replserver.ReplServer(replserver.ReplServer.Config(
             path=self._config.path,
         )) as rs:
             thread = threading.Thread(target=rs.run, name='replserver')
@@ -212,5 +203,5 @@ class DebugBootstrap(ContextBootstrap['DebugBootstrap.Config']):
         if not self._config.enable:
             return
 
-        with ddbg.debugging_on_exception():
+        with d_debug.debugging_on_exception():
             yield

omlish/bootstrap/main.py

@@ -1,9 +1,9 @@
+# ruff: noqa: UP006 UP007 UP045
 """
 TODO:
  - -x / --exec - os.exec entrypoint
  - refuse to install non-exec-relevant Bootstraps when chosen
 """
-# ruff: noqa: UP006 UP007 UP045
 import argparse
 import dataclasses as dc
 import io
@@ -19,12 +19,9 @@ from .harness import BOOTSTRAP_TYPES_BY_NAME
 from .harness import bootstrap
 
 
-if ta.TYPE_CHECKING:
+with lang.auto_proxy_import(globals()):
     import runpy
 
-else:
-    runpy = lang.proxy_import('runpy')
-
 
 ##
 

omlish/bootstrap/sys.py

@@ -2,14 +2,10 @@
 import contextlib
 import dataclasses as dc
 import enum
-import faulthandler
 import gc
 import importlib
 import logging
 import os
-import pwd
-import resource
-import signal
 import sys
 import typing as ta
 
@@ -19,18 +15,17 @@ from .base import ContextBootstrap
 from .base import SimpleBootstrap
 
 
-if ta.TYPE_CHECKING:
+with lang.auto_proxy_import(globals()):
+    import faulthandler
+    import pwd
+    import resource
+    import signal
+
     from .. import libc
     from ..formats import dotenv
     from ..logs import all as logs
     from ..os.pidfiles import pidfile
 
-else:
-    libc = lang.proxy_import('..libc', __package__)
-    logs = lang.proxy_import('..logs', __package__)
-    dotenv = lang.proxy_import('..formats.dotenv', __package__)
-    pidfile = lang.proxy_import('..os.pidfiles.pidfile', __package__)
-
 
 ##
 
@@ -184,14 +179,16 @@ class FaulthandlerBootstrap(ContextBootstrap['FaulthandlerBootstrap.Config']):
 ##
 
 
-SIGNALS_BY_NAME = {
-    a[len('SIG'):]: v  # noqa
-    for a in dir(signal)
-    if a.startswith('SIG')
-    and not a.startswith('SIG_')
-    and a == a.upper()
-    and isinstance((v := getattr(signal, a)), int)
-}
+@lang.cached_function
+def signals_by_name() -> ta.Mapping[str, int]:
+    return {
+        a[len('SIG'):]: v  # noqa
+        for a in dir(signal)
+        if a.startswith('SIG')
+        and not a.startswith('SIG_')
+        and a == a.upper()
+        and isinstance((v := getattr(signal, a)), int)
+    }
 
 
 class PrctlBootstrap(SimpleBootstrap['PrctlBootstrap.Config']):
@@ -208,20 +205,22 @@ class PrctlBootstrap(SimpleBootstrap['PrctlBootstrap.Config']):
             if isinstance(self._config.deathsig, int):
                 sig = self._config.deathsig
             else:
-                sig = SIGNALS_BY_NAME[self._config.deathsig.upper()]
+                sig = signals_by_name()[self._config.deathsig.upper()]
             libc.prctl(libc.PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
 
 
 ##
 
 
-RLIMITS_BY_NAME = {
-    a[len('RLIMIT_'):]: v  # noqa
-    for a in dir(resource)
-    if a.startswith('RLIMIT_')
-    and a == a.upper()
-    and isinstance((v := getattr(resource, a)), int)
-}
+@lang.cached_function
+def rlimits_by_name() -> ta.Mapping[str, int]:
+    return {
+        a[len('RLIMIT_'):]: v  # noqa
+        for a in dir(resource)
+        if a.startswith('RLIMIT_')
+        and a == a.upper()
+        and isinstance((v := getattr(resource, a)), int)
+    }
 
 
 class RlimitBootstrap(ContextBootstrap['RlimitBootstrap.Config']):
@@ -240,7 +239,7 @@ class RlimitBootstrap(ContextBootstrap['RlimitBootstrap.Config']):
 
         prev = {}
         for k, (s, h) in self._config.limits.items():
-            i = RLIMITS_BY_NAME[k.upper()]
+            i = rlimits_by_name()[k.upper()]
             prev[i] = resource.getrlimit(i)
             resource.setrlimit(i, (or_infin(s), or_infin(h)))
 

omlish/cexts/include/omlish/omlish.hh

@@ -0,0 +1 @@
+#pragma once

omlish/collections/__init__.py

@@ -6,12 +6,21 @@ from .. import lang as _lang
 with _lang.auto_proxy_init(globals()):
     ##
 
+    from .attrregistry import (  # noqa
+        AttrRegistry,
+
+        AttrRegistryCache,
+        SimpleAttrRegistryCache,
+    )
+
     from .bimap import (  # noqa
         BiMap,
 
         make_bi_map,
     )
 
+    from . import cache  # noqa
+
     from .coerce import (  # noqa
         abs_set,
         abs_set_of,
@@ -64,9 +73,10 @@ with _lang.auto_proxy_init(globals()):
     from . import kv  # noqa
 
     from .mappings import (  # noqa
+        DynamicTypeMap,
         MissingDict,
         TypeMap,
-        DynamicTypeMap,
+        dict_factory,
         guarded_map_update,
         multikey_dict,
     )
@@ -145,6 +155,8 @@ with _lang.auto_proxy_init(globals()):
         key_cmp,
         make_map,
         make_map_by,
+        map_keys,
+        map_values,
         multi_map,
         multi_map_by,
         partition,

omlish/collections/attrregistry.py

@@ -0,0 +1,210 @@
+"""
+TODO:
+ - lock?
+"""
+import dataclasses as dc
+import typing as ta
+import weakref
+
+from .. import check
+from .. import lang
+from .mappings import dict_factory
+
+
+T = ta.TypeVar('T')
+K = ta.TypeVar('K')
+V = ta.TypeVar('V')
+
+
+##
+
+
+class AttrRegistry(ta.Generic[K, V]):
+    """
+    MRO-honoring class member registry. There are many ways to do this, and this one is attr name based: a class is
+    considered to have objects registered to it based on whether or not they are accessible by a non-shadowed,
+    MRO-resolved named attribute on that class.
+
+    Care must be taken when overriding registered objects from superclasses in subclasses - shadowing the attribute name
+    of the superclass member will not automatically register the new member with the same name to the registry - it must
+    be explicitly registered itself. This is a feature, allowing for selective de-registration of objects in subclasses
+    via name shadowing.
+    """
+
+    def __init__(
+            self,
+            *,
+            requires_override: bool = False,
+            is_override: ta.Callable[[ta.Any], bool] | None = None,
+            forbid_duplicates: bool = False,
+            identity: bool = False,
+            weak: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._requires_override = requires_override
+        self._is_override = is_override
+        self._forbid_duplicates = forbid_duplicates
+        self._identity = identity
+        self._weak = weak
+
+        self._objs: ta.MutableMapping[K, V] = dict_factory(identity=identity, weak=weak)()
+        self._invalidate_callbacks: list[ta.Callable[[], None]] = []
+
+    def add_invalidate_callback(self, callback: ta.Callable[[], None]) -> None:
+        self._invalidate_callbacks.append(callback)
+
+    @ta.overload
+    def register(self, obj: K, val: V) -> None:
+        ...
+
+    @ta.overload
+    def register(self, val: V) -> ta.Callable[[T], T]:
+        ...
+
+    def register(self, *args):
+        def inner(obj, val):
+            check.not_in(obj, self._objs)
+
+            self._objs[obj] = val
+
+            for iv in self._invalidate_callbacks:
+                iv()
+
+            return obj
+
+        if len(args) == 1:
+            return lambda obj: inner(obj, args[0])
+        elif len(args) == 2:
+            inner = inner(*args)
+            return None
+        else:
+            raise TypeError(args)
+
+    def _lookup(self, obj: ta.Any) -> lang.Maybe[V]:
+        if not self._identity:
+            try:
+                hash(obj)
+            except TypeError:
+                return lang.empty()
+
+        try:
+            val = self._objs[obj]
+        except KeyError:
+            return lang.empty()
+        else:
+            return lang.just(val)
+
+    @dc.dataclass()
+    class DuplicatesForbiddenError(Exception):
+        owner_cls: type
+        instance_cls: type
+        att: str
+        ex_att: str
+
+    def collect(self, instance_cls: type, owner_cls: type | None = None) -> dict[str, tuple[K, V]]:
+        if owner_cls is None:
+            owner_cls = instance_cls
+
+        mro = instance_cls.__mro__[-2::-1]
+        try:
+            mro_pos = mro.index(owner_cls)
+        except ValueError:
+            raise TypeError(f'Owner class {owner_cls} not in mro of instance class {instance_cls}') from None
+
+        mro_dct: dict[str, list[tuple[type, ta.Any]]] = {}
+        for cur_cls in mro[:mro_pos + 1]:
+            for att, obj in cur_cls.__dict__.items():
+                if att not in mro_dct:
+                    if not self._lookup(obj).present:
+                        continue
+
+                try:
+                    lst = mro_dct[att]
+                except KeyError:
+                    lst = mro_dct[att] = []
+                lst.append((cur_cls, obj))
+
+        #
+
+        seen: ta.MutableMapping[ta.Any, str] | None = None
+        if self._forbid_duplicates:
+            seen = dict_factory(identity=self._identity)()
+
+        out: dict[str, tuple[K, V]] = {}
+        for att, lst in mro_dct.items():
+            if not lst:
+                raise RuntimeError
+            _, obj = lst[-1]
+
+            if len(lst) > 1:
+                if self._requires_override and not (self._is_override or lang.is_override)(obj):
+                    raise lang.RequiresOverrideError(
+                        att,
+                        instance_cls,
+                        lst[-1][0],
+                        lst[0][0],
+                    )
+
+            if not (mv := self._lookup(obj)).present:
+                continue
+
+            if seen is not None:
+                try:
+                    ex_att = seen[obj]
+                except KeyError:
+                    pass
+                else:
+                    raise AttrRegistry.DuplicatesForbiddenError(owner_cls, instance_cls, att, ex_att)  # noqa
+                seen[obj] = att
+
+            out[att] = (obj, mv.must())
+
+        return out
+
+
+##
+
+
+class AttrRegistryCache(ta.Generic[K, V, T]):
+    def __init__(
+            self,
+            registry: AttrRegistry[K, V],
+            prepare: ta.Callable[[type, dict[str, tuple[K, V]]], T],
+    ) -> None:
+        super().__init__()
+
+        self._registry = registry
+        self._prepare = prepare
+
+        self._cache: dict[ta.Any, T] = {}
+
+        def cache_remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                cache = ref_self._cache  # noqa
+                try:
+                    del cache[k]
+                except KeyError:
+                    pass
+
+        self._cache_remove = cache_remove
+
+        registry.add_invalidate_callback(self._cache.clear)
+
+    def get(self, instance_cls: type) -> T:
+        cls_ref = weakref.ref(instance_cls)
+        try:
+            return self._cache[cls_ref]
+        except KeyError:
+            pass
+        del cls_ref
+
+        collected = self._registry.collect(instance_cls)
+        out = self._prepare(instance_cls, collected)
+        self._cache[weakref.ref(instance_cls, self._cache_remove)] = out
+        return out
+
+
+class SimpleAttrRegistryCache(AttrRegistryCache[K, V, dict[str, tuple[K, V]]], ta.Generic[K, V]):
+    def __init__(self, registry: AttrRegistry[K, V]) -> None:
+        super().__init__(registry, lambda _, dct: dct)

omlish/collections/cache/impl.py

@@ -41,6 +41,7 @@ def LRU(cache: 'Cache') -> None:  # noqa
     cache._kill(cache._root.lru_next)  # type: ignore  # noqa
 
 
+# aka FIFO
 def LRI(cache: 'Cache') -> None:  # noqa
     cache._kill(cache._root.ins_next)  # type: ignore  # noqa
 

omlish/collections/identity.py

@@ -105,6 +105,7 @@ class IdentityWeakKeyDictionary(ta.MutableMapping[K, V]):
 
     TODO:
      - audit for freethreaded
+      - leans on stdlib impls so shouldn't matter?
     """
 
     def __init__(self, *args: ta.Any, **kwargs: ta.Any) -> None:

omlish/collections/mappings.py

@@ -1,6 +1,12 @@
 import typing as ta
 import weakref
 
+from .. import lang
+
+
+with lang.auto_proxy_import(globals()):
+    from . import identity as _identity
+
 
 T = ta.TypeVar('T')
 K = ta.TypeVar('K')
@@ -63,6 +69,9 @@ class TypeMap(ta.Generic[T]):
     def __iter__(self) -> ta.Iterator[T]:
         return iter(self._items)
 
+    def __contains__(self, ty: type[T]) -> bool:
+        return ty in self._items
+
     def get(self, ty: type[T]) -> T | None:
         return self._dct.get(ty)
 
@@ -138,3 +147,22 @@ class MissingDict(dict[K, V]):
     def __missing__(self, key):
         v = self[key] = self._missing_fn(key)
         return v
+
+
+##
+
+
+def dict_factory[K, V](
+        *,
+        identity: bool = False,
+        weak: bool = False,
+) -> ta.Callable[..., ta.MutableMapping[K, V]]:
+    if identity:
+        if weak:
+            return _identity.IdentityWeakKeyDictionary
+        else:
+            return _identity.IdentityKeyDict
+    elif weak:
+        return weakref.WeakKeyDictionary
+    else:
+        return dict

omlish/collections/trie.py

@@ -37,12 +37,16 @@ class Trie(ta.MutableMapping[ta.Sequence[K], V], ta.Generic[K, V]):
         def children(self) -> ta.Mapping[K2, 'Trie.Node[K2, V2]']:
             return self._children
 
-    def __init__(self) -> None:
+    def __init__(self, items: ta.Iterable[tuple[ta.Iterable[K], V]] | None = None) -> None:
         super().__init__()
 
         self._len = 0
         self._root: Trie.Node[K, V] = Trie.Node()
 
+        if items is not None:
+            for k, v in items:
+                self[k] = v
+
     @property
     def root(self) -> Node[K, V]:
         return self._root

omlish/collections/utils.py

@@ -7,7 +7,9 @@ from .identity import IdentitySet
 
 T = ta.TypeVar('T')
 K = ta.TypeVar('K')
+K2 = ta.TypeVar('K2')
 V = ta.TypeVar('V')
+V2 = ta.TypeVar('V2')
 
 
 ##
@@ -135,6 +137,81 @@ def make_map_by(
 ##
 
 
+@ta.overload
+def map_keys(
+        fn: ta.Callable[[K], K2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: ta.Literal[False] = False,
+        strict: bool = False,
+) -> dict[K2, V]:
+    return {}
+
+
+@ta.overload
+def map_keys(
+        fn: ta.Callable[[K], K2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: bool = False,
+        strict: bool = False,
+) -> ta.MutableMapping[K2, V]:
+    return {}
+
+
+def map_keys(
+        fn,
+        dct,
+        *,
+        identity=False,
+        strict=False,
+):
+    return make_map(
+        ((fn(k), v) for k, v in (dct.items() if isinstance(dct, ta.Mapping) else dct)),
+        identity=identity,
+        strict=strict,
+    )
+
+
+@ta.overload
+def map_values(
+        fn: ta.Callable[[V], V2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: ta.Literal[False] = False,
+        strict: bool = False,
+) -> dict[K, V2]:
+    return {}
+
+
+@ta.overload
+def map_values(
+        fn: ta.Callable[[V], V2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: bool = False,
+        strict: bool = False,
+) -> ta.MutableMapping[K2, V]:
+    return {}
+
+
+def map_values(
+        fn,
+        dct,
+        *,
+        identity=False,
+        strict=False,
+):
+    return make_map(
+        ((k, fn(v)) for k, v in (dct.items() if isinstance(dct, ta.Mapping) else dct)),
+        identity=identity,
+        strict=strict,
+    )
+
+
+##
+
+
 @ta.overload
 def multi_map(
         kvs: ta.Iterable[tuple[K, V]],

omlish/concurrent/all.py

@@ -12,6 +12,7 @@ with _lang.auto_proxy_init(globals()):
     from .futures import (  # noqa
         FutureError,
         FutureTimeoutError,
-        wait_futures,
+        wait_all_futures_or_raise,
         wait_dependent_futures,
+        wait_futures,
     )

omlish/concurrent/futures.py

@@ -48,6 +48,13 @@ def wait_futures(
         raise_exceptions: bool = False,
         cancel_on_exception: bool = False,
 ) -> bool:
+    # TODO:
+    #  - raise_exceptions 'at_end', ExceptionGroup
+    #  - more responsive than tick_interval_s - semaphore, add callbacks to each fut to decrement, sleep with a timed
+    #    wait on sem
+    #  - cancel_on_timeout
+    #  - obviate wait_all_futures_or_raise
+
     start = time.time()
 
     not_done = set(futures)
@@ -72,6 +79,24 @@ def wait_futures(
     return False
 
 
+def wait_all_futures_or_raise(futures: ta.Sequence[cf.Future]) -> None:
+    done, not_done = cf.wait(futures, return_when=cf.ALL_COMPLETED)
+    if not_done:
+        raise RuntimeError(f'Not all futures finished: {not_done}')
+
+    excs: list[Exception] = []
+    for f in done:
+        try:
+            f.result()
+        except Exception as e:  # noqa
+            excs.append(e)
+
+    if len(excs) == 1:
+        raise excs[0]
+    elif excs:
+        raise ExceptionGroup('One or more futures failed', excs)
+
+
 def wait_dependent_futures(
         executor: cf.Executor,
         dependency_sets_by_fn: ta.Mapping[ta.Callable, ta.AbstractSet[ta.Callable]],