omlish 0.0.0.dev423__py3-none-any.whl → 0.0.0.dev484__py3-none-any.whl

omlish/bootstrap/sys.py

@@ -2,14 +2,10 @@
 import contextlib
 import dataclasses as dc
 import enum
-import faulthandler
 import gc
 import importlib
 import logging
 import os
-import pwd
-import resource
-import signal
 import sys
 import typing as ta
 
@@ -19,18 +15,17 @@ from .base import ContextBootstrap
 from .base import SimpleBootstrap
 
 
-if ta.TYPE_CHECKING:
+with lang.auto_proxy_import(globals()):
+    import faulthandler
+    import pwd
+    import resource
+    import signal
+
     from .. import libc
     from ..formats import dotenv
     from ..logs import all as logs
     from ..os.pidfiles import pidfile
 
-else:
-    libc = lang.proxy_import('..libc', __package__)
-    logs = lang.proxy_import('..logs', __package__)
-    dotenv = lang.proxy_import('..formats.dotenv', __package__)
-    pidfile = lang.proxy_import('..os.pidfiles.pidfile', __package__)
-
 
 ##
 
@@ -184,14 +179,16 @@ class FaulthandlerBootstrap(ContextBootstrap['FaulthandlerBootstrap.Config']):
 ##
 
 
-SIGNALS_BY_NAME = {
-    a[len('SIG'):]: v  # noqa
-    for a in dir(signal)
-    if a.startswith('SIG')
-    and not a.startswith('SIG_')
-    and a == a.upper()
-    and isinstance((v := getattr(signal, a)), int)
-}
+@lang.cached_function
+def signals_by_name() -> ta.Mapping[str, int]:
+    return {
+        a[len('SIG'):]: v  # noqa
+        for a in dir(signal)
+        if a.startswith('SIG')
+        and not a.startswith('SIG_')
+        and a == a.upper()
+        and isinstance((v := getattr(signal, a)), int)
+    }
 
 
 class PrctlBootstrap(SimpleBootstrap['PrctlBootstrap.Config']):
@@ -208,20 +205,22 @@ class PrctlBootstrap(SimpleBootstrap['PrctlBootstrap.Config']):
             if isinstance(self._config.deathsig, int):
                 sig = self._config.deathsig
             else:
-                sig = SIGNALS_BY_NAME[self._config.deathsig.upper()]
+                sig = signals_by_name()[self._config.deathsig.upper()]
             libc.prctl(libc.PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
 
 
 ##
 
 
-RLIMITS_BY_NAME = {
-    a[len('RLIMIT_'):]: v  # noqa
-    for a in dir(resource)
-    if a.startswith('RLIMIT_')
-    and a == a.upper()
-    and isinstance((v := getattr(resource, a)), int)
-}
+@lang.cached_function
+def rlimits_by_name() -> ta.Mapping[str, int]:
+    return {
+        a[len('RLIMIT_'):]: v  # noqa
+        for a in dir(resource)
+        if a.startswith('RLIMIT_')
+        and a == a.upper()
+        and isinstance((v := getattr(resource, a)), int)
+    }
 
 
 class RlimitBootstrap(ContextBootstrap['RlimitBootstrap.Config']):
@@ -240,7 +239,7 @@ class RlimitBootstrap(ContextBootstrap['RlimitBootstrap.Config']):
 
         prev = {}
         for k, (s, h) in self._config.limits.items():
-            i = RLIMITS_BY_NAME[k.upper()]
+            i = rlimits_by_name()[k.upper()]
             prev[i] = resource.getrlimit(i)
             resource.setrlimit(i, (or_infin(s), or_infin(h)))
 

omlish/c3.py

@@ -33,6 +33,7 @@
 #
 # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
 # License Agreement.
+import abc
 import functools
 import operator
 import typing as ta
@@ -75,7 +76,9 @@ def merge(seqs: ta.MutableSequence[list[T]]) -> list[T]:
 
 
 def _default_is_abstract(obj: ta.Any) -> bool:
-    return hasattr(obj, '__abstractmethods__')
+    # .lite.abstracts.Abstract has '__abstractmethods__' but is not an ABCMeta, and thus for the purposes of c3 mro are
+    # not considered 'abstract'. What 'abstract' means in this context is 'can have virtual subclasses'.
+    return hasattr(obj, '__abstractmethods__') and isinstance(obj, abc.ABCMeta)
 
 
 def mro(

omlish/cexts/include/omlish/omlish.hh

@@ -0,0 +1 @@
+#pragma once

omlish/collections/__init__.py

@@ -6,12 +6,21 @@ from .. import lang as _lang
 with _lang.auto_proxy_init(globals()):
     ##
 
+    from .attrregistry import (  # noqa
+        AttrRegistry,
+
+        AttrRegistryCache,
+        SimpleAttrRegistryCache,
+    )
+
     from .bimap import (  # noqa
         BiMap,
 
         make_bi_map,
     )
 
+    from . import cache  # noqa
+
     from .coerce import (  # noqa
         abs_set,
         abs_set_of,
@@ -64,9 +73,10 @@ with _lang.auto_proxy_init(globals()):
     from . import kv  # noqa
 
     from .mappings import (  # noqa
+        DynamicTypeMap,
         MissingDict,
         TypeMap,
-        DynamicTypeMap,
+        dict_factory,
         guarded_map_update,
         multikey_dict,
     )
@@ -145,6 +155,8 @@ with _lang.auto_proxy_init(globals()):
         key_cmp,
         make_map,
         make_map_by,
+        map_keys,
+        map_values,
         multi_map,
         multi_map_by,
         partition,

omlish/collections/attrregistry.py

@@ -0,0 +1,210 @@
+"""
+TODO:
+ - lock?
+"""
+import dataclasses as dc
+import typing as ta
+import weakref
+
+from .. import check
+from .. import lang
+from .mappings import dict_factory
+
+
+T = ta.TypeVar('T')
+K = ta.TypeVar('K')
+V = ta.TypeVar('V')
+
+
+##
+
+
+class AttrRegistry(ta.Generic[K, V]):
+    """
+    MRO-honoring class member registry. There are many ways to do this, and this one is attr name based: a class is
+    considered to have objects registered to it based on whether or not they are accessible by a non-shadowed,
+    MRO-resolved named attribute on that class.
+
+    Care must be taken when overriding registered objects from superclasses in subclasses - shadowing the attribute name
+    of the superclass member will not automatically register the new member with the same name to the registry - it must
+    be explicitly registered itself. This is a feature, allowing for selective de-registration of objects in subclasses
+    via name shadowing.
+    """
+
+    def __init__(
+            self,
+            *,
+            requires_override: bool = False,
+            is_override: ta.Callable[[ta.Any], bool] | None = None,
+            forbid_duplicates: bool = False,
+            identity: bool = False,
+            weak: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._requires_override = requires_override
+        self._is_override = is_override
+        self._forbid_duplicates = forbid_duplicates
+        self._identity = identity
+        self._weak = weak
+
+        self._objs: ta.MutableMapping[K, V] = dict_factory(identity=identity, weak=weak)()
+        self._invalidate_callbacks: list[ta.Callable[[], None]] = []
+
+    def add_invalidate_callback(self, callback: ta.Callable[[], None]) -> None:
+        self._invalidate_callbacks.append(callback)
+
+    @ta.overload
+    def register(self, obj: K, val: V) -> None:
+        ...
+
+    @ta.overload
+    def register(self, val: V) -> ta.Callable[[T], T]:
+        ...
+
+    def register(self, *args):
+        def inner(obj, val):
+            check.not_in(obj, self._objs)
+
+            self._objs[obj] = val
+
+            for iv in self._invalidate_callbacks:
+                iv()
+
+            return obj
+
+        if len(args) == 1:
+            return lambda obj: inner(obj, args[0])
+        elif len(args) == 2:
+            inner = inner(*args)
+            return None
+        else:
+            raise TypeError(args)
+
+    def _lookup(self, obj: ta.Any) -> lang.Maybe[V]:
+        if not self._identity:
+            try:
+                hash(obj)
+            except TypeError:
+                return lang.empty()
+
+        try:
+            val = self._objs[obj]
+        except KeyError:
+            return lang.empty()
+        else:
+            return lang.just(val)
+
+    @dc.dataclass()
+    class DuplicatesForbiddenError(Exception):
+        owner_cls: type
+        instance_cls: type
+        att: str
+        ex_att: str
+
+    def collect(self, instance_cls: type, owner_cls: type | None = None) -> dict[str, tuple[K, V]]:
+        if owner_cls is None:
+            owner_cls = instance_cls
+
+        mro = instance_cls.__mro__[-2::-1]
+        try:
+            mro_pos = mro.index(owner_cls)
+        except ValueError:
+            raise TypeError(f'Owner class {owner_cls} not in mro of instance class {instance_cls}') from None
+
+        mro_dct: dict[str, list[tuple[type, ta.Any]]] = {}
+        for cur_cls in mro[:mro_pos + 1]:
+            for att, obj in cur_cls.__dict__.items():
+                if att not in mro_dct:
+                    if not self._lookup(obj).present:
+                        continue
+
+                try:
+                    lst = mro_dct[att]
+                except KeyError:
+                    lst = mro_dct[att] = []
+                lst.append((cur_cls, obj))
+
+        #
+
+        seen: ta.MutableMapping[ta.Any, str] | None = None
+        if self._forbid_duplicates:
+            seen = dict_factory(identity=self._identity)()
+
+        out: dict[str, tuple[K, V]] = {}
+        for att, lst in mro_dct.items():
+            if not lst:
+                raise RuntimeError
+            _, obj = lst[-1]
+
+            if len(lst) > 1:
+                if self._requires_override and not (self._is_override or lang.is_override)(obj):
+                    raise lang.RequiresOverrideError(
+                        att,
+                        instance_cls,
+                        lst[-1][0],
+                        lst[0][0],
+                    )
+
+            if not (mv := self._lookup(obj)).present:
+                continue
+
+            if seen is not None:
+                try:
+                    ex_att = seen[obj]
+                except KeyError:
+                    pass
+                else:
+                    raise AttrRegistry.DuplicatesForbiddenError(owner_cls, instance_cls, att, ex_att)  # noqa
+                seen[obj] = att
+
+            out[att] = (obj, mv.must())
+
+        return out
+
+
+##
+
+
+class AttrRegistryCache(ta.Generic[K, V, T]):
+    def __init__(
+            self,
+            registry: AttrRegistry[K, V],
+            prepare: ta.Callable[[type, dict[str, tuple[K, V]]], T],
+    ) -> None:
+        super().__init__()
+
+        self._registry = registry
+        self._prepare = prepare
+
+        self._cache: dict[ta.Any, T] = {}
+
+        def cache_remove(k, self_ref=weakref.ref(self)):
+            if (ref_self := self_ref()) is not None:
+                cache = ref_self._cache  # noqa
+                try:
+                    del cache[k]
+                except KeyError:
+                    pass
+
+        self._cache_remove = cache_remove
+
+        registry.add_invalidate_callback(self._cache.clear)
+
+    def get(self, instance_cls: type) -> T:
+        cls_ref = weakref.ref(instance_cls)
+        try:
+            return self._cache[cls_ref]
+        except KeyError:
+            pass
+        del cls_ref
+
+        collected = self._registry.collect(instance_cls)
+        out = self._prepare(instance_cls, collected)
+        self._cache[weakref.ref(instance_cls, self._cache_remove)] = out
+        return out
+
+
+class SimpleAttrRegistryCache(AttrRegistryCache[K, V, dict[str, tuple[K, V]]], ta.Generic[K, V]):
+    def __init__(self, registry: AttrRegistry[K, V]) -> None:
+        super().__init__(registry, lambda _, dct: dct)

omlish/collections/cache/impl.py

@@ -10,12 +10,12 @@ TODO:
 import collections
 import contextlib
 import functools
-import logging
 import time
 import typing as ta
 import weakref
 
 from ... import lang
+from ...logs import all as logs
 from .types import Cache
 from .types import Eviction
 from .types import OverweightError
@@ -27,7 +27,7 @@ V = ta.TypeVar('V')
 V2 = ta.TypeVar('V2')
 
 
-log = logging.getLogger(__name__)
+log = logs.get_module_logger(globals())
 
 
 ##
@@ -41,6 +41,7 @@ def LRU(cache: 'Cache') -> None:  # noqa
     cache._kill(cache._root.lru_next)  # type: ignore  # noqa
 
 
+# aka FIFO
 def LRI(cache: 'Cache') -> None:  # noqa
     cache._kill(cache._root.ins_next)  # type: ignore  # noqa
 

omlish/collections/identity.py

@@ -105,6 +105,7 @@ class IdentityWeakKeyDictionary(ta.MutableMapping[K, V]):
 
     TODO:
      - audit for freethreaded
+      - leans on stdlib impls so shouldn't matter?
     """
 
     def __init__(self, *args: ta.Any, **kwargs: ta.Any) -> None:

omlish/collections/mappings.py

@@ -1,6 +1,12 @@
 import typing as ta
 import weakref
 
+from .. import lang
+
+
+with lang.auto_proxy_import(globals()):
+    from . import identity as _identity
+
 
 T = ta.TypeVar('T')
 K = ta.TypeVar('K')
@@ -63,6 +69,9 @@ class TypeMap(ta.Generic[T]):
     def __iter__(self) -> ta.Iterator[T]:
         return iter(self._items)
 
+    def __contains__(self, ty: type[T]) -> bool:
+        return ty in self._items
+
     def get(self, ty: type[T]) -> T | None:
         return self._dct.get(ty)
 
@@ -138,3 +147,22 @@ class MissingDict(dict[K, V]):
     def __missing__(self, key):
         v = self[key] = self._missing_fn(key)
         return v
+
+
+##
+
+
+def dict_factory[K, V](
+        *,
+        identity: bool = False,
+        weak: bool = False,
+) -> ta.Callable[..., ta.MutableMapping[K, V]]:
+    if identity:
+        if weak:
+            return _identity.IdentityWeakKeyDictionary
+        else:
+            return _identity.IdentityKeyDict
+    elif weak:
+        return weakref.WeakKeyDictionary
+    else:
+        return dict

omlish/collections/trie.py

@@ -37,12 +37,16 @@ class Trie(ta.MutableMapping[ta.Sequence[K], V], ta.Generic[K, V]):
         def children(self) -> ta.Mapping[K2, 'Trie.Node[K2, V2]']:
             return self._children
 
-    def __init__(self) -> None:
+    def __init__(self, items: ta.Iterable[tuple[ta.Iterable[K], V]] | None = None) -> None:
         super().__init__()
 
         self._len = 0
         self._root: Trie.Node[K, V] = Trie.Node()
 
+        if items is not None:
+            for k, v in items:
+                self[k] = v
+
     @property
     def root(self) -> Node[K, V]:
         return self._root

omlish/collections/utils.py

@@ -7,7 +7,9 @@ from .identity import IdentitySet
 
 T = ta.TypeVar('T')
 K = ta.TypeVar('K')
+K2 = ta.TypeVar('K2')
 V = ta.TypeVar('V')
+V2 = ta.TypeVar('V2')
 
 
 ##
@@ -135,6 +137,81 @@ def make_map_by(
 ##
 
 
+@ta.overload
+def map_keys(
+        fn: ta.Callable[[K], K2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: ta.Literal[False] = False,
+        strict: bool = False,
+) -> dict[K2, V]:
+    return {}
+
+
+@ta.overload
+def map_keys(
+        fn: ta.Callable[[K], K2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: bool = False,
+        strict: bool = False,
+) -> ta.MutableMapping[K2, V]:
+    return {}
+
+
+def map_keys(
+        fn,
+        dct,
+        *,
+        identity=False,
+        strict=False,
+):
+    return make_map(
+        ((fn(k), v) for k, v in (dct.items() if isinstance(dct, ta.Mapping) else dct)),
+        identity=identity,
+        strict=strict,
+    )
+
+
+@ta.overload
+def map_values(
+        fn: ta.Callable[[V], V2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: ta.Literal[False] = False,
+        strict: bool = False,
+) -> dict[K, V2]:
+    return {}
+
+
+@ta.overload
+def map_values(
+        fn: ta.Callable[[V], V2],
+        dct: ta.Mapping[K, V] | ta.Iterable[tuple[K, V]],
+        *,
+        identity: bool = False,
+        strict: bool = False,
+) -> ta.MutableMapping[K2, V]:
+    return {}
+
+
+def map_values(
+        fn,
+        dct,
+        *,
+        identity=False,
+        strict=False,
+):
+    return make_map(
+        ((k, fn(v)) for k, v in (dct.items() if isinstance(dct, ta.Mapping) else dct)),
+        identity=identity,
+        strict=strict,
+    )
+
+
+##
+
+
 @ta.overload
 def multi_map(
         kvs: ta.Iterable[tuple[K, V]],

omlish/concurrent/__init__.py

@@ -1,11 +0,0 @@
-from .executors import (  # noqa
-    ImmediateExecutor,
-    new_executor,
-)
-
-from .futures import (  # noqa
-    FutureError,
-    FutureTimeoutError,
-    wait_futures,
-    wait_dependent_futures,
-)

omlish/concurrent/all.py

@@ -0,0 +1,18 @@
+from .. import lang as _lang
+
+
+with _lang.auto_proxy_init(globals()):
+    ##
+
+    from .executors import (  # noqa
+        ImmediateExecutor,
+        new_executor,
+    )
+
+    from .futures import (  # noqa
+        FutureError,
+        FutureTimeoutError,
+        wait_all_futures_or_raise,
+        wait_dependent_futures,
+        wait_futures,
+    )

omlish/concurrent/futures.py

@@ -48,6 +48,13 @@ def wait_futures(
         raise_exceptions: bool = False,
         cancel_on_exception: bool = False,
 ) -> bool:
+    # TODO:
+    #  - raise_exceptions 'at_end', ExceptionGroup
+    #  - more responsive than tick_interval_s - semaphore, add callbacks to each fut to decrement, sleep with a timed
+    #    wait on sem
+    #  - cancel_on_timeout
+    #  - obviate wait_all_futures_or_raise
+
     start = time.time()
 
     not_done = set(futures)
@@ -72,6 +79,24 @@ def wait_futures(
     return False
 
 
+def wait_all_futures_or_raise(futures: ta.Sequence[cf.Future]) -> None:
+    done, not_done = cf.wait(futures, return_when=cf.ALL_COMPLETED)
+    if not_done:
+        raise RuntimeError(f'Not all futures finished: {not_done}')
+
+    excs: list[Exception] = []
+    for f in done:
+        try:
+            f.result()
+        except Exception as e:  # noqa
+            excs.append(e)
+
+    if len(excs) == 1:
+        raise excs[0]
+    elif excs:
+        raise ExceptionGroup('One or more futures failed', excs)
+
+
 def wait_dependent_futures(
         executor: cf.Executor,
         dependency_sets_by_fn: ta.Mapping[ta.Callable, ta.AbstractSet[ta.Callable]],

omlish/concurrent/threadlets.py

@@ -75,7 +75,7 @@ class GreenletThreadlet(Threadlet):
 
     @property
     def parent(self) -> ta.Optional['GreenletThreadlet']:
-        return GreenletThreadlet(self.g.parent)
+        return GreenletThreadlet(p) if (p := self.g.parent) is not None else None
 
     @property
     def dead(self) -> bool:

omlish/configs/processing/flattening.py

@@ -119,7 +119,7 @@ class ConfigFlattening:
     def unflatten(self, flattened: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.Any]:
         root = ConfigFlattening.UnflattenDict()
 
-        def split_keys(fkey: str) -> ta.Iterable[ta.Union[str, int]]:
+        def split_keys(fkey: str) -> ta.Iterator[ta.Union[str, int]]:
             for part in fkey.split(self._delimiter):
                 if self._index_open in part:
                     check.state(part.endswith(self._index_close))

omlish/configs/processing/merging.py

@@ -3,6 +3,7 @@
 """
 TODO:
  - smarter merging than just dumb dict-squashing
+  - at least sequence merging
 """
 import typing as ta
 
@@ -19,13 +20,14 @@ def merge_configs(*ms: ConfigMap) -> ConfigMap:
                 e = o[k]
             except KeyError:
                 o[k] = v
+                continue
+
+            if isinstance(e, ta.Mapping) and isinstance(v, ta.Mapping):
+                rec(e, v)  # noqa
             else:
-                if isinstance(e, ta.Mapping) and isinstance(v, ta.Mapping):
-                    rec(e, v)  # noqa
-                else:
-                    if isinstance(v, ta.Mapping):
-                        v = dict(v)
-                    o[k] = v
+                if isinstance(v, ta.Mapping):
+                    v = dict(v)
+                o[k] = v
 
     o: dict = {}
     for i in ms: