omlish 0.0.0.dev228__tar.gz → 0.0.0.dev230__tar.gz

{omlish-0.0.0.dev228/omlish.egg-info → omlish-0.0.0.dev230}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev228
+Version: 0.0.0.dev230
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev228'
-__revision__ = '00e3d1141f5caccfbfa74099183b2d12485e1b0d'
+__version__ = '0.0.0.dev230'
+__revision__ = 'ed2672ad6a2fb4718c57e281c5ce91fdc7998a6f'
 
 
 #

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/__init__.py

@@ -76,39 +76,39 @@ from .ordered import (  # noqa
     OrderedSet,
 )
 
-from .persistent import (  # noqa
+from .persistent.persistent import (  # noqa
     PersistentMap,
 )
 
 if _ta.TYPE_CHECKING:
-    from .skiplist import (  # noqa
+    from .persistent.treapmap import (  # noqa
+        TreapMap,
+        new_treap_map,
+    )
+else:
+    _lang.proxy_init(globals(), '.persistent.treapmap', [
+        'TreapMap',
+        'new_treap_map',
+    ])
+
+if _ta.TYPE_CHECKING:
+    from .sorted.skiplist import (  # noqa
         SkipList,
         SkipListDict,
     )
 else:
-    _lang.proxy_init(globals(), '.skiplist', [
+    _lang.proxy_init(globals(), '.sorted.skiplist', [
         'SkipList',
         'SkipListDict',
     ])
 
-from .sorted import (  # noqa
+from .sorted.sorted import (  # noqa
     SortedCollection,
     SortedListDict,
     SortedMapping,
     SortedMutableMapping,
 )
 
-if _ta.TYPE_CHECKING:
-    from .treapmap import (  # noqa
-        TreapMap,
-        new_treap_map,
-    )
-else:
-    _lang.proxy_init(globals(), '.treapmap', [
-        'TreapMap',
-        'new_treap_map',
-    ])
-
 from .unmodifiable import (  # noqa
     Unmodifiable,
     UnmodifiableMapping,

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/frozen.py

@@ -16,7 +16,6 @@ class Frozen(ta.Hashable, abc.ABC):
 
 
 class FrozenDict(ta.Mapping[K, V], Frozen):
-
     def __new__(cls, *args: ta.Any, **kwargs: ta.Any) -> 'FrozenDict[K, V]':  # noqa
         if len(args) == 1 and Frozen in type(args[0]).__bases__:
             return args[0]
@@ -73,7 +72,6 @@ class FrozenDict(ta.Mapping[K, V], Frozen):
 
 
 class FrozenList(ta.Sequence[T], Frozen):
-
     def __init__(self, it: ta.Iterable[T] | None = None) -> None:
         super().__init__()
 

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/identity.py

@@ -13,7 +13,6 @@ V = ta.TypeVar('V')
 
 
 class IdentityWrapper(ta.Generic[T]):
-
     def __init__(self, value: T) -> None:
         super().__init__()
         self._value = value
@@ -36,7 +35,6 @@ class IdentityWrapper(ta.Generic[T]):
 
 
 class IdentityKeyDict(ta.MutableMapping[K, V]):
-
     def __init__(self, *args, **kwargs) -> None:
         super().__init__()
         self._dict: dict[int, tuple[K, V]] = {}
@@ -70,7 +68,6 @@ class IdentityKeyDict(ta.MutableMapping[K, V]):
 
 
 class IdentitySet(ta.MutableSet[T]):
-
     def __init__(self, init: ta.Iterable[T] | None = None) -> None:
         super().__init__()
         self._dict: dict[int, T] = {}

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/indexed.py

@@ -8,7 +8,6 @@ T = ta.TypeVar('T')
 
 
 class IndexedSeq(ta.Sequence[T]):
-
     def __init__(self, it: ta.Iterable[T], *, identity: bool = False) -> None:
         super().__init__()
 
@@ -42,7 +41,6 @@ class IndexedSeq(ta.Sequence[T]):
 
 
 class IndexedSetSeq(ta.Sequence[ta.AbstractSet[T]]):
-
     def __init__(self, it: ta.Iterable[ta.Iterable[T]], *, identity: bool = False) -> None:
         super().__init__()
 

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/mappings.py

@@ -49,7 +49,6 @@ def yield_dict_init(*args, **kwargs) -> ta.Iterable[tuple[ta.Any, ta.Any]]:
 
 
 class TypeMap(ta.Generic[T]):
-
     def __init__(self, items: ta.Iterable[T] = ()) -> None:
         super().__init__()
 
@@ -79,7 +78,6 @@ class TypeMap(ta.Generic[T]):
 
 
 class DynamicTypeMap(ta.Generic[V]):
-
     def __init__(self, items: ta.Iterable[V] = (), *, weak: bool = False) -> None:
         super().__init__()
 
@@ -115,7 +113,6 @@ class DynamicTypeMap(ta.Generic[V]):
 
 
 class MissingDict(dict[K, V]):
-
     def __init__(self, missing_fn: ta.Callable[[K], V]) -> None:
         if not callable(missing_fn):
             raise TypeError(missing_fn)

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/ordered.py

@@ -5,7 +5,6 @@ T = ta.TypeVar('T')
 
 
 class OrderedSet(ta.MutableSet[T]):
-
     def __init__(self, iterable: ta.Iterable[T] | None = None) -> None:
         super().__init__()
         self._dct: dict[T, ta.Any] = {}
@@ -56,7 +55,6 @@ class OrderedSet(ta.MutableSet[T]):
 
 
 class OrderedFrozenSet(ta.FrozenSet[T]):  # noqa
-
     _list: ta.Sequence[T]
 
     def __new__(cls, items: ta.Iterable[T]) -> frozenset[T]:  # type: ignore

{omlish-0.0.0.dev228/omlish/collections → omlish-0.0.0.dev230/omlish/collections/sorted}/skiplist.py

@@ -188,6 +188,5 @@ class SkipList(SortedCollection[T]):
 
 
 class SkipListDict(SortedListDict[K, V]):
-
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(SkipList(comparator=SortedListDict._item_comparator), *args, **kwargs)  # noqa

{omlish-0.0.0.dev228/omlish/collections → omlish-0.0.0.dev230/omlish/collections/sorted}/sorted.py

@@ -1,8 +1,8 @@
 import abc
 import typing as ta
 
-from .. import lang
-from .mappings import yield_dict_init
+from ... import lang
+from ..mappings import yield_dict_init
 
 
 T = ta.TypeVar('T')
@@ -12,7 +12,6 @@ V = ta.TypeVar('V')
 
 
 class SortedCollection(lang.Abstract, ta.Collection[T]):
-
     Comparator = ta.Callable[[U, U], int]
 
     @staticmethod
@@ -55,7 +54,6 @@ class SortedCollection(lang.Abstract, ta.Collection[T]):
 
 
 class SortedMapping(ta.Mapping[K, V]):
-
     @abc.abstractmethod
     def items(self) -> ta.Iterator[tuple[K, V]]:  # type: ignore
         raise NotImplementedError
@@ -78,7 +76,6 @@ class SortedMutableMapping(ta.MutableMapping[K, V], SortedMapping[K, V]):
 
 
 class SortedListDict(SortedMutableMapping[K, V]):
-
     @staticmethod
     def _item_comparator(a: tuple[K, V], b: tuple[K, V]) -> int:
         return SortedCollection.default_comparator(a[0], b[0])

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/collections/unmodifiable.py

@@ -13,7 +13,6 @@ class Unmodifiable(lang.Abstract):
 
 
 class UnmodifiableSequence(ta.Sequence[T], Unmodifiable, lang.Final):
-
     def __init__(self, target: ta.Sequence[T]) -> None:
         super().__init__()
 
@@ -65,7 +64,6 @@ class UnmodifiableSequence(ta.Sequence[T], Unmodifiable, lang.Final):
 
 
 class UnmodifiableSet(ta.AbstractSet[T], Unmodifiable, lang.Final):
-
     def __init__(self, target: ta.AbstractSet[T]) -> None:
         super().__init__()
 

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/dispatch/dispatch.py

@@ -31,7 +31,10 @@ def get_impl_func_cls_set(func: ta.Callable) -> frozenset[type]:
         else:
             return check.isinstance(a, type)
 
-    _, cls = next(iter(ta.get_type_hints(func).items()))
+    # Exclude 'return' to support difficult to handle return types - they are unimportant.
+    # TODO: only get hints for first arg - requires inspection, which requires chopping off `self`, which can be tricky.
+    _, cls = next(iter(rfl.get_filtered_type_hints(func, exclude=['return']).items()))
+
     rty = rfl.type_(cls)
     if isinstance(rty, rfl.Union):
         ret = frozenset(erase(arg) for arg in rty.args)

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/dispatch/methods.py

@@ -81,6 +81,10 @@ class Method:
         mro_dct = lang.build_mro_dict(instance_cls, owner_cls)
         seen: ta.Mapping[ta.Any, str] = {}
         for nam, att in mro_dct.items():
+            try:
+                hash(att)
+            except TypeError:
+                continue
             if att in self._impls:
                 try:
                     ex_nam = seen[att]

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/formats/json/__init__.py

@@ -1,4 +1,9 @@
 # ruff: noqa: I001
+"""
+TODO:
+ - delimited.py / jsonl
+  - + record separators ala https://en.wikipedia.org/wiki/JSON_streaming
+"""
 import typing as _ta
 
 from ... import lang as _lang

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/io/compress/brotli.py

@@ -27,9 +27,9 @@ class BrotliCompression(Compression):
         return brotli.compress(
             d,
             **(dict(mode=self.mode) if self.mode is not None else {}),
-            **(dict(mode=self.quality) if self.quality is not None else {}),
-            **(dict(mode=self.lgwin) if self.lgwin is not None else {}),
-            **(dict(mode=self.lgblock) if self.lgblock is not None else {}),
+            **(dict(quality=self.quality) if self.quality is not None else {}),
+            **(dict(lgwin=self.lgwin) if self.lgwin is not None else {}),
+            **(dict(lgblock=self.lgblock) if self.lgblock is not None else {}),
         )
 
     def decompress(self, d: bytes) -> bytes:

omlish-0.0.0.dev230/omlish/io/fdio/__init__.py

@@ -0,0 +1,4 @@
+# @omlish-lite
+"""
+Basically the old asyncore system, which still has usecases over asyncio (such as single-threaded, forking code).
+"""

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/marshal/__init__.py

@@ -34,11 +34,6 @@ from .exceptions import (  # noqa
     UnhandledTypeError,
 )
 
-from .forbidden import (  # noqa
-    ForbiddenTypeMarshalerFactory,
-    ForbiddenTypeUnmarshalerFactory,
-)
-
 from .global_ import (  # noqa
     GLOBAL_REGISTRY,
 
@@ -54,11 +49,6 @@ from .naming import (  # noqa
     translate_name,
 )
 
-from .nop import (  # noqa
-    NOP_MARSHALER_UNMARSHALER,
-    NopMarshalerUnmarshaler,
-)
-
 from .objects.helpers import (  # noqa
     update_field_metadata,
     update_fields_metadata,
@@ -127,6 +117,16 @@ from .standard import (  # noqa
     new_standard_unmarshaler_factory,
 )
 
+from .trivial.forbidden import (  # noqa
+    ForbiddenTypeMarshalerFactory,
+    ForbiddenTypeUnmarshalerFactory,
+)
+
+from .trivial.nop import (  # noqa
+    NOP_MARSHALER_UNMARSHALER,
+    NopMarshalerUnmarshaler,
+)
+
 from .values import (  # noqa
     Value,
 )

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/marshal/base.py

@@ -95,9 +95,9 @@ from .exceptions import UnhandledTypeError
 from .factories import RecursiveTypeFactory
 from .factories import TypeCacheFactory
 from .factories import TypeMapFactory
+from .proxy import _Proxy
 from .registries import Registry
 from .registries import RegistryItem
-from .utils import _Proxy
 from .values import Value
 
 

{omlish-0.0.0.dev228 → omlish-0.0.0.dev230}/omlish/marshal/standard.py

@@ -23,8 +23,6 @@ from .objects.namedtuples import NamedtupleMarshalerFactory
 from .objects.namedtuples import NamedtupleUnmarshalerFactory
 from .polymorphism.unions import PrimitiveUnionMarshalerFactory
 from .polymorphism.unions import PrimitiveUnionUnmarshalerFactory
-from .singular.any import ANY_MARSHALER_FACTORY
-from .singular.any import ANY_UNMARSHALER_FACTORY
 from .singular.base64 import BASE64_MARSHALER_FACTORY
 from .singular.base64 import BASE64_UNMARSHALER_FACTORY
 from .singular.datetimes import DATETIME_MARSHALER_FACTORY
@@ -37,6 +35,8 @@ from .singular.primitives import PRIMITIVE_MARSHALER_FACTORY
 from .singular.primitives import PRIMITIVE_UNMARSHALER_FACTORY
 from .singular.uuids import UUID_MARSHALER_FACTORY
 from .singular.uuids import UUID_UNMARSHALER_FACTORY
+from .trivial.any import ANY_MARSHALER_FACTORY
+from .trivial.any import ANY_UNMARSHALER_FACTORY
 
 
 ##

{omlish-0.0.0.dev228/omlish/marshal → omlish-0.0.0.dev230/omlish/marshal/trivial}/forbidden.py

@@ -1,13 +1,13 @@
 import dataclasses as dc
 import typing as ta
 
-from .. import reflect as rfl
-from ..funcs import match as mfs
-from .base import MarshalContext
-from .base import Marshaler
-from .base import UnmarshalContext
-from .base import Unmarshaler
-from .exceptions import ForbiddenTypeError
+from ... import reflect as rfl
+from ...funcs import match as mfs
+from ..base import MarshalContext
+from ..base import Marshaler
+from ..base import UnmarshalContext
+from ..base import Unmarshaler
+from ..exceptions import ForbiddenTypeError
 
 
 C = ta.TypeVar('C')

{omlish-0.0.0.dev228/omlish/marshal → omlish-0.0.0.dev230/omlish/marshal/trivial}/nop.py

@@ -1,10 +1,10 @@
 import typing as ta
 
-from .base import MarshalContext
-from .base import Marshaler
-from .base import UnmarshalContext
-from .base import Unmarshaler
-from .values import Value
+from ..base import MarshalContext
+from ..base import Marshaler
+from ..base import UnmarshalContext
+from ..base import Unmarshaler
+from ..values import Value
 
 
 class NopMarshalerUnmarshaler(Marshaler, Unmarshaler):

omlish-0.0.0.dev230/omlish/os/deathpacts/__init__.py

@@ -0,0 +1,15 @@
+from .base import (  # noqa
+    Deathpact,
+    NopDeathpact,
+
+    BaseDeathpact,
+)
+
+from .heartbeatfile import (  # noqa
+    HeartbeatFileDeathpact,
+)
+
+from .pipe import (  # noqa
+    PipeDeathpact,
+    ForkAwarePipeDeathpact,
+)

omlish-0.0.0.dev230/omlish/os/deathpacts/base.py

@@ -0,0 +1,76 @@
+import abc
+import os
+import signal
+import sys
+import time
+import typing as ta
+
+
+##
+
+
+class Deathpact(abc.ABC):
+    @abc.abstractmethod
+    def poll(self) -> None:
+        raise NotImplementedError
+
+
+class NopDeathpact(Deathpact):
+    def poll(self) -> None:
+        pass
+
+
+##
+
+
+class BaseDeathpact(Deathpact, abc.ABC):
+    def __init__(
+            self,
+            *,
+            interval_s: float = .5,
+            signal: int | None = signal.SIGTERM,  # noqa
+            output: ta.Literal['stdout', 'stderr'] | None = 'stderr',
+            on_die: ta.Callable[[], None] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._interval_s = interval_s
+        self._signal = signal
+        self._output = output
+        self._on_die = on_die
+
+        self._last_check_t: float | None = None
+
+    def _print(self, msg: str) -> None:
+        match self._output:
+            case 'stdout':
+                f = sys.stdout
+            case 'stderr':
+                f = sys.stderr
+            case _:
+                return
+        print(f'{self} pid={os.getpid()}: {msg}', file=f)
+
+    def die(self) -> None:
+        self._print('Triggered! Process terminating!')
+
+        if self._on_die is not None:
+            self._on_die()
+
+        if self._signal is not None:
+            os.kill(os.getpid(), self._signal)
+
+        sys.exit(1)
+
+    @abc.abstractmethod
+    def should_die(self) -> bool:
+        raise NotImplementedError
+
+    def maybe_die(self) -> None:
+        if self.should_die():
+            self.die()
+
+    def poll(self) -> None:
+        if self._last_check_t is None or (time.monotonic() - self._last_check_t) >= self._interval_s:
+            self.maybe_die()
+            self._last_check_t = time.monotonic()

omlish-0.0.0.dev230/omlish/os/deathpacts/heartbeatfile.py

@@ -0,0 +1,85 @@
+"""
+TODO:
+ - chaining
+"""
+import os
+import time
+import typing as ta
+
+from ... import check
+from ..forkhooks import ProcessOriginTracker
+from ..temp import make_temp_file
+from .base import BaseDeathpact
+
+
+##
+
+
+class HeartbeatFileDeathpact(BaseDeathpact):
+    def __init__(
+            self,
+            path: str,
+            ttl_s: float = 10.,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        self._path = path
+        self._ttl_s = ttl_s
+
+        self._process_origin = ProcessOriginTracker()
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    def is_parent(self) -> bool:
+        return self._process_origin.is_in_origin_process()
+
+    #
+
+    def __enter__(self) -> ta.Self:
+        check.state(self.is_parent())
+
+        self.update()
+
+        return self
+
+    def close(self) -> None:
+        if self.is_parent():
+            try:
+                os.unlink(self._path)
+            except FileNotFoundError:
+                pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    #
+
+    @classmethod
+    def _now(cls) -> float:
+        return time.monotonic()
+
+    def update(self) -> None:
+        check.state(self.is_parent())
+
+        new = make_temp_file()
+        with open(new, 'w') as f:
+            f.write(str(self._now()))
+
+        # FIXME: same filesystem
+        os.replace(new, self._path)
+
+    def read(self) -> float:
+        try:
+            with open(self._path) as f:
+                return float(f.read())
+        except FileNotFoundError:
+            return float('-inf')
+
+    def age(self) -> float:
+        return self._now() - self.read()
+
+    def should_die(self) -> bool:
+        return self.age() >= self._ttl_s