omlish 0.0.0.dev227__py3-none-any.whl → 0.0.0.dev229__py3-none-any.whl

omlish/marshal/standard.py

@@ -1,42 +1,42 @@
 from ..funcs import match as mfs
-from .any import ANY_MARSHALER_FACTORY
-from .any import ANY_UNMARSHALER_FACTORY
 from .base import MarshalerFactory
 from .base import RecursiveMarshalerFactory
 from .base import RecursiveUnmarshalerFactory
 from .base import TypeCacheMarshalerFactory
 from .base import TypeCacheUnmarshalerFactory
 from .base import UnmarshalerFactory
-from .base64 import BASE64_MARSHALER_FACTORY
-from .base64 import BASE64_UNMARSHALER_FACTORY
-from .dataclasses import DataclassMarshalerFactory
-from .dataclasses import DataclassUnmarshalerFactory
-from .datetimes import DATETIME_MARSHALER_FACTORY
-from .datetimes import DATETIME_UNMARSHALER_FACTORY
-from .enums import EnumMarshalerFactory
-from .enums import EnumUnmarshalerFactory
-from .iterables import IterableMarshalerFactory
-from .iterables import IterableUnmarshalerFactory
-from .literals import LiteralMarshalerFactory
-from .literals import LiteralUnmarshalerFactory
-from .mappings import MappingMarshalerFactory
-from .mappings import MappingUnmarshalerFactory
-from .maybes import MaybeMarshalerFactory
-from .maybes import MaybeUnmarshalerFactory
-from .namedtuples import NamedtupleMarshalerFactory
-from .namedtuples import NamedtupleUnmarshalerFactory
-from .newtypes import NewtypeMarshalerFactory
-from .newtypes import NewtypeUnmarshalerFactory
-from .numbers import NUMBERS_MARSHALER_FACTORY
-from .numbers import NUMBERS_UNMARSHALER_FACTORY
-from .optionals import OptionalMarshalerFactory
-from .optionals import OptionalUnmarshalerFactory
-from .primitives import PRIMITIVE_MARSHALER_FACTORY
-from .primitives import PRIMITIVE_UNMARSHALER_FACTORY
-from .unions import PrimitiveUnionMarshalerFactory
-from .unions import PrimitiveUnionUnmarshalerFactory
-from .uuids import UUID_MARSHALER_FACTORY
-from .uuids import UUID_UNMARSHALER_FACTORY
+from .composite.iterables import IterableMarshalerFactory
+from .composite.iterables import IterableUnmarshalerFactory
+from .composite.literals import LiteralMarshalerFactory
+from .composite.literals import LiteralUnmarshalerFactory
+from .composite.mappings import MappingMarshalerFactory
+from .composite.mappings import MappingUnmarshalerFactory
+from .composite.maybes import MaybeMarshalerFactory
+from .composite.maybes import MaybeUnmarshalerFactory
+from .composite.newtypes import NewtypeMarshalerFactory
+from .composite.newtypes import NewtypeUnmarshalerFactory
+from .composite.optionals import OptionalMarshalerFactory
+from .composite.optionals import OptionalUnmarshalerFactory
+from .objects.dataclasses import DataclassMarshalerFactory
+from .objects.dataclasses import DataclassUnmarshalerFactory
+from .objects.namedtuples import NamedtupleMarshalerFactory
+from .objects.namedtuples import NamedtupleUnmarshalerFactory
+from .polymorphism.unions import PrimitiveUnionMarshalerFactory
+from .polymorphism.unions import PrimitiveUnionUnmarshalerFactory
+from .singular.any import ANY_MARSHALER_FACTORY
+from .singular.any import ANY_UNMARSHALER_FACTORY
+from .singular.base64 import BASE64_MARSHALER_FACTORY
+from .singular.base64 import BASE64_UNMARSHALER_FACTORY
+from .singular.datetimes import DATETIME_MARSHALER_FACTORY
+from .singular.datetimes import DATETIME_UNMARSHALER_FACTORY
+from .singular.enums import EnumMarshalerFactory
+from .singular.enums import EnumUnmarshalerFactory
+from .singular.numbers import NUMBERS_MARSHALER_FACTORY
+from .singular.numbers import NUMBERS_UNMARSHALER_FACTORY
+from .singular.primitives import PRIMITIVE_MARSHALER_FACTORY
+from .singular.primitives import PRIMITIVE_UNMARSHALER_FACTORY
+from .singular.uuids import UUID_MARSHALER_FACTORY
+from .singular.uuids import UUID_UNMARSHALER_FACTORY
 
 
 ##

omlish/os/death.py

@@ -4,8 +4,11 @@ import signal
 import sys
 import time
 import typing as ta
+import weakref
 
 from .. import check
+from .forkhooks import ForkHook
+from .forkhooks import get_fork_depth
 
 
 ##
@@ -82,12 +85,24 @@ class BaseDeathpact(Deathpact, abc.ABC):
 
 
 class PipeDeathpact(BaseDeathpact):
+    """
+    NOTE: Closes write side in children lazily on poll - does not proactively close write sides on fork. This means
+          parents which fork children into codepaths unaware of live PipeDeathpacts will leave write sides open in those
+          children, potentially leading to zombies (if those children outlast the parent). Use ForkAwarePipeDeathpact to
+          handle such cases.
+    """
+
+    _COOKIE: ta.ClassVar[bytes] = os.urandom(16)
+
     def __init__(self, **kwargs: ta.Any) -> None:
         super().__init__(**kwargs)
 
         self._rfd: int | None = None
         self._wfd: int | None = None
 
+        self._cookie: bytes | None = self._COOKIE
+        self._fork_depth: int | None = get_fork_depth()
+
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}(rfd={self._rfd}, wfd={self._wfd})'
 
@@ -95,26 +110,54 @@ class PipeDeathpact(BaseDeathpact):
     def pass_fd(self) -> int:
         return check.not_none(self._rfd)
 
+    def is_parent(self) -> bool:
+        return (self._COOKIE, get_fork_depth()) == (self._cookie, self._fork_depth)
+
+    #
+
     def __enter__(self) -> ta.Self:
         check.none(self._rfd)
         check.none(self._wfd)
 
         self._rfd, self._wfd = os.pipe()
 
-        os.set_inheritable(self._rfd, True)
         os.set_blocking(self._rfd, False)
 
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._rfd is not None:
-            os.close(check.not_none(self._wfd))
+    def _close_wfd_if_not_parent(self) -> None:
+        if self._wfd is not None:
+            if not self.is_parent():
+                os.close(check.not_none(self._wfd))
             self._wfd = None
 
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._rfd is not None:
             os.close(self._rfd)
             self._rfd = None
 
+        self._close_wfd_if_not_parent()
+
+    #
+
+    def __getstate__(self):
+        return {
+            **self.__dict__,
+            **dict(
+                _wfd=None,
+                _cookie=None,
+                _fork_depth=None,
+            ),
+        }
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+
+    #
+
     def should_die(self) -> bool:
+        self._close_wfd_if_not_parent()
+
         try:
             buf = os.read(check.not_none(self._rfd), 1)
         except BlockingIOError:
@@ -125,3 +168,28 @@ class PipeDeathpact(BaseDeathpact):
             self.die()
 
         return True
+
+
+#
+
+
+class ForkAwarePipeDeathpact(PipeDeathpact):
+    """
+    TODO:
+     - Despite no correct way to do threads+forks, still audit thread-safety. Is WeakSet threadsafe? Probably not..
+    """
+
+    _PARENTS: ta.ClassVar[ta.MutableSet['ForkAwarePipeDeathpact']] = weakref.WeakSet()
+
+    def __init__(self, **kwargs: ta.Any) -> None:
+        super().__init__(**kwargs)
+
+        self._ForkHook.install()
+        self._PARENTS.add(self)
+
+    class _ForkHook(ForkHook):
+        @classmethod
+        def _after_fork_in_child(cls) -> None:
+            for pdp in ForkAwarePipeDeathpact._PARENTS:
+                pdp._close_wfd_if_not_parent()  # noqa
+            ForkAwarePipeDeathpact._PARENTS.clear()

omlish/os/fcntl.py

@@ -19,7 +19,7 @@ class FcntlLockData:
 
     #
 
-    _STRUCT_PACK_BY_PLATFORM: ta.ClassVar[ta.Mapping[str, ta.Sequence[ta.Tuple[str, str]]]] = {
+    _STRUCT_PACKING_BY_PLATFORM: ta.ClassVar[ta.Mapping[str, ta.Sequence[ta.Tuple[str, str]]]] = {
         'linux': [
             ('type', 'h'),
             ('whence', 'h'),
@@ -36,24 +36,23 @@ class FcntlLockData:
         ],
     }
 
-    def pack(self) -> bytes:
+    @classmethod
+    def _struct_packing(cls) -> ta.Sequence[ta.Tuple[str, str]]:
         try:
-            pack = self._STRUCT_PACK_BY_PLATFORM[sys.platform]
+            return cls._STRUCT_PACKING_BY_PLATFORM[sys.platform]
         except KeyError:
             raise OSError from None
 
-        fmt = ''.join(f for _, f in pack)
-        tup = [getattr(self, a) for a, _ in pack]
+    def pack(self) -> bytes:
+        packing = self._struct_packing()
+        fmt = ''.join(f for _, f in packing)
+        tup = [getattr(self, a) for a, _ in packing]
         return struct.pack(fmt, *tup)
 
     @classmethod
     def unpack(cls, data: bytes) -> 'FcntlLockData':
-        try:
-            pack = cls._STRUCT_PACK_BY_PLATFORM[sys.platform]
-        except KeyError:
-            raise OSError from None
-
-        fmt = ''.join(f for _, f in pack)
+        packing = cls._struct_packing()
+        fmt = ''.join(f for _, f in packing)
         tup = struct.unpack(fmt, data)
-        kw = {a: v for (a, _), v in zip(pack, tup)}
+        kw = {a: v for (a, _), v in zip(packing, tup)}
         return FcntlLockData(**kw)

omlish/os/files.py

@@ -1,24 +1,39 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
 import contextlib
+import errno
+import fcntl
 import os
 import typing as ta
 
 
-def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
+def is_fd_open(fd: int) -> bool:
+    try:
+        fcntl.fcntl(fd, fcntl.F_GETFD)
+    except OSError as e:
+        if e.errno == errno.EBADF:
+            return False
+        raise
+    else:
+        return True
+
+
+def touch(path: str, mode: int = 0o666, exist_ok: bool = True) -> None:
     if exist_ok:
         # First try to bump modification time
         # Implementation note: GNU touch uses the UTIME_NOW option of the utimensat() / futimens() functions.
         try:
-            os.utime(self, None)
+            os.utime(path, None)
         except OSError:
             pass
         else:
             return
+
     flags = os.O_CREAT | os.O_WRONLY
     if not exist_ok:
         flags |= os.O_EXCL
-    fd = os.open(self, flags, mode)
+
+    fd = os.open(path, flags, mode)
     os.close(fd)
 
 

omlish/os/forkhooks.py

@@ -0,0 +1,215 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - ForkHook base class? all classmethods? prevents pickling
+"""
+import abc
+import os
+import threading
+import typing as ta
+
+from ..lite.check import check
+
+
+##
+
+
+class _ForkHookManager:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    def __init_subclass__(cls, **kwargs):  # noqa
+        raise TypeError
+
+    #
+
+    # Intentionally not an RLock - do not nest calls.
+    _lock: ta.ClassVar[threading.Lock] = threading.Lock()
+
+    #
+
+    _installed: ta.ClassVar[bool] = False
+
+    @classmethod
+    def _install(cls) -> bool:
+        if cls._installed:
+            return False
+
+        check.state(not cls._installed)
+
+        os.register_at_fork(
+            before=cls._before_fork,
+            after_in_parent=cls._after_fork_in_parent,
+            after_in_child=cls._after_fork_in_child,
+        )
+
+        cls._installed = True
+        return True
+
+    @classmethod
+    def install(cls) -> bool:
+        with cls._lock:
+            return cls._install()
+
+    #
+
+    class Hook(ta.NamedTuple):
+        key: ta.Any
+        priority: int
+
+        # NOTE: these are called inside the global, non-reentrant manager lock
+        before_fork: ta.Optional[ta.Callable[[], None]] = None
+        after_fork_in_parent: ta.Optional[ta.Callable[[], None]] = None
+        after_fork_in_child: ta.Optional[ta.Callable[[], None]] = None
+
+    #
+
+    _hooks_by_key: ta.ClassVar[ta.Dict[str, Hook]] = {}
+
+    _hook_keys: ta.ClassVar[ta.FrozenSet[str]] = frozenset()
+    _priority_ordered_hooks: ta.ClassVar[ta.List[Hook]] = []
+
+    @classmethod
+    def _rebuild_hook_collections(cls) -> None:
+        cls._hook_keys = frozenset(cls._hooks_by_key)
+
+        # Uses on dict order preservation for insertion-order of hooks of equal priority (although that shouldn't be
+        # depended upon for usecase correctness.
+        cls._priority_ordered_hooks = sorted(cls._hooks_by_key.values(), key=lambda h: h.priority)
+
+    #
+
+    class HookAlreadyPresentError(Exception):
+        pass
+
+    @classmethod
+    def add_hook(cls, hook: Hook) -> None:
+        with cls._lock:
+            if hook.key in cls._hooks_by_key:
+                raise cls.HookAlreadyPresentError(hook.key)
+
+            check.isinstance(hook.priority, int)
+
+            cls._hooks_by_key[hook.key] = hook
+            cls._rebuild_hook_collections()
+
+            cls._install()
+
+    @classmethod
+    def try_add_hook(cls, hook: Hook) -> bool:
+        if hook.key in cls._hook_keys:
+            return False
+
+        try:
+            cls.add_hook(hook)
+        except cls.HookAlreadyPresentError:
+            return False
+        else:
+            return True
+
+    @classmethod
+    def contains_hook(cls, key: ta.Any) -> bool:
+        return key in cls._hook_keys
+
+    @classmethod
+    def remove_hook(cls, key: ta.Any) -> None:
+        with cls._lock:
+            del cls._hooks_by_key[key]
+
+            cls._rebuild_hook_collections()
+
+    #
+
+    @classmethod
+    def _before_fork(cls) -> None:
+        cls._lock.acquire()
+
+        for hook in cls._priority_ordered_hooks:
+            if (fn := hook.before_fork) is not None:
+                fn()
+
+    @classmethod
+    def _after_fork_in_parent(cls) -> None:
+        for hook in cls._priority_ordered_hooks:
+            if (fn := hook.after_fork_in_parent) is not None:
+                fn()
+
+        cls._lock.release()
+
+    @classmethod
+    def _after_fork_in_child(cls) -> None:
+        for hook in cls._priority_ordered_hooks:
+            if (fn := hook.after_fork_in_child) is not None:
+                fn()
+
+        cls._lock.release()
+
+
+#
+
+
+class ForkHook(abc.ABC):  # noqa
+    @ta.final
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    @classmethod
+    @ta.final
+    def install(cls) -> bool:
+        if _ForkHookManager.contains_hook(cls):
+            return False
+
+        return _ForkHookManager.try_add_hook(_ForkHookManager.Hook(
+            key=cls,
+            priority=cls._hook_priority,
+
+            before_fork=cls._before_fork,
+            after_fork_in_parent=cls._after_fork_in_parent,
+            after_fork_in_child=cls._after_fork_in_child,
+        ))
+
+    def __init_subclass__(cls, install: bool = False, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        if install:
+            cls.install()
+
+    #
+
+    _hook_priority: ta.ClassVar[int] = 0
+
+    @classmethod  # noqa
+    def _before_fork(cls) -> None:
+        pass
+
+    @classmethod  # noqa
+    def _after_fork_in_parent(cls) -> None:
+        pass
+
+    @classmethod  # noqa
+    def _after_fork_in_child(cls) -> None:
+        pass
+
+
+##
+
+
+class _ForkDepthTracker(ForkHook):
+    _hook_priority = -1000
+
+    _fork_depth: ta.ClassVar[int] = 0
+
+    @classmethod
+    def _after_fork_in_child(cls) -> None:
+        cls._fork_depth += 1
+
+    @classmethod
+    def get_fork_depth(cls) -> int:
+        cls.install()
+
+        return cls._fork_depth
+
+
+def get_fork_depth() -> int:
+    return _ForkDepthTracker.get_fork_depth()

omlish/os/pidfiles/pinning.py

@@ -1,6 +1,10 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
 """
+Notes:
+ - still racy as to if it's a different actual process as initial check, just with same pid, but due to 'identity' /
+   semantic meaning of the named pidfile the processes are considered equivalent
+
 Strategies:
  - linux
   - get pid of owner (lslocks or F_GETLK)

omlish/sockets/bind.py

@@ -15,10 +15,10 @@ import socket as socket_
 import stat
 import typing as ta
 
-from omlish.lite.check import check
-from omlish.lite.dataclasses import dataclass_maybe_post_init
-from omlish.sockets.addresses import SocketAddress
-from omlish.sockets.addresses import SocketAndAddress
+from ..lite.check import check
+from ..lite.dataclasses import dataclass_maybe_post_init
+from .addresses import SocketAddress
+from .addresses import SocketAndAddress
 
 
 SocketBinderT = ta.TypeVar('SocketBinderT', bound='SocketBinder')

{omlish-0.0.0.dev227.dist-info → omlish-0.0.0.dev229.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev227
+Version: 0.0.0.dev229
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -32,7 +32,7 @@ Requires-Dist: cbor2~=5.6; extra == "all"
 Requires-Dist: cloudpickle~=3.1; extra == "all"
 Requires-Dist: httpx[http2]~=0.28; extra == "all"
 Requires-Dist: wrapt~=1.17; extra == "all"
-Requires-Dist: cryptography~=43.0; extra == "all"
+Requires-Dist: cryptography~=44.0; extra == "all"
 Requires-Dist: sqlalchemy[asyncio]~=2.0; extra == "all"
 Requires-Dist: pg8000~=1.31; extra == "all"
 Requires-Dist: pymysql~=1.1; extra == "all"
@@ -76,7 +76,7 @@ Requires-Dist: httpx[http2]~=0.28; extra == "http"
 Provides-Extra: misc
 Requires-Dist: wrapt~=1.17; extra == "misc"
 Provides-Extra: secrets
-Requires-Dist: cryptography~=43.0; extra == "secrets"
+Requires-Dist: cryptography~=44.0; extra == "secrets"
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy[asyncio]~=2.0; extra == "sqlalchemy"
 Provides-Extra: sqldrivers