omlish 0.0.0.dev229__tar.gz → 0.0.0.dev231__tar.gz

{omlish-0.0.0.dev229/omlish.egg-info → omlish-0.0.0.dev231}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev229
+Version: 0.0.0.dev231
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev229'
-__revision__ = '646f2d8caaef32d279a6ebcfd121449752fbae93'
+__version__ = '0.0.0.dev231'
+__revision__ = '2eb4140ac802af2814bc4b4d5a8d37613083ae32'
 
 
 #

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/__init__.py

@@ -76,39 +76,39 @@ from .ordered import (  # noqa
     OrderedSet,
 )
 
-from .persistent import (  # noqa
+from .persistent.persistent import (  # noqa
     PersistentMap,
 )
 
 if _ta.TYPE_CHECKING:
-    from .skiplist import (  # noqa
+    from .persistent.treapmap import (  # noqa
+        TreapMap,
+        new_treap_map,
+    )
+else:
+    _lang.proxy_init(globals(), '.persistent.treapmap', [
+        'TreapMap',
+        'new_treap_map',
+    ])
+
+if _ta.TYPE_CHECKING:
+    from .sorted.skiplist import (  # noqa
         SkipList,
         SkipListDict,
     )
 else:
-    _lang.proxy_init(globals(), '.skiplist', [
+    _lang.proxy_init(globals(), '.sorted.skiplist', [
         'SkipList',
         'SkipListDict',
     ])
 
-from .sorted import (  # noqa
+from .sorted.sorted import (  # noqa
     SortedCollection,
     SortedListDict,
     SortedMapping,
     SortedMutableMapping,
 )
 
-if _ta.TYPE_CHECKING:
-    from .treapmap import (  # noqa
-        TreapMap,
-        new_treap_map,
-    )
-else:
-    _lang.proxy_init(globals(), '.treapmap', [
-        'TreapMap',
-        'new_treap_map',
-    ])
-
 from .unmodifiable import (  # noqa
     Unmodifiable,
     UnmodifiableMapping,

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/frozen.py

@@ -16,7 +16,6 @@ class Frozen(ta.Hashable, abc.ABC):
 
 
 class FrozenDict(ta.Mapping[K, V], Frozen):
-
     def __new__(cls, *args: ta.Any, **kwargs: ta.Any) -> 'FrozenDict[K, V]':  # noqa
         if len(args) == 1 and Frozen in type(args[0]).__bases__:
             return args[0]
@@ -73,7 +72,6 @@ class FrozenDict(ta.Mapping[K, V], Frozen):
 
 
 class FrozenList(ta.Sequence[T], Frozen):
-
     def __init__(self, it: ta.Iterable[T] | None = None) -> None:
         super().__init__()
 

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/identity.py

@@ -13,7 +13,6 @@ V = ta.TypeVar('V')
 
 
 class IdentityWrapper(ta.Generic[T]):
-
     def __init__(self, value: T) -> None:
         super().__init__()
         self._value = value
@@ -36,7 +35,6 @@ class IdentityWrapper(ta.Generic[T]):
 
 
 class IdentityKeyDict(ta.MutableMapping[K, V]):
-
     def __init__(self, *args, **kwargs) -> None:
         super().__init__()
         self._dict: dict[int, tuple[K, V]] = {}
@@ -70,7 +68,6 @@ class IdentityKeyDict(ta.MutableMapping[K, V]):
 
 
 class IdentitySet(ta.MutableSet[T]):
-
     def __init__(self, init: ta.Iterable[T] | None = None) -> None:
         super().__init__()
         self._dict: dict[int, T] = {}

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/indexed.py

@@ -8,7 +8,6 @@ T = ta.TypeVar('T')
 
 
 class IndexedSeq(ta.Sequence[T]):
-
     def __init__(self, it: ta.Iterable[T], *, identity: bool = False) -> None:
         super().__init__()
 
@@ -42,7 +41,6 @@ class IndexedSeq(ta.Sequence[T]):
 
 
 class IndexedSetSeq(ta.Sequence[ta.AbstractSet[T]]):
-
     def __init__(self, it: ta.Iterable[ta.Iterable[T]], *, identity: bool = False) -> None:
         super().__init__()
 

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/mappings.py

@@ -49,7 +49,6 @@ def yield_dict_init(*args, **kwargs) -> ta.Iterable[tuple[ta.Any, ta.Any]]:
 
 
 class TypeMap(ta.Generic[T]):
-
     def __init__(self, items: ta.Iterable[T] = ()) -> None:
         super().__init__()
 
@@ -79,7 +78,6 @@ class TypeMap(ta.Generic[T]):
 
 
 class DynamicTypeMap(ta.Generic[V]):
-
     def __init__(self, items: ta.Iterable[V] = (), *, weak: bool = False) -> None:
         super().__init__()
 
@@ -115,7 +113,6 @@ class DynamicTypeMap(ta.Generic[V]):
 
 
 class MissingDict(dict[K, V]):
-
     def __init__(self, missing_fn: ta.Callable[[K], V]) -> None:
         if not callable(missing_fn):
             raise TypeError(missing_fn)

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/ordered.py

@@ -5,7 +5,6 @@ T = ta.TypeVar('T')
 
 
 class OrderedSet(ta.MutableSet[T]):
-
     def __init__(self, iterable: ta.Iterable[T] | None = None) -> None:
         super().__init__()
         self._dct: dict[T, ta.Any] = {}
@@ -56,7 +55,6 @@ class OrderedSet(ta.MutableSet[T]):
 
 
 class OrderedFrozenSet(ta.FrozenSet[T]):  # noqa
-
     _list: ta.Sequence[T]
 
     def __new__(cls, items: ta.Iterable[T]) -> frozenset[T]:  # type: ignore

{omlish-0.0.0.dev229/omlish/collections → omlish-0.0.0.dev231/omlish/collections/sorted}/skiplist.py

@@ -188,6 +188,5 @@ class SkipList(SortedCollection[T]):
 
 
 class SkipListDict(SortedListDict[K, V]):
-
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(SkipList(comparator=SortedListDict._item_comparator), *args, **kwargs)  # noqa

{omlish-0.0.0.dev229/omlish/collections → omlish-0.0.0.dev231/omlish/collections/sorted}/sorted.py

@@ -1,8 +1,8 @@
 import abc
 import typing as ta
 
-from .. import lang
-from .mappings import yield_dict_init
+from ... import lang
+from ..mappings import yield_dict_init
 
 
 T = ta.TypeVar('T')
@@ -12,7 +12,6 @@ V = ta.TypeVar('V')
 
 
 class SortedCollection(lang.Abstract, ta.Collection[T]):
-
     Comparator = ta.Callable[[U, U], int]
 
     @staticmethod
@@ -55,7 +54,6 @@ class SortedCollection(lang.Abstract, ta.Collection[T]):
 
 
 class SortedMapping(ta.Mapping[K, V]):
-
     @abc.abstractmethod
     def items(self) -> ta.Iterator[tuple[K, V]]:  # type: ignore
         raise NotImplementedError
@@ -78,7 +76,6 @@ class SortedMutableMapping(ta.MutableMapping[K, V], SortedMapping[K, V]):
 
 
 class SortedListDict(SortedMutableMapping[K, V]):
-
     @staticmethod
     def _item_comparator(a: tuple[K, V], b: tuple[K, V]) -> int:
         return SortedCollection.default_comparator(a[0], b[0])

{omlish-0.0.0.dev229 → omlish-0.0.0.dev231}/omlish/collections/unmodifiable.py

@@ -13,7 +13,6 @@ class Unmodifiable(lang.Abstract):
 
 
 class UnmodifiableSequence(ta.Sequence[T], Unmodifiable, lang.Final):
-
     def __init__(self, target: ta.Sequence[T]) -> None:
         super().__init__()
 
@@ -65,7 +64,6 @@ class UnmodifiableSequence(ta.Sequence[T], Unmodifiable, lang.Final):
 
 
 class UnmodifiableSet(ta.AbstractSet[T], Unmodifiable, lang.Final):
-
     def __init__(self, target: ta.AbstractSet[T]) -> None:
         super().__init__()
 

omlish-0.0.0.dev231/omlish/daemons/daemon.py

@@ -0,0 +1,200 @@
+"""
+TODO:
+ - OK.. this is useful for non-system-daemons too... which have no pidfile..
+ - split out pidfile concern to.. identity?
+ - async[io] support, really just waiting
+ - helpers for http, json, etc
+ - heartbeat? status checks? heartbeat file?
+ - zmq
+ - cli, cfg reload
+ - bootstrap
+  - rsrc limit
+  - logs
+ - https://github.com/Homebrew/homebrew-services
+ - deathpacts
+ - timebomb
+ - pickle protocol, revision / venv check, multiprocessing manager support
+"""
+import contextlib
+import functools
+import logging
+import os.path
+import threading
+import time
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..os.pidfiles.manager import _PidfileManager  # noqa
+from ..os.pidfiles.manager import open_inheritable_pidfile
+from ..os.pidfiles.pidfile import Pidfile
+from .reparent import reparent_process
+from .spawning import InProcessSpawner
+from .spawning import Spawn
+from .spawning import Spawner
+from .spawning import Spawning
+from .spawning import spawner_for
+from .targets import Target
+from .targets import target_runner_for
+from .waiting import Wait
+from .waiting import waiter_for
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Daemon:
+    @dc.dataclass(frozen=True, kw_only=True)
+    class Config:
+        target: Target
+        spawning: Spawning
+
+        #
+
+        reparent_process: bool = False
+
+        pid_file: str | None = None
+
+        #
+
+        wait: Wait | None = None
+
+        wait_timeout: lang.TimeoutLike = 10.
+        wait_sleep_s: float = .1
+
+        launched_timeout_s: float = 5.
+
+        #
+
+        def __post_init__(self) -> None:
+            check.isinstance(self.pid_file, (str, None))
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+
+        self._config = config
+
+    @property
+    def config(self) -> Config:
+        return self._config
+
+    #
+
+    @property
+    def has_pidfile(self) -> bool:
+        return self._config.pid_file is not None
+
+    def _non_inheritable_pidfile(self) -> Pidfile:
+        check.state(self.has_pidfile)
+        return Pidfile(
+            check.non_empty_str(self._config.pid_file),
+            inheritable=False,
+        )
+
+    def is_running(self) -> bool:
+        check.state(self.has_pidfile)
+
+        if not os.path.isfile(check.non_empty_str(self._config.pid_file)):
+            return False
+
+        with self._non_inheritable_pidfile() as pf:
+            return not pf.try_acquire_lock()
+
+    #
+
+    def _inner_launch(
+            self,
+            *,
+            pidfile_manager: ta.ContextManager | None,
+            launched_callback: ta.Callable[[], None] | None = None,
+    ) -> None:
+        try:
+            if self._config.reparent_process:
+                log.info('Reparenting')
+                reparent_process()
+
+            with contextlib.ExitStack() as es:
+                pidfile: Pidfile | None = None  # noqa
+                if pidfile_manager is not None:
+                    pidfile = check.isinstance(es.enter_context(pidfile_manager), Pidfile)
+                    pidfile.write()
+
+                if launched_callback is not None:
+                    launched_callback()
+
+                runner = target_runner_for(self._config.target)
+                runner.run()
+
+        finally:
+            if launched_callback is not None:
+                launched_callback()
+
+    def launch_no_wait(self) -> bool:
+        with contextlib.ExitStack() as es:
+            spawner: Spawner = es.enter_context(spawner_for(self._config.spawning))
+
+            #
+
+            inherit_fds: set[int] = set()
+            launched_event: threading.Event | None = None
+
+            pidfile: Pidfile | None = None  # noqa
+            pidfile_manager: ta.ContextManager | None = None
+
+            if (pid_file := self._config.pid_file) is not None:
+                if not isinstance(spawner, InProcessSpawner):
+                    pidfile = es.enter_context(open_inheritable_pidfile(pid_file))
+                    pidfile_manager = lang.NopContextManager(pidfile)
+
+                else:
+                    check.state(not self._config.reparent_process)
+                    pidfile = es.enter_context(Pidfile(pid_file))
+                    pidfile_manager = pidfile.dup()
+                    launched_event = threading.Event()
+
+                if not pidfile.try_acquire_lock():
+                    return False
+
+                inherit_fds.add(check.isinstance(pidfile.fileno(), int))
+
+            #
+
+            spawner.spawn(Spawn(
+                functools.partial(
+                    self._inner_launch,
+                    pidfile_manager=pidfile_manager,
+                    launched_callback=launched_event.set if launched_event is not None else None,
+                ),
+                inherit_fds=inherit_fds,
+            ))
+
+            if launched_event is not None:
+                check.state(launched_event.wait(timeout=self._config.launched_timeout_s))
+
+            return True
+
+    #
+
+    def wait_sync(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        if self._config.wait is None:
+            return
+
+        timeout = lang.Timeout.of(timeout, self._config.wait_timeout)
+        waiter = waiter_for(self._config.wait)
+        while not waiter.do_wait():
+            timeout()
+            time.sleep(self._config.wait_sleep_s or 0.)
+
+    #
+
+    class _NOT_SET(lang.Marker):  # noqa
+        pass
+
+    def launch(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        self.launch_no_wait()
+
+        self.wait_sync(timeout)

omlish-0.0.0.dev231/omlish/daemons/reparent.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+
+def reparent_process() -> None:
+    if (pid := os.fork()):  # noqa
+        sys.exit(0)
+        raise RuntimeError('Unreachable')  # noqa
+
+    os.setsid()
+
+    if (pid := os.fork()):  # noqa
+        sys.exit(0)
+
+    sys.stdout.flush()
+    sys.stderr.flush()

omlish-0.0.0.dev231/omlish/daemons/spawning.py

@@ -0,0 +1,166 @@
+import abc
+import functools
+import os
+import sys
+import threading
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..diag import pydevd
+
+
+if ta.TYPE_CHECKING:
+    import multiprocessing as mp
+    import multiprocessing.context
+    import multiprocessing.process  # noqa
+
+    from ..multiprocessing import spawn as omp_spawn
+
+else:
+    mp = lang.proxy_import('multiprocessing', extras=['context', 'process'])
+    subprocess = lang.proxy_import('subprocess')
+
+    omp_spawn = lang.proxy_import('..multiprocessing.spawn', __package__)
+
+
+##
+
+
+class Spawning(dc.Case):
+    pass
+
+
+class Spawn(dc.Frozen, final=True):
+    fn: ta.Callable[[], None]
+
+    _: dc.KW_ONLY
+
+    inherit_fds: ta.Collection[int] | None = None
+
+
+class Spawner(lang.ContextManaged, abc.ABC):
+    @abc.abstractmethod
+    def spawn(self, spawn: Spawn) -> None:
+        raise NotImplementedError
+
+
+class InProcessSpawner(Spawner, abc.ABC):
+    pass
+
+
+@functools.singledispatch
+def spawner_for(spawning: Spawning) -> Spawner:
+    raise TypeError(spawning)
+
+
+##
+
+
+class MultiprocessingSpawning(Spawning, kw_only=True):
+    # Defaults to 'fork' if under pydevd, else 'spawn'
+    start_method: str | None = None
+
+    non_daemon: bool = False
+
+
+class MultiprocessingSpawner(Spawner):
+    def __init__(self, spawning: MultiprocessingSpawning) -> None:
+        super().__init__()
+
+        self._spawning = spawning
+        self._process: ta.Optional['mp.process.BaseProcess'] = None  # noqa
+
+    def _process_cls(self, spawn: Spawn) -> type['mp.process.BaseProcess']:
+        if (start_method := self._spawning.start_method) is None:
+            # Unfortunately, pydevd forces the use of the 'fork' start_method, which cannot be mixed with 'spawn':
+            #   https://github.com/python/cpython/blob/a7427f2db937adb4c787754deb4c337f1894fe86/Lib/multiprocessing/spawn.py#L102  # noqa
+            if pydevd.is_running():
+                start_method = 'fork'
+            else:
+                start_method = 'spawn'
+
+        ctx: 'mp.context.BaseContext'  # noqa
+        if start_method == 'fork':
+            ctx = mp.get_context(check.non_empty_str(start_method))
+
+        elif start_method == 'spawn':
+            ctx = omp_spawn.ExtrasSpawnContext(omp_spawn.SpawnExtras(
+                pass_fds=frozenset(spawn.inherit_fds) if spawn.inherit_fds is not None else None,
+            ))
+
+        else:
+            raise ValueError(start_method)
+
+        return ctx.Process  # type: ignore
+
+    def spawn(self, spawn: Spawn) -> None:
+        check.none(self._process)
+        self._process = self._process_cls(spawn)(
+            target=spawn.fn,
+            daemon=not self._spawning.non_daemon,
+        )
+        self._process.start()
+
+
+@spawner_for.register
+def _(spawning: MultiprocessingSpawning) -> MultiprocessingSpawner:
+    return MultiprocessingSpawner(spawning)
+
+
+##
+
+
+class ForkSpawning(Spawning):
+    pass
+
+
+class ForkSpawner(Spawner, dc.Frozen):
+    spawning: ForkSpawning
+
+    def spawn(self, spawn: Spawn) -> None:
+        if (pid := os.fork()):  # noqa
+            return
+
+        try:
+            spawn.fn()
+        except BaseException:  # noqa
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
+        raise RuntimeError('Unreachable')  # noqa
+
+
+@spawner_for.register
+def _(spawning: ForkSpawning) -> ForkSpawner:
+    return ForkSpawner(spawning)
+
+
+##
+
+
+class ThreadSpawning(Spawning, kw_only=True):
+    non_daemon: bool = False
+
+
+class ThreadSpawner(InProcessSpawner):
+    def __init__(self, spawning: ThreadSpawning) -> None:
+        super().__init__()
+
+        self._spawning = spawning
+        self._thread: threading.Thread | None = None
+
+    def spawn(self, spawn: Spawn) -> None:
+        check.none(self._thread)
+        self._thread = threading.Thread(
+            target=spawn.fn,
+            daemon=not self._spawning.non_daemon,
+        )
+        self._thread.start()
+
+
+@spawner_for.register
+def _(spawning: ThreadSpawning) -> ThreadSpawner:
+    return ThreadSpawner(spawning)

omlish-0.0.0.dev231/omlish/daemons/targets.py

@@ -0,0 +1,89 @@
+import abc
+import functools
+import os
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import runpy
+else:
+    runpy = lang.proxy_import('runpy')
+
+
+##
+
+
+class Target(dc.Case):
+    pass
+
+
+class TargetRunner(abc.ABC):
+    @abc.abstractmethod
+    def run(self) -> None:
+        raise NotImplementedError
+
+
+@functools.singledispatch
+def target_runner_for(target: Target) -> TargetRunner:
+    raise TypeError(target)
+
+
+##
+
+
+class FnTarget(Target):
+    fn: ta.Callable[[], None]
+
+
+class FnTargetRunner(TargetRunner, dc.Frozen):
+    target: FnTarget
+
+    def run(self) -> None:
+        self.target.fn()
+
+
+@target_runner_for.register
+def _(target: FnTarget) -> FnTargetRunner:
+    return FnTargetRunner(target)
+
+
+##
+
+
+class NameTarget(Target):
+    name: str
+
+
+class NameTargetRunner(TargetRunner, dc.Frozen):
+    target: NameTarget
+
+    def run(self) -> None:
+        name = self.target.name
+        if lang.can_import(name):
+            runpy._run_module_as_main(name)  # type: ignore  # noqa
+        else:
+            obj = lang.import_attr(self.target.name)
+            obj()
+
+
+##
+
+
+class ExecTarget(Target):
+    cmd: ta.Sequence[str] = dc.xfield(coerce=check.of_not_isinstance(str))
+
+
+class ExecTargetRunner(TargetRunner, dc.Frozen):
+    target: ExecTarget
+
+    def run(self) -> None:
+        os.execl(*self.target.cmd)
+
+
+@target_runner_for.register
+def _(target: ExecTarget) -> ExecTargetRunner:
+    return ExecTargetRunner(target)