omlish 0.0.0.dev230__tar.gz → 0.0.0.dev231__tar.gz

{omlish-0.0.0.dev230/omlish.egg-info → omlish-0.0.0.dev231}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev230
+Version: 0.0.0.dev231
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev230 → omlish-0.0.0.dev231}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev230'
-__revision__ = 'ed2672ad6a2fb4718c57e281c5ce91fdc7998a6f'
+__version__ = '0.0.0.dev231'
+__revision__ = '2eb4140ac802af2814bc4b4d5a8d37613083ae32'
 
 
 #

omlish-0.0.0.dev231/omlish/daemons/daemon.py

@@ -0,0 +1,200 @@
+"""
+TODO:
+ - OK.. this is useful for non-system-daemons too... which have no pidfile..
+ - split out pidfile concern to.. identity?
+ - async[io] support, really just waiting
+ - helpers for http, json, etc
+ - heartbeat? status checks? heartbeat file?
+ - zmq
+ - cli, cfg reload
+ - bootstrap
+  - rsrc limit
+  - logs
+ - https://github.com/Homebrew/homebrew-services
+ - deathpacts
+ - timebomb
+ - pickle protocol, revision / venv check, multiprocessing manager support
+"""
+import contextlib
+import functools
+import logging
+import os.path
+import threading
+import time
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..os.pidfiles.manager import _PidfileManager  # noqa
+from ..os.pidfiles.manager import open_inheritable_pidfile
+from ..os.pidfiles.pidfile import Pidfile
+from .reparent import reparent_process
+from .spawning import InProcessSpawner
+from .spawning import Spawn
+from .spawning import Spawner
+from .spawning import Spawning
+from .spawning import spawner_for
+from .targets import Target
+from .targets import target_runner_for
+from .waiting import Wait
+from .waiting import waiter_for
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Daemon:
+    @dc.dataclass(frozen=True, kw_only=True)
+    class Config:
+        target: Target
+        spawning: Spawning
+
+        #
+
+        reparent_process: bool = False
+
+        pid_file: str | None = None
+
+        #
+
+        wait: Wait | None = None
+
+        wait_timeout: lang.TimeoutLike = 10.
+        wait_sleep_s: float = .1
+
+        launched_timeout_s: float = 5.
+
+        #
+
+        def __post_init__(self) -> None:
+            check.isinstance(self.pid_file, (str, None))
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+
+        self._config = config
+
+    @property
+    def config(self) -> Config:
+        return self._config
+
+    #
+
+    @property
+    def has_pidfile(self) -> bool:
+        return self._config.pid_file is not None
+
+    def _non_inheritable_pidfile(self) -> Pidfile:
+        check.state(self.has_pidfile)
+        return Pidfile(
+            check.non_empty_str(self._config.pid_file),
+            inheritable=False,
+        )
+
+    def is_running(self) -> bool:
+        check.state(self.has_pidfile)
+
+        if not os.path.isfile(check.non_empty_str(self._config.pid_file)):
+            return False
+
+        with self._non_inheritable_pidfile() as pf:
+            return not pf.try_acquire_lock()
+
+    #
+
+    def _inner_launch(
+            self,
+            *,
+            pidfile_manager: ta.ContextManager | None,
+            launched_callback: ta.Callable[[], None] | None = None,
+    ) -> None:
+        try:
+            if self._config.reparent_process:
+                log.info('Reparenting')
+                reparent_process()
+
+            with contextlib.ExitStack() as es:
+                pidfile: Pidfile | None = None  # noqa
+                if pidfile_manager is not None:
+                    pidfile = check.isinstance(es.enter_context(pidfile_manager), Pidfile)
+                    pidfile.write()
+
+                if launched_callback is not None:
+                    launched_callback()
+
+                runner = target_runner_for(self._config.target)
+                runner.run()
+
+        finally:
+            if launched_callback is not None:
+                launched_callback()
+
+    def launch_no_wait(self) -> bool:
+        with contextlib.ExitStack() as es:
+            spawner: Spawner = es.enter_context(spawner_for(self._config.spawning))
+
+            #
+
+            inherit_fds: set[int] = set()
+            launched_event: threading.Event | None = None
+
+            pidfile: Pidfile | None = None  # noqa
+            pidfile_manager: ta.ContextManager | None = None
+
+            if (pid_file := self._config.pid_file) is not None:
+                if not isinstance(spawner, InProcessSpawner):
+                    pidfile = es.enter_context(open_inheritable_pidfile(pid_file))
+                    pidfile_manager = lang.NopContextManager(pidfile)
+
+                else:
+                    check.state(not self._config.reparent_process)
+                    pidfile = es.enter_context(Pidfile(pid_file))
+                    pidfile_manager = pidfile.dup()
+                    launched_event = threading.Event()
+
+                if not pidfile.try_acquire_lock():
+                    return False
+
+                inherit_fds.add(check.isinstance(pidfile.fileno(), int))
+
+            #
+
+            spawner.spawn(Spawn(
+                functools.partial(
+                    self._inner_launch,
+                    pidfile_manager=pidfile_manager,
+                    launched_callback=launched_event.set if launched_event is not None else None,
+                ),
+                inherit_fds=inherit_fds,
+            ))
+
+            if launched_event is not None:
+                check.state(launched_event.wait(timeout=self._config.launched_timeout_s))
+
+            return True
+
+    #
+
+    def wait_sync(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        if self._config.wait is None:
+            return
+
+        timeout = lang.Timeout.of(timeout, self._config.wait_timeout)
+        waiter = waiter_for(self._config.wait)
+        while not waiter.do_wait():
+            timeout()
+            time.sleep(self._config.wait_sleep_s or 0.)
+
+    #
+
+    class _NOT_SET(lang.Marker):  # noqa
+        pass
+
+    def launch(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        self.launch_no_wait()
+
+        self.wait_sync(timeout)

omlish-0.0.0.dev231/omlish/daemons/reparent.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+
+def reparent_process() -> None:
+    if (pid := os.fork()):  # noqa
+        sys.exit(0)
+        raise RuntimeError('Unreachable')  # noqa
+
+    os.setsid()
+
+    if (pid := os.fork()):  # noqa
+        sys.exit(0)
+
+    sys.stdout.flush()
+    sys.stderr.flush()

omlish-0.0.0.dev231/omlish/daemons/spawning.py

@@ -0,0 +1,166 @@
+import abc
+import functools
+import os
+import sys
+import threading
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..diag import pydevd
+
+
+if ta.TYPE_CHECKING:
+    import multiprocessing as mp
+    import multiprocessing.context
+    import multiprocessing.process  # noqa
+
+    from ..multiprocessing import spawn as omp_spawn
+
+else:
+    mp = lang.proxy_import('multiprocessing', extras=['context', 'process'])
+    subprocess = lang.proxy_import('subprocess')
+
+    omp_spawn = lang.proxy_import('..multiprocessing.spawn', __package__)
+
+
+##
+
+
+class Spawning(dc.Case):
+    pass
+
+
+class Spawn(dc.Frozen, final=True):
+    fn: ta.Callable[[], None]
+
+    _: dc.KW_ONLY
+
+    inherit_fds: ta.Collection[int] | None = None
+
+
+class Spawner(lang.ContextManaged, abc.ABC):
+    @abc.abstractmethod
+    def spawn(self, spawn: Spawn) -> None:
+        raise NotImplementedError
+
+
+class InProcessSpawner(Spawner, abc.ABC):
+    pass
+
+
+@functools.singledispatch
+def spawner_for(spawning: Spawning) -> Spawner:
+    raise TypeError(spawning)
+
+
+##
+
+
+class MultiprocessingSpawning(Spawning, kw_only=True):
+    # Defaults to 'fork' if under pydevd, else 'spawn'
+    start_method: str | None = None
+
+    non_daemon: bool = False
+
+
+class MultiprocessingSpawner(Spawner):
+    def __init__(self, spawning: MultiprocessingSpawning) -> None:
+        super().__init__()
+
+        self._spawning = spawning
+        self._process: ta.Optional['mp.process.BaseProcess'] = None  # noqa
+
+    def _process_cls(self, spawn: Spawn) -> type['mp.process.BaseProcess']:
+        if (start_method := self._spawning.start_method) is None:
+            # Unfortunately, pydevd forces the use of the 'fork' start_method, which cannot be mixed with 'spawn':
+            #   https://github.com/python/cpython/blob/a7427f2db937adb4c787754deb4c337f1894fe86/Lib/multiprocessing/spawn.py#L102  # noqa
+            if pydevd.is_running():
+                start_method = 'fork'
+            else:
+                start_method = 'spawn'
+
+        ctx: 'mp.context.BaseContext'  # noqa
+        if start_method == 'fork':
+            ctx = mp.get_context(check.non_empty_str(start_method))
+
+        elif start_method == 'spawn':
+            ctx = omp_spawn.ExtrasSpawnContext(omp_spawn.SpawnExtras(
+                pass_fds=frozenset(spawn.inherit_fds) if spawn.inherit_fds is not None else None,
+            ))
+
+        else:
+            raise ValueError(start_method)
+
+        return ctx.Process  # type: ignore
+
+    def spawn(self, spawn: Spawn) -> None:
+        check.none(self._process)
+        self._process = self._process_cls(spawn)(
+            target=spawn.fn,
+            daemon=not self._spawning.non_daemon,
+        )
+        self._process.start()
+
+
+@spawner_for.register
+def _(spawning: MultiprocessingSpawning) -> MultiprocessingSpawner:
+    return MultiprocessingSpawner(spawning)
+
+
+##
+
+
+class ForkSpawning(Spawning):
+    pass
+
+
+class ForkSpawner(Spawner, dc.Frozen):
+    spawning: ForkSpawning
+
+    def spawn(self, spawn: Spawn) -> None:
+        if (pid := os.fork()):  # noqa
+            return
+
+        try:
+            spawn.fn()
+        except BaseException:  # noqa
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
+        raise RuntimeError('Unreachable')  # noqa
+
+
+@spawner_for.register
+def _(spawning: ForkSpawning) -> ForkSpawner:
+    return ForkSpawner(spawning)
+
+
+##
+
+
+class ThreadSpawning(Spawning, kw_only=True):
+    non_daemon: bool = False
+
+
+class ThreadSpawner(InProcessSpawner):
+    def __init__(self, spawning: ThreadSpawning) -> None:
+        super().__init__()
+
+        self._spawning = spawning
+        self._thread: threading.Thread | None = None
+
+    def spawn(self, spawn: Spawn) -> None:
+        check.none(self._thread)
+        self._thread = threading.Thread(
+            target=spawn.fn,
+            daemon=not self._spawning.non_daemon,
+        )
+        self._thread.start()
+
+
+@spawner_for.register
+def _(spawning: ThreadSpawning) -> ThreadSpawner:
+    return ThreadSpawner(spawning)

omlish-0.0.0.dev231/omlish/daemons/targets.py

@@ -0,0 +1,89 @@
+import abc
+import functools
+import os
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import runpy
+else:
+    runpy = lang.proxy_import('runpy')
+
+
+##
+
+
+class Target(dc.Case):
+    pass
+
+
+class TargetRunner(abc.ABC):
+    @abc.abstractmethod
+    def run(self) -> None:
+        raise NotImplementedError
+
+
+@functools.singledispatch
+def target_runner_for(target: Target) -> TargetRunner:
+    raise TypeError(target)
+
+
+##
+
+
+class FnTarget(Target):
+    fn: ta.Callable[[], None]
+
+
+class FnTargetRunner(TargetRunner, dc.Frozen):
+    target: FnTarget
+
+    def run(self) -> None:
+        self.target.fn()
+
+
+@target_runner_for.register
+def _(target: FnTarget) -> FnTargetRunner:
+    return FnTargetRunner(target)
+
+
+##
+
+
+class NameTarget(Target):
+    name: str
+
+
+class NameTargetRunner(TargetRunner, dc.Frozen):
+    target: NameTarget
+
+    def run(self) -> None:
+        name = self.target.name
+        if lang.can_import(name):
+            runpy._run_module_as_main(name)  # type: ignore  # noqa
+        else:
+            obj = lang.import_attr(self.target.name)
+            obj()
+
+
+##
+
+
+class ExecTarget(Target):
+    cmd: ta.Sequence[str] = dc.xfield(coerce=check.of_not_isinstance(str))
+
+
+class ExecTargetRunner(TargetRunner, dc.Frozen):
+    target: ExecTarget
+
+    def run(self) -> None:
+        os.execl(*self.target.cmd)
+
+
+@target_runner_for.register
+def _(target: ExecTarget) -> ExecTargetRunner:
+    return ExecTargetRunner(target)

omlish-0.0.0.dev231/omlish/daemons/waiting.py

@@ -0,0 +1,95 @@
+import abc
+import functools
+import socket
+import typing as ta
+
+from .. import dataclasses as dc
+
+
+##
+
+
+class Wait(dc.Case):
+    pass
+
+
+class Waiter(abc.ABC):
+    @abc.abstractmethod
+    def do_wait(self) -> bool:
+        raise NotImplementedError
+
+
+@functools.singledispatch
+def waiter_for(wait: Wait) -> Waiter:
+    raise TypeError(wait)
+
+
+##
+
+
+class SequentialWait(Wait):
+    waits: ta.Sequence[Wait]
+
+
+class SequentialWaiter(Waiter):
+    def __init__(self, waiters: ta.Sequence[Waiter]) -> None:
+        super().__init__()
+
+        self._waiters = waiters
+        self._idx = 0
+
+    def do_wait(self) -> bool:
+        while self._idx < len(self._waiters):
+            if not self._waiters[self._idx].do_wait():
+                return False
+            self._idx += 1
+        return True
+
+
+@waiter_for.register
+def _(wait: SequentialWait) -> SequentialWaiter:
+    return SequentialWaiter([waiter_for(c) for c in wait.waits])
+
+
+##
+
+
+class FnWait(Wait):
+    fn: ta.Callable[[], bool]
+
+
+class FnWaiter(Waiter, dc.Frozen):
+    wait: FnWait
+
+    def do_wait(self) -> bool:
+        return self.wait.fn()
+
+
+@waiter_for.register
+def _(wait: FnWait) -> FnWaiter:
+    return FnWaiter(wait)
+
+
+##
+
+
+class ConnectWait(Wait):
+    address: ta.Any
+
+
+class ConnectWaiter(Waiter, dc.Frozen):
+    wait: ConnectWait
+
+    def do_wait(self) -> bool:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.connect(self.wait.address)
+            except ConnectionRefusedError:
+                return False
+            else:
+                return True
+
+
+@waiter_for.register
+def _(wait: ConnectWait) -> ConnectWaiter:
+    return ConnectWaiter(wait)

{omlish-0.0.0.dev230 → omlish-0.0.0.dev231}/omlish/lang/__init__.py

@@ -61,9 +61,7 @@ from .contextmanagers import (  # noqa
     DefaultLockable,
     ExitStacked,
     Lockable,
-    NOP_CONTEXT_MANAGED,
     NOP_CONTEXT_MANAGER,
-    NopContextManaged,
     NopContextManager,
     Timer,
     a_defer,

{omlish-0.0.0.dev230 → omlish-0.0.0.dev231}/omlish/lang/contextmanagers.py

@@ -18,6 +18,11 @@ T = ta.TypeVar('T')
 ##
 
 
+class _NOT_SET:  # noqa
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+
 class ContextManaged:
     def __enter__(self) -> ta.Self:
         return self
@@ -31,21 +36,21 @@ class ContextManaged:
         return None
 
 
-class NopContextManaged(ContextManaged):
-    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
-        raise TypeError
-
+class NopContextManager(ContextManaged):
+    def __init__(self, /, value: ta.Any = _NOT_SET) -> None:
+        super().__init__()
 
-NOP_CONTEXT_MANAGED = NopContextManaged()
+        self._value = value
 
+    def __enter__(self):
+        if (value := self._value) is _NOT_SET:
+            return self
+        else:
+            return value
 
-class NopContextManager:
     def __init_subclass__(cls, **kwargs: ta.Any) -> None:
         raise TypeError
 
-    def __call__(self, *args, **kwargs):
-        return NOP_CONTEXT_MANAGED
-
 
 NOP_CONTEXT_MANAGER = NopContextManager()
 
@@ -350,7 +355,7 @@ def default_lock(value: DefaultLockable, default: DefaultLockable = None) -> Loc
         return lambda: lock
 
     elif value is False or value is None:
-        return NOP_CONTEXT_MANAGER
+        return lambda: NOP_CONTEXT_MANAGER
 
     elif callable(value):
         return value

{omlish-0.0.0.dev230 → omlish-0.0.0.dev231}/omlish/libc.py

@@ -62,7 +62,6 @@ libc.free.argtypes = [ct.c_void_p]
 
 
 class Malloc:
-
     def __init__(self, sz: int) -> None:
         super().__init__()
 
@@ -174,7 +173,6 @@ if LINUX:
 
 
 class Mmap:
-
     def __init__(
             self,
             length: int,
@@ -537,14 +535,14 @@ elif DARWIN:
 
 
 if LINUX:
-    def gettid():
+    def gettid() -> int:
         syscalls = {
             'i386': 224,  # unistd_32.h: #define __NR_gettid 224
             'x86_64': 186,  # unistd_64.h: #define __NR_gettid 186
             'aarch64': 178,  # asm-generic/unistd.h: #define __NR_gettid 178
         }
         try:
-            tid = ct.CDLL('libc.so.6').syscall(syscalls[platform.machine()])
+            tid = libc.syscall(syscalls[platform.machine()])
         except Exception:  # noqa
             tid = -1
         return tid

{omlish-0.0.0.dev230 → omlish-0.0.0.dev231}/omlish/lite/timeouts.py

@@ -8,7 +8,7 @@ import time
 import typing as ta
 
 
-TimeoutLike = ta.Union['Timeout', 'Timeout.Default', ta.Iterable['TimeoutLike'], float]  # ta.TypeAlias
+TimeoutLike = ta.Union['Timeout', ta.Type['Timeout.Default'], ta.Iterable['TimeoutLike'], float]  # ta.TypeAlias
 
 
 ##