omlish 0.0.0.dev230__tar.gz → 0.0.0.dev232__tar.gz

{omlish-0.0.0.dev230/omlish.egg-info → omlish-0.0.0.dev232}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev230
+Version: 0.0.0.dev232
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -16,7 +16,7 @@ Provides-Extra: all
 Requires-Dist: anyio~=4.8; extra == "all"
 Requires-Dist: sniffio~=1.3; extra == "all"
 Requires-Dist: greenlet~=3.1; extra == "all"
-Requires-Dist: trio~=0.27; extra == "all"
+Requires-Dist: trio~=0.29; extra == "all"
 Requires-Dist: trio-asyncio~=0.15; extra == "all"
 Requires-Dist: lz4~=4.4; extra == "all"
 Requires-Dist: python-snappy~=0.7; extra == "all"
@@ -24,7 +24,7 @@ Requires-Dist: zstandard~=0.23; extra == "all"
 Requires-Dist: brotli~=1.1; extra == "all"
 Requires-Dist: asttokens~=3.0; extra == "all"
 Requires-Dist: executing~=2.2; extra == "all"
-Requires-Dist: psutil~=6.0; extra == "all"
+Requires-Dist: psutil~=7.0; extra == "all"
 Requires-Dist: orjson~=3.10; extra == "all"
 Requires-Dist: ujson~=5.10; extra == "all"
 Requires-Dist: pyyaml~=6.0; extra == "all"
@@ -54,7 +54,7 @@ Provides-Extra: async
 Requires-Dist: anyio~=4.8; extra == "async"
 Requires-Dist: sniffio~=1.3; extra == "async"
 Requires-Dist: greenlet~=3.1; extra == "async"
-Requires-Dist: trio~=0.27; extra == "async"
+Requires-Dist: trio~=0.29; extra == "async"
 Requires-Dist: trio-asyncio~=0.15; extra == "async"
 Provides-Extra: compress
 Requires-Dist: lz4~=4.4; extra == "compress"
@@ -64,7 +64,7 @@ Requires-Dist: brotli~=1.1; extra == "compress"
 Provides-Extra: diag
 Requires-Dist: asttokens~=3.0; extra == "diag"
 Requires-Dist: executing~=2.2; extra == "diag"
-Requires-Dist: psutil~=6.0; extra == "diag"
+Requires-Dist: psutil~=7.0; extra == "diag"
 Provides-Extra: formats
 Requires-Dist: orjson~=3.10; extra == "formats"
 Requires-Dist: ujson~=5.10; extra == "formats"

{omlish-0.0.0.dev230 → omlish-0.0.0.dev232}/omlish/.manifests.json

@@ -307,6 +307,18 @@
       }
     }
   },
+  {
+    "module": ".os.pidfiles.__main__",
+    "attr": "_CLI_MODULE",
+    "file": "omlish/os/pidfiles/__main__.py",
+    "line": 1,
+    "value": {
+      "$omdev.cli.types.CliModule": {
+        "cmd_name": "pidfiles",
+        "mod_name": "omlish.os.pidfiles.__main__"
+      }
+    }
+  },
   {
     "module": ".secrets.pwgen",
     "attr": "_CLI_MODULE",

{omlish-0.0.0.dev230 → omlish-0.0.0.dev232}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev230'
-__revision__ = 'ed2672ad6a2fb4718c57e281c5ce91fdc7998a6f'
+__version__ = '0.0.0.dev232'
+__revision__ = '2f0435f277c869f9ecf426be2aeaf08eeb9792de'
 
 
 #
@@ -37,7 +37,7 @@ class Project(ProjectBase):
 
             'greenlet ~= 3.1',
 
-            'trio ~= 0.27',
+            'trio ~= 0.29',
             'trio-asyncio ~= 0.15',
         ],
 
@@ -56,7 +56,7 @@ class Project(ProjectBase):
             'asttokens ~= 3.0',
             'executing ~= 2.2',
 
-            'psutil ~= 6.0',
+            'psutil ~= 7.0',
         ],
 
         'formats': [

omlish-0.0.0.dev232/omlish/daemons/daemon.py

@@ -0,0 +1,126 @@
+"""
+TODO:
+ - OK.. this is useful for non-system-daemons too... which have no pidfile..
+ - split out pidfile concern to.. identity?
+ - async[io] support, really just waiting
+ - helpers for http, json, etc
+ - heartbeat? status checks? heartbeat file?
+ - zmq
+ - cli, cfg reload
+ - bootstrap
+  - rsrc limit
+  - logs
+ - https://github.com/Homebrew/homebrew-services
+ - deathpacts
+ - timebomb
+ - pickle protocol, revision / venv check, multiprocessing manager support
+"""
+import logging
+import os.path
+import time
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..os.pidfiles.pidfile import Pidfile
+from .launching import Launcher
+from .spawning import Spawning
+from .targets import Target
+from .waiting import Wait
+from .waiting import waiter_for
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Daemon:
+    @dc.dataclass(frozen=True, kw_only=True)
+    class Config:
+        target: Target
+        spawning: Spawning
+
+        #
+
+        pid_file: str | None = None
+
+        #
+
+        reparent_process: bool = False
+        launched_timeout_s: float = 5.
+
+        #
+
+        wait: Wait | None = None
+
+        wait_timeout: lang.TimeoutLike = 10.
+        wait_sleep_s: float = .1
+
+        #
+
+        def __post_init__(self) -> None:
+            check.isinstance(self.pid_file, (str, None))
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+
+        self._config = config
+
+    @property
+    def config(self) -> Config:
+        return self._config
+
+    #
+
+    @property
+    def has_pidfile(self) -> bool:
+        return self._config.pid_file is not None
+
+    def _non_inheritable_pidfile(self) -> Pidfile:
+        check.state(self.has_pidfile)
+        return Pidfile(
+            check.non_empty_str(self._config.pid_file),
+            inheritable=False,
+        )
+
+    def is_running(self) -> bool:
+        check.state(self.has_pidfile)
+
+        if not os.path.isfile(check.non_empty_str(self._config.pid_file)):
+            return False
+
+        with self._non_inheritable_pidfile() as pf:
+            return not pf.try_acquire_lock()
+
+    #
+
+    def wait_sync(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        if self._config.wait is None:
+            return
+
+        timeout = lang.Timeout.of(timeout, self._config.wait_timeout)
+        waiter = waiter_for(self._config.wait)
+        while not waiter.do_wait():
+            timeout()
+            time.sleep(self._config.wait_sleep_s or 0.)
+
+    #
+
+    def launch_no_wait(self) -> None:
+        launcher = Launcher(
+            target=self._config.target,
+            spawning=self._config.spawning,
+
+            pid_file=self._config.pid_file,
+            reparent_process=self._config.reparent_process,
+            launched_timeout_s=self._config.launched_timeout_s,
+        )
+
+        launcher.launch()
+
+    def launch(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        self.launch_no_wait()
+
+        self.wait_sync(timeout)

omlish-0.0.0.dev232/omlish/daemons/launching.py

@@ -0,0 +1,121 @@
+"""
+TODO:
+ - Config? dedupe defaults with Daemon
+ - ExitStacked? hold ref to Spawner, which holds refs to thread/proc - which will likely outlive it, but still
+"""
+import contextlib
+import functools
+import logging
+import threading
+import typing as ta
+
+from .. import check
+from .. import lang
+from ..os.pidfiles.manager import open_inheritable_pidfile
+from ..os.pidfiles.pidfile import Pidfile
+from .reparent import reparent_process
+from .spawning import InProcessSpawner
+from .spawning import Spawn
+from .spawning import Spawner
+from .spawning import Spawning
+from .spawning import spawner_for
+from .targets import Target
+from .targets import target_runner_for
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Launcher:
+    def __init__(
+            self,
+            *,
+            target: Target,
+            spawning: Spawning,
+
+            pid_file: str | None = None,
+            reparent_process: bool = False,  # noqa
+            launched_timeout_s: float = 5.,
+    ) -> None:
+        super().__init__()
+
+        self._target = target
+        self._spawning = spawning
+
+        self._pid_file = pid_file
+        self._reparent_process = reparent_process
+        self._launched_timeout_s = launched_timeout_s
+
+    def _inner_launch(
+            self,
+            *,
+            pidfile_manager: ta.ContextManager | None,
+            launched_callback: ta.Callable[[], None] | None = None,
+    ) -> None:
+        try:
+            if self._reparent_process:
+                log.info('Reparenting')
+                reparent_process()
+
+            with contextlib.ExitStack() as es:
+                pidfile: Pidfile | None = None  # noqa
+                if pidfile_manager is not None:
+                    pidfile = check.isinstance(es.enter_context(pidfile_manager), Pidfile)
+                    pidfile.write()
+
+                if launched_callback is not None:
+                    launched_callback()
+
+                runner = target_runner_for(self._target)
+                runner.run()
+
+        finally:
+            if launched_callback is not None:
+                launched_callback()
+
+    def launch(self) -> bool:
+        with contextlib.ExitStack() as es:
+            spawner: Spawner = es.enter_context(spawner_for(self._spawning))
+
+            #
+
+            inherit_fds: set[int] = set()
+            launched_event: threading.Event | None = None
+
+            pidfile: Pidfile | None = None  # noqa
+            pidfile_manager: ta.ContextManager | None = None
+
+            if (pid_file := self._pid_file) is not None:
+                if not isinstance(spawner, InProcessSpawner):
+                    pidfile = es.enter_context(open_inheritable_pidfile(pid_file))
+                    pidfile_manager = lang.NopContextManager(pidfile)
+
+                else:
+                    check.state(not self._reparent_process)
+                    pidfile = es.enter_context(Pidfile(pid_file))
+                    pidfile_manager = pidfile.dup()
+                    launched_event = threading.Event()
+
+                if not pidfile.try_acquire_lock():
+                    return False
+
+                inherit_fds.add(check.isinstance(pidfile.fileno(), int))
+
+            #
+
+            spawner.spawn(Spawn(
+                functools.partial(
+                    self._inner_launch,
+                    pidfile_manager=pidfile_manager,
+                    launched_callback=launched_event.set if launched_event is not None else None,
+                ),
+                inherit_fds=inherit_fds,
+            ))
+
+            if launched_event is not None:
+                check.state(launched_event.wait(timeout=self._launched_timeout_s))
+
+            return True

omlish-0.0.0.dev232/omlish/daemons/reparent.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+
+def reparent_process() -> None:
+    if (pid := os.fork()):  # noqa
+        sys.exit(0)
+        raise RuntimeError('Unreachable')  # noqa
+
+    os.setsid()
+
+    if (pid := os.fork()):  # noqa
+        sys.exit(0)
+
+    sys.stdout.flush()
+    sys.stderr.flush()

omlish-0.0.0.dev232/omlish/daemons/spawning.py

@@ -0,0 +1,166 @@
+import abc
+import functools
+import os
+import sys
+import threading
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..diag import pydevd
+
+
+if ta.TYPE_CHECKING:
+    import multiprocessing as mp
+    import multiprocessing.context
+    import multiprocessing.process  # noqa
+
+    from ..multiprocessing import spawn as omp_spawn
+
+else:
+    mp = lang.proxy_import('multiprocessing', extras=['context', 'process'])
+    subprocess = lang.proxy_import('subprocess')
+
+    omp_spawn = lang.proxy_import('..multiprocessing.spawn', __package__)
+
+
+##
+
+
+class Spawning(dc.Case):
+    pass
+
+
+class Spawn(dc.Frozen, final=True):
+    fn: ta.Callable[[], None]
+
+    _: dc.KW_ONLY
+
+    inherit_fds: ta.Collection[int] | None = None
+
+
+class Spawner(lang.ContextManaged, abc.ABC):
+    @abc.abstractmethod
+    def spawn(self, spawn: Spawn) -> None:
+        raise NotImplementedError
+
+
+class InProcessSpawner(Spawner, abc.ABC):
+    pass
+
+
+@functools.singledispatch
+def spawner_for(spawning: Spawning) -> Spawner:
+    raise TypeError(spawning)
+
+
+##
+
+
+class MultiprocessingSpawning(Spawning, kw_only=True):
+    # Defaults to 'fork' if under pydevd, else 'spawn'
+    start_method: str | None = None
+
+    non_daemon: bool = False
+
+
+class MultiprocessingSpawner(Spawner):
+    def __init__(self, spawning: MultiprocessingSpawning) -> None:
+        super().__init__()
+
+        self._spawning = spawning
+        self._process: ta.Optional['mp.process.BaseProcess'] = None  # noqa
+
+    def _process_cls(self, spawn: Spawn) -> type['mp.process.BaseProcess']:
+        if (start_method := self._spawning.start_method) is None:
+            # Unfortunately, pydevd forces the use of the 'fork' start_method, which cannot be mixed with 'spawn':
+            #   https://github.com/python/cpython/blob/a7427f2db937adb4c787754deb4c337f1894fe86/Lib/multiprocessing/spawn.py#L102  # noqa
+            if pydevd.is_running():
+                start_method = 'fork'
+            else:
+                start_method = 'spawn'
+
+        ctx: 'mp.context.BaseContext'  # noqa
+        if start_method == 'fork':
+            ctx = mp.get_context(check.non_empty_str(start_method))
+
+        elif start_method == 'spawn':
+            ctx = omp_spawn.ExtrasSpawnContext(omp_spawn.SpawnExtras(
+                pass_fds=frozenset(spawn.inherit_fds) if spawn.inherit_fds is not None else None,
+            ))
+
+        else:
+            raise ValueError(start_method)
+
+        return ctx.Process  # type: ignore
+
+    def spawn(self, spawn: Spawn) -> None:
+        check.none(self._process)
+        self._process = self._process_cls(spawn)(
+            target=spawn.fn,
+            daemon=not self._spawning.non_daemon,
+        )
+        self._process.start()
+
+
+@spawner_for.register
+def _(spawning: MultiprocessingSpawning) -> MultiprocessingSpawner:
+    return MultiprocessingSpawner(spawning)
+
+
+##
+
+
+class ForkSpawning(Spawning):
+    pass
+
+
+class ForkSpawner(Spawner, dc.Frozen):
+    spawning: ForkSpawning
+
+    def spawn(self, spawn: Spawn) -> None:
+        if (pid := os.fork()):  # noqa
+            return
+
+        try:
+            spawn.fn()
+        except BaseException:  # noqa
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
+        raise RuntimeError('Unreachable')  # noqa
+
+
+@spawner_for.register
+def _(spawning: ForkSpawning) -> ForkSpawner:
+    return ForkSpawner(spawning)
+
+
+##
+
+
+class ThreadSpawning(Spawning, kw_only=True):
+    non_daemon: bool = False
+
+
+class ThreadSpawner(InProcessSpawner):
+    def __init__(self, spawning: ThreadSpawning) -> None:
+        super().__init__()
+
+        self._spawning = spawning
+        self._thread: threading.Thread | None = None
+
+    def spawn(self, spawn: Spawn) -> None:
+        check.none(self._thread)
+        self._thread = threading.Thread(
+            target=spawn.fn,
+            daemon=not self._spawning.non_daemon,
+        )
+        self._thread.start()
+
+
+@spawner_for.register
+def _(spawning: ThreadSpawning) -> ThreadSpawner:
+    return ThreadSpawner(spawning)

omlish-0.0.0.dev232/omlish/daemons/targets.py

@@ -0,0 +1,89 @@
+import abc
+import functools
+import os
+import typing as ta
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import runpy
+else:
+    runpy = lang.proxy_import('runpy')
+
+
+##
+
+
+class Target(dc.Case):
+    pass
+
+
+class TargetRunner(abc.ABC):
+    @abc.abstractmethod
+    def run(self) -> None:
+        raise NotImplementedError
+
+
+@functools.singledispatch
+def target_runner_for(target: Target) -> TargetRunner:
+    raise TypeError(target)
+
+
+##
+
+
+class FnTarget(Target):
+    fn: ta.Callable[[], None]
+
+
+class FnTargetRunner(TargetRunner, dc.Frozen):
+    target: FnTarget
+
+    def run(self) -> None:
+        self.target.fn()
+
+
+@target_runner_for.register
+def _(target: FnTarget) -> FnTargetRunner:
+    return FnTargetRunner(target)
+
+
+##
+
+
+class NameTarget(Target):
+    name: str
+
+
+class NameTargetRunner(TargetRunner, dc.Frozen):
+    target: NameTarget
+
+    def run(self) -> None:
+        name = self.target.name
+        if lang.can_import(name):
+            runpy._run_module_as_main(name)  # type: ignore  # noqa
+        else:
+            obj = lang.import_attr(self.target.name)
+            obj()
+
+
+##
+
+
+class ExecTarget(Target):
+    cmd: ta.Sequence[str] = dc.xfield(coerce=check.of_not_isinstance(str))
+
+
+class ExecTargetRunner(TargetRunner, dc.Frozen):
+    target: ExecTarget
+
+    def run(self) -> None:
+        os.execl(*self.target.cmd)
+
+
+@target_runner_for.register
+def _(target: ExecTarget) -> ExecTargetRunner:
+    return ExecTargetRunner(target)

omlish-0.0.0.dev232/omlish/daemons/waiting.py

@@ -0,0 +1,95 @@
+import abc
+import functools
+import socket
+import typing as ta
+
+from .. import dataclasses as dc
+
+
+##
+
+
+class Wait(dc.Case):
+    pass
+
+
+class Waiter(abc.ABC):
+    @abc.abstractmethod
+    def do_wait(self) -> bool:
+        raise NotImplementedError
+
+
+@functools.singledispatch
+def waiter_for(wait: Wait) -> Waiter:
+    raise TypeError(wait)
+
+
+##
+
+
+class SequentialWait(Wait):
+    waits: ta.Sequence[Wait]
+
+
+class SequentialWaiter(Waiter):
+    def __init__(self, waiters: ta.Sequence[Waiter]) -> None:
+        super().__init__()
+
+        self._waiters = waiters
+        self._idx = 0
+
+    def do_wait(self) -> bool:
+        while self._idx < len(self._waiters):
+            if not self._waiters[self._idx].do_wait():
+                return False
+            self._idx += 1
+        return True
+
+
+@waiter_for.register
+def _(wait: SequentialWait) -> SequentialWaiter:
+    return SequentialWaiter([waiter_for(c) for c in wait.waits])
+
+
+##
+
+
+class FnWait(Wait):
+    fn: ta.Callable[[], bool]
+
+
+class FnWaiter(Waiter, dc.Frozen):
+    wait: FnWait
+
+    def do_wait(self) -> bool:
+        return self.wait.fn()
+
+
+@waiter_for.register
+def _(wait: FnWait) -> FnWaiter:
+    return FnWaiter(wait)
+
+
+##
+
+
+class ConnectWait(Wait):
+    address: ta.Any
+
+
+class ConnectWaiter(Waiter, dc.Frozen):
+    wait: ConnectWait
+
+    def do_wait(self) -> bool:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.connect(self.wait.address)
+            except ConnectionRefusedError:
+                return False
+            else:
+                return True
+
+
+@waiter_for.register
+def _(wait: ConnectWait) -> ConnectWaiter:
+    return ConnectWaiter(wait)

{omlish-0.0.0.dev230 → omlish-0.0.0.dev232}/omlish/lang/__init__.py

@@ -61,9 +61,7 @@ from .contextmanagers import (  # noqa
     DefaultLockable,
     ExitStacked,
     Lockable,
-    NOP_CONTEXT_MANAGED,
     NOP_CONTEXT_MANAGER,
-    NopContextManaged,
     NopContextManager,
     Timer,
     a_defer,