omlish 0.0.0.dev231__tar.gz → 0.0.0.dev233__tar.gz

{omlish-0.0.0.dev231/omlish.egg-info → omlish-0.0.0.dev233}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev231
+Version: 0.0.0.dev233
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -16,7 +16,7 @@ Provides-Extra: all
 Requires-Dist: anyio~=4.8; extra == "all"
 Requires-Dist: sniffio~=1.3; extra == "all"
 Requires-Dist: greenlet~=3.1; extra == "all"
-Requires-Dist: trio~=0.27; extra == "all"
+Requires-Dist: trio~=0.29; extra == "all"
 Requires-Dist: trio-asyncio~=0.15; extra == "all"
 Requires-Dist: lz4~=4.4; extra == "all"
 Requires-Dist: python-snappy~=0.7; extra == "all"
@@ -24,7 +24,7 @@ Requires-Dist: zstandard~=0.23; extra == "all"
 Requires-Dist: brotli~=1.1; extra == "all"
 Requires-Dist: asttokens~=3.0; extra == "all"
 Requires-Dist: executing~=2.2; extra == "all"
-Requires-Dist: psutil~=6.0; extra == "all"
+Requires-Dist: psutil~=7.0; extra == "all"
 Requires-Dist: orjson~=3.10; extra == "all"
 Requires-Dist: ujson~=5.10; extra == "all"
 Requires-Dist: pyyaml~=6.0; extra == "all"
@@ -54,7 +54,7 @@ Provides-Extra: async
 Requires-Dist: anyio~=4.8; extra == "async"
 Requires-Dist: sniffio~=1.3; extra == "async"
 Requires-Dist: greenlet~=3.1; extra == "async"
-Requires-Dist: trio~=0.27; extra == "async"
+Requires-Dist: trio~=0.29; extra == "async"
 Requires-Dist: trio-asyncio~=0.15; extra == "async"
 Provides-Extra: compress
 Requires-Dist: lz4~=4.4; extra == "compress"
@@ -64,7 +64,7 @@ Requires-Dist: brotli~=1.1; extra == "compress"
 Provides-Extra: diag
 Requires-Dist: asttokens~=3.0; extra == "diag"
 Requires-Dist: executing~=2.2; extra == "diag"
-Requires-Dist: psutil~=6.0; extra == "diag"
+Requires-Dist: psutil~=7.0; extra == "diag"
 Provides-Extra: formats
 Requires-Dist: orjson~=3.10; extra == "formats"
 Requires-Dist: ujson~=5.10; extra == "formats"

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/.manifests.json

@@ -307,6 +307,18 @@
       }
     }
   },
+  {
+    "module": ".os.pidfiles.__main__",
+    "attr": "_CLI_MODULE",
+    "file": "omlish/os/pidfiles/__main__.py",
+    "line": 1,
+    "value": {
+      "$omdev.cli.types.CliModule": {
+        "cmd_name": "pidfiles",
+        "mod_name": "omlish.os.pidfiles.__main__"
+      }
+    }
+  },
   {
     "module": ".secrets.pwgen",
     "attr": "_CLI_MODULE",

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev231'
-__revision__ = '2eb4140ac802af2814bc4b4d5a8d37613083ae32'
+__version__ = '0.0.0.dev233'
+__revision__ = '104fc39d3a0d5ea92881a45333cbcb1810f6b7ff'
 
 
 #
@@ -37,7 +37,7 @@ class Project(ProjectBase):
 
             'greenlet ~= 3.1',
 
-            'trio ~= 0.27',
+            'trio ~= 0.29',
             'trio-asyncio ~= 0.15',
         ],
 
@@ -56,7 +56,7 @@ class Project(ProjectBase):
             'asttokens ~= 3.0',
             'executing ~= 2.2',
 
-            'psutil ~= 6.0',
+            'psutil ~= 7.0',
         ],
 
         'formats': [

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/codecs/registry.py

@@ -105,7 +105,7 @@ def _install_standard_codecs(registry: CodecRegistry) -> None:
 @cached.function
 def _build_manifest_lazy_loaded_codecs() -> ta.Sequence[LazyLoadedCodec]:
     ldr = manifest_load.MANIFEST_LOADER
-    pkgs = {__package__.split('.')[0], *ldr.discover()}
+    pkgs = {__package__.split('.')[0], *ldr.discover_pkgs()}
     mns = ldr.load(*pkgs, only=[LazyLoadedCodec])
     return [m.value for m in mns]
 

omlish-0.0.0.dev233/omlish/daemons/daemon.py

@@ -0,0 +1,126 @@
+"""
+TODO:
+ - OK.. this is useful for non-system-daemons too... which have no pidfile..
+ - split out pidfile concern to.. identity?
+ - async[io] support, really just waiting
+ - helpers for http, json, etc
+ - heartbeat? status checks? heartbeat file?
+ - zmq
+ - cli, cfg reload
+ - bootstrap
+  - rsrc limit
+  - logs
+ - https://github.com/Homebrew/homebrew-services
+ - deathpacts
+ - timebomb
+ - pickle protocol, revision / venv check, multiprocessing manager support
+"""
+import logging
+import os.path
+import time
+
+from .. import check
+from .. import dataclasses as dc
+from .. import lang
+from ..os.pidfiles.pidfile import Pidfile
+from .launching import Launcher
+from .spawning import Spawning
+from .targets import Target
+from .waiting import Wait
+from .waiting import waiter_for
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Daemon:
+    @dc.dataclass(frozen=True, kw_only=True)
+    class Config:
+        target: Target
+        spawning: Spawning
+
+        #
+
+        pid_file: str | None = None
+
+        #
+
+        reparent_process: bool = False
+        launched_timeout_s: float = 5.
+
+        #
+
+        wait: Wait | None = None
+
+        wait_timeout: lang.TimeoutLike = 10.
+        wait_sleep_s: float = .1
+
+        #
+
+        def __post_init__(self) -> None:
+            check.isinstance(self.pid_file, (str, None))
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+
+        self._config = config
+
+    @property
+    def config(self) -> Config:
+        return self._config
+
+    #
+
+    @property
+    def has_pidfile(self) -> bool:
+        return self._config.pid_file is not None
+
+    def _non_inheritable_pidfile(self) -> Pidfile:
+        check.state(self.has_pidfile)
+        return Pidfile(
+            check.non_empty_str(self._config.pid_file),
+            inheritable=False,
+        )
+
+    def is_running(self) -> bool:
+        check.state(self.has_pidfile)
+
+        if not os.path.isfile(check.non_empty_str(self._config.pid_file)):
+            return False
+
+        with self._non_inheritable_pidfile() as pf:
+            return not pf.try_acquire_lock()
+
+    #
+
+    def wait_sync(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        if self._config.wait is None:
+            return
+
+        timeout = lang.Timeout.of(timeout, self._config.wait_timeout)
+        waiter = waiter_for(self._config.wait)
+        while not waiter.do_wait():
+            timeout()
+            time.sleep(self._config.wait_sleep_s or 0.)
+
+    #
+
+    def launch_no_wait(self) -> None:
+        launcher = Launcher(
+            target=self._config.target,
+            spawning=self._config.spawning,
+
+            pid_file=self._config.pid_file,
+            reparent_process=self._config.reparent_process,
+            launched_timeout_s=self._config.launched_timeout_s,
+        )
+
+        launcher.launch()
+
+    def launch(self, timeout: lang.TimeoutLike = lang.Timeout.Default) -> None:
+        self.launch_no_wait()
+
+        self.wait_sync(timeout)

omlish-0.0.0.dev233/omlish/daemons/launching.py

@@ -0,0 +1,121 @@
+"""
+TODO:
+ - Config? dedupe defaults with Daemon
+ - ExitStacked? hold ref to Spawner, which holds refs to thread/proc - which will likely outlive it, but still
+"""
+import contextlib
+import functools
+import logging
+import threading
+import typing as ta
+
+from .. import check
+from .. import lang
+from ..os.pidfiles.manager import open_inheritable_pidfile
+from ..os.pidfiles.pidfile import Pidfile
+from .reparent import reparent_process
+from .spawning import InProcessSpawner
+from .spawning import Spawn
+from .spawning import Spawner
+from .spawning import Spawning
+from .spawning import spawner_for
+from .targets import Target
+from .targets import target_runner_for
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class Launcher:
+    def __init__(
+            self,
+            *,
+            target: Target,
+            spawning: Spawning,
+
+            pid_file: str | None = None,
+            reparent_process: bool = False,  # noqa
+            launched_timeout_s: float = 5.,
+    ) -> None:
+        super().__init__()
+
+        self._target = target
+        self._spawning = spawning
+
+        self._pid_file = pid_file
+        self._reparent_process = reparent_process
+        self._launched_timeout_s = launched_timeout_s
+
+    def _inner_launch(
+            self,
+            *,
+            pidfile_manager: ta.ContextManager | None,
+            launched_callback: ta.Callable[[], None] | None = None,
+    ) -> None:
+        try:
+            if self._reparent_process:
+                log.info('Reparenting')
+                reparent_process()
+
+            with contextlib.ExitStack() as es:
+                pidfile: Pidfile | None = None  # noqa
+                if pidfile_manager is not None:
+                    pidfile = check.isinstance(es.enter_context(pidfile_manager), Pidfile)
+                    pidfile.write()
+
+                if launched_callback is not None:
+                    launched_callback()
+
+                runner = target_runner_for(self._target)
+                runner.run()
+
+        finally:
+            if launched_callback is not None:
+                launched_callback()
+
+    def launch(self) -> bool:
+        with contextlib.ExitStack() as es:
+            spawner: Spawner = es.enter_context(spawner_for(self._spawning))
+
+            #
+
+            inherit_fds: set[int] = set()
+            launched_event: threading.Event | None = None
+
+            pidfile: Pidfile | None = None  # noqa
+            pidfile_manager: ta.ContextManager | None = None
+
+            if (pid_file := self._pid_file) is not None:
+                if not isinstance(spawner, InProcessSpawner):
+                    pidfile = es.enter_context(open_inheritable_pidfile(pid_file))
+                    pidfile_manager = lang.NopContextManager(pidfile)
+
+                else:
+                    check.state(not self._reparent_process)
+                    pidfile = es.enter_context(Pidfile(pid_file))
+                    pidfile_manager = pidfile.dup()
+                    launched_event = threading.Event()
+
+                if not pidfile.try_acquire_lock():
+                    return False
+
+                inherit_fds.add(check.isinstance(pidfile.fileno(), int))
+
+            #
+
+            spawner.spawn(Spawn(
+                functools.partial(
+                    self._inner_launch,
+                    pidfile_manager=pidfile_manager,
+                    launched_callback=launched_event.set if launched_event is not None else None,
+                ),
+                inherit_fds=inherit_fds,
+            ))
+
+            if launched_event is not None:
+                check.state(launched_event.wait(timeout=self._launched_timeout_s))
+
+            return True

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/lang/functions.py

@@ -190,6 +190,16 @@ class Args:
         self.args = args
         self.kwargs = kwargs
 
+    def update(self, *args: ta.Any, **kwargs: ta.Any) -> 'Args':
+        return Args(
+            *self.args,
+            *args,
+            **{
+                **self.kwargs,
+                **kwargs,
+            },
+        )
+
     def __call__(self, fn: ta.Callable[..., T]) -> T:
         return fn(*self.args, **self.kwargs)
 

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/manifests/load.py

@@ -9,6 +9,7 @@ import dataclasses as dc
 import importlib.machinery
 import importlib.resources
 import json
+import os.path
 import threading
 import typing as ta
 
@@ -193,7 +194,7 @@ class ManifestLoader:
 
     ENTRY_POINT_GROUP = 'omlish.manifests'
 
-    def discover(self) -> ta.Sequence[str]:
+    def discover_pkgs(self) -> ta.Sequence[str]:
         # This is a fat dep so do it late.
         import importlib.metadata
 
@@ -202,6 +203,35 @@ class ManifestLoader:
             for ep in importlib.metadata.entry_points(group=self.ENTRY_POINT_GROUP)
         ]
 
+    def scan_pkg_root(self, root: str) -> ta.Sequence[str]:
+        pkgs: ta.List[str] = []
+        for n in os.listdir(root):
+            if os.path.isdir(p := os.path.join(root, n)) and os.path.exists(os.path.join(p, '__init__.py')):
+                pkgs.append(n)
+        return pkgs
+
+    def scan_or_discover_pkgs(
+            self,
+            *,
+            specified_roots: ta.Optional[ta.Sequence[str]] = None,
+            fallback_root: ta.Optional[str] = None,
+    ) -> ta.Sequence[str]:
+        pkgs: list[str] = []
+
+        if specified_roots is not None:
+            if isinstance(specified_roots, str):
+                raise TypeError(specified_roots)
+            for r in specified_roots:
+                pkgs.extend(self.scan_pkg_root(r))
+
+        else:
+            pkgs.extend(self.discover_pkgs())
+
+            if not pkgs and fallback_root is not None:
+                pkgs.extend(self.scan_pkg_root(fallback_root))
+
+        return pkgs
+
 
 ##
 

omlish-0.0.0.dev233/omlish/os/pidfiles/__main__.py

@@ -0,0 +1,11 @@
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': 'pidfiles',
+    'mod_name': __name__,
+}}
+
+
+if __name__ == '__main__':
+    from .cli import _main  # noqa
+
+    _main()

omlish-0.0.0.dev233/omlish/os/pidfiles/cli.py

@@ -0,0 +1,88 @@
+"""
+TODO:
+ - F_SETLK mode
+"""
+import os
+import typing as ta
+
+from ... import lang
+from ...argparse import all as ap
+from ..pidfiles.pidfile import Pidfile
+from ..pidfiles.pinning import PidfilePinner
+from ..signals import parse_signal
+
+
+class Cli(ap.Cli):
+    _PIDFILE_ARGS: ta.ClassVar[ta.Sequence[ap.Arg]] = [
+        ap.arg('pid-file'),
+        ap.arg('--create', action='store_true'),
+    ]
+
+    def _pidfile_args(self) -> lang.Args:
+        return lang.Args(
+            self.args.pid_file,
+            inheritable=False,
+            no_create=not self._args.create,
+        )
+
+    def _args_pidfile(self) -> Pidfile:
+        return self._pidfile_args()(Pidfile)
+
+    #
+
+    @ap.cmd(*_PIDFILE_ARGS)
+    def read_no_verify(self) -> None:
+        with self._args_pidfile() as pidfile:
+            print(pidfile.read())
+
+    @ap.cmd(*_PIDFILE_ARGS)
+    def lock(self) -> None:
+        with self._args_pidfile() as pidfile:
+            pidfile.acquire_lock()
+            print(os.getpid())
+            input()
+
+    #
+
+    _PIDFILE_PINNER_ARGS: ta.ClassVar[ta.Sequence[ap.Arg]] = [
+        *_PIDFILE_ARGS,
+        ap.arg('--timeout', type=float),
+    ]
+
+    def _pidfile_pinner_args(self) -> lang.Args:
+        return self._pidfile_args().update(
+            timeout=self._args.timeout,
+        )
+
+    def _args_pidfile_pinner(self) -> ta.ContextManager[int]:
+        return self._pidfile_pinner_args()(PidfilePinner.default_impl()().pin_pidfile_owner)
+
+    #
+
+    @ap.cmd(*_PIDFILE_PINNER_ARGS)
+    def read(self) -> None:
+        with self._args_pidfile_pinner() as pid:
+            print(pid)
+
+    @ap.cmd(*_PIDFILE_PINNER_ARGS)
+    def pin(self) -> None:
+        with self._args_pidfile_pinner() as pid:
+            print(pid)
+            input()
+
+    @ap.cmd(
+        *_PIDFILE_PINNER_ARGS,
+        ap.arg('signal'),
+    )
+    def kill(self) -> None:
+        sig = parse_signal(self._args.signal)
+        with self._args_pidfile_pinner() as pid:
+            os.kill(pid, sig)
+
+
+def _main() -> None:
+    Cli().cli_run_and_exit()
+
+
+if __name__ == '__main__':
+    _main()

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/os/pidfiles/pidfile.py

@@ -27,11 +27,13 @@ class Pidfile:
             path: str,
             *,
             inheritable: bool = True,
+            no_create: bool = False,
     ) -> None:
         super().__init__()
 
         self._path = path
         self._inheritable = inheritable
+        self._no_create = no_create
 
     @property
     def path(self) -> str:
@@ -74,7 +76,10 @@ class Pidfile:
             fd = os.dup(self._fd_to_dup)
             del self._fd_to_dup
         else:
-            fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
+            ofl = os.O_RDWR
+            if not self._no_create:
+                ofl |= os.O_CREAT
+            fd = os.open(self._path, ofl, 0o600)
 
         try:
             if self._inheritable:

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish/os/pidfiles/pinning.py

@@ -27,6 +27,7 @@ import typing as ta
 
 from ...diag.lslocks import LslocksCommand
 from ...diag.lsof import LsofCommand
+from ...lite.check import check
 from ...lite.timeouts import Timeout
 from ...lite.timeouts import TimeoutLike
 from ...subprocesses.sync import subprocesses  # noqa
@@ -64,13 +65,21 @@ class PidfilePinner(abc.ABC):
             path: str,
             *,
             timeout: ta.Optional[TimeoutLike] = None,
+            inheritable: bool = False,  # Present to match Pidfile kwargs for convenience, but enforced to be False.
+            **kwargs: ta.Any,
     ) -> ta.Iterator[int]:
+        check.arg(not inheritable)
+
         timeout = Timeout.of(timeout)
 
         if not os.path.isfile(path):
             raise self.NoOwnerError
 
-        with Pidfile(path, inheritable=False) as pf:
+        with Pidfile(
+                path,
+                inheritable=False,
+                **kwargs,
+        ) as pf:
             try:
                 with self._pin_pidfile_owner(pf, timeout) as pid:
                     yield pid

omlish-0.0.0.dev233/omlish/os/signals.py

@@ -0,0 +1,15 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import signal
+
+
+def parse_signal(s: str) -> int:
+    try:
+        return int(s)
+    except ValueError:
+        pass
+
+    s = s.upper()
+    if not s.startswith('SIG'):
+        s = 'SIG' + s
+    return signal.Signals[s]  # noqa

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev231
+Version: 0.0.0.dev233
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -16,7 +16,7 @@ Provides-Extra: all
 Requires-Dist: anyio~=4.8; extra == "all"
 Requires-Dist: sniffio~=1.3; extra == "all"
 Requires-Dist: greenlet~=3.1; extra == "all"
-Requires-Dist: trio~=0.27; extra == "all"
+Requires-Dist: trio~=0.29; extra == "all"
 Requires-Dist: trio-asyncio~=0.15; extra == "all"
 Requires-Dist: lz4~=4.4; extra == "all"
 Requires-Dist: python-snappy~=0.7; extra == "all"
@@ -24,7 +24,7 @@ Requires-Dist: zstandard~=0.23; extra == "all"
 Requires-Dist: brotli~=1.1; extra == "all"
 Requires-Dist: asttokens~=3.0; extra == "all"
 Requires-Dist: executing~=2.2; extra == "all"
-Requires-Dist: psutil~=6.0; extra == "all"
+Requires-Dist: psutil~=7.0; extra == "all"
 Requires-Dist: orjson~=3.10; extra == "all"
 Requires-Dist: ujson~=5.10; extra == "all"
 Requires-Dist: pyyaml~=6.0; extra == "all"
@@ -54,7 +54,7 @@ Provides-Extra: async
 Requires-Dist: anyio~=4.8; extra == "async"
 Requires-Dist: sniffio~=1.3; extra == "async"
 Requires-Dist: greenlet~=3.1; extra == "async"
-Requires-Dist: trio~=0.27; extra == "async"
+Requires-Dist: trio~=0.29; extra == "async"
 Requires-Dist: trio-asyncio~=0.15; extra == "async"
 Provides-Extra: compress
 Requires-Dist: lz4~=4.4; extra == "compress"
@@ -64,7 +64,7 @@ Requires-Dist: brotli~=1.1; extra == "compress"
 Provides-Extra: diag
 Requires-Dist: asttokens~=3.0; extra == "diag"
 Requires-Dist: executing~=2.2; extra == "diag"
-Requires-Dist: psutil~=6.0; extra == "diag"
+Requires-Dist: psutil~=7.0; extra == "diag"
 Provides-Extra: formats
 Requires-Dist: orjson~=3.10; extra == "formats"
 Requires-Dist: ujson~=5.10; extra == "formats"

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish.egg-info/SOURCES.txt

@@ -187,6 +187,7 @@ omlish/configs/processing/rewriting.py
 omlish/configs/processing/strings.py
 omlish/daemons/__init__.py
 omlish/daemons/daemon.py
+omlish/daemons/launching.py
 omlish/daemons/reparent.py
 omlish/daemons/spawning.py
 omlish/daemons/targets.py
@@ -528,6 +529,7 @@ omlish/os/forkhooks.py
 omlish/os/journald.py
 omlish/os/linux.py
 omlish/os/paths.py
+omlish/os/signals.py
 omlish/os/sizes.py
 omlish/os/temp.py
 omlish/os/deathpacts/__init__.py
@@ -535,6 +537,8 @@ omlish/os/deathpacts/base.py
 omlish/os/deathpacts/heartbeatfile.py
 omlish/os/deathpacts/pipe.py
 omlish/os/pidfiles/__init__.py
+omlish/os/pidfiles/__main__.py
+omlish/os/pidfiles/cli.py
 omlish/os/pidfiles/manager.py
 omlish/os/pidfiles/pidfile.py
 omlish/os/pidfiles/pinning.py

{omlish-0.0.0.dev231 → omlish-0.0.0.dev233}/omlish.egg-info/requires.txt

@@ -3,7 +3,7 @@
 anyio~=4.8
 sniffio~=1.3
 greenlet~=3.1
-trio~=0.27
+trio~=0.29
 trio-asyncio~=0.15
 lz4~=4.4
 python-snappy~=0.7
@@ -11,7 +11,7 @@ zstandard~=0.23
 brotli~=1.1
 asttokens~=3.0
 executing~=2.2
-psutil~=6.0
+psutil~=7.0
 orjson~=3.10
 ujson~=5.10
 pyyaml~=6.0
@@ -35,7 +35,7 @@ pytest~=8.0
 anyio~=4.8
 sniffio~=1.3
 greenlet~=3.1
-trio~=0.27
+trio~=0.29
 trio-asyncio~=0.15
 
 [compress]
@@ -47,7 +47,7 @@ brotli~=1.1
 [diag]
 asttokens~=3.0
 executing~=2.2
-psutil~=6.0
+psutil~=7.0
 
 [formats]
 orjson~=3.10