ominfra 0.0.0.dev118__py3-none-any.whl → 0.0.0.dev119__py3-none-any.whl

ominfra/supervisor/__main__.py

@@ -1,4 +1,4 @@
 if __name__ == '__main__':
-    from .supervisor import main
+    from .main import main
 
     main()

ominfra/supervisor/context.py

@@ -2,7 +2,6 @@
 import errno
 import fcntl
 import grp
-import logging
 import os
 import pwd
 import re
@@ -12,6 +11,8 @@ import stat
 import typing as ta
 import warnings
 
+from omlish.lite.logs import log
+
 from .compat import SignalReceiver
 from .compat import close_fd
 from .compat import mktempfile
@@ -23,6 +24,7 @@ from .datatypes import name_to_uid
 from .exceptions import NoPermissionError
 from .exceptions import NotExecutableError
 from .exceptions import NotFoundError
+from .poller import BasePoller
 from .poller import Poller
 from .states import SupervisorState
 from .states import SupervisorStates
@@ -30,41 +32,47 @@ from .types import AbstractServerContext
 from .types import AbstractSubprocess
 
 
-log = logging.getLogger(__name__)
-
-
 class ServerContext(AbstractServerContext):
-    first = False
-    test = False
-
-    ##
-
-    def __init__(self, config: ServerConfig) -> None:
+    def __init__(
+            self,
+            config: ServerConfig,
+            *,
+            epoch: int = 0,
+    ) -> None:
         super().__init__()
 
         self._config = config
+        self._epoch = epoch
 
         self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
         self._state: SupervisorState = SupervisorStates.RUNNING
 
-        self.signal_receiver = SignalReceiver()
+        self._signal_receiver = SignalReceiver()
 
-        self.poller = Poller()
+        self._poller: BasePoller = Poller()
 
-        if self.config.user is not None:
-            uid = name_to_uid(self.config.user)
-            self.uid = uid
-            self.gid = gid_for_uid(uid)
+        if config.user is not None:
+            uid = name_to_uid(config.user)
+            self._uid: ta.Optional[int] = uid
+            self._gid: ta.Optional[int] = gid_for_uid(uid)
         else:
-            self.uid = None
-            self.gid = None
+            self._uid = None
+            self._gid = None
 
-        self.unlink_pidfile = False
+        self._unlink_pidfile = False
 
     @property
     def config(self) -> ServerConfig:
         return self._config
 
+    @property
+    def epoch(self) -> int:
+        return self._epoch
+
+    @property
+    def first(self) -> bool:
+        return not self._epoch
+
     @property
     def state(self) -> SupervisorState:
         return self._state
@@ -72,17 +80,26 @@ class ServerContext(AbstractServerContext):
     def set_state(self, state: SupervisorState) -> None:
         self._state = state
 
+    @property
+    def poller(self) -> BasePoller:
+        return self._poller
+
     @property
     def pid_history(self) -> ta.Dict[int, AbstractSubprocess]:
         return self._pid_history
 
-    uid: ta.Optional[int]
-    gid: ta.Optional[int]
+    @property
+    def uid(self) -> ta.Optional[int]:
+        return self._uid
+
+    @property
+    def gid(self) -> ta.Optional[int]:
+        return self._gid
 
     ##
 
     def set_signals(self) -> None:
-        self.signal_receiver.install(
+        self._signal_receiver.install(
             signal.SIGTERM,
             signal.SIGINT,
             signal.SIGQUIT,
@@ -193,7 +210,7 @@ class ServerContext(AbstractServerContext):
                     ))
 
     def cleanup(self) -> None:
-        if self.unlink_pidfile:
+        if self._unlink_pidfile:
             try_unlink(self.config.pidfile)
         self.poller.close()
 
@@ -246,6 +263,7 @@ class ServerContext(AbstractServerContext):
             # Parent
             log.debug('supervisord forked; parent exiting')
             real_exit(0)
+
         # Child
         log.info('daemonizing the supervisord process')
         if self.config.directory:
@@ -255,11 +273,15 @@ class ServerContext(AbstractServerContext):
                 log.critical("can't chdir into %r: %s", self.config.directory, err)
             else:
                 log.info('set current directory: %r', self.config.directory)
+
         os.dup2(0, os.open('/dev/null', os.O_RDONLY))
         os.dup2(1, os.open('/dev/null', os.O_WRONLY))
         os.dup2(2, os.open('/dev/null', os.O_WRONLY))
+
         os.setsid()
+
         os.umask(self.config.umask)
+
         # XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
         # file descriptors.  In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
         # again after the setsid() call, for obscure SVR4 reasons.
@@ -274,7 +296,7 @@ class ServerContext(AbstractServerContext):
         return logfile
 
     def get_signal(self) -> ta.Optional[int]:
-        return self.signal_receiver.get_signal()
+        return self._signal_receiver.get_signal()
 
     def write_pidfile(self) -> None:
         pid = os.getpid()
@@ -284,7 +306,7 @@ class ServerContext(AbstractServerContext):
         except OSError:
             log.critical('could not write pidfile %s', self.config.pidfile)
         else:
-            self.unlink_pidfile = True
+            self._unlink_pidfile = True
             log.info('supervisord started with pid %s', pid)
 
 
@@ -337,11 +359,14 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
             os.setgroups(groups)
         except OSError:
             return 'Could not set groups of effective user'
+
     try:
         os.setgid(gid)
     except OSError:
         return 'Could not set group id of effective user'
+
     os.setuid(uid)
+
     return None
 
 

ominfra/supervisor/dispatchers.py

@@ -5,6 +5,8 @@ import logging
 import os
 import typing as ta
 
+from omlish.lite.logs import log
+
 from .compat import as_bytes
 from .compat import compact_traceback
 from .compat import find_prefix_at_end
@@ -17,9 +19,6 @@ from .events import notify_event
 from .types import AbstractSubprocess
 
 
-log = logging.getLogger(__name__)
-
-
 class Dispatcher(abc.ABC):
 
     def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
@@ -176,16 +175,16 @@ class OutputDispatcher(Dispatcher):
             # )
 
     def remove_logs(self):
-        for log in (self._normal_log, self._capture_log):
-            if log is not None:
-                for handler in log.handlers:
+        for l in (self._normal_log, self._capture_log):
+            if l is not None:
+                for handler in l.handlers:
                     handler.remove()  # type: ignore
                     handler.reopen()  # type: ignore
 
     def reopen_logs(self):
-        for log in (self._normal_log, self._capture_log):
-            if log is not None:
-                for handler in log.handlers:
+        for l in (self._normal_log, self._capture_log):
+            if l is not None:
+                for handler in l.handlers:
                     handler.reopen()  # type: ignore
 
     def _log(self, data):

ominfra/supervisor/main.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# ruff: noqa: UP006 UP007
+# @omlish-amalg ../scripts/supervisor.py
+import itertools
+import json
+import typing as ta
+
+from omlish.lite.journald import journald_log_handler_factory
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.marshal import unmarshal_obj
+
+from .compat import ExitNow
+from .configs import ServerConfig
+from .context import ServerContext
+from .states import SupervisorStates
+from .supervisor import Supervisor
+
+
+def main(
+        argv: ta.Optional[ta.Sequence[str]] = None,
+        *,
+        no_logging: bool = False,
+) -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config_file', metavar='config-file')
+    parser.add_argument('--no-journald', action='store_true')
+    args = parser.parse_args(argv)
+
+    #
+
+    if not (cf := args.config_file):
+        raise RuntimeError('No config file specified')
+
+    if not no_logging:
+        configure_standard_logging(
+            'INFO',
+            handler_factory=journald_log_handler_factory if not args.no_journald else None,
+        )
+
+    #
+
+    # if we hup, restart by making a new Supervisor()
+    for epoch in itertools.count():
+        with open(cf) as f:
+            config_src = f.read()
+
+        config_dct = json.loads(config_src)
+        config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
+
+        context = ServerContext(
+            config,
+            epoch=epoch,
+        )
+
+        supervisor = Supervisor(context)
+        try:
+            supervisor.main()
+        except ExitNow:
+            pass
+
+        if context.state < SupervisorStates.RESTARTING:
+            break
+
+
+if __name__ == '__main__':
+    main()

ominfra/supervisor/poller.py

@@ -1,13 +1,11 @@
 # ruff: noqa: UP006 UP007
 import abc
 import errno
-import logging
 import select
 import sys
 import typing as ta
 
-
-log = logging.getLogger(__name__)
+from omlish.lite.logs import log
 
 
 class BasePoller(abc.ABC):

ominfra/supervisor/process.py

@@ -1,7 +1,6 @@
 # ruff: noqa: UP006 UP007
 import errno
 import functools
-import logging
 import os
 import shlex
 import signal
@@ -9,6 +8,8 @@ import time
 import traceback
 import typing as ta
 
+from omlish.lite.logs import log
+
 from .compat import as_bytes
 from .compat import as_string
 from .compat import close_fd
@@ -54,9 +55,6 @@ from .types import AbstractServerContext
 from .types import AbstractSubprocess
 
 
-log = logging.getLogger(__name__)
-
-
 @functools.total_ordering
 class Subprocess(AbstractSubprocess):
     """A class to manage a subprocess."""

ominfra/supervisor/supervisor.py

@@ -1,22 +1,17 @@
-#!/usr/bin/env python3
 # ruff: noqa: UP006 UP007
-# @omlish-amalg ../scripts/supervisor.py
-import json
-import logging
 import signal
 import time
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check_not_none
-from omlish.lite.logs import configure_standard_logging
-from omlish.lite.marshal import unmarshal_obj
+from omlish.lite.logs import log
 
 from .compat import ExitNow
 from .compat import as_string
 from .compat import decode_wait_status
 from .compat import signame
 from .configs import ProcessGroupConfig
-from .configs import ServerConfig
 from .context import ServerContext
 from .dispatchers import Dispatcher
 from .events import TICK_EVENTS
@@ -33,7 +28,8 @@ from .states import SupervisorStates
 from .states import get_process_state_description
 
 
-log = logging.getLogger(__name__)
+def timeslice(period, when):
+    return int(when - (when % period))
 
 
 class Supervisor:
@@ -56,6 +52,11 @@ class Supervisor:
         return self._context.state
 
     def main(self) -> None:
+        self.setup()
+        self.run()
+
+    @cached_nullary
+    def setup(self) -> None:
         if not self._context.first:
             # prevent crash on libdispatch-based systems, at least for the first request
             self._context.cleanup_fds()
@@ -70,9 +71,11 @@ class Supervisor:
             # clean up old automatic logs
             self._context.clear_auto_child_logdir()
 
-        self.run()
-
-    def run(self) -> None:
+    def run(
+            self,
+            *,
+            callback: ta.Optional[ta.Callable[['Supervisor'], bool]] = None,
+    ) -> None:
         self._process_groups = {}  # clear
         self._stop_groups = None  # clear
 
@@ -90,12 +93,23 @@ class Supervisor:
             # writing pid file needs to come *after* daemonizing or pid will be wrong
             self._context.write_pidfile()
 
-            self.runforever()
+            notify_event(SupervisorRunningEvent())
+
+            while True:
+                if callback is not None and not callback(self):
+                    break
+
+                self._run_once()
 
         finally:
             self._context.cleanup()
 
-    def diff_to_active(self):
+    class DiffToActive(ta.NamedTuple):
+        added: ta.List[ProcessGroupConfig]
+        changed: ta.List[ProcessGroupConfig]
+        removed: ta.List[ProcessGroupConfig]
+
+    def diff_to_active(self) -> DiffToActive:
         new = self._context.config.groups or []
         cur = [group.config for group in self._process_groups.values()]
 
@@ -107,7 +121,7 @@ class Supervisor:
 
         changed = [cand for cand in new if cand != curdict.get(cand.name, cand)]
 
-        return added, changed, removed
+        return Supervisor.DiffToActive(added, changed, removed)
 
     def add_process_group(self, config: ProcessGroupConfig) -> bool:
         name = config.name
@@ -173,90 +187,84 @@ class Supervisor:
                 # down, so push it back on to the end of the stop group queue
                 self._stop_groups.append(group)
 
-    def runforever(self) -> None:
-        notify_event(SupervisorRunningEvent())
-        timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
+    def _run_once(self) -> None:
+        combined_map = {}
+        combined_map.update(self.get_process_map())
+
+        pgroups = list(self._process_groups.values())
+        pgroups.sort()
+
+        if self._context.state < SupervisorStates.RUNNING:
+            if not self._stopping:
+                # first time, set the stopping flag, do a notification and set stop_groups
+                self._stopping = True
+                self._stop_groups = pgroups[:]
+                notify_event(SupervisorStoppingEvent())
+
+            self._ordered_stop_groups_phase_1()
+
+            if not self.shutdown_report():
+                # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
+                raise ExitNow
 
-        while True:
-            combined_map = {}
-            combined_map.update(self.get_process_map())
-
-            pgroups = list(self._process_groups.values())
-            pgroups.sort()
-
-            if self._context.state < SupervisorStates.RUNNING:
-                if not self._stopping:
-                    # first time, set the stopping flag, do a notification and set stop_groups
-                    self._stopping = True
-                    self._stop_groups = pgroups[:]
-                    notify_event(SupervisorStoppingEvent())
-
-                self._ordered_stop_groups_phase_1()
-
-                if not self.shutdown_report():
-                    # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
-                    raise ExitNow
-
-            for fd, dispatcher in combined_map.items():
-                if dispatcher.readable():
-                    self._context.poller.register_readable(fd)
-                if dispatcher.writable():
-                    self._context.poller.register_writable(fd)
-
-            r, w = self._context.poller.poll(timeout)
-
-            for fd in r:
-                if fd in combined_map:
-                    try:
-                        dispatcher = combined_map[fd]
-                        log.debug('read event caused by %r', dispatcher)
-                        dispatcher.handle_read_event()
-                        if not dispatcher.readable():
-                            self._context.poller.unregister_readable(fd)
-                    except ExitNow:
-                        raise
-                    except Exception:  # noqa
-                        combined_map[fd].handle_error()
-                else:
-                    # if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
-                    # time, which may cause 100% cpu usage
-                    log.debug('unexpected read event from fd %r', fd)
-                    try:
+        for fd, dispatcher in combined_map.items():
+            if dispatcher.readable():
+                self._context.poller.register_readable(fd)
+            if dispatcher.writable():
+                self._context.poller.register_writable(fd)
+
+        timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
+        r, w = self._context.poller.poll(timeout)
+
+        for fd in r:
+            if fd in combined_map:
+                try:
+                    dispatcher = combined_map[fd]
+                    log.debug('read event caused by %r', dispatcher)
+                    dispatcher.handle_read_event()
+                    if not dispatcher.readable():
                         self._context.poller.unregister_readable(fd)
-                    except Exception:  # noqa
-                        pass
-
-            for fd in w:
-                if fd in combined_map:
-                    try:
-                        dispatcher = combined_map[fd]
-                        log.debug('write event caused by %r', dispatcher)
-                        dispatcher.handle_write_event()
-                        if not dispatcher.writable():
-                            self._context.poller.unregister_writable(fd)
-                    except ExitNow:
-                        raise
-                    except Exception:  # noqa
-                        combined_map[fd].handle_error()
-                else:
-                    log.debug('unexpected write event from fd %r', fd)
-                    try:
+                except ExitNow:
+                    raise
+                except Exception:  # noqa
+                    combined_map[fd].handle_error()
+            else:
+                # if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
+                # time, which may cause 100% cpu usage
+                log.debug('unexpected read event from fd %r', fd)
+                try:
+                    self._context.poller.unregister_readable(fd)
+                except Exception:  # noqa
+                    pass
+
+        for fd in w:
+            if fd in combined_map:
+                try:
+                    dispatcher = combined_map[fd]
+                    log.debug('write event caused by %r', dispatcher)
+                    dispatcher.handle_write_event()
+                    if not dispatcher.writable():
                         self._context.poller.unregister_writable(fd)
-                    except Exception:  # noqa
-                        pass
-
-            for group in pgroups:
-                group.transition()
+                except ExitNow:
+                    raise
+                except Exception:  # noqa
+                    combined_map[fd].handle_error()
+            else:
+                log.debug('unexpected write event from fd %r', fd)
+                try:
+                    self._context.poller.unregister_writable(fd)
+                except Exception:  # noqa
+                    pass
 
-            self._reap()
-            self._handle_signal()
-            self._tick()
+        for group in pgroups:
+            group.transition()
 
-            if self._context.state < SupervisorStates.RUNNING:
-                self._ordered_stop_groups_phase_2()
+        self._reap()
+        self._handle_signal()
+        self._tick()
 
-            if self._context.test:
-                break
+        if self._context.state < SupervisorStates.RUNNING:
+            self._ordered_stop_groups_phase_2()
 
     def _tick(self, now: ta.Optional[float] = None) -> None:
         """Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
@@ -325,52 +333,3 @@ class Supervisor:
 
         else:
             log.debug('received %s indicating nothing', signame(sig))
-
-
-def timeslice(period, when):
-    return int(when - (when % period))
-
-
-def main(args=None, test=False):
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('config_file', metavar='config-file')
-    args = parser.parse_args()
-
-    configure_standard_logging('INFO')
-
-    if not (cf := args.config_file):
-        raise RuntimeError('No config file specified')
-
-    # if we hup, restart by making a new Supervisor()
-    first = True
-    while True:
-        with open(cf) as f:
-            config_src = f.read()
-        config_dct = json.loads(config_src)
-        config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
-
-        context = ServerContext(
-            config,
-        )
-
-        context.first = first
-        context.test = test
-        go(context)
-        # options.close_logger()
-        first = False
-        if test or (context.state < SupervisorStates.RESTARTING):
-            break
-
-
-def go(context):  # pragma: no cover
-    d = Supervisor(context)
-    try:
-        d.main()
-    except ExitNow:
-        pass
-
-
-if __name__ == '__main__':
-    main()

{ominfra-0.0.0.dev118.dist-info → ominfra-0.0.0.dev119.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev118
+Version: 0.0.0.dev119
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev ==0.0.0.dev118
-Requires-Dist: omlish ==0.0.0.dev118
+Requires-Dist: omdev ==0.0.0.dev119
+Requires-Dist: omlish ==0.0.0.dev119
 Provides-Extra: all
 Requires-Dist: paramiko ~=3.5 ; extra == 'all'
 Requires-Dist: asyncssh ~=2.18 ; extra == 'all'