ominfra 0.0.0.dev128__py3-none-any.whl → 0.0.0.dev130__py3-none-any.whl

ominfra/supervisor/processimpl.py

@@ -22,10 +22,10 @@ from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .states import ProcessState
 from .states import SupervisorState
-from .types import InputDispatcher
 from .types import Process
 from .types import ProcessGroup
-from .types import ServerContext
+from .types import ProcessInputDispatcher
+from .types import SupervisorStateManager
 from .utils.os import decode_wait_status
 from .utils.ostypes import Pid
 from .utils.ostypes import Rc
@@ -47,7 +47,7 @@ class ProcessImpl(Process):
             config: ProcessConfig,
             group: ProcessGroup,
             *,
-            context: ServerContext,
+            supervisor_states: SupervisorStateManager,
             event_callbacks: EventCallbacks,
             process_spawning_factory: ProcessSpawningFactory,
     ) -> None:
@@ -56,7 +56,7 @@ class ProcessImpl(Process):
         self._config = config
         self._group = group
 
-        self._context = context
+        self._supervisor_states = supervisor_states
         self._event_callbacks = event_callbacks
 
         self._spawning = process_spawning_factory(self)
@@ -87,7 +87,7 @@ class ProcessImpl(Process):
     #
 
     def __repr__(self) -> str:
-        return f'<Subprocess at {id(self)} with name {self._config.name} in state {self.get_state().name}>'
+        return f'<Subprocess at {id(self)} with name {self._config.name} in state {self._state.name}>'
 
     #
 
@@ -109,10 +109,6 @@ class ProcessImpl(Process):
 
     #
 
-    @property
-    def context(self) -> ServerContext:
-        return self._context
-
     @property
     def state(self) -> ProcessState:
         return self._state
@@ -175,7 +171,7 @@ class ProcessImpl(Process):
         if stdin_fd is None:
             raise OSError(errno.EPIPE, 'Process has no stdin channel')
 
-        dispatcher = check_isinstance(self._dispatchers[stdin_fd], InputDispatcher)
+        dispatcher = check_isinstance(self._dispatchers[stdin_fd], ProcessInputDispatcher)
         if dispatcher.closed:
             raise OSError(errno.EPIPE, "Process' stdin channel is closed")
 
@@ -382,6 +378,7 @@ class ProcessImpl(Process):
         self._last_stop = now
 
         if now > self._last_start:
+            log.info(f'{now - self._last_start=}')  # noqa
             too_quickly = now - self._last_start < self._config.startsecs
         else:
             too_quickly = False
@@ -445,18 +442,13 @@ class ProcessImpl(Process):
         self._pipes = ProcessPipes()
         self._dispatchers = Dispatchers([])
 
-    def get_state(self) -> ProcessState:
-        return self._state
-
     def transition(self) -> None:
         now = time.time()
         state = self._state
 
         self._check_and_adjust_for_system_clock_rollback(now)
 
-        logger = log
-
-        if self.context.state > SupervisorState.RESTARTING:
+        if self._supervisor_states.state > SupervisorState.RESTARTING:
             # dont start any processes if supervisor is shutting down
             if state == ProcessState.EXITED:
                 if self._config.autorestart:
@@ -487,14 +479,14 @@ class ProcessImpl(Process):
                 self.check_in_state(ProcessState.STARTING)
                 self.change_state(ProcessState.RUNNING)
                 msg = ('entered RUNNING state, process has stayed up for > than %s seconds (startsecs)' % self._config.startsecs)  # noqa
-                logger.info('success: %s %s', self.name, msg)
+                log.info('success: %s %s', self.name, msg)
 
         if state == ProcessState.BACKOFF:
             if self._backoff > self._config.startretries:
                 # BACKOFF -> FATAL if the proc has exceeded its number of retries
                 self.give_up()
                 msg = ('entered FATAL state, too many start retries too quickly')
-                logger.info('gave up: %s %s', self.name, msg)
+                log.info('gave up: %s %s', self.name, msg)
 
         elif state == ProcessState.STOPPING:
             time_left = self._delay - now

ominfra/supervisor/signals.py

@@ -0,0 +1,66 @@
+# ruff: noqa: UP006 UP007
+import signal
+
+from omlish.lite.logs import log
+
+from .groups import ProcessGroupManager
+from .states import SupervisorState
+from .types import ProcessOutputDispatcher
+from .types import SupervisorStateManager
+from .utils.signals import SignalReceiver
+from .utils.signals import sig_name
+
+
+class SignalHandler:
+    def __init__(
+            self,
+            *,
+            states: SupervisorStateManager,
+            signal_receiver: SignalReceiver,
+            process_groups: ProcessGroupManager,
+    ) -> None:
+        super().__init__()
+
+        self._states = states
+        self._signal_receiver = signal_receiver
+        self._process_groups = process_groups
+
+    def set_signals(self) -> None:
+        self._signal_receiver.install(
+            signal.SIGTERM,
+            signal.SIGINT,
+            signal.SIGQUIT,
+            signal.SIGHUP,
+            signal.SIGCHLD,
+            signal.SIGUSR2,
+        )
+
+    def handle_signals(self) -> None:
+        sig = self._signal_receiver.get_signal()
+        if not sig:
+            return
+
+        if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
+            log.warning('received %s indicating exit request', sig_name(sig))
+            self._states.set_state(SupervisorState.SHUTDOWN)
+
+        elif sig == signal.SIGHUP:
+            if self._states.state == SupervisorState.SHUTDOWN:
+                log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig))  # noqa
+            else:
+                log.warning('received %s indicating restart request', sig_name(sig))  # noqa
+                self._states.set_state(SupervisorState.RESTARTING)
+
+        elif sig == signal.SIGCHLD:
+            log.debug('received %s indicating a child quit', sig_name(sig))
+
+        elif sig == signal.SIGUSR2:
+            log.info('received %s indicating log reopen request', sig_name(sig))
+
+            for p in self._process_groups.all_processes():
+                for d in p.get_dispatchers():
+                    if isinstance(d, ProcessOutputDispatcher):
+                        d.reopen_logs()
+
+        else:
+            log.debug('received %s indicating nothing', sig_name(sig))

ominfra/supervisor/spawningimpl.py

@@ -7,6 +7,7 @@ import typing as ta
 
 from omlish.lite.check import check_isinstance
 from omlish.lite.check import check_not_none
+from omlish.lite.fdio.handlers import FdIoHandler
 from omlish.lite.typing import Func3
 
 from .configs import ProcessConfig
@@ -29,11 +30,10 @@ from .process import PidHistory
 from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .spawning import SpawnedProcess
-from .types import Dispatcher
-from .types import InputDispatcher
-from .types import OutputDispatcher
 from .types import Process
 from .types import ProcessGroup
+from .types import ProcessInputDispatcher
+from .types import ProcessOutputDispatcher
 from .utils.diag import compact_traceback
 from .utils.fds import close_fd
 from .utils.fs import get_path
@@ -44,11 +44,11 @@ from .utils.ostypes import Rc
 from .utils.strings import as_bytes
 
 
-class OutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], Fd, OutputDispatcher]):
+class ProcessOutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], Fd, ProcessOutputDispatcher]):
     pass
 
 
-class InputDispatcherFactory(Func3[Process, str, Fd, InputDispatcher]):
+class ProcessInputDispatcherFactory(Func3[Process, str, Fd, ProcessInputDispatcher]):
     pass
 
 
@@ -66,8 +66,8 @@ class ProcessSpawningImpl(ProcessSpawning):
             server_config: ServerConfig,
             pid_history: PidHistory,
 
-            output_dispatcher_factory: OutputDispatcherFactory,
-            input_dispatcher_factory: InputDispatcherFactory,
+            output_dispatcher_factory: ProcessOutputDispatcherFactory,
+            input_dispatcher_factory: ProcessInputDispatcherFactory,
 
             inherited_fds: ta.Optional[InheritedFds] = None,
     ) -> None:
@@ -200,28 +200,28 @@ class ProcessSpawningImpl(ProcessSpawning):
         return exe, args
 
     def _make_dispatchers(self, pipes: ProcessPipes) -> Dispatchers:
-        dispatchers: ta.List[Dispatcher] = []
+        dispatchers: ta.List[FdIoHandler] = []
 
         if pipes.stdout is not None:
             dispatchers.append(check_isinstance(self._output_dispatcher_factory(
                 self.process,
                 ProcessCommunicationStdoutEvent,
                 pipes.stdout,
-            ), OutputDispatcher))
+            ), ProcessOutputDispatcher))
 
         if pipes.stderr is not None:
             dispatchers.append(check_isinstance(self._output_dispatcher_factory(
                 self.process,
                 ProcessCommunicationStderrEvent,
                 pipes.stderr,
-            ), OutputDispatcher))
+            ), ProcessOutputDispatcher))
 
         if pipes.stdin is not None:
             dispatchers.append(check_isinstance(self._input_dispatcher_factory(
                 self.process,
                 'stdin',
                 pipes.stdin,
-            ), InputDispatcher))
+            ), ProcessInputDispatcher))
 
         return Dispatchers(dispatchers)
 

ominfra/supervisor/supervisor.py

@@ -1,40 +1,38 @@
 # ruff: noqa: UP006 UP007
-import signal
+import errno
+import os
 import time
 import typing as ta
 
 from omlish.lite.check import check_isinstance
-from omlish.lite.check import check_not_none
+from omlish.lite.fdio.pollers import FdIoPoller
 from omlish.lite.logs import log
 from omlish.lite.typing import Func1
 
 from .configs import ProcessGroupConfig
-from .context import ServerContextImpl
-from .dispatchers import Dispatchers
+from .configs import ServerConfig
 from .events import TICK_EVENTS
 from .events import EventCallbacks
 from .events import SupervisorRunningEvent
 from .events import SupervisorStoppingEvent
 from .groups import ProcessGroup
 from .groups import ProcessGroupManager
-from .poller import Poller
+from .io import IoManager
 from .process import PidHistory
 from .setup import SupervisorSetup
+from .signals import SignalHandler
 from .states import SupervisorState
-from .types import OutputDispatcher
+from .types import ExitNow
 from .types import Process
+from .types import SupervisorStateManager
 from .utils.os import decode_wait_status
-from .utils.signals import SignalReceiver
-from .utils.signals import sig_name
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
 
 
 ##
 
 
-class ExitNow(Exception):  # noqa
-    pass
-
-
 def timeslice(period: int, when: float) -> int:
     return int(when - (when % period))
 
@@ -42,59 +40,18 @@ def timeslice(period: int, when: float) -> int:
 ##
 
 
-class SignalHandler:
-    def __init__(
-            self,
-            *,
-            context: ServerContextImpl,
-            signal_receiver: SignalReceiver,
-            process_groups: ProcessGroupManager,
-    ) -> None:
+class SupervisorStateManagerImpl(SupervisorStateManager):
+    def __init__(self) -> None:
         super().__init__()
 
-        self._context = context
-        self._signal_receiver = signal_receiver
-        self._process_groups = process_groups
-
-    def set_signals(self) -> None:
-        self._signal_receiver.install(
-            signal.SIGTERM,
-            signal.SIGINT,
-            signal.SIGQUIT,
-            signal.SIGHUP,
-            signal.SIGCHLD,
-            signal.SIGUSR2,
-        )
-
-    def handle_signals(self) -> None:
-        sig = self._signal_receiver.get_signal()
-        if not sig:
-            return
-
-        if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
-            log.warning('received %s indicating exit request', sig_name(sig))
-            self._context.set_state(SupervisorState.SHUTDOWN)
-
-        elif sig == signal.SIGHUP:
-            if self._context.state == SupervisorState.SHUTDOWN:
-                log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig))  # noqa
-            else:
-                log.warning('received %s indicating restart request', sig_name(sig))  # noqa
-                self._context.set_state(SupervisorState.RESTARTING)
+        self._state: SupervisorState = SupervisorState.RUNNING
 
-        elif sig == signal.SIGCHLD:
-            log.debug('received %s indicating a child quit', sig_name(sig))
-
-        elif sig == signal.SIGUSR2:
-            log.info('received %s indicating log reopen request', sig_name(sig))
-
-            for p in self._process_groups.all_processes():
-                for d in p.get_dispatchers():
-                    if isinstance(d, OutputDispatcher):
-                        d.reopen_logs()
+    @property
+    def state(self) -> SupervisorState:
+        return self._state
 
-        else:
-            log.debug('received %s indicating nothing', sig_name(sig))
+    def set_state(self, state: SupervisorState) -> None:
+        self._state = state
 
 
 ##
@@ -108,18 +65,20 @@ class Supervisor:
     def __init__(
             self,
             *,
-            context: ServerContextImpl,
-            poller: Poller,
+            config: ServerConfig,
+            poller: FdIoPoller,
             process_groups: ProcessGroupManager,
             signal_handler: SignalHandler,
             event_callbacks: EventCallbacks,
             process_group_factory: ProcessGroupFactory,
             pid_history: PidHistory,
             setup: SupervisorSetup,
+            states: SupervisorStateManager,
+            io: IoManager,
     ) -> None:
         super().__init__()
 
-        self._context = context
+        self._config = config
         self._poller = poller
         self._process_groups = process_groups
         self._signal_handler = signal_handler
@@ -127,6 +86,8 @@ class Supervisor:
         self._process_group_factory = process_group_factory
         self._pid_history = pid_history
         self._setup = setup
+        self._states = states
+        self._io = io
 
         self._ticks: ta.Dict[int, float] = {}
         self._stop_groups: ta.Optional[ta.List[ProcessGroup]] = None  # list used for priority ordered shutdown
@@ -136,11 +97,8 @@ class Supervisor:
     #
 
     @property
-    def context(self) -> ServerContextImpl:
-        return self._context
-
-    def get_state(self) -> SupervisorState:
-        return self._context.state
+    def state(self) -> SupervisorState:
+        return self._states.state
 
     #
 
@@ -181,7 +139,7 @@ class Supervisor:
                 log.info('waiting for %s to die', namestr)
                 self._last_shutdown_report = now
                 for proc in unstopped:
-                    log.debug('%s state: %s', proc.config.name, proc.get_state().name)
+                    log.debug('%s state: %s', proc.config.name, proc.state.name)
 
         return unstopped
 
@@ -205,7 +163,7 @@ class Supervisor:
         self._event_callbacks.clear()
 
         try:
-            for config in self._context.config.groups or []:
+            for config in self._config.groups or []:
                 self.add_process_group(config)
 
             self._signal_handler.set_signals()
@@ -224,12 +182,14 @@ class Supervisor:
     #
 
     def _run_once(self) -> None:
+        now = time.time()
         self._poll()
+        log.info(f'Poll took {time.time() - now}')  # noqa
         self._reap()
         self._signal_handler.handle_signals()
         self._tick()
 
-        if self._context.state < SupervisorState.RUNNING:
+        if self._states.state < SupervisorState.RUNNING:
             self._ordered_stop_groups_phase_2()
 
     def _ordered_stop_groups_phase_1(self) -> None:
@@ -248,20 +208,11 @@ class Supervisor:
                 # down, so push it back on to the end of the stop group queue
                 self._stop_groups.append(group)
 
-    def get_dispatchers(self) -> Dispatchers:
-        return Dispatchers(
-            d
-            for p in self._process_groups.all_processes()
-            for d in p.get_dispatchers()
-        )
-
     def _poll(self) -> None:
-        dispatchers = self.get_dispatchers()
-
         sorted_groups = list(self._process_groups)
         sorted_groups.sort()
 
-        if self._context.state < SupervisorState.RUNNING:
+        if self._states.state < SupervisorState.RUNNING:
             if not self._stopping:
                 # first time, set the stopping flag, do a notification and set stop_groups
                 self._stopping = True
@@ -274,54 +225,7 @@ class Supervisor:
                 # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
                 raise ExitNow
 
-        for fd, dispatcher in dispatchers.items():
-            if dispatcher.readable():
-                self._poller.register_readable(fd)
-            if dispatcher.writable():
-                self._poller.register_writable(fd)
-
-        timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
-        r, w = self._poller.poll(timeout)
-
-        for fd in r:
-            if fd in dispatchers:
-                try:
-                    dispatcher = dispatchers[fd]
-                    log.debug('read event caused by %r', dispatcher)
-                    dispatcher.handle_read_event()
-                    if not dispatcher.readable():
-                        self._poller.unregister_readable(fd)
-                except ExitNow:
-                    raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
-            else:
-                # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
-                # time, which may cause 100% cpu usage
-                log.debug('unexpected read event from fd %r', fd)
-                try:
-                    self._poller.unregister_readable(fd)
-                except Exception:  # noqa
-                    pass
-
-        for fd in w:
-            if fd in dispatchers:
-                try:
-                    dispatcher = dispatchers[fd]
-                    log.debug('write event caused by %r', dispatcher)
-                    dispatcher.handle_write_event()
-                    if not dispatcher.writable():
-                        self._poller.unregister_writable(fd)
-                except ExitNow:
-                    raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
-            else:
-                log.debug('unexpected write event from fd %r', fd)
-                try:
-                    self._poller.unregister_writable(fd)
-                except Exception:  # noqa
-                    pass
+        self._io.poll()
 
         for group in sorted_groups:
             for process in group:
@@ -331,17 +235,18 @@ class Supervisor:
         if depth >= 100:
             return
 
-        pid, sts = self._context.waitpid()
-        if not pid:
+        wp = waitpid()
+        log.info(f'Waited pid: {wp}')  # noqa
+        if wp is None or not wp.pid:
             return
 
-        process = self._pid_history.get(pid, None)
+        process = self._pid_history.get(wp.pid, None)
         if process is None:
-            _, msg = decode_wait_status(check_not_none(sts))
-            log.info('reaped unknown pid %s (%s)', pid, msg)
+            _, msg = decode_wait_status(wp.sts)
+            log.info('reaped unknown pid %s (%s)', wp.pid, msg)
         else:
-            process.finish(check_not_none(sts))
-            del self._pid_history[pid]
+            process.finish(wp.sts)
+            del self._pid_history[wp.pid]
 
         if not once:
             # keep reaping until no more kids to reap, but don't recurse infinitely
@@ -366,3 +271,31 @@ class Supervisor:
             if this_tick != last_tick:
                 self._ticks[period] = this_tick
                 self._event_callbacks.notify(event(this_tick, self))
+
+
+##
+
+
+class WaitedPid(ta.NamedTuple):
+    pid: Pid
+    sts: Rc
+
+
+def waitpid() -> ta.Optional[WaitedPid]:
+    # Need pthread_sigmask here to avoid concurrent sigchld, but Python doesn't offer in Python < 3.4.  There is
+    # still a race condition here; we can get a sigchld while we're sitting in the waitpid call. However, AFAICT, if
+    # waitpid is interrupted by SIGCHLD, as long as we call waitpid again (which happens every so often during the
+    # normal course in the mainloop), we'll eventually reap the child that we tried to reap during the interrupted
+    # call. At least on Linux, this appears to be true, or at least stopping 50 processes at once never left zombies
+    # lying around.
+    try:
+        pid, sts = os.waitpid(-1, os.WNOHANG)
+    except OSError as exc:
+        code = exc.args[0]
+        if code not in (errno.ECHILD, errno.EINTR):
+            log.critical('waitpid error %r; a process may not be cleaned up properly', code)
+        if code == errno.EINTR:
+            log.debug('EINTR during reap')
+        return None
+    else:
+        return WaitedPid(pid, sts)  # type: ignore