ominfra 0.0.0.dev127__py3-none-any.whl → 0.0.0.dev129__py3-none-any.whl

ominfra/supervisor/spawning.py

@@ -5,11 +5,12 @@ import dataclasses as dc
 from .dispatchers import Dispatchers
 from .pipes import ProcessPipes
 from .types import Process
+from .utils.ostypes import Pid
 
 
 @dc.dataclass(frozen=True)
 class SpawnedProcess:
-    pid: int
+    pid: Pid
     pipes: ProcessPipes
     dispatchers: Dispatchers
 

ominfra/supervisor/spawningimpl.py

@@ -25,31 +25,34 @@ from .pipes import close_child_pipes
 from .pipes import close_pipes
 from .pipes import make_process_pipes
 from .privileges import drop_privileges
-from .processes import PidHistory
+from .process import PidHistory
 from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .spawning import SpawnedProcess
 from .types import Dispatcher
-from .types import InputDispatcher
-from .types import OutputDispatcher
 from .types import Process
 from .types import ProcessGroup
-from .utils import as_bytes
-from .utils import close_fd
-from .utils import compact_traceback
-from .utils import get_path
-from .utils import real_exit
-
-
-class OutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], int, OutputDispatcher]):
+from .types import ProcessInputDispatcher
+from .types import ProcessOutputDispatcher
+from .utils.diag import compact_traceback
+from .utils.fds import close_fd
+from .utils.fs import get_path
+from .utils.os import real_exit
+from .utils.ostypes import Fd
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
+from .utils.strings import as_bytes
+
+
+class ProcessOutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], Fd, ProcessOutputDispatcher]):
     pass
 
 
-class InputDispatcherFactory(Func3[Process, str, int, InputDispatcher]):
+class ProcessInputDispatcherFactory(Func3[Process, str, Fd, ProcessInputDispatcher]):
     pass
 
 
-InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[int])
+InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[Fd])
 
 
 ##
@@ -63,8 +66,8 @@ class ProcessSpawningImpl(ProcessSpawning):
             server_config: ServerConfig,
             pid_history: PidHistory,
 
-            output_dispatcher_factory: OutputDispatcherFactory,
-            input_dispatcher_factory: InputDispatcherFactory,
+            output_dispatcher_factory: ProcessOutputDispatcherFactory,
+            input_dispatcher_factory: ProcessInputDispatcherFactory,
 
             inherited_fds: ta.Optional[InheritedFds] = None,
     ) -> None:
@@ -120,7 +123,7 @@ class ProcessSpawningImpl(ProcessSpawning):
             raise ProcessSpawnError(f"Unknown error making dispatchers for '{self.process.name}': {exc}") from exc
 
         try:
-            pid = os.fork()
+            pid = Pid(os.fork())
         except OSError as exc:
             code = exc.args[0]
             if code == errno.EAGAIN:
@@ -204,21 +207,21 @@ class ProcessSpawningImpl(ProcessSpawning):
                 self.process,
                 ProcessCommunicationStdoutEvent,
                 pipes.stdout,
-            ), OutputDispatcher))
+            ), ProcessOutputDispatcher))
 
         if pipes.stderr is not None:
             dispatchers.append(check_isinstance(self._output_dispatcher_factory(
                 self.process,
                 ProcessCommunicationStderrEvent,
                 pipes.stderr,
-            ), OutputDispatcher))
+            ), ProcessOutputDispatcher))
 
         if pipes.stdin is not None:
             dispatchers.append(check_isinstance(self._input_dispatcher_factory(
                 self.process,
                 'stdin',
                 pipes.stdin,
-            ), InputDispatcher))
+            ), ProcessInputDispatcher))
 
         return Dispatchers(dispatchers)
 
@@ -299,7 +302,7 @@ class ProcessSpawningImpl(ProcessSpawning):
 
         finally:
             os.write(2, as_bytes('supervisor: child process was not spawned\n'))
-            real_exit(127)  # exit process with code for spawn failure
+            real_exit(Rc(127))  # exit process with code for spawn failure
 
         raise RuntimeError('Unreachable')
 
@@ -316,7 +319,7 @@ class ProcessSpawningImpl(ProcessSpawning):
         for i in range(3, self._server_config.minfds):
             if i in self._inherited_fds:
                 continue
-            close_fd(i)
+            close_fd(Fd(i))
 
     def _set_uid(self) -> ta.Optional[str]:
         if self.config.uid is None:

ominfra/supervisor/supervisor.py

@@ -1,92 +1,57 @@
 # ruff: noqa: UP006 UP007
-import signal
+import errno
+import os
 import time
 import typing as ta
 
 from omlish.lite.check import check_isinstance
-from omlish.lite.check import check_not_none
 from omlish.lite.logs import log
 from omlish.lite.typing import Func1
 
 from .configs import ProcessGroupConfig
-from .context import ServerContextImpl
-from .dispatchers import Dispatchers
+from .configs import ServerConfig
 from .events import TICK_EVENTS
 from .events import EventCallbacks
 from .events import SupervisorRunningEvent
 from .events import SupervisorStoppingEvent
 from .groups import ProcessGroup
 from .groups import ProcessGroupManager
+from .io import IoManager
 from .poller import Poller
-from .processes import PidHistory
+from .process import PidHistory
 from .setup import SupervisorSetup
-from .signals import SignalReceiver
-from .signals import sig_name
+from .signals import SignalHandler
 from .states import SupervisorState
-from .types import OutputDispatcher
+from .types import ExitNow
 from .types import Process
-from .utils import ExitNow
-from .utils import as_string
-from .utils import decode_wait_status
-from .utils import timeslice
+from .types import SupervisorStateManager
+from .utils.os import decode_wait_status
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
 
 
 ##
 
 
-class SignalHandler:
-    def __init__(
-            self,
-            *,
-            context: ServerContextImpl,
-            signal_receiver: SignalReceiver,
-            process_groups: ProcessGroupManager,
-    ) -> None:
-        super().__init__()
+def timeslice(period: int, when: float) -> int:
+    return int(when - (when % period))
 
-        self._context = context
-        self._signal_receiver = signal_receiver
-        self._process_groups = process_groups
-
-    def set_signals(self) -> None:
-        self._signal_receiver.install(
-            signal.SIGTERM,
-            signal.SIGINT,
-            signal.SIGQUIT,
-            signal.SIGHUP,
-            signal.SIGCHLD,
-            signal.SIGUSR2,
-        )
-
-    def handle_signals(self) -> None:
-        sig = self._signal_receiver.get_signal()
-        if not sig:
-            return
 
-        if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
-            log.warning('received %s indicating exit request', sig_name(sig))
-            self._context.set_state(SupervisorState.SHUTDOWN)
+##
 
-        elif sig == signal.SIGHUP:
-            if self._context.state == SupervisorState.SHUTDOWN:
-                log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig))  # noqa
-            else:
-                log.warning('received %s indicating restart request', sig_name(sig))  # noqa
-                self._context.set_state(SupervisorState.RESTARTING)
 
-        elif sig == signal.SIGCHLD:
-            log.debug('received %s indicating a child quit', sig_name(sig))
+class SupervisorStateManagerImpl(SupervisorStateManager):
+    def __init__(self) -> None:
+        super().__init__()
 
-        elif sig == signal.SIGUSR2:
-            log.info('received %s indicating log reopen request', sig_name(sig))
+        self._state: SupervisorState = SupervisorState.RUNNING
 
-            for p in self._process_groups.all_processes():
-                for d in p.get_dispatchers():
-                    if isinstance(d, OutputDispatcher):
-                        d.reopen_logs()
+    @property
+    def state(self) -> SupervisorState:
+        return self._state
 
-        else:
-            log.debug('received %s indicating nothing', sig_name(sig))
+    def set_state(self, state: SupervisorState) -> None:
+        self._state = state
 
 
 ##
@@ -100,7 +65,7 @@ class Supervisor:
     def __init__(
             self,
             *,
-            context: ServerContextImpl,
+            config: ServerConfig,
             poller: Poller,
             process_groups: ProcessGroupManager,
             signal_handler: SignalHandler,
@@ -108,10 +73,12 @@ class Supervisor:
             process_group_factory: ProcessGroupFactory,
             pid_history: PidHistory,
             setup: SupervisorSetup,
+            states: SupervisorStateManager,
+            io: IoManager,
     ) -> None:
         super().__init__()
 
-        self._context = context
+        self._config = config
         self._poller = poller
         self._process_groups = process_groups
         self._signal_handler = signal_handler
@@ -119,6 +86,8 @@ class Supervisor:
         self._process_group_factory = process_group_factory
         self._pid_history = pid_history
         self._setup = setup
+        self._states = states
+        self._io = io
 
         self._ticks: ta.Dict[int, float] = {}
         self._stop_groups: ta.Optional[ta.List[ProcessGroup]] = None  # list used for priority ordered shutdown
@@ -128,11 +97,8 @@ class Supervisor:
     #
 
     @property
-    def context(self) -> ServerContextImpl:
-        return self._context
-
-    def get_state(self) -> SupervisorState:
-        return self._context.state
+    def state(self) -> SupervisorState:
+        return self._states.state
 
     #
 
@@ -168,12 +134,12 @@ class Supervisor:
             # throttle 'waiting for x to die' reports
             now = time.time()
             if now > (self._last_shutdown_report + 3):  # every 3 secs
-                names = [as_string(p.config.name) for p in unstopped]
+                names = [p.config.name for p in unstopped]
                 namestr = ', '.join(names)
                 log.info('waiting for %s to die', namestr)
                 self._last_shutdown_report = now
                 for proc in unstopped:
-                    log.debug('%s state: %s', proc.config.name, proc.get_state().name)
+                    log.debug('%s state: %s', proc.config.name, proc.state.name)
 
         return unstopped
 
@@ -197,7 +163,7 @@ class Supervisor:
         self._event_callbacks.clear()
 
         try:
-            for config in self._context.config.groups or []:
+            for config in self._config.groups or []:
                 self.add_process_group(config)
 
             self._signal_handler.set_signals()
@@ -221,7 +187,7 @@ class Supervisor:
         self._signal_handler.handle_signals()
         self._tick()
 
-        if self._context.state < SupervisorState.RUNNING:
+        if self._states.state < SupervisorState.RUNNING:
             self._ordered_stop_groups_phase_2()
 
     def _ordered_stop_groups_phase_1(self) -> None:
@@ -240,20 +206,11 @@ class Supervisor:
                 # down, so push it back on to the end of the stop group queue
                 self._stop_groups.append(group)
 
-    def get_dispatchers(self) -> Dispatchers:
-        return Dispatchers(
-            d
-            for p in self._process_groups.all_processes()
-            for d in p.get_dispatchers()
-        )
-
     def _poll(self) -> None:
-        dispatchers = self.get_dispatchers()
-
         sorted_groups = list(self._process_groups)
         sorted_groups.sort()
 
-        if self._context.state < SupervisorState.RUNNING:
+        if self._states.state < SupervisorState.RUNNING:
             if not self._stopping:
                 # first time, set the stopping flag, do a notification and set stop_groups
                 self._stopping = True
@@ -266,54 +223,7 @@ class Supervisor:
                 # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
                 raise ExitNow
 
-        for fd, dispatcher in dispatchers.items():
-            if dispatcher.readable():
-                self._poller.register_readable(fd)
-            if dispatcher.writable():
-                self._poller.register_writable(fd)
-
-        timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
-        r, w = self._poller.poll(timeout)
-
-        for fd in r:
-            if fd in dispatchers:
-                try:
-                    dispatcher = dispatchers[fd]
-                    log.debug('read event caused by %r', dispatcher)
-                    dispatcher.handle_read_event()
-                    if not dispatcher.readable():
-                        self._poller.unregister_readable(fd)
-                except ExitNow:
-                    raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
-            else:
-                # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
-                # time, which may cause 100% cpu usage
-                log.debug('unexpected read event from fd %r', fd)
-                try:
-                    self._poller.unregister_readable(fd)
-                except Exception:  # noqa
-                    pass
-
-        for fd in w:
-            if fd in dispatchers:
-                try:
-                    dispatcher = dispatchers[fd]
-                    log.debug('write event caused by %r', dispatcher)
-                    dispatcher.handle_write_event()
-                    if not dispatcher.writable():
-                        self._poller.unregister_writable(fd)
-                except ExitNow:
-                    raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
-            else:
-                log.debug('unexpected write event from fd %r', fd)
-                try:
-                    self._poller.unregister_writable(fd)
-                except Exception:  # noqa
-                    pass
+        self._io.poll()
 
         for group in sorted_groups:
             for process in group:
@@ -323,17 +233,17 @@ class Supervisor:
         if depth >= 100:
             return
 
-        pid, sts = self._context.waitpid()
-        if not pid:
+        wp = waitpid()
+        if wp is None or not wp.pid:
             return
 
-        process = self._pid_history.get(pid, None)
+        process = self._pid_history.get(wp.pid, None)
         if process is None:
-            _, msg = decode_wait_status(check_not_none(sts))
-            log.info('reaped unknown pid %s (%s)', pid, msg)
+            _, msg = decode_wait_status(wp.sts)
+            log.info('reaped unknown pid %s (%s)', wp.pid, msg)
         else:
-            process.finish(check_not_none(sts))
-            del self._pid_history[pid]
+            process.finish(wp.sts)
+            del self._pid_history[wp.pid]
 
         if not once:
             # keep reaping until no more kids to reap, but don't recurse infinitely
@@ -358,3 +268,31 @@ class Supervisor:
             if this_tick != last_tick:
                 self._ticks[period] = this_tick
                 self._event_callbacks.notify(event(this_tick, self))
+
+
+##
+
+
+class WaitedPid(ta.NamedTuple):
+    pid: Pid
+    sts: Rc
+
+
+def waitpid() -> ta.Optional[WaitedPid]:
+    # Need pthread_sigmask here to avoid concurrent sigchld, but Python doesn't offer in Python < 3.4.  There is
+    # still a race condition here; we can get a sigchld while we're sitting in the waitpid call. However, AFAICT, if
+    # waitpid is interrupted by SIGCHLD, as long as we call waitpid again (which happens every so often during the
+    # normal course in the mainloop), we'll eventually reap the child that we tried to reap during the interrupted
+    # call. At least on Linux, this appears to be true, or at least stopping 50 processes at once never left zombies
+    # lying around.
+    try:
+        pid, sts = os.waitpid(-1, os.WNOHANG)
+    except OSError as exc:
+        code = exc.args[0]
+        if code not in (errno.ECHILD, errno.EINTR):
+            log.critical('waitpid error %r; a process may not be cleaned up properly', code)
+        if code == errno.EINTR:
+            log.debug('EINTR during reap')
+        return None
+    else:
+        return WaitedPid(pid, sts)  # type: ignore

ominfra/supervisor/types.py

@@ -3,12 +3,14 @@ import abc
 import functools
 import typing as ta
 
-from .collections import KeyedCollectionAccessors
 from .configs import ProcessConfig
 from .configs import ProcessGroupConfig
-from .configs import ServerConfig
 from .states import ProcessState
 from .states import SupervisorState
+from .utils.collections import KeyedCollectionAccessors
+from .utils.ostypes import Fd
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
 
 
 if ta.TYPE_CHECKING:
@@ -18,6 +20,10 @@ if ta.TYPE_CHECKING:
 ##
 
 
+class ExitNow(Exception):  # noqa
+    pass
+
+
 ServerEpoch = ta.NewType('ServerEpoch', int)
 
 
@@ -41,12 +47,7 @@ class ConfigPriorityOrdered(abc.ABC):
 ##
 
 
-class ServerContext(abc.ABC):
-    @property
-    @abc.abstractmethod
-    def config(self) -> ServerConfig:
-        raise NotImplementedError
-
+class SupervisorStateManager(abc.ABC):
     @property
     @abc.abstractmethod
     def state(self) -> SupervisorState:
@@ -56,21 +57,11 @@ class ServerContext(abc.ABC):
     def set_state(self, state: SupervisorState) -> None:
         raise NotImplementedError
 
-    @property
-    @abc.abstractmethod
-    def pid_history(self) -> ta.Dict[int, 'Process']:
-        raise NotImplementedError
-
 
 ##
 
 
 class Dispatcher(abc.ABC):
-    @property
-    @abc.abstractmethod
-    def process(self) -> 'Process':
-        raise NotImplementedError
-
     @property
     @abc.abstractmethod
     def channel(self) -> str:
@@ -78,7 +69,7 @@ class Dispatcher(abc.ABC):
 
     @property
     @abc.abstractmethod
-    def fd(self) -> int:
+    def fd(self) -> Fd:
         raise NotImplementedError
 
     @property
@@ -114,8 +105,32 @@ class Dispatcher(abc.ABC):
     def handle_write_event(self) -> None:
         raise TypeError
 
+    #
+
+    def handle_connect(self) -> None:
+        raise TypeError
+
+    def handle_close(self) -> None:
+        raise TypeError
+
+    def handle_accepted(self, sock, addr) -> None:
+        raise TypeError
+
 
-class OutputDispatcher(Dispatcher, abc.ABC):
+class HasDispatchers(abc.ABC):
+    @abc.abstractmethod
+    def get_dispatchers(self) -> 'Dispatchers':
+        raise NotImplementedError
+
+
+class ProcessDispatcher(Dispatcher, abc.ABC):
+    @property
+    @abc.abstractmethod
+    def process(self) -> 'Process':
+        raise NotImplementedError
+
+
+class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
     @abc.abstractmethod
     def remove_logs(self) -> None:
         raise NotImplementedError
@@ -125,7 +140,7 @@ class OutputDispatcher(Dispatcher, abc.ABC):
         raise NotImplementedError
 
 
-class InputDispatcher(Dispatcher, abc.ABC):
+class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
     @abc.abstractmethod
     def write(self, chars: ta.Union[bytes, str]) -> None:
         raise NotImplementedError
@@ -138,7 +153,11 @@ class InputDispatcher(Dispatcher, abc.ABC):
 ##
 
 
-class Process(ConfigPriorityOrdered, abc.ABC):
+class Process(
+    ConfigPriorityOrdered,
+    HasDispatchers,
+    abc.ABC,
+):
     @property
     @abc.abstractmethod
     def name(self) -> str:
@@ -156,18 +175,13 @@ class Process(ConfigPriorityOrdered, abc.ABC):
 
     @property
     @abc.abstractmethod
-    def pid(self) -> int:
+    def pid(self) -> Pid:
         raise NotImplementedError
 
     #
 
-    @property
-    @abc.abstractmethod
-    def context(self) -> ServerContext:
-        raise NotImplementedError
-
     @abc.abstractmethod
-    def finish(self, sts: int) -> None:
+    def finish(self, sts: Rc) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
@@ -182,18 +196,15 @@ class Process(ConfigPriorityOrdered, abc.ABC):
     def transition(self) -> None:
         raise NotImplementedError
 
+    @property
     @abc.abstractmethod
-    def get_state(self) -> ProcessState:
+    def state(self) -> ProcessState:
         raise NotImplementedError
 
     @abc.abstractmethod
     def after_setuid(self) -> None:
         raise NotImplementedError
 
-    @abc.abstractmethod
-    def get_dispatchers(self) -> 'Dispatchers':
-        raise NotImplementedError
-
 
 ##
 

ominfra/supervisor/utils/diag.py

@@ -0,0 +1,31 @@
+# ruff: noqa: UP006 UP007
+import sys
+import types
+import typing as ta
+
+
+def compact_traceback() -> ta.Tuple[
+    ta.Tuple[str, str, int],
+    ta.Type[BaseException],
+    BaseException,
+    types.TracebackType,
+]:
+    t, v, tb = sys.exc_info()
+    if not tb:
+        raise RuntimeError('No traceback')
+
+    tbinfo = []
+    while tb:
+        tbinfo.append((
+            tb.tb_frame.f_code.co_filename,
+            tb.tb_frame.f_code.co_name,
+            str(tb.tb_lineno),
+        ))
+        tb = tb.tb_next
+
+    # just to be safe
+    del tb
+
+    file, function, line = tbinfo[-1]
+    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])  # noqa
+    return (file, function, line), t, v, info  # type: ignore

ominfra/supervisor/utils/fds.py

@@ -0,0 +1,46 @@
+# ruff: noqa: UP006 UP007
+import errno
+import os
+import typing as ta
+
+from .ostypes import Fd
+
+
+class PipeFds(ta.NamedTuple):
+    r: Fd
+    w: Fd
+
+
+def make_pipe() -> PipeFds:
+    return PipeFds(*os.pipe())  # type: ignore
+
+
+def read_fd(fd: Fd) -> bytes:
+    try:
+        data = os.read(fd, 2 << 16)  # 128K
+    except OSError as why:
+        if why.args[0] not in (errno.EWOULDBLOCK, errno.EBADF, errno.EINTR):
+            raise
+        data = b''
+    return data
+
+
+def close_fd(fd: Fd) -> bool:
+    try:
+        os.close(fd)
+    except OSError:
+        return False
+    return True
+
+
+def is_fd_open(fd: Fd) -> bool:
+    try:
+        n = os.dup(fd)
+    except OSError:
+        return False
+    os.close(n)
+    return True
+
+
+def get_open_fds(limit: int) -> ta.FrozenSet[Fd]:
+    return frozenset(fd for i in range(limit) if is_fd_open(fd := Fd(i)))