ominfra 0.0.0.dev128__py3-none-any.whl → 0.0.0.dev129__py3-none-any.whl

ominfra/supervisor/dispatchers.py

@@ -1,6 +1,6 @@
 # ruff: noqa: UP006 UP007
 from .types import Dispatcher
-from .types import OutputDispatcher
+from .types import ProcessOutputDispatcher
 from .utils.collections import KeyedCollection
 from .utils.ostypes import Fd
 
@@ -24,10 +24,10 @@ class Dispatchers(KeyedCollection[Fd, Dispatcher]):
 
     def remove_logs(self) -> None:
         for d in self:
-            if isinstance(d, OutputDispatcher):
+            if isinstance(d, ProcessOutputDispatcher):
                 d.remove_logs()
 
     def reopen_logs(self) -> None:
         for d in self:
-            if isinstance(d, OutputDispatcher):
+            if isinstance(d, ProcessOutputDispatcher):
                 d.reopen_logs()

ominfra/supervisor/dispatchersimpl.py

@@ -8,14 +8,15 @@ import typing as ta
 from omlish.lite.logs import log
 
 from .configs import ProcessConfig
+from .configs import ServerConfig
 from .events import EventCallbacks
 from .events import ProcessCommunicationEvent
 from .events import ProcessLogStderrEvent
 from .events import ProcessLogStdoutEvent
-from .types import Dispatcher
-from .types import InputDispatcher
-from .types import OutputDispatcher
 from .types import Process
+from .types import ProcessDispatcher
+from .types import ProcessInputDispatcher
+from .types import ProcessOutputDispatcher
 from .utils.diag import compact_traceback
 from .utils.fds import read_fd
 from .utils.ostypes import Fd
@@ -24,7 +25,7 @@ from .utils.strings import find_prefix_at_end
 from .utils.strings import strip_escapes
 
 
-class BaseDispatcherImpl(Dispatcher, abc.ABC):
+class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
     def __init__(
             self,
             process: Process,
@@ -32,6 +33,7 @@ class BaseDispatcherImpl(Dispatcher, abc.ABC):
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,
+            server_config: ServerConfig,
     ) -> None:
         super().__init__()
 
@@ -39,6 +41,7 @@ class BaseDispatcherImpl(Dispatcher, abc.ABC):
         self._channel = channel  # 'stderr' or 'stdout'
         self._fd = fd
         self._event_callbacks = event_callbacks
+        self._server_config = server_config
 
         self._closed = False  # True if close() has been called
 
@@ -79,7 +82,7 @@ class BaseDispatcherImpl(Dispatcher, abc.ABC):
         self.close()
 
 
-class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
+class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispatcher):
     """
     Dispatcher for one channel (stdout or stderr) of one process. Serves several purposes:
 
@@ -95,12 +98,14 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,
+            server_config: ServerConfig,
     ) -> None:
         super().__init__(
             process,
             event_type.channel,
             fd,
             event_callbacks=event_callbacks,
+            server_config=server_config,
         )
 
         self._event_type = event_type
@@ -124,11 +129,10 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
 
         self._main_log_level = logging.DEBUG
 
-        self._log_to_main_log = process.context.config.loglevel <= self._main_log_level
+        self._log_to_main_log = self._server_config.loglevel <= self._main_log_level
 
-        config = self._process.config
-        self._stdout_events_enabled = config.stdout.events_enabled
-        self._stderr_events_enabled = config.stderr.events_enabled
+        self._stdout_events_enabled = self._process.config.stdout.events_enabled
+        self._stderr_events_enabled = self._process.config.stderr.events_enabled
 
     _child_log: ta.Optional[logging.Logger] = None  # the current logger (normal_log or capture_log)
     _normal_log: ta.Optional[logging.Logger] = None  # the "normal" (non-capture) logger
@@ -199,7 +203,7 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
         if not data:
             return
 
-        if self._process.context.config.strip_ansi:
+        if self._server_config.strip_ansi:
             data = strip_escapes(as_bytes(data))
 
         if self._child_log:
@@ -297,7 +301,7 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
             self.close()
 
 
-class InputDispatcherImpl(BaseDispatcherImpl, InputDispatcher):
+class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatcher):
     def __init__(
             self,
             process: Process,
@@ -305,12 +309,14 @@ class InputDispatcherImpl(BaseDispatcherImpl, InputDispatcher):
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,
+            server_config: ServerConfig,
     ) -> None:
         super().__init__(
             process,
             channel,
             fd,
             event_callbacks=event_callbacks,
+            server_config=server_config,
         )
 
         self._input_buffer = b''

ominfra/supervisor/groupsimpl.py

@@ -61,7 +61,7 @@ class ProcessGroupImpl(ProcessGroup):
     #
 
     def get_unstopped_processes(self) -> ta.List[Process]:
-        return [x for x in self if not x.get_state().stopped]
+        return [x for x in self if not x.state.stopped]
 
     def stop_all(self) -> None:
         processes = list(self._by_name.values())
@@ -69,7 +69,7 @@ class ProcessGroupImpl(ProcessGroup):
         processes.reverse()  # stop in desc priority order
 
         for proc in processes:
-            state = proc.get_state()
+            state = proc.state
             if state == ProcessState.RUNNING:
                 # RUNNING -> STOPPING
                 proc.stop()

ominfra/supervisor/inject.py

@@ -6,13 +6,13 @@ from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
 
 from .configs import ServerConfig
-from .context import ServerContextImpl
-from .dispatchersimpl import InputDispatcherImpl
-from .dispatchersimpl import OutputDispatcherImpl
+from .dispatchersimpl import ProcessInputDispatcherImpl
+from .dispatchersimpl import ProcessOutputDispatcherImpl
 from .events import EventCallbacks
 from .groups import ProcessGroupManager
 from .groupsimpl import ProcessFactory
 from .groupsimpl import ProcessGroupImpl
+from .io import IoManager
 from .poller import Poller
 from .poller import get_poller_impl
 from .process import PidHistory
@@ -23,15 +23,16 @@ from .setup import DaemonizeListeners
 from .setup import SupervisorUser
 from .setupimpl import SupervisorSetup
 from .setupimpl import SupervisorSetupImpl
+from .signals import SignalHandler
 from .spawningimpl import InheritedFds
-from .spawningimpl import InputDispatcherFactory
-from .spawningimpl import OutputDispatcherFactory
+from .spawningimpl import ProcessInputDispatcherFactory
+from .spawningimpl import ProcessOutputDispatcherFactory
 from .spawningimpl import ProcessSpawningImpl
 from .supervisor import ProcessGroupFactory
-from .supervisor import SignalHandler
 from .supervisor import Supervisor
-from .types import ServerContext
+from .supervisor import SupervisorStateManagerImpl
 from .types import ServerEpoch
+from .types import SupervisorStateManager
 from .utils.signals import SignalReceiver
 from .utils.users import get_user
 
@@ -52,17 +53,21 @@ def bind_server(
 
         inj.bind(DaemonizeListener, array=True, to_key=Poller),
 
-        inj.bind(ServerContextImpl, singleton=True),
-        inj.bind(ServerContext, to_key=ServerContextImpl),
-
         inj.bind(EventCallbacks, singleton=True),
 
         inj.bind(SignalReceiver, singleton=True),
 
+        inj.bind(IoManager, singleton=True),
+
         inj.bind(SignalHandler, singleton=True),
+
         inj.bind(ProcessGroupManager, singleton=True),
+
         inj.bind(Supervisor, singleton=True),
 
+        inj.bind(SupervisorStateManagerImpl, singleton=True),
+        inj.bind(SupervisorStateManager, to_key=SupervisorStateManagerImpl),
+
         inj.bind(PidHistory()),
 
         inj.bind_factory(ProcessGroupImpl, ProcessGroupFactory),
@@ -70,8 +75,8 @@ def bind_server(
 
         inj.bind_factory(ProcessSpawningImpl, ProcessSpawningFactory),
 
-        inj.bind_factory(OutputDispatcherImpl, OutputDispatcherFactory),
-        inj.bind_factory(InputDispatcherImpl, InputDispatcherFactory),
+        inj.bind_factory(ProcessOutputDispatcherImpl, ProcessOutputDispatcherFactory),
+        inj.bind_factory(ProcessInputDispatcherImpl, ProcessInputDispatcherFactory),
     ]
 
     #

ominfra/supervisor/io.py

@@ -0,0 +1,82 @@
+# ruff: noqa: UP006 UP007
+from omlish.lite.logs import log
+
+from .dispatchers import Dispatchers
+from .groups import ProcessGroupManager
+from .poller import Poller
+from .types import ExitNow
+
+
+##
+
+
+class IoManager:
+    def __init__(
+            self,
+            *,
+            poller: Poller,
+            process_groups: ProcessGroupManager,
+    ) -> None:
+        super().__init__()
+
+        self._poller = poller
+        self._process_groups = process_groups
+
+    def get_dispatchers(self) -> Dispatchers:
+        return Dispatchers(
+            d
+            for p in self._process_groups.all_processes()
+            for d in p.get_dispatchers()
+        )
+
+    def poll(self) -> None:
+        dispatchers = self.get_dispatchers()
+
+        for fd, dispatcher in dispatchers.items():
+            if dispatcher.readable():
+                self._poller.register_readable(fd)
+            if dispatcher.writable():
+                self._poller.register_writable(fd)
+
+        timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
+        r, w = self._poller.poll(timeout)
+
+        for fd in r:
+            if fd in dispatchers:
+                try:
+                    dispatcher = dispatchers[fd]
+                    log.debug('read event caused by %r', dispatcher)
+                    dispatcher.handle_read_event()
+                    if not dispatcher.readable():
+                        self._poller.unregister_readable(fd)
+                except ExitNow:
+                    raise
+                except Exception:  # noqa
+                    dispatchers[fd].handle_error()
+            else:
+                # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
+                # time, which may cause 100% cpu usage
+                log.debug('unexpected read event from fd %r', fd)
+                try:
+                    self._poller.unregister_readable(fd)
+                except Exception:  # noqa
+                    pass
+
+        for fd in w:
+            if fd in dispatchers:
+                try:
+                    dispatcher = dispatchers[fd]
+                    log.debug('write event caused by %r', dispatcher)
+                    dispatcher.handle_write_event()
+                    if not dispatcher.writable():
+                        self._poller.unregister_writable(fd)
+                except ExitNow:
+                    raise
+                except Exception:  # noqa
+                    dispatchers[fd].handle_error()
+            else:
+                log.debug('unexpected write event from fd %r', fd)
+                try:
+                    self._poller.unregister_writable(fd)
+                except Exception:  # noqa
+                    pass

ominfra/supervisor/main.py

@@ -37,17 +37,17 @@ from omlish.lite.http.coroserver import CoroHttpServer
 from omlish.lite.inject import inj
 from omlish.lite.journald import journald_log_handler_factory
 from omlish.lite.logs import configure_standard_logging
+from omlish.lite.runtime import is_debugger_attached
 
 from ..configs import read_config_file
 from .configs import ServerConfig
 from .configs import prepare_server_config
-from .context import ServerContextImpl
-from .context import ServerEpoch
 from .inject import bind_server
 from .spawningimpl import InheritedFds
 from .states import SupervisorState
-from .supervisor import ExitNow
 from .supervisor import Supervisor
+from .types import ExitNow
+from .types import ServerEpoch
 from .utils.fds import get_open_fds
 
 
@@ -79,7 +79,7 @@ def main(
     if not no_logging:
         configure_standard_logging(
             'INFO',
-            handler_factory=journald_log_handler_factory if not args.no_journald else None,
+            handler_factory=journald_log_handler_factory if not (args.no_journald or is_debugger_attached()) else None,
         )
 
     #
@@ -102,7 +102,6 @@ def main(
             inherited_fds=inherited_fds,
         ))
 
-        context = injector[ServerContextImpl]
         supervisor = injector[Supervisor]
 
         try:
@@ -110,7 +109,7 @@ def main(
         except ExitNow:
             pass
 
-        if context.state < SupervisorState.RESTARTING:
+        if supervisor.state < SupervisorState.RESTARTING:
             break
 
 

ominfra/supervisor/processimpl.py

@@ -22,10 +22,10 @@ from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .states import ProcessState
 from .states import SupervisorState
-from .types import InputDispatcher
 from .types import Process
 from .types import ProcessGroup
-from .types import ServerContext
+from .types import ProcessInputDispatcher
+from .types import SupervisorStateManager
 from .utils.os import decode_wait_status
 from .utils.ostypes import Pid
 from .utils.ostypes import Rc
@@ -47,7 +47,7 @@ class ProcessImpl(Process):
             config: ProcessConfig,
             group: ProcessGroup,
             *,
-            context: ServerContext,
+            supervisor_states: SupervisorStateManager,
             event_callbacks: EventCallbacks,
             process_spawning_factory: ProcessSpawningFactory,
     ) -> None:
@@ -56,7 +56,7 @@ class ProcessImpl(Process):
         self._config = config
         self._group = group
 
-        self._context = context
+        self._supervisor_states = supervisor_states
         self._event_callbacks = event_callbacks
 
         self._spawning = process_spawning_factory(self)
@@ -87,7 +87,7 @@ class ProcessImpl(Process):
     #
 
     def __repr__(self) -> str:
-        return f'<Subprocess at {id(self)} with name {self._config.name} in state {self.get_state().name}>'
+        return f'<Subprocess at {id(self)} with name {self._config.name} in state {self._state.name}>'
 
     #
 
@@ -109,10 +109,6 @@ class ProcessImpl(Process):
 
     #
 
-    @property
-    def context(self) -> ServerContext:
-        return self._context
-
     @property
     def state(self) -> ProcessState:
         return self._state
@@ -175,7 +171,7 @@ class ProcessImpl(Process):
         if stdin_fd is None:
             raise OSError(errno.EPIPE, 'Process has no stdin channel')
 
-        dispatcher = check_isinstance(self._dispatchers[stdin_fd], InputDispatcher)
+        dispatcher = check_isinstance(self._dispatchers[stdin_fd], ProcessInputDispatcher)
         if dispatcher.closed:
             raise OSError(errno.EPIPE, "Process' stdin channel is closed")
 
@@ -445,9 +441,6 @@ class ProcessImpl(Process):
         self._pipes = ProcessPipes()
         self._dispatchers = Dispatchers([])
 
-    def get_state(self) -> ProcessState:
-        return self._state
-
     def transition(self) -> None:
         now = time.time()
         state = self._state
@@ -456,7 +449,7 @@ class ProcessImpl(Process):
 
         logger = log
 
-        if self.context.state > SupervisorState.RESTARTING:
+        if self._supervisor_states.state > SupervisorState.RESTARTING:
             # dont start any processes if supervisor is shutting down
             if state == ProcessState.EXITED:
                 if self._config.autorestart:

ominfra/supervisor/signals.py

@@ -0,0 +1,66 @@
+# ruff: noqa: UP006 UP007
+import signal
+
+from omlish.lite.logs import log
+
+from .groups import ProcessGroupManager
+from .states import SupervisorState
+from .types import ProcessOutputDispatcher
+from .types import SupervisorStateManager
+from .utils.signals import SignalReceiver
+from .utils.signals import sig_name
+
+
+class SignalHandler:
+    def __init__(
+            self,
+            *,
+            states: SupervisorStateManager,
+            signal_receiver: SignalReceiver,
+            process_groups: ProcessGroupManager,
+    ) -> None:
+        super().__init__()
+
+        self._states = states
+        self._signal_receiver = signal_receiver
+        self._process_groups = process_groups
+
+    def set_signals(self) -> None:
+        self._signal_receiver.install(
+            signal.SIGTERM,
+            signal.SIGINT,
+            signal.SIGQUIT,
+            signal.SIGHUP,
+            signal.SIGCHLD,
+            signal.SIGUSR2,
+        )
+
+    def handle_signals(self) -> None:
+        sig = self._signal_receiver.get_signal()
+        if not sig:
+            return
+
+        if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
+            log.warning('received %s indicating exit request', sig_name(sig))
+            self._states.set_state(SupervisorState.SHUTDOWN)
+
+        elif sig == signal.SIGHUP:
+            if self._states.state == SupervisorState.SHUTDOWN:
+                log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig))  # noqa
+            else:
+                log.warning('received %s indicating restart request', sig_name(sig))  # noqa
+                self._states.set_state(SupervisorState.RESTARTING)
+
+        elif sig == signal.SIGCHLD:
+            log.debug('received %s indicating a child quit', sig_name(sig))
+
+        elif sig == signal.SIGUSR2:
+            log.info('received %s indicating log reopen request', sig_name(sig))
+
+            for p in self._process_groups.all_processes():
+                for d in p.get_dispatchers():
+                    if isinstance(d, ProcessOutputDispatcher):
+                        d.reopen_logs()
+
+        else:
+            log.debug('received %s indicating nothing', sig_name(sig))

ominfra/supervisor/spawningimpl.py

@@ -30,10 +30,10 @@ from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .spawning import SpawnedProcess
 from .types import Dispatcher
-from .types import InputDispatcher
-from .types import OutputDispatcher
 from .types import Process
 from .types import ProcessGroup
+from .types import ProcessInputDispatcher
+from .types import ProcessOutputDispatcher
 from .utils.diag import compact_traceback
 from .utils.fds import close_fd
 from .utils.fs import get_path
@@ -44,11 +44,11 @@ from .utils.ostypes import Rc
 from .utils.strings import as_bytes
 
 
-class OutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], Fd, OutputDispatcher]):
+class ProcessOutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], Fd, ProcessOutputDispatcher]):
     pass
 
 
-class InputDispatcherFactory(Func3[Process, str, Fd, InputDispatcher]):
+class ProcessInputDispatcherFactory(Func3[Process, str, Fd, ProcessInputDispatcher]):
     pass
 
 
@@ -66,8 +66,8 @@ class ProcessSpawningImpl(ProcessSpawning):
             server_config: ServerConfig,
             pid_history: PidHistory,
 
-            output_dispatcher_factory: OutputDispatcherFactory,
-            input_dispatcher_factory: InputDispatcherFactory,
+            output_dispatcher_factory: ProcessOutputDispatcherFactory,
+            input_dispatcher_factory: ProcessInputDispatcherFactory,
 
             inherited_fds: ta.Optional[InheritedFds] = None,
     ) -> None:
@@ -207,21 +207,21 @@ class ProcessSpawningImpl(ProcessSpawning):
                 self.process,
                 ProcessCommunicationStdoutEvent,
                 pipes.stdout,
-            ), OutputDispatcher))
+            ), ProcessOutputDispatcher))
 
         if pipes.stderr is not None:
             dispatchers.append(check_isinstance(self._output_dispatcher_factory(
                 self.process,
                 ProcessCommunicationStderrEvent,
                 pipes.stderr,
-            ), OutputDispatcher))
+            ), ProcessOutputDispatcher))
 
         if pipes.stdin is not None:
             dispatchers.append(check_isinstance(self._input_dispatcher_factory(
                 self.process,
                 'stdin',
                 pipes.stdin,
-            ), InputDispatcher))
+            ), ProcessInputDispatcher))
 
         return Dispatchers(dispatchers)