ominfra 0.0.0.dev122__py3-none-any.whl → 0.0.0.dev124__py3-none-any.whl

ominfra/supervisor/LICENSE.txt

@@ -0,0 +1,28 @@
+Supervisor is licensed under the following license:
+
+  A copyright notice accompanies this license document that identifies the copyright holders.
+
+  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+  following conditions are met:
+
+  1.  Redistributions in source code must retain the accompanying copyright notice, this list of conditions, and the
+      following disclaimer.
+
+  2.  Redistributions in binary form must reproduce the accompanying copyright notice, this list of conditions, and the
+      following disclaimer in the documentation and/or other materials provided with the distribution.
+
+  3.  Names of the copyright holders must not be used to endorse or promote products derived from this software without
+      prior written permission from the copyright holders.
+
+  4.  If any files are modified, you must cause the modified files to carry prominent notices stating that you changed
+      the files and the date of any change.
+
+  Disclaimer
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT
+    NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+    EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+    STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+    EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ominfra/supervisor/context.py

@@ -6,57 +6,47 @@ import os
 import pwd
 import re
 import resource
-import signal
 import stat
 import typing as ta
 import warnings
 
 from omlish.lite.logs import log
 
-from .compat import SignalReceiver
-from .compat import close_fd
-from .compat import mktempfile
-from .compat import real_exit
-from .compat import try_unlink
 from .configs import ServerConfig
 from .datatypes import gid_for_uid
 from .datatypes import name_to_uid
 from .exceptions import NoPermissionError
 from .exceptions import NotExecutableError
 from .exceptions import NotFoundError
-from .poller import BasePoller
 from .poller import Poller
 from .states import SupervisorState
-from .states import SupervisorStates
 from .types import AbstractServerContext
 from .types import AbstractSubprocess
+from .utils import close_fd
+from .utils import mktempfile
+from .utils import real_exit
+from .utils import try_unlink
 
 
 ServerEpoch = ta.NewType('ServerEpoch', int)
 
-InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[int])
-
 
 class ServerContext(AbstractServerContext):
     def __init__(
             self,
             config: ServerConfig,
+            poller: Poller,
             *,
             epoch: ServerEpoch = ServerEpoch(0),
-            inherited_fds: ta.Optional[InheritedFds] = None,
     ) -> None:
         super().__init__()
 
         self._config = config
+        self._poller = poller
         self._epoch = epoch
-        self._inherited_fds = InheritedFds(frozenset(inherited_fds or []))
 
         self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
-        self._state: SupervisorState = SupervisorStates.RUNNING
-
-        self._signal_receiver = SignalReceiver()
-
-        self._poller: BasePoller = Poller()
+        self._state: SupervisorState = SupervisorState.RUNNING
 
         if config.user is not None:
             uid = name_to_uid(config.user)
@@ -87,10 +77,6 @@ class ServerContext(AbstractServerContext):
     def set_state(self, state: SupervisorState) -> None:
         self._state = state
 
-    @property
-    def poller(self) -> BasePoller:
-        return self._poller
-
     @property
     def pid_history(self) -> ta.Dict[int, AbstractSubprocess]:
         return self._pid_history
@@ -103,22 +89,8 @@ class ServerContext(AbstractServerContext):
     def gid(self) -> ta.Optional[int]:
         return self._gid
 
-    @property
-    def inherited_fds(self) -> InheritedFds:
-        return self._inherited_fds
-
     ##
 
-    def set_signals(self) -> None:
-        self._signal_receiver.install(
-            signal.SIGTERM,
-            signal.SIGINT,
-            signal.SIGQUIT,
-            signal.SIGHUP,
-            signal.SIGCHLD,
-            signal.SIGUSR2,
-        )
-
     def waitpid(self) -> ta.Tuple[ta.Optional[int], ta.Optional[int]]:
         # Need pthread_sigmask here to avoid concurrent sigchld, but Python doesn't offer in Python < 3.4.  There is
         # still a race condition here; we can get a sigchld while we're sitting in the waitpid call. However, AFAICT, if
@@ -223,7 +195,7 @@ class ServerContext(AbstractServerContext):
     def cleanup(self) -> None:
         if self._unlink_pidfile:
             try_unlink(self.config.pidfile)
-        self.poller.close()
+        self._poller.close()
 
     def cleanup_fds(self) -> None:
         # try to close any leaked file descriptors (for reload)
@@ -249,9 +221,9 @@ class ServerContext(AbstractServerContext):
                     log.warning('Failed to clean up %r', pathname)
 
     def daemonize(self) -> None:
-        self.poller.before_daemonize()
+        self._poller.before_daemonize()
         self._daemonize()
-        self.poller.after_daemonize()
+        self._poller.after_daemonize()
 
     def _daemonize(self) -> None:
         # To daemonize, we need to become the leader of our own session (process) group.  If we do not, signals sent to
@@ -306,9 +278,6 @@ class ServerContext(AbstractServerContext):
         )
         return logfile
 
-    def get_signal(self) -> ta.Optional[int]:
-        return self._signal_receiver.get_signal()
-
     def write_pidfile(self) -> None:
         pid = os.getpid()
         try:
@@ -395,19 +364,25 @@ def make_pipes(stderr=True) -> ta.Mapping[str, int]:
         'stderr': None,
         'child_stderr': None,
     }
+
     try:
         stdin, child_stdin = os.pipe()
         pipes['child_stdin'], pipes['stdin'] = stdin, child_stdin
+
         stdout, child_stdout = os.pipe()
         pipes['stdout'], pipes['child_stdout'] = stdout, child_stdout
+
         if stderr:
             stderr, child_stderr = os.pipe()
             pipes['stderr'], pipes['child_stderr'] = stderr, child_stderr
+
         for fd in (pipes['stdout'], pipes['stderr'], pipes['stdin']):
             if fd is not None:
                 flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NDELAY
                 fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
         return pipes  # type: ignore
+
     except OSError:
         for fd in pipes.values():
             if fd is not None:

ominfra/supervisor/dispatchers.py

@@ -1,4 +1,4 @@
-# ruff: noqa: UP007
+# ruff: noqa: UP006 UP007
 import abc
 import errno
 import logging
@@ -7,26 +7,35 @@ import typing as ta
 
 from omlish.lite.logs import log
 
-from .compat import as_bytes
-from .compat import compact_traceback
-from .compat import find_prefix_at_end
-from .compat import readfd
-from .compat import strip_escapes
 from .configs import ProcessConfig
-from .events import EVENT_CALLBACKS
+from .events import EventCallbacks
+from .events import ProcessCommunicationEvent
 from .events import ProcessLogStderrEvent
 from .events import ProcessLogStdoutEvent
 from .types import AbstractSubprocess
+from .utils import as_bytes
+from .utils import compact_traceback
+from .utils import find_prefix_at_end
+from .utils import read_fd
+from .utils import strip_escapes
 
 
 class Dispatcher(abc.ABC):
-
-    def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
+    def __init__(
+            self,
+            process: AbstractSubprocess,
+            channel: str,
+            fd: int,
+            *,
+            event_callbacks: EventCallbacks,
+    ) -> None:
         super().__init__()
 
         self._process = process  # process which "owns" this dispatcher
         self._channel = channel  # 'stderr' or 'stdout'
         self._fd = fd
+        self._event_callbacks = event_callbacks
+
         self._closed = False  # True if close() has been called
 
     def __repr__(self) -> str:
@@ -86,18 +95,23 @@ class OutputDispatcher(Dispatcher):
     - route the output to the appropriate log handlers as specified in the config.
     """
 
-    def __init__(self, process: AbstractSubprocess, event_type, fd):
-        """
-        Initialize the dispatcher.
-
-        `event_type` should be one of ProcessLogStdoutEvent or ProcessLogStderrEvent
-        """
-
-        super().__init__(process, event_type.channel, fd)
+    def __init__(
+            self,
+            process: AbstractSubprocess,
+            event_type: ta.Type[ProcessCommunicationEvent],
+            fd: int,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(
+            process,
+            event_type.channel,
+            fd,
+            **kwargs,
+        )
 
-        self.event_type = event_type
+        self._event_type = event_type
 
-        self.lc: ProcessConfig.Log = getattr(process.config, self._channel)
+        self._lc: ProcessConfig.Log = getattr(process.config, self._channel)
 
         self._init_normal_log()
         self._init_capture_log()
@@ -109,8 +123,8 @@ class OutputDispatcher(Dispatcher):
 
         # all code below is purely for minor speedups
 
-        begin_token = self.event_type.BEGIN_TOKEN
-        end_token = self.event_type.END_TOKEN
+        begin_token = self._event_type.BEGIN_TOKEN
+        end_token = self._event_type.END_TOKEN
         self._begin_token_data = (begin_token, len(begin_token))
         self._end_token_data = (end_token, len(end_token))
 
@@ -135,10 +149,10 @@ class OutputDispatcher(Dispatcher):
         config = self._process.config  # noqa
         channel = self._channel  # noqa
 
-        logfile = self.lc.file
-        maxbytes = self.lc.maxbytes  # noqa
-        backups = self.lc.backups  # noqa
-        to_syslog = self.lc.syslog
+        logfile = self._lc.file
+        maxbytes = self._lc.maxbytes  # noqa
+        backups = self._lc.backups  # noqa
+        to_syslog = self._lc.syslog
 
         if logfile or to_syslog:
             self._normal_log = logging.getLogger(__name__)
@@ -165,7 +179,7 @@ class OutputDispatcher(Dispatcher):
         is detected. Sets self.capture_log if capturing is enabled.
         """
 
-        capture_maxbytes = self.lc.capture_maxbytes
+        capture_maxbytes = self._lc.capture_maxbytes
         if capture_maxbytes:
             self._capture_log = logging.getLogger(__name__)
             # loggers.handle_boundIO(
@@ -174,45 +188,47 @@ class OutputDispatcher(Dispatcher):
             #     maxbytes=capture_maxbytes,
             # )
 
-    def remove_logs(self):
+    def remove_logs(self) -> None:
         for l in (self._normal_log, self._capture_log):
             if l is not None:
                 for handler in l.handlers:
                     handler.remove()  # type: ignore
                     handler.reopen()  # type: ignore
 
-    def reopen_logs(self):
+    def reopen_logs(self) -> None:
         for l in (self._normal_log, self._capture_log):
             if l is not None:
                 for handler in l.handlers:
                     handler.reopen()  # type: ignore
 
-    def _log(self, data):
-        if data:
-            if self._process.context.config.strip_ansi:
-                data = strip_escapes(data)
+    def _log(self, data: ta.Union[str, bytes, None]) -> None:
+        if not data:
+            return
 
-            if self._child_log:
-                self._child_log.info(data)
+        if self._process.context.config.strip_ansi:
+            data = strip_escapes(as_bytes(data))
 
-            if self._log_to_main_log:
-                if not isinstance(data, bytes):
-                    text = data
-                else:
-                    try:
-                        text = data.decode('utf-8')
-                    except UnicodeDecodeError:
-                        text = f'Undecodable: {data!r}'
-                log.log(self._main_log_level, '%r %s output:\n%s', self._process.config.name, self._channel, text)  # noqa
+        if self._child_log:
+            self._child_log.info(data)
 
-            if self._channel == 'stdout':
-                if self._stdout_events_enabled:
-                    EVENT_CALLBACKS.notify(ProcessLogStdoutEvent(self._process, self._process.pid, data))
+        if self._log_to_main_log:
+            if not isinstance(data, bytes):
+                text = data
+            else:
+                try:
+                    text = data.decode('utf-8')
+                except UnicodeDecodeError:
+                    text = f'Undecodable: {data!r}'
+            log.log(self._main_log_level, '%r %s output:\n%s', self._process.config.name, self._channel, text)  # noqa
+
+        if self._channel == 'stdout':
+            if self._stdout_events_enabled:
+                self._event_callbacks.notify(ProcessLogStdoutEvent(self._process, self._process.pid, data))
 
-            elif self._stderr_events_enabled:
-                EVENT_CALLBACKS.notify(ProcessLogStderrEvent(self._process, self._process.pid, data))
+        elif self._stderr_events_enabled:
+            self._event_callbacks.notify(ProcessLogStderrEvent(self._process, self._process.pid, data))
 
-    def record_output(self):
+    def record_output(self) -> None:
         if self._capture_log is None:
             # shortcut trying to find capture data
             data = self._output_buffer
@@ -221,11 +237,11 @@ class OutputDispatcher(Dispatcher):
             return
 
         if self._capture_mode:
-            token, tokenlen = self._end_token_data
+            token, token_len = self._end_token_data
         else:
-            token, tokenlen = self._begin_token_data
+            token, token_len = self._begin_token_data
 
-        if len(self._output_buffer) <= tokenlen:
+        if len(self._output_buffer) <= token_len:
             return  # not enough data
 
         data = self._output_buffer
@@ -248,7 +264,7 @@ class OutputDispatcher(Dispatcher):
         if after:
             self.record_output()
 
-    def toggle_capture_mode(self):
+    def toggle_capture_mode(self) -> None:
         self._capture_mode = not self._capture_mode
 
         if self._capture_log is not None:
@@ -260,8 +276,8 @@ class OutputDispatcher(Dispatcher):
                 data = self._capture_log.getvalue()  # type: ignore
                 channel = self._channel
                 procname = self._process.config.name
-                event = self.event_type(self._process, self._process.pid, data)
-                EVENT_CALLBACKS.notify(event)
+                event = self._event_type(self._process, self._process.pid, data)
+                self._event_callbacks.notify(event)
 
                 log.debug('%r %s emitted a comm event', procname, channel)
                 for handler in self._capture_log.handlers:
@@ -278,7 +294,7 @@ class OutputDispatcher(Dispatcher):
         return True
 
     def handle_read_event(self) -> None:
-        data = readfd(self._fd)
+        data = read_fd(self._fd)
         self._output_buffer += data
         self.record_output()
         if not data:
@@ -288,11 +304,25 @@ class OutputDispatcher(Dispatcher):
 
 
 class InputDispatcher(Dispatcher):
+    def __init__(
+            self,
+            process: AbstractSubprocess,
+            channel: str,
+            fd: int,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(
+            process,
+            channel,
+            fd,
+            **kwargs,
+        )
 
-    def __init__(self, process: AbstractSubprocess, channel: str, fd: int) -> None:
-        super().__init__(process, channel, fd)
         self._input_buffer = b''
 
+    def write(self, chars: ta.Union[bytes, str]) -> None:
+        self._input_buffer += as_bytes(chars)
+
     def writable(self) -> bool:
         if self._input_buffer and not self._closed:
             return True

ominfra/supervisor/events.py

@@ -2,36 +2,44 @@
 import abc
 import typing as ta
 
-from .compat import as_string
-from .states import get_process_state_description
+from .states import ProcessState
+
+
+##
+
+
+class Event(abc.ABC):  # noqa
+    """Abstract event type."""
+
+
+##
+
+
+EventCallback = ta.Callable[['Event'], None]
 
 
 class EventCallbacks:
     def __init__(self) -> None:
         super().__init__()
 
-        self._callbacks: ta.List[ta.Tuple[type, ta.Callable]] = []
+        self._callbacks: ta.List[ta.Tuple[ta.Type[Event], EventCallback]] = []
 
-    def subscribe(self, type, callback):  # noqa
+    def subscribe(self, type: ta.Type[Event], callback: EventCallback) -> None:  # noqa
         self._callbacks.append((type, callback))
 
-    def unsubscribe(self, type, callback):  # noqa
+    def unsubscribe(self, type: ta.Type[Event], callback: EventCallback) -> None:  # noqa
         self._callbacks.remove((type, callback))
 
-    def notify(self, event):
+    def notify(self, event: Event) -> None:
         for type, callback in self._callbacks:  # noqa
             if isinstance(event, type):
                 callback(event)
 
-    def clear(self):
+    def clear(self) -> None:
         self._callbacks[:] = []
 
 
-EVENT_CALLBACKS = EventCallbacks()
-
-
-class Event(abc.ABC):  # noqa
-    """Abstract event type."""
+##
 
 
 class ProcessLogEvent(Event, abc.ABC):
@@ -43,24 +51,6 @@ class ProcessLogEvent(Event, abc.ABC):
         self.pid = pid
         self.data = data
 
-    def payload(self):
-        groupname = ''
-        if self.process.group is not None:
-            groupname = self.process.group.config.name
-        try:
-            data = as_string(self.data)
-        except UnicodeDecodeError:
-            data = f'Undecodable: {self.data!r}'
-
-        result = 'processname:%s groupname:%s pid:%s channel:%s\n%s' % (  # noqa
-            as_string(self.process.config.name),
-            as_string(groupname),
-            self.pid,
-            as_string(self.channel),  # type: ignore
-            data,
-        )
-        return result
-
 
 class ProcessLogStdoutEvent(ProcessLogEvent):
     channel = 'stdout'
@@ -70,27 +60,22 @@ class ProcessLogStderrEvent(ProcessLogEvent):
     channel = 'stderr'
 
 
+#
+
+
 class ProcessCommunicationEvent(Event, abc.ABC):
     # event mode tokens
     BEGIN_TOKEN = b'<!--XSUPERVISOR:BEGIN-->'
     END_TOKEN = b'<!--XSUPERVISOR:END-->'
 
+    channel: ta.ClassVar[str]
+
     def __init__(self, process, pid, data):
         super().__init__()
         self.process = process
         self.pid = pid
         self.data = data
 
-    def payload(self):
-        groupname = ''
-        if self.process.group is not None:
-            groupname = self.process.group.config.name
-        try:
-            data = as_string(self.data)
-        except UnicodeDecodeError:
-            data = f'Undecodable: {self.data!r}'
-        return f'processname:{self.process.config.name} groupname:{groupname} pid:{self.pid}\n{data}'
-
 
 class ProcessCommunicationStdoutEvent(ProcessCommunicationEvent):
     channel = 'stdout'
@@ -100,22 +85,22 @@ class ProcessCommunicationStderrEvent(ProcessCommunicationEvent):
     channel = 'stderr'
 
 
+#
+
+
 class RemoteCommunicationEvent(Event):
     def __init__(self, type, data):  # noqa
         super().__init__()
         self.type = type
         self.data = data
 
-    def payload(self):
-        return f'type:{self.type}\n{self.data}'
+
+#
 
 
 class SupervisorStateChangeEvent(Event):
     """Abstract class."""
 
-    def payload(self):
-        return ''
-
 
 class SupervisorRunningEvent(SupervisorStateChangeEvent):
     pass
@@ -125,6 +110,9 @@ class SupervisorStoppingEvent(SupervisorStateChangeEvent):
     pass
 
 
+#
+
+
 class EventRejectedEvent:  # purposely does not subclass Event
     def __init__(self, process, event):
         super().__init__()
@@ -132,6 +120,9 @@ class EventRejectedEvent:  # purposely does not subclass Event
         self.event = event
 
 
+#
+
+
 class ProcessStateEvent(Event):
     """Abstract class, never raised directly."""
     frm = None
@@ -146,19 +137,6 @@ class ProcessStateEvent(Event):
         # us, we stash the values at the time the event was sent
         self.extra_values = self.get_extra_values()
 
-    def payload(self):
-        groupname = ''
-        if self.process.group is not None:
-            groupname = self.process.group.config.name
-        l = [
-            ('processname', self.process.config.name),
-            ('groupname', groupname),
-            ('from_state', get_process_state_description(self.from_state)),
-        ]
-        l.extend(self.extra_values)
-        s = ' '.join([f'{name}:{val}' for name, val in l])
-        return s
-
     def get_extra_values(self):
         return []
 
@@ -204,14 +182,26 @@ class ProcessStateStoppedEvent(ProcessStateEvent):
         return [('pid', self.process.pid)]
 
 
+PROCESS_STATE_EVENT_MAP: ta.Mapping[ProcessState, ta.Type[ProcessStateEvent]] = {
+    ProcessState.BACKOFF: ProcessStateBackoffEvent,
+    ProcessState.FATAL: ProcessStateFatalEvent,
+    ProcessState.UNKNOWN: ProcessStateUnknownEvent,
+    ProcessState.STOPPED: ProcessStateStoppedEvent,
+    ProcessState.EXITED: ProcessStateExitedEvent,
+    ProcessState.RUNNING: ProcessStateRunningEvent,
+    ProcessState.STARTING: ProcessStateStartingEvent,
+    ProcessState.STOPPING: ProcessStateStoppingEvent,
+}
+
+
+#
+
+
 class ProcessGroupEvent(Event):
     def __init__(self, group):
         super().__init__()
         self.group = group
 
-    def payload(self):
-        return f'groupname:{self.group}\n'
-
 
 class ProcessGroupAddedEvent(ProcessGroupEvent):
     pass
@@ -221,6 +211,9 @@ class ProcessGroupRemovedEvent(ProcessGroupEvent):
     pass
 
 
+#
+
+
 class TickEvent(Event):
     """Abstract."""
 
@@ -229,9 +222,6 @@ class TickEvent(Event):
         self.when = when
         self.supervisord = supervisord
 
-    def payload(self):
-        return f'when:{self.when}'
-
 
 class Tick5Event(TickEvent):
     period = 5
@@ -245,11 +235,14 @@ class Tick3600Event(TickEvent):
     period = 3600
 
 
-TICK_EVENTS = [  # imported elsewhere
+TICK_EVENTS = (  # imported elsewhere
     Tick5Event,
     Tick60Event,
     Tick3600Event,
-]
+)
+
+
+##
 
 
 class EventTypes:
@@ -294,7 +287,3 @@ def get_event_name_by_type(requested):
         if typ is requested:
             return name
     return None
-
-
-def register(name, event):
-    setattr(EventTypes, name, event)