ominfra 0.0.0.dev119__py3-none-any.whl → 0.0.0.dev121__py3-none-any.whl

ominfra/supervisor/compat.py

@@ -161,6 +161,19 @@ def close_fd(fd: int) -> bool:
     return True
 
 
+def is_fd_open(fd: int) -> bool:
+    try:
+        n = os.dup(fd)
+    except OSError:
+        return False
+    os.close(n)
+    return True
+
+
+def get_open_fds(limit: int) -> ta.FrozenSet[int]:
+    return frozenset(filter(is_fd_open, range(limit)))
+
+
 def mktempfile(suffix: str, prefix: str, dir: str) -> str:  # noqa
     fd, filename = tempfile.mkstemp(suffix, prefix, dir)
     os.close(fd)

ominfra/supervisor/configs.py

@@ -5,6 +5,8 @@ import signal
 import tempfile
 import typing as ta
 
+from ..configs import ConfigMapping
+from ..configs import build_config_named_children
 from .datatypes import byte_size
 from .datatypes import existing_directory
 from .datatypes import existing_dirpath
@@ -12,6 +14,9 @@ from .datatypes import logging_level
 from .datatypes import octal_type
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class ProcessConfig:
     name: str
@@ -108,3 +113,19 @@ class ServerConfig:
             child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
             **kwargs,
         )
+
+
+##
+
+
+def prepare_process_group_config(dct: ConfigMapping) -> ConfigMapping:
+    out = dict(dct)
+    out['processes'] = build_config_named_children(out.get('processes'))
+    return out
+
+
+def prepare_server_config(dct: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.Any]:
+    out = dict(dct)
+    group_dcts = build_config_named_children(out.get('groups'))
+    out['groups'] = [prepare_process_group_config(group_dct) for group_dct in group_dcts or []]
+    return out

ominfra/supervisor/context.py

@@ -32,17 +32,24 @@ from .types import AbstractServerContext
 from .types import AbstractSubprocess
 
 
+ServerEpoch = ta.NewType('ServerEpoch', int)
+
+InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[int])
+
+
 class ServerContext(AbstractServerContext):
     def __init__(
             self,
             config: ServerConfig,
             *,
-            epoch: int = 0,
+            epoch: ServerEpoch = ServerEpoch(0),
+            inherited_fds: ta.Optional[InheritedFds] = None,
     ) -> None:
         super().__init__()
 
         self._config = config
         self._epoch = epoch
+        self._inherited_fds = InheritedFds(frozenset(inherited_fds or []))
 
         self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
         self._state: SupervisorState = SupervisorStates.RUNNING
@@ -66,7 +73,7 @@ class ServerContext(AbstractServerContext):
         return self._config
 
     @property
-    def epoch(self) -> int:
+    def epoch(self) -> ServerEpoch:
         return self._epoch
 
     @property
@@ -96,6 +103,10 @@ class ServerContext(AbstractServerContext):
     def gid(self) -> ta.Optional[int]:
         return self._gid
 
+    @property
+    def inherited_fds(self) -> InheritedFds:
+        return self._inherited_fds
+
     ##
 
     def set_signals(self) -> None:

ominfra/supervisor/dispatchers.py

@@ -13,9 +13,9 @@ from .compat import find_prefix_at_end
 from .compat import readfd
 from .compat import strip_escapes
 from .configs import ProcessConfig
+from .events import EVENT_CALLBACKS
 from .events import ProcessLogStderrEvent
 from .events import ProcessLogStdoutEvent
-from .events import notify_event
 from .types import AbstractSubprocess
 
 
@@ -207,10 +207,10 @@ class OutputDispatcher(Dispatcher):
 
             if self._channel == 'stdout':
                 if self._stdout_events_enabled:
-                    notify_event(ProcessLogStdoutEvent(self._process, self._process.pid, data))
+                    EVENT_CALLBACKS.notify(ProcessLogStdoutEvent(self._process, self._process.pid, data))
 
             elif self._stderr_events_enabled:
-                notify_event(ProcessLogStderrEvent(self._process, self._process.pid, data))
+                EVENT_CALLBACKS.notify(ProcessLogStderrEvent(self._process, self._process.pid, data))
 
     def record_output(self):
         if self._capture_log is None:
@@ -261,7 +261,7 @@ class OutputDispatcher(Dispatcher):
                 channel = self._channel
                 procname = self._process.config.name
                 event = self.event_type(self._process, self._process.pid, data)
-                notify_event(event)
+                EVENT_CALLBACKS.notify(event)
 
                 log.debug('%r %s emitted a comm event', procname, channel)
                 for handler in self._capture_log.handlers:

ominfra/supervisor/events.py

@@ -29,12 +29,9 @@ class EventCallbacks:
 
 EVENT_CALLBACKS = EventCallbacks()
 
-notify_event = EVENT_CALLBACKS.notify
-clear_events = EVENT_CALLBACKS.clear
-
 
 class Event(abc.ABC):  # noqa
-    """Abstract event type """
+    """Abstract event type."""
 
 
 class ProcessLogEvent(Event, abc.ABC):
@@ -114,7 +111,7 @@ class RemoteCommunicationEvent(Event):
 
 
 class SupervisorStateChangeEvent(Event):
-    """ Abstract class """
+    """Abstract class."""
 
     def payload(self):
         return ''
@@ -136,7 +133,7 @@ class EventRejectedEvent:  # purposely does not subclass Event
 
 
 class ProcessStateEvent(Event):
-    """ Abstract class, never raised directly """
+    """Abstract class, never raised directly."""
     frm = None
     to = None
 
@@ -225,7 +222,7 @@ class ProcessGroupRemovedEvent(ProcessGroupEvent):
 
 
 class TickEvent(Event):
-    """ Abstract """
+    """Abstract."""
 
     def __init__(self, when, supervisord):
         super().__init__()

ominfra/supervisor/main.py

@@ -1,19 +1,82 @@
 #!/usr/bin/env python3
 # ruff: noqa: UP006 UP007
 # @omlish-amalg ../scripts/supervisor.py
+import functools
 import itertools
-import json
+import os.path
 import typing as ta
 
+from omlish.lite.inject import Injector
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
 from omlish.lite.journald import journald_log_handler_factory
 from omlish.lite.logs import configure_standard_logging
-from omlish.lite.marshal import unmarshal_obj
 
+from ..configs import read_config_file
 from .compat import ExitNow
+from .compat import get_open_fds
+from .configs import ProcessConfig
+from .configs import ProcessGroupConfig
 from .configs import ServerConfig
+from .configs import prepare_server_config
+from .context import InheritedFds
 from .context import ServerContext
+from .context import ServerEpoch
+from .process import ProcessGroup
+from .process import Subprocess
+from .process import SubprocessFactory
 from .states import SupervisorStates
+from .supervisor import ProcessGroupFactory
 from .supervisor import Supervisor
+from .types import AbstractServerContext
+
+
+##
+
+
+def build_server_bindings(
+        config: ServerConfig,
+        *,
+        server_epoch: ta.Optional[ServerEpoch] = None,
+        inherited_fds: ta.Optional[InheritedFds] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(config),
+
+        inj.bind(ServerContext, singleton=True),
+        inj.bind(AbstractServerContext, to_key=ServerContext),
+
+        inj.bind(Supervisor, singleton=True),
+    ]
+
+    #
+
+    def make_process_group_factory(injector: Injector) -> ProcessGroupFactory:
+        def inner(group_config: ProcessGroupConfig) -> ProcessGroup:
+            return injector.inject(functools.partial(ProcessGroup, group_config))
+        return ProcessGroupFactory(inner)
+    lst.append(inj.bind(make_process_group_factory))
+
+    def make_subprocess_factory(injector: Injector) -> SubprocessFactory:
+        def inner(process_config: ProcessConfig, group: ProcessGroup) -> Subprocess:
+            return injector.inject(functools.partial(Subprocess, process_config, group))
+        return SubprocessFactory(inner)
+    lst.append(inj.bind(make_subprocess_factory))
+
+    #
+
+    if server_epoch is not None:
+        lst.append(inj.bind(server_epoch, key=ServerEpoch))
+    if inherited_fds is not None:
+        lst.append(inj.bind(inherited_fds, key=InheritedFds))
+
+    #
+
+    return inj.as_bindings(*lst)
+
+
+##
 
 
 def main(
@@ -26,6 +89,7 @@ def main(
     parser = argparse.ArgumentParser()
     parser.add_argument('config_file', metavar='config-file')
     parser.add_argument('--no-journald', action='store_true')
+    parser.add_argument('--inherit-initial-fds', action='store_true')
     args = parser.parse_args(argv)
 
     #
@@ -41,20 +105,27 @@ def main(
 
     #
 
+    initial_fds: ta.Optional[InheritedFds] = None
+    if args.inherit_initial_fds:
+        initial_fds = InheritedFds(get_open_fds(0x10000))
+
     # if we hup, restart by making a new Supervisor()
     for epoch in itertools.count():
-        with open(cf) as f:
-            config_src = f.read()
-
-        config_dct = json.loads(config_src)
-        config: ServerConfig = unmarshal_obj(config_dct, ServerConfig)
+        config = read_config_file(
+            os.path.expanduser(cf),
+            ServerConfig,
+            prepare=prepare_server_config,
+        )
 
-        context = ServerContext(
+        injector = inj.create_injector(build_server_bindings(
             config,
-            epoch=epoch,
-        )
+            server_epoch=ServerEpoch(epoch),
+            inherited_fds=initial_fds,
+        ))
+
+        context = injector.provide(ServerContext)
+        supervisor = injector.provide(Supervisor)
 
-        supervisor = Supervisor(context)
         try:
             supervisor.main()
         except ExitNow:

ominfra/supervisor/process.py

@@ -1,7 +1,8 @@
 # ruff: noqa: UP006 UP007
+import dataclasses as dc
 import errno
 import functools
-import os
+import os.path
 import shlex
 import signal
 import time
@@ -30,6 +31,7 @@ from .datatypes import RestartUnconditionally
 from .dispatchers import Dispatcher
 from .dispatchers import InputDispatcher
 from .dispatchers import OutputDispatcher
+from .events import EVENT_CALLBACKS
 from .events import EventRejectedEvent
 from .events import ProcessCommunicationEvent
 from .events import ProcessCommunicationStderrEvent
@@ -43,7 +45,6 @@ from .events import ProcessStateStartingEvent
 from .events import ProcessStateStoppedEvent
 from .events import ProcessStateStoppingEvent
 from .events import ProcessStateUnknownEvent
-from .events import notify_event
 from .exceptions import BadCommandError
 from .exceptions import ProcessError
 from .states import STOPPED_STATES
@@ -55,6 +56,9 @@ from .types import AbstractServerContext
 from .types import AbstractSubprocess
 
 
+##
+
+
 @functools.total_ordering
 class Subprocess(AbstractSubprocess):
     """A class to manage a subprocess."""
@@ -80,7 +84,12 @@ class Subprocess(AbstractSubprocess):
     spawn_err = None  # error message attached by spawn() if any
     group = None  # ProcessGroup instance if process is in the group
 
-    def __init__(self, config: ProcessConfig, group: 'ProcessGroup', context: AbstractServerContext) -> None:
+    def __init__(
+            self,
+            config: ProcessConfig,
+            group: 'ProcessGroup',
+            context: AbstractServerContext,
+    ) -> None:
         super().__init__()
         self._config = config
         self.group = group
@@ -207,7 +216,7 @@ class Subprocess(AbstractSubprocess):
         event_class = self.event_map.get(new_state)
         if event_class is not None:
             event = event_class(self, old_state, expected)
-            notify_event(event)
+            EVENT_CALLBACKS.notify(event)
 
         return True
 
@@ -324,7 +333,10 @@ class Subprocess(AbstractSubprocess):
             os.dup2(self._pipes['child_stdout'], 2)
         else:
             os.dup2(self._pipes['child_stderr'], 2)
+
         for i in range(3, self.context.config.minfds):
+            if i in self.context.inherited_fds:
+                continue
             close_fd(i)
 
     def _spawn_as_child(self, filename: str, argv: ta.Sequence[str]) -> None:
@@ -359,7 +371,7 @@ class Subprocess(AbstractSubprocess):
             cwd = self.config.directory
             try:
                 if cwd is not None:
-                    os.chdir(cwd)
+                    os.chdir(os.path.expanduser(cwd))
             except OSError as why:
                 code = errno.errorcode.get(why.args[0], why.args[0])
                 msg = f"couldn't chdir to {cwd}: {code}\n"
@@ -415,7 +427,7 @@ class Subprocess(AbstractSubprocess):
         return self.kill(self.config.stopsignal)
 
     def stop_report(self) -> None:
-        """ Log a 'waiting for x to stop' message with throttling. """
+        """Log a 'waiting for x to stop' message with throttling."""
         if self.state == ProcessStates.STOPPING:
             now = time.time()
 
@@ -545,7 +557,7 @@ class Subprocess(AbstractSubprocess):
         return None
 
     def finish(self, sts: int) -> None:
-        """ The process was reaped and we need to report and manage its state """
+        """The process was reaped and we need to report and manage its state."""
 
         self.drain()
 
@@ -626,7 +638,7 @@ class Subprocess(AbstractSubprocess):
         # system that this event was rejected so it can be processed again.
         if self.event is not None:
             # Note: this should only be true if we were in the BUSY state when finish() was called.
-            notify_event(EventRejectedEvent(self, self.event))  # type: ignore
+            EVENT_CALLBACKS.notify(EventRejectedEvent(self, self.event))  # type: ignore
             self.event = None
 
     def set_uid(self) -> ta.Optional[str]:
@@ -718,15 +730,39 @@ class Subprocess(AbstractSubprocess):
         pass
 
 
+##
+
+
+@dc.dataclass(frozen=True)
+class SubprocessFactory:
+    fn: ta.Callable[[ProcessConfig, 'ProcessGroup'], Subprocess]
+
+    def __call__(self, config: ProcessConfig, group: 'ProcessGroup') -> Subprocess:
+        return self.fn(config, group)
+
+
 @functools.total_ordering
 class ProcessGroup:
-    def __init__(self, config: ProcessGroupConfig, context: ServerContext):
+    def __init__(
+            self,
+            config: ProcessGroupConfig,
+            context: ServerContext,
+            *,
+            subprocess_factory: ta.Optional[SubprocessFactory] = None,
+    ):
         super().__init__()
         self.config = config
         self.context = context
+
+        if subprocess_factory is None:
+            def make_subprocess(config: ProcessConfig, group: ProcessGroup) -> Subprocess:
+                return Subprocess(config, group, self.context)
+            subprocess_factory = SubprocessFactory(make_subprocess)
+        self._subprocess_factory = subprocess_factory
+
         self.processes = {}
         for pconfig in self.config.processes or []:
-            process = Subprocess(pconfig, self, self.context)
+            process = self._subprocess_factory(pconfig, self)
             self.processes[pconfig.name] = process
 
     def __lt__(self, other):