ominfra 0.0.0.dev121__py3-none-any.whl → 0.0.0.dev123__py3-none-any.whl

ominfra/supervisor/groups.py

@@ -0,0 +1,162 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import typing as ta
+
+from .configs import ProcessConfig
+from .configs import ProcessGroupConfig
+from .context import ServerContext
+from .dispatchers import Dispatcher
+from .events import EventCallbacks
+from .events import ProcessGroupAddedEvent
+from .events import ProcessGroupRemovedEvent
+from .states import ProcessState
+from .types import AbstractProcessGroup
+from .types import AbstractServerContext
+from .types import AbstractSubprocess
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SubprocessFactory:
+    fn: ta.Callable[[ProcessConfig, AbstractProcessGroup], AbstractSubprocess]
+
+    def __call__(self, config: ProcessConfig, group: AbstractProcessGroup) -> AbstractSubprocess:
+        return self.fn(config, group)
+
+
+class ProcessGroup(AbstractProcessGroup):
+    def __init__(
+            self,
+            config: ProcessGroupConfig,
+            context: ServerContext,
+            *,
+            subprocess_factory: SubprocessFactory,
+    ):
+        super().__init__()
+
+        self._config = config
+        self._context = context
+        self._subprocess_factory = subprocess_factory
+
+        self._processes = {}
+        for pconfig in self._config.processes or []:
+            process = self._subprocess_factory(pconfig, self)
+            self._processes[pconfig.name] = process
+
+    @property
+    def config(self) -> ProcessGroupConfig:
+        return self._config
+
+    @property
+    def name(self) -> str:
+        return self._config.name
+
+    @property
+    def context(self) -> AbstractServerContext:
+        return self._context
+
+    def __repr__(self):
+        # repr can't return anything other than a native string, but the name might be unicode - a problem on Python 2.
+        name = self._config.name
+        return f'<{self.__class__.__name__} instance at {id(self)} named {name}>'
+
+    def remove_logs(self) -> None:
+        for process in self._processes.values():
+            process.remove_logs()
+
+    def reopen_logs(self) -> None:
+        for process in self._processes.values():
+            process.reopen_logs()
+
+    def stop_all(self) -> None:
+        processes = list(self._processes.values())
+        processes.sort()
+        processes.reverse()  # stop in desc priority order
+
+        for proc in processes:
+            state = proc.get_state()
+            if state == ProcessState.RUNNING:
+                # RUNNING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.STARTING:
+                # STARTING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.BACKOFF:
+                # BACKOFF -> FATAL
+                proc.give_up()
+
+    def get_unstopped_processes(self) -> ta.List[AbstractSubprocess]:
+        return [x for x in self._processes.values() if not x.get_state().stopped]
+
+    def get_dispatchers(self) -> ta.Dict[int, Dispatcher]:
+        dispatchers: dict = {}
+        for process in self._processes.values():
+            dispatchers.update(process.get_dispatchers())
+        return dispatchers
+
+    def before_remove(self) -> None:
+        pass
+
+    def transition(self) -> None:
+        for proc in self._processes.values():
+            proc.transition()
+
+    def after_setuid(self) -> None:
+        for proc in self._processes.values():
+            proc.create_auto_child_logs()
+
+
+##
+
+
+class ProcessGroups:
+    def __init__(
+            self,
+            *,
+            event_callbacks: EventCallbacks,
+    ) -> None:
+        super().__init__()
+
+        self._event_callbacks = event_callbacks
+
+        self._by_name: ta.Dict[str, ProcessGroup] = {}
+
+    def get(self, name: str) -> ta.Optional[ProcessGroup]:
+        return self._by_name.get(name)
+
+    def __getitem__(self, name: str) -> ProcessGroup:
+        return self._by_name[name]
+
+    def __len__(self) -> int:
+        return len(self._by_name)
+
+    def __iter__(self) -> ta.Iterator[ProcessGroup]:
+        return iter(self._by_name.values())
+
+    def all(self) -> ta.Mapping[str, ProcessGroup]:
+        return self._by_name
+
+    def add(self, group: ProcessGroup) -> None:
+        if (name := group.name) in self._by_name:
+            raise KeyError(f'Process group already exists: {name}')
+
+        self._by_name[name] = group
+
+        self._event_callbacks.notify(ProcessGroupAddedEvent(name))
+
+    def remove(self, name: str) -> None:
+        group = self._by_name[name]
+
+        group.before_remove()
+
+        del self._by_name[name]
+
+        self._event_callbacks.notify(ProcessGroupRemovedEvent(name))
+
+    def clear(self) -> None:
+        # FIXME: events?
+        self._by_name.clear()

ominfra/supervisor/main.py

@@ -14,22 +14,30 @@ from omlish.lite.journald import journald_log_handler_factory
 from omlish.lite.logs import configure_standard_logging
 
 from ..configs import read_config_file
-from .compat import ExitNow
-from .compat import get_open_fds
 from .configs import ProcessConfig
 from .configs import ProcessGroupConfig
 from .configs import ServerConfig
 from .configs import prepare_server_config
-from .context import InheritedFds
 from .context import ServerContext
 from .context import ServerEpoch
-from .process import ProcessGroup
+from .events import EventCallbacks
+from .groups import ProcessGroup
+from .groups import SubprocessFactory
+from .poller import Poller
+from .poller import get_poller_impl
+from .process import InheritedFds
 from .process import Subprocess
-from .process import SubprocessFactory
-from .states import SupervisorStates
+from .signals import SignalReceiver
+from .states import SupervisorState
 from .supervisor import ProcessGroupFactory
+from .supervisor import ProcessGroups
+from .supervisor import SignalHandler
 from .supervisor import Supervisor
+from .types import AbstractProcessGroup
 from .types import AbstractServerContext
+from .types import AbstractSubprocess
+from .utils import ExitNow
+from .utils import get_open_fds
 
 
 ##
@@ -44,9 +52,17 @@ def build_server_bindings(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
+        inj.bind(get_poller_impl(), key=Poller, singleton=True),
+
         inj.bind(ServerContext, singleton=True),
         inj.bind(AbstractServerContext, to_key=ServerContext),
 
+        inj.bind(EventCallbacks, singleton=True),
+
+        inj.bind(SignalReceiver, singleton=True),
+
+        inj.bind(SignalHandler, singleton=True),
+        inj.bind(ProcessGroups, singleton=True),
         inj.bind(Supervisor, singleton=True),
     ]
 
@@ -59,7 +75,7 @@ def build_server_bindings(
     lst.append(inj.bind(make_process_group_factory))
 
     def make_subprocess_factory(injector: Injector) -> SubprocessFactory:
-        def inner(process_config: ProcessConfig, group: ProcessGroup) -> Subprocess:
+        def inner(process_config: ProcessConfig, group: AbstractProcessGroup) -> AbstractSubprocess:
             return injector.inject(functools.partial(Subprocess, process_config, group))
         return SubprocessFactory(inner)
     lst.append(inj.bind(make_subprocess_factory))
@@ -105,9 +121,9 @@ def main(
 
     #
 
-    initial_fds: ta.Optional[InheritedFds] = None
+    inherited_fds: ta.Optional[InheritedFds] = None
     if args.inherit_initial_fds:
-        initial_fds = InheritedFds(get_open_fds(0x10000))
+        inherited_fds = InheritedFds(get_open_fds(0x10000))
 
     # if we hup, restart by making a new Supervisor()
     for epoch in itertools.count():
@@ -120,7 +136,7 @@ def main(
         injector = inj.create_injector(build_server_bindings(
             config,
             server_epoch=ServerEpoch(epoch),
-            inherited_fds=initial_fds,
+            inherited_fds=inherited_fds,
         ))
 
         context = injector.provide(ServerContext)
@@ -131,7 +147,7 @@ def main(
         except ExitNow:
             pass
 
-        if context.state < SupervisorStates.RESTARTING:
+        if context.state < SupervisorState.RESTARTING:
             break
 
 

ominfra/supervisor/poller.py

@@ -8,8 +8,7 @@ import typing as ta
 from omlish.lite.logs import log
 
 
-class BasePoller(abc.ABC):
-
+class Poller(abc.ABC):
     def __init__(self) -> None:
         super().__init__()
 
@@ -43,35 +42,34 @@ class BasePoller(abc.ABC):
         pass
 
 
-class SelectPoller(BasePoller):
-
+class SelectPoller(Poller):
     def __init__(self) -> None:
         super().__init__()
 
-        self._readables: ta.Set[int] = set()
-        self._writables: ta.Set[int] = set()
+        self._readable: ta.Set[int] = set()
+        self._writable: ta.Set[int] = set()
 
     def register_readable(self, fd: int) -> None:
-        self._readables.add(fd)
+        self._readable.add(fd)
 
     def register_writable(self, fd: int) -> None:
-        self._writables.add(fd)
+        self._writable.add(fd)
 
     def unregister_readable(self, fd: int) -> None:
-        self._readables.discard(fd)
+        self._readable.discard(fd)
 
     def unregister_writable(self, fd: int) -> None:
-        self._writables.discard(fd)
+        self._writable.discard(fd)
 
     def unregister_all(self) -> None:
-        self._readables.clear()
-        self._writables.clear()
+        self._readable.clear()
+        self._writable.clear()
 
     def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
         try:
             r, w, x = select.select(
-                self._readables,
-                self._writables,
+                self._readable,
+                self._writable,
                 [], timeout,
             )
         except OSError as err:
@@ -86,7 +84,7 @@ class SelectPoller(BasePoller):
         return r, w
 
 
-class PollPoller(BasePoller):
+class PollPoller(Poller):
     _READ = select.POLLIN | select.POLLPRI | select.POLLHUP
     _WRITE = select.POLLOUT
 
@@ -94,40 +92,40 @@ class PollPoller(BasePoller):
         super().__init__()
 
         self._poller = select.poll()
-        self._readables: set[int] = set()
-        self._writables: set[int] = set()
+        self._readable: set[int] = set()
+        self._writable: set[int] = set()
 
     def register_readable(self, fd: int) -> None:
         self._poller.register(fd, self._READ)
-        self._readables.add(fd)
+        self._readable.add(fd)
 
     def register_writable(self, fd: int) -> None:
         self._poller.register(fd, self._WRITE)
-        self._writables.add(fd)
+        self._writable.add(fd)
 
     def unregister_readable(self, fd: int) -> None:
-        self._readables.discard(fd)
+        self._readable.discard(fd)
         self._poller.unregister(fd)
-        if fd in self._writables:
+        if fd in self._writable:
             self._poller.register(fd, self._WRITE)
 
     def unregister_writable(self, fd: int) -> None:
-        self._writables.discard(fd)
+        self._writable.discard(fd)
         self._poller.unregister(fd)
-        if fd in self._readables:
+        if fd in self._readable:
             self._poller.register(fd, self._READ)
 
     def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
         fds = self._poll_fds(timeout)  # type: ignore
-        readables, writables = [], []
+        readable, writable = [], []
         for fd, eventmask in fds:
             if self._ignore_invalid(fd, eventmask):
                 continue
             if eventmask & self._READ:
-                readables.append(fd)
+                readable.append(fd)
             if eventmask & self._WRITE:
-                writables.append(fd)
-        return readables, writables
+                writable.append(fd)
+        return readable, writable
 
     def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
         try:
@@ -143,41 +141,41 @@ class PollPoller(BasePoller):
             # POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
             # more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
             self._poller.unregister(fd)
-            self._readables.discard(fd)
-            self._writables.discard(fd)
+            self._readable.discard(fd)
+            self._writable.discard(fd)
             return True
         return False
 
 
 if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
-    class KqueuePoller(BasePoller):
+    class KqueuePoller(Poller):
         max_events = 1000
 
         def __init__(self) -> None:
             super().__init__()
 
             self._kqueue: ta.Optional[ta.Any] = select.kqueue()
-            self._readables: set[int] = set()
-            self._writables: set[int] = set()
+            self._readable: set[int] = set()
+            self._writable: set[int] = set()
 
         def register_readable(self, fd: int) -> None:
-            self._readables.add(fd)
+            self._readable.add(fd)
             kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
             self._kqueue_control(fd, kevent)
 
         def register_writable(self, fd: int) -> None:
-            self._writables.add(fd)
+            self._writable.add(fd)
             kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
             self._kqueue_control(fd, kevent)
 
         def unregister_readable(self, fd: int) -> None:
             kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
-            self._readables.discard(fd)
+            self._readable.discard(fd)
             self._kqueue_control(fd, kevent)
 
         def unregister_writable(self, fd: int) -> None:
             kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
-            self._writables.discard(fd)
+            self._writable.discard(fd)
             self._kqueue_control(fd, kevent)
 
         def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
@@ -190,32 +188,32 @@ if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
                     raise
 
         def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
-            readables, writables = [], []  # type: ignore
+            readable, writable = [], []  # type: ignore
 
             try:
                 kevents = self._kqueue.control(None, self.max_events, timeout)  # type: ignore
             except OSError as error:
                 if error.errno == errno.EINTR:
                     log.debug('EINTR encountered in poll')
-                    return readables, writables
+                    return readable, writable
                 raise
 
             for kevent in kevents:
                 if kevent.filter == select.KQ_FILTER_READ:
-                    readables.append(kevent.ident)
+                    readable.append(kevent.ident)
                 if kevent.filter == select.KQ_FILTER_WRITE:
-                    writables.append(kevent.ident)
+                    writable.append(kevent.ident)
 
-            return readables, writables
+            return readable, writable
 
         def before_daemonize(self) -> None:
             self.close()
 
         def after_daemonize(self) -> None:
             self._kqueue = select.kqueue()
-            for fd in self._readables:
+            for fd in self._readable:
                 self.register_readable(fd)
-            for fd in self._writables:
+            for fd in self._writable:
                 self.register_writable(fd)
 
         def close(self) -> None:
@@ -226,13 +224,13 @@ else:
     KqueuePoller = None
 
 
-Poller: ta.Type[BasePoller]
-if (
-        sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
-        hasattr(select, 'kqueue') and KqueuePoller is not None
-):
-    Poller = KqueuePoller
-elif hasattr(select, 'poll'):
-    Poller = PollPoller
-else:
-    Poller = SelectPoller
+def get_poller_impl() -> ta.Type[Poller]:
+    if (
+            sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
+            hasattr(select, 'kqueue') and KqueuePoller is not None
+    ):
+        return KqueuePoller
+    elif hasattr(select, 'poll'):
+        return PollPoller
+    else:
+        return SelectPoller