ominfra 0.0.0.dev122__py3-none-any.whl → 0.0.0.dev124__py3-none-any.whl

ominfra/supervisor/groups.py

@@ -0,0 +1,162 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import typing as ta
+
+from .configs import ProcessConfig
+from .configs import ProcessGroupConfig
+from .context import ServerContext
+from .dispatchers import Dispatcher
+from .events import EventCallbacks
+from .events import ProcessGroupAddedEvent
+from .events import ProcessGroupRemovedEvent
+from .states import ProcessState
+from .types import AbstractProcessGroup
+from .types import AbstractServerContext
+from .types import AbstractSubprocess
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SubprocessFactory:
+    fn: ta.Callable[[ProcessConfig, AbstractProcessGroup], AbstractSubprocess]
+
+    def __call__(self, config: ProcessConfig, group: AbstractProcessGroup) -> AbstractSubprocess:
+        return self.fn(config, group)
+
+
+class ProcessGroup(AbstractProcessGroup):
+    def __init__(
+            self,
+            config: ProcessGroupConfig,
+            context: ServerContext,
+            *,
+            subprocess_factory: SubprocessFactory,
+    ):
+        super().__init__()
+
+        self._config = config
+        self._context = context
+        self._subprocess_factory = subprocess_factory
+
+        self._processes = {}
+        for pconfig in self._config.processes or []:
+            process = self._subprocess_factory(pconfig, self)
+            self._processes[pconfig.name] = process
+
+    @property
+    def config(self) -> ProcessGroupConfig:
+        return self._config
+
+    @property
+    def name(self) -> str:
+        return self._config.name
+
+    @property
+    def context(self) -> AbstractServerContext:
+        return self._context
+
+    def __repr__(self):
+        # repr can't return anything other than a native string, but the name might be unicode - a problem on Python 2.
+        name = self._config.name
+        return f'<{self.__class__.__name__} instance at {id(self)} named {name}>'
+
+    def remove_logs(self) -> None:
+        for process in self._processes.values():
+            process.remove_logs()
+
+    def reopen_logs(self) -> None:
+        for process in self._processes.values():
+            process.reopen_logs()
+
+    def stop_all(self) -> None:
+        processes = list(self._processes.values())
+        processes.sort()
+        processes.reverse()  # stop in desc priority order
+
+        for proc in processes:
+            state = proc.get_state()
+            if state == ProcessState.RUNNING:
+                # RUNNING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.STARTING:
+                # STARTING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.BACKOFF:
+                # BACKOFF -> FATAL
+                proc.give_up()
+
+    def get_unstopped_processes(self) -> ta.List[AbstractSubprocess]:
+        return [x for x in self._processes.values() if not x.get_state().stopped]
+
+    def get_dispatchers(self) -> ta.Dict[int, Dispatcher]:
+        dispatchers: dict = {}
+        for process in self._processes.values():
+            dispatchers.update(process.get_dispatchers())
+        return dispatchers
+
+    def before_remove(self) -> None:
+        pass
+
+    def transition(self) -> None:
+        for proc in self._processes.values():
+            proc.transition()
+
+    def after_setuid(self) -> None:
+        for proc in self._processes.values():
+            proc.create_auto_child_logs()
+
+
+##
+
+
+class ProcessGroups:
+    def __init__(
+            self,
+            *,
+            event_callbacks: EventCallbacks,
+    ) -> None:
+        super().__init__()
+
+        self._event_callbacks = event_callbacks
+
+        self._by_name: ta.Dict[str, ProcessGroup] = {}
+
+    def get(self, name: str) -> ta.Optional[ProcessGroup]:
+        return self._by_name.get(name)
+
+    def __getitem__(self, name: str) -> ProcessGroup:
+        return self._by_name[name]
+
+    def __len__(self) -> int:
+        return len(self._by_name)
+
+    def __iter__(self) -> ta.Iterator[ProcessGroup]:
+        return iter(self._by_name.values())
+
+    def all(self) -> ta.Mapping[str, ProcessGroup]:
+        return self._by_name
+
+    def add(self, group: ProcessGroup) -> None:
+        if (name := group.name) in self._by_name:
+            raise KeyError(f'Process group already exists: {name}')
+
+        self._by_name[name] = group
+
+        self._event_callbacks.notify(ProcessGroupAddedEvent(name))
+
+    def remove(self, name: str) -> None:
+        group = self._by_name[name]
+
+        group.before_remove()
+
+        del self._by_name[name]
+
+        self._event_callbacks.notify(ProcessGroupRemovedEvent(name))
+
+    def clear(self) -> None:
+        # FIXME: events?
+        self._by_name.clear()

ominfra/supervisor/inject.py

@@ -0,0 +1,81 @@
+# ruff: noqa: UP006 UP007
+import functools
+import typing as ta
+
+from omlish.lite.inject import Injector
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from .configs import ProcessConfig
+from .configs import ProcessGroupConfig
+from .configs import ServerConfig
+from .context import ServerContext
+from .context import ServerEpoch
+from .events import EventCallbacks
+from .groups import ProcessGroup
+from .groups import SubprocessFactory
+from .poller import Poller
+from .poller import get_poller_impl
+from .process import InheritedFds
+from .process import Subprocess
+from .signals import SignalReceiver
+from .supervisor import ProcessGroupFactory
+from .supervisor import ProcessGroups
+from .supervisor import SignalHandler
+from .supervisor import Supervisor
+from .types import AbstractProcessGroup
+from .types import AbstractServerContext
+from .types import AbstractSubprocess
+
+
+##
+
+
+def bind_server(
+        config: ServerConfig,
+        *,
+        server_epoch: ta.Optional[ServerEpoch] = None,
+        inherited_fds: ta.Optional[InheritedFds] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(config),
+
+        inj.bind(get_poller_impl(), key=Poller, singleton=True),
+
+        inj.bind(ServerContext, singleton=True),
+        inj.bind(AbstractServerContext, to_key=ServerContext),
+
+        inj.bind(EventCallbacks, singleton=True),
+
+        inj.bind(SignalReceiver, singleton=True),
+
+        inj.bind(SignalHandler, singleton=True),
+        inj.bind(ProcessGroups, singleton=True),
+        inj.bind(Supervisor, singleton=True),
+    ]
+
+    #
+
+    def make_process_group_factory(injector: Injector) -> ProcessGroupFactory:
+        def inner(group_config: ProcessGroupConfig) -> ProcessGroup:
+            return injector.inject(functools.partial(ProcessGroup, group_config))
+        return ProcessGroupFactory(inner)
+    lst.append(inj.bind(make_process_group_factory))
+
+    def make_subprocess_factory(injector: Injector) -> SubprocessFactory:
+        def inner(process_config: ProcessConfig, group: AbstractProcessGroup) -> AbstractSubprocess:
+            return injector.inject(functools.partial(Subprocess, process_config, group))
+        return SubprocessFactory(inner)
+    lst.append(inj.bind(make_subprocess_factory))
+
+    #
+
+    if server_epoch is not None:
+        lst.append(inj.bind(server_epoch, key=ServerEpoch))
+    if inherited_fds is not None:
+        lst.append(inj.bind(inherited_fds, key=InheritedFds))
+
+    #
+
+    return inj.as_bindings(*lst)

ominfra/supervisor/main.py

@@ -1,79 +1,26 @@
 #!/usr/bin/env python3
 # ruff: noqa: UP006 UP007
 # @omlish-amalg ../scripts/supervisor.py
-import functools
 import itertools
 import os.path
 import typing as ta
 
-from omlish.lite.inject import Injector
-from omlish.lite.inject import InjectorBindingOrBindings
-from omlish.lite.inject import InjectorBindings
+from omlish.lite.http.coroserver import CoroHttpServer
 from omlish.lite.inject import inj
 from omlish.lite.journald import journald_log_handler_factory
 from omlish.lite.logs import configure_standard_logging
 
 from ..configs import read_config_file
-from .compat import ExitNow
-from .compat import get_open_fds
-from .configs import ProcessConfig
-from .configs import ProcessGroupConfig
 from .configs import ServerConfig
 from .configs import prepare_server_config
-from .context import InheritedFds
 from .context import ServerContext
 from .context import ServerEpoch
-from .process import ProcessGroup
-from .process import Subprocess
-from .process import SubprocessFactory
-from .states import SupervisorStates
-from .supervisor import ProcessGroupFactory
+from .inject import bind_server
+from .process import InheritedFds
+from .states import SupervisorState
 from .supervisor import Supervisor
-from .types import AbstractServerContext
-
-
-##
-
-
-def build_server_bindings(
-        config: ServerConfig,
-        *,
-        server_epoch: ta.Optional[ServerEpoch] = None,
-        inherited_fds: ta.Optional[InheritedFds] = None,
-) -> InjectorBindings:
-    lst: ta.List[InjectorBindingOrBindings] = [
-        inj.bind(config),
-
-        inj.bind(ServerContext, singleton=True),
-        inj.bind(AbstractServerContext, to_key=ServerContext),
-
-        inj.bind(Supervisor, singleton=True),
-    ]
-
-    #
-
-    def make_process_group_factory(injector: Injector) -> ProcessGroupFactory:
-        def inner(group_config: ProcessGroupConfig) -> ProcessGroup:
-            return injector.inject(functools.partial(ProcessGroup, group_config))
-        return ProcessGroupFactory(inner)
-    lst.append(inj.bind(make_process_group_factory))
-
-    def make_subprocess_factory(injector: Injector) -> SubprocessFactory:
-        def inner(process_config: ProcessConfig, group: ProcessGroup) -> Subprocess:
-            return injector.inject(functools.partial(Subprocess, process_config, group))
-        return SubprocessFactory(inner)
-    lst.append(inj.bind(make_subprocess_factory))
-
-    #
-
-    if server_epoch is not None:
-        lst.append(inj.bind(server_epoch, key=ServerEpoch))
-    if inherited_fds is not None:
-        lst.append(inj.bind(inherited_fds, key=InheritedFds))
-
-    #
-
-    return inj.as_bindings(*lst)
+from .utils import ExitNow
+from .utils import get_open_fds
 
 
 ##
@@ -84,6 +31,10 @@ def main(
         *,
         no_logging: bool = False,
 ) -> None:
+    server_cls = CoroHttpServer  # noqa
+
+    #
+
     import argparse
 
     parser = argparse.ArgumentParser()
@@ -105,9 +56,9 @@ def main(
 
     #
 
-    initial_fds: ta.Optional[InheritedFds] = None
+    inherited_fds: ta.Optional[InheritedFds] = None
     if args.inherit_initial_fds:
-        initial_fds = InheritedFds(get_open_fds(0x10000))
+        inherited_fds = InheritedFds(get_open_fds(0x10000))
 
     # if we hup, restart by making a new Supervisor()
     for epoch in itertools.count():
@@ -117,21 +68,21 @@ def main(
             prepare=prepare_server_config,
         )
 
-        injector = inj.create_injector(build_server_bindings(
+        injector = inj.create_injector(bind_server(
             config,
             server_epoch=ServerEpoch(epoch),
-            inherited_fds=initial_fds,
+            inherited_fds=inherited_fds,
         ))
 
-        context = injector.provide(ServerContext)
-        supervisor = injector.provide(Supervisor)
+        context = injector[ServerContext]
+        supervisor = injector[Supervisor]
 
         try:
             supervisor.main()
         except ExitNow:
             pass
 
-        if context.state < SupervisorStates.RESTARTING:
+        if context.state < SupervisorState.RESTARTING:
             break
 
 

ominfra/supervisor/poller.py

@@ -8,8 +8,7 @@ import typing as ta
 from omlish.lite.logs import log
 
 
-class BasePoller(abc.ABC):
-
+class Poller(abc.ABC):
     def __init__(self) -> None:
         super().__init__()
 
@@ -43,35 +42,34 @@ class BasePoller(abc.ABC):
         pass
 
 
-class SelectPoller(BasePoller):
-
+class SelectPoller(Poller):
     def __init__(self) -> None:
         super().__init__()
 
-        self._readables: ta.Set[int] = set()
-        self._writables: ta.Set[int] = set()
+        self._readable: ta.Set[int] = set()
+        self._writable: ta.Set[int] = set()
 
     def register_readable(self, fd: int) -> None:
-        self._readables.add(fd)
+        self._readable.add(fd)
 
     def register_writable(self, fd: int) -> None:
-        self._writables.add(fd)
+        self._writable.add(fd)
 
     def unregister_readable(self, fd: int) -> None:
-        self._readables.discard(fd)
+        self._readable.discard(fd)
 
     def unregister_writable(self, fd: int) -> None:
-        self._writables.discard(fd)
+        self._writable.discard(fd)
 
     def unregister_all(self) -> None:
-        self._readables.clear()
-        self._writables.clear()
+        self._readable.clear()
+        self._writable.clear()
 
     def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
         try:
             r, w, x = select.select(
-                self._readables,
-                self._writables,
+                self._readable,
+                self._writable,
                 [], timeout,
             )
         except OSError as err:
@@ -86,7 +84,7 @@ class SelectPoller(BasePoller):
         return r, w
 
 
-class PollPoller(BasePoller):
+class PollPoller(Poller):
     _READ = select.POLLIN | select.POLLPRI | select.POLLHUP
     _WRITE = select.POLLOUT
 
@@ -94,40 +92,40 @@ class PollPoller(BasePoller):
         super().__init__()
 
         self._poller = select.poll()
-        self._readables: set[int] = set()
-        self._writables: set[int] = set()
+        self._readable: set[int] = set()
+        self._writable: set[int] = set()
 
     def register_readable(self, fd: int) -> None:
         self._poller.register(fd, self._READ)
-        self._readables.add(fd)
+        self._readable.add(fd)
 
     def register_writable(self, fd: int) -> None:
         self._poller.register(fd, self._WRITE)
-        self._writables.add(fd)
+        self._writable.add(fd)
 
     def unregister_readable(self, fd: int) -> None:
-        self._readables.discard(fd)
+        self._readable.discard(fd)
         self._poller.unregister(fd)
-        if fd in self._writables:
+        if fd in self._writable:
             self._poller.register(fd, self._WRITE)
 
     def unregister_writable(self, fd: int) -> None:
-        self._writables.discard(fd)
+        self._writable.discard(fd)
         self._poller.unregister(fd)
-        if fd in self._readables:
+        if fd in self._readable:
             self._poller.register(fd, self._READ)
 
     def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
         fds = self._poll_fds(timeout)  # type: ignore
-        readables, writables = [], []
+        readable, writable = [], []
         for fd, eventmask in fds:
             if self._ignore_invalid(fd, eventmask):
                 continue
             if eventmask & self._READ:
-                readables.append(fd)
+                readable.append(fd)
             if eventmask & self._WRITE:
-                writables.append(fd)
-        return readables, writables
+                writable.append(fd)
+        return readable, writable
 
     def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
         try:
@@ -143,41 +141,41 @@ class PollPoller(BasePoller):
             # POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
             # more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
             self._poller.unregister(fd)
-            self._readables.discard(fd)
-            self._writables.discard(fd)
+            self._readable.discard(fd)
+            self._writable.discard(fd)
             return True
         return False
 
 
 if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
-    class KqueuePoller(BasePoller):
+    class KqueuePoller(Poller):
         max_events = 1000
 
         def __init__(self) -> None:
             super().__init__()
 
             self._kqueue: ta.Optional[ta.Any] = select.kqueue()
-            self._readables: set[int] = set()
-            self._writables: set[int] = set()
+            self._readable: set[int] = set()
+            self._writable: set[int] = set()
 
         def register_readable(self, fd: int) -> None:
-            self._readables.add(fd)
+            self._readable.add(fd)
             kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
             self._kqueue_control(fd, kevent)
 
         def register_writable(self, fd: int) -> None:
-            self._writables.add(fd)
+            self._writable.add(fd)
             kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
             self._kqueue_control(fd, kevent)
 
         def unregister_readable(self, fd: int) -> None:
             kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
-            self._readables.discard(fd)
+            self._readable.discard(fd)
             self._kqueue_control(fd, kevent)
 
         def unregister_writable(self, fd: int) -> None:
             kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
-            self._writables.discard(fd)
+            self._writable.discard(fd)
             self._kqueue_control(fd, kevent)
 
         def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
@@ -190,32 +188,32 @@ if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
                     raise
 
         def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
-            readables, writables = [], []  # type: ignore
+            readable, writable = [], []  # type: ignore
 
             try:
                 kevents = self._kqueue.control(None, self.max_events, timeout)  # type: ignore
             except OSError as error:
                 if error.errno == errno.EINTR:
                     log.debug('EINTR encountered in poll')
-                    return readables, writables
+                    return readable, writable
                 raise
 
             for kevent in kevents:
                 if kevent.filter == select.KQ_FILTER_READ:
-                    readables.append(kevent.ident)
+                    readable.append(kevent.ident)
                 if kevent.filter == select.KQ_FILTER_WRITE:
-                    writables.append(kevent.ident)
+                    writable.append(kevent.ident)
 
-            return readables, writables
+            return readable, writable
 
         def before_daemonize(self) -> None:
             self.close()
 
         def after_daemonize(self) -> None:
             self._kqueue = select.kqueue()
-            for fd in self._readables:
+            for fd in self._readable:
                 self.register_readable(fd)
-            for fd in self._writables:
+            for fd in self._writable:
                 self.register_writable(fd)
 
         def close(self) -> None:
@@ -226,13 +224,13 @@ else:
     KqueuePoller = None
 
 
-Poller: ta.Type[BasePoller]
-if (
-        sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
-        hasattr(select, 'kqueue') and KqueuePoller is not None
-):
-    Poller = KqueuePoller
-elif hasattr(select, 'poll'):
-    Poller = PollPoller
-else:
-    Poller = SelectPoller
+def get_poller_impl() -> ta.Type[Poller]:
+    if (
+            sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
+            hasattr(select, 'kqueue') and KqueuePoller is not None
+    ):
+        return KqueuePoller
+    elif hasattr(select, 'poll'):
+        return PollPoller
+    else:
+        return SelectPoller