ominfra 0.0.0.dev126__py3-none-any.whl → 0.0.0.dev128__py3-none-any.whl

ominfra/supervisor/groups.py

@@ -1,114 +1,16 @@
 # ruff: noqa: UP006 UP007
 import typing as ta
 
-from omlish.lite.check import check_isinstance
-from omlish.lite.typing import Func
-
 from .configs import ProcessGroupConfig
 from .events import EventCallbacks
 from .events import ProcessGroupAddedEvent
 from .events import ProcessGroupRemovedEvent
-from .states import ProcessState
-from .types import Dispatcher
 from .types import Process
 from .types import ProcessGroup
-from .types import ServerContext
-
-
-##
+from .utils.collections import KeyedCollectionAccessors
 
 
-ProcessFactory = ta.NewType('ProcessFactory', Func[Process])  # (config: ProcessConfig, group: ProcessGroup)
-
-
-class ProcessGroupImpl(ProcessGroup):
-    def __init__(
-            self,
-            config: ProcessGroupConfig,
-            context: ServerContext,
-            *,
-            process_factory: ProcessFactory,
-    ):
-        super().__init__()
-
-        self._config = config
-        self._context = context
-        self._process_factory = process_factory
-
-        self._processes = {}
-        for pconfig in self._config.processes or []:
-            process = check_isinstance(self._process_factory(pconfig, self), Process)
-            self._processes[pconfig.name] = process
-
-    @property
-    def config(self) -> ProcessGroupConfig:
-        return self._config
-
-    @property
-    def name(self) -> str:
-        return self._config.name
-
-    @property
-    def context(self) -> ServerContext:
-        return self._context
-
-    def __repr__(self):
-        # repr can't return anything other than a native string, but the name might be unicode - a problem on Python 2.
-        name = self._config.name
-        return f'<{self.__class__.__name__} instance at {id(self)} named {name}>'
-
-    def remove_logs(self) -> None:
-        for process in self._processes.values():
-            process.remove_logs()
-
-    def reopen_logs(self) -> None:
-        for process in self._processes.values():
-            process.reopen_logs()
-
-    def stop_all(self) -> None:
-        processes = list(self._processes.values())
-        processes.sort()
-        processes.reverse()  # stop in desc priority order
-
-        for proc in processes:
-            state = proc.get_state()
-            if state == ProcessState.RUNNING:
-                # RUNNING -> STOPPING
-                proc.stop()
-
-            elif state == ProcessState.STARTING:
-                # STARTING -> STOPPING
-                proc.stop()
-
-            elif state == ProcessState.BACKOFF:
-                # BACKOFF -> FATAL
-                proc.give_up()
-
-    def get_unstopped_processes(self) -> ta.List[Process]:
-        return [x for x in self._processes.values() if not x.get_state().stopped]
-
-    def get_dispatchers(self) -> ta.Dict[int, Dispatcher]:
-        dispatchers: dict = {}
-        for process in self._processes.values():
-            dispatchers.update(process.get_dispatchers())
-        return dispatchers
-
-    def before_remove(self) -> None:
-        pass
-
-    def transition(self) -> None:
-        for proc in self._processes.values():
-            proc.transition()
-
-    def after_setuid(self) -> None:
-        for proc in self._processes.values():
-            proc.create_auto_child_logs()
-
-
-##
-
-
-class ProcessGroups:
+class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
     def __init__(
             self,
             *,
@@ -120,20 +22,17 @@ class ProcessGroups:
 
         self._by_name: ta.Dict[str, ProcessGroup] = {}
 
-    def get(self, name: str) -> ta.Optional[ProcessGroup]:
-        return self._by_name.get(name)
+    @property
+    def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
+        return self._by_name
 
-    def __getitem__(self, name: str) -> ProcessGroup:
-        return self._by_name[name]
+    #
 
-    def __len__(self) -> int:
-        return len(self._by_name)
+    def all_processes(self) -> ta.Iterator[Process]:
+        for g in self:
+            yield from g
 
-    def __iter__(self) -> ta.Iterator[ProcessGroup]:
-        return iter(self._by_name.values())
-
-    def all(self) -> ta.Mapping[str, ProcessGroup]:
-        return self._by_name
+    #
 
     def add(self, group: ProcessGroup) -> None:
         if (name := group.name) in self._by_name:
@@ -155,3 +54,26 @@ class ProcessGroups:
     def clear(self) -> None:
         # FIXME: events?
         self._by_name.clear()
+
+    #
+
+    class Diff(ta.NamedTuple):
+        added: ta.List[ProcessGroupConfig]
+        changed: ta.List[ProcessGroupConfig]
+        removed: ta.List[ProcessGroupConfig]
+
+    def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
+        cur = [group.config for group in self]
+
+        cur_by_name = {cfg.name: cfg for cfg in cur}
+        new_by_name = {cfg.name: cfg for cfg in new}
+
+        added = [cand for cand in new if cand.name not in cur_by_name]
+        removed = [cand for cand in cur if cand.name not in new_by_name]
+        changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
+
+        return ProcessGroupManager.Diff(
+            added,
+            changed,
+            removed,
+        )

ominfra/supervisor/groupsimpl.py

@@ -0,0 +1,86 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.typing import Func2
+
+from .configs import ProcessConfig
+from .configs import ProcessGroupConfig
+from .states import ProcessState
+from .types import Process
+from .types import ProcessGroup
+
+
+class ProcessFactory(Func2[ProcessConfig, ProcessGroup, Process]):
+    pass
+
+
+class ProcessGroupImpl(ProcessGroup):
+    def __init__(
+            self,
+            config: ProcessGroupConfig,
+            *,
+            process_factory: ProcessFactory,
+    ):
+        super().__init__()
+
+        self._config = config
+        self._process_factory = process_factory
+
+        by_name: ta.Dict[str, Process] = {}
+        for pconfig in self._config.processes or []:
+            p = check_isinstance(self._process_factory(pconfig, self), Process)
+            if p.name in by_name:
+                raise KeyError(f'name {p.name} of process {p} already registered by {by_name[p.name]}')
+            by_name[pconfig.name] = p
+        self._by_name = by_name
+
+    @property
+    def _by_key(self) -> ta.Mapping[str, Process]:
+        return self._by_name
+
+    #
+
+    def __repr__(self) -> str:
+        return f'<{self.__class__.__name__} instance at {id(self)} named {self._config.name}>'
+
+    #
+
+    @property
+    def name(self) -> str:
+        return self._config.name
+
+    @property
+    def config(self) -> ProcessGroupConfig:
+        return self._config
+
+    @property
+    def by_name(self) -> ta.Mapping[str, Process]:
+        return self._by_name
+
+    #
+
+    def get_unstopped_processes(self) -> ta.List[Process]:
+        return [x for x in self if not x.get_state().stopped]
+
+    def stop_all(self) -> None:
+        processes = list(self._by_name.values())
+        processes.sort()
+        processes.reverse()  # stop in desc priority order
+
+        for proc in processes:
+            state = proc.get_state()
+            if state == ProcessState.RUNNING:
+                # RUNNING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.STARTING:
+                # STARTING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.BACKOFF:
+                # BACKOFF -> FATAL
+                proc.give_up()
+
+    def before_remove(self) -> None:
+        pass

ominfra/supervisor/inject.py

@@ -7,27 +7,33 @@ from omlish.lite.inject import inj
 
 from .configs import ServerConfig
 from .context import ServerContextImpl
-from .context import ServerEpoch
-from .dispatchers import InputDispatcherImpl
-from .dispatchers import OutputDispatcherImpl
+from .dispatchersimpl import InputDispatcherImpl
+from .dispatchersimpl import OutputDispatcherImpl
 from .events import EventCallbacks
-from .groups import ProcessFactory
-from .groups import ProcessGroupImpl
+from .groups import ProcessGroupManager
+from .groupsimpl import ProcessFactory
+from .groupsimpl import ProcessGroupImpl
 from .poller import Poller
 from .poller import get_poller_impl
-from .process import InheritedFds
-from .process import InputDispatcherFactory
-from .process import OutputDispatcherFactory
-from .process import ProcessImpl
-from .signals import SignalReceiver
+from .process import PidHistory
+from .processimpl import ProcessImpl
+from .processimpl import ProcessSpawningFactory
+from .setup import DaemonizeListener
+from .setup import DaemonizeListeners
+from .setup import SupervisorUser
+from .setupimpl import SupervisorSetup
+from .setupimpl import SupervisorSetupImpl
+from .spawningimpl import InheritedFds
+from .spawningimpl import InputDispatcherFactory
+from .spawningimpl import OutputDispatcherFactory
+from .spawningimpl import ProcessSpawningImpl
 from .supervisor import ProcessGroupFactory
-from .supervisor import ProcessGroups
 from .supervisor import SignalHandler
 from .supervisor import Supervisor
 from .types import ServerContext
-
-
-##
+from .types import ServerEpoch
+from .utils.signals import SignalReceiver
+from .utils.users import get_user
 
 
 def bind_server(
@@ -39,7 +45,12 @@ def bind_server(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
-        inj.bind(get_poller_impl(), key=Poller, singleton=True),
+        inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
+
+        inj.bind(SupervisorSetupImpl, singleton=True),
+        inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
+
+        inj.bind(DaemonizeListener, array=True, to_key=Poller),
 
         inj.bind(ServerContextImpl, singleton=True),
         inj.bind(ServerContext, to_key=ServerContextImpl),
@@ -49,14 +60,18 @@ def bind_server(
         inj.bind(SignalReceiver, singleton=True),
 
         inj.bind(SignalHandler, singleton=True),
-        inj.bind(ProcessGroups, singleton=True),
+        inj.bind(ProcessGroupManager, singleton=True),
         inj.bind(Supervisor, singleton=True),
 
-        inj.bind_factory(ProcessGroupFactory, ProcessGroupImpl),
-        inj.bind_factory(ProcessFactory, ProcessImpl),
+        inj.bind(PidHistory()),
+
+        inj.bind_factory(ProcessGroupImpl, ProcessGroupFactory),
+        inj.bind_factory(ProcessImpl, ProcessFactory),
+
+        inj.bind_factory(ProcessSpawningImpl, ProcessSpawningFactory),
 
-        inj.bind_factory(OutputDispatcherFactory, OutputDispatcherImpl),
-        inj.bind_factory(InputDispatcherFactory, InputDispatcherImpl),
+        inj.bind_factory(OutputDispatcherImpl, OutputDispatcherFactory),
+        inj.bind_factory(InputDispatcherImpl, InputDispatcherFactory),
     ]
 
     #
@@ -68,4 +83,14 @@ def bind_server(
 
     #
 
+    if config.user is not None:
+        user = get_user(config.user)
+        lst.append(inj.bind(user, key=SupervisorUser))
+
+    #
+
+    lst.append(inj.bind(get_poller_impl(), key=Poller, singleton=True))
+
+    #
+
     return inj.as_bindings(*lst)

ominfra/supervisor/main.py

@@ -44,11 +44,11 @@ from .configs import prepare_server_config
 from .context import ServerContextImpl
 from .context import ServerEpoch
 from .inject import bind_server
-from .process import InheritedFds
+from .spawningimpl import InheritedFds
 from .states import SupervisorState
+from .supervisor import ExitNow
 from .supervisor import Supervisor
-from .utils import ExitNow
-from .utils import get_open_fds
+from .utils.fds import get_open_fds
 
 
 ##

ominfra/supervisor/pipes.py

@@ -0,0 +1,85 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import fcntl
+import os
+import typing as ta
+
+from .utils.fds import close_fd
+from .utils.fds import make_pipe
+from .utils.ostypes import Fd
+
+
+@dc.dataclass(frozen=True)
+class ProcessPipes:
+    child_stdin: ta.Optional[Fd] = None
+    stdin: ta.Optional[Fd] = None
+
+    stdout: ta.Optional[Fd] = None
+    child_stdout: ta.Optional[Fd] = None
+
+    stderr: ta.Optional[Fd] = None
+    child_stderr: ta.Optional[Fd] = None
+
+    def child_fds(self) -> ta.List[Fd]:
+        return [fd for fd in [self.child_stdin, self.child_stdout, self.child_stderr] if fd is not None]
+
+    def parent_fds(self) -> ta.List[Fd]:
+        return [fd for fd in [self.stdin, self.stdout, self.stderr] if fd is not None]
+
+
+def make_process_pipes(stderr=True) -> ProcessPipes:
+    """
+    Create pipes for parent to child stdin/stdout/stderr communications.  Open fd in non-blocking mode so we can
+    read them in the mainloop without blocking.  If stderr is False, don't create a pipe for stderr.
+    """
+
+    pipes: ta.Dict[str, ta.Optional[Fd]] = {
+        'child_stdin': None,
+        'stdin': None,
+
+        'stdout': None,
+        'child_stdout': None,
+
+        'stderr': None,
+        'child_stderr': None,
+    }
+
+    try:
+        pipes['child_stdin'], pipes['stdin'] = make_pipe()
+        pipes['stdout'], pipes['child_stdout'] = make_pipe()
+
+        if stderr:
+            pipes['stderr'], pipes['child_stderr'] = make_pipe()
+
+        for fd in (
+                pipes['stdout'],
+                pipes['stderr'],
+                pipes['stdin'],
+        ):
+            if fd is not None:
+                flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NDELAY
+                fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+        return ProcessPipes(**pipes)
+
+    except OSError:
+        for fd in pipes.values():
+            if fd is not None:
+                close_fd(fd)
+
+        raise
+
+
+def close_pipes(pipes: ProcessPipes) -> None:
+    close_parent_pipes(pipes)
+    close_child_pipes(pipes)
+
+
+def close_parent_pipes(pipes: ProcessPipes) -> None:
+    for fd in pipes.parent_fds():
+        close_fd(fd)
+
+
+def close_child_pipes(pipes: ProcessPipes) -> None:
+    for fd in pipes.child_fds():
+        close_fd(fd)

ominfra/supervisor/poller.py

@@ -7,29 +7,32 @@ import typing as ta
 
 from omlish.lite.logs import log
 
+from .setup import DaemonizeListener
+from .utils.ostypes import Fd
 
-class Poller(abc.ABC):
+
+class Poller(DaemonizeListener, abc.ABC):
     def __init__(self) -> None:
         super().__init__()
 
     @abc.abstractmethod
-    def register_readable(self, fd: int) -> None:
+    def register_readable(self, fd: Fd) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def register_writable(self, fd: int) -> None:
+    def register_writable(self, fd: Fd) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def unregister_readable(self, fd: int) -> None:
+    def unregister_readable(self, fd: Fd) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def unregister_writable(self, fd: int) -> None:
+    def unregister_writable(self, fd: Fd) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
+    def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
         raise NotImplementedError
 
     def before_daemonize(self) -> None:  # noqa
@@ -46,37 +49,37 @@ class SelectPoller(Poller):
     def __init__(self) -> None:
         super().__init__()
 
-        self._readable: ta.Set[int] = set()
-        self._writable: ta.Set[int] = set()
+        self._readable: ta.Set[Fd] = set()
+        self._writable: ta.Set[Fd] = set()
 
-    def register_readable(self, fd: int) -> None:
+    def register_readable(self, fd: Fd) -> None:
         self._readable.add(fd)
 
-    def register_writable(self, fd: int) -> None:
+    def register_writable(self, fd: Fd) -> None:
         self._writable.add(fd)
 
-    def unregister_readable(self, fd: int) -> None:
+    def unregister_readable(self, fd: Fd) -> None:
         self._readable.discard(fd)
 
-    def unregister_writable(self, fd: int) -> None:
+    def unregister_writable(self, fd: Fd) -> None:
         self._writable.discard(fd)
 
     def unregister_all(self) -> None:
         self._readable.clear()
         self._writable.clear()
 
-    def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
+    def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
         try:
             r, w, x = select.select(
                 self._readable,
                 self._writable,
                 [], timeout,
             )
-        except OSError as err:
-            if err.args[0] == errno.EINTR:
+        except OSError as exc:
+            if exc.args[0] == errno.EINTR:
                 log.debug('EINTR encountered in poll')
                 return [], []
-            if err.args[0] == errno.EBADF:
+            if exc.args[0] == errno.EBADF:
                 log.debug('EBADF encountered in poll')
                 self.unregister_all()
                 return [], []
@@ -92,30 +95,30 @@ class PollPoller(Poller):
         super().__init__()
 
         self._poller = select.poll()
-        self._readable: set[int] = set()
-        self._writable: set[int] = set()
+        self._readable: set[Fd] = set()
+        self._writable: set[Fd] = set()
 
-    def register_readable(self, fd: int) -> None:
+    def register_readable(self, fd: Fd) -> None:
         self._poller.register(fd, self._READ)
         self._readable.add(fd)
 
-    def register_writable(self, fd: int) -> None:
+    def register_writable(self, fd: Fd) -> None:
         self._poller.register(fd, self._WRITE)
         self._writable.add(fd)
 
-    def unregister_readable(self, fd: int) -> None:
+    def unregister_readable(self, fd: Fd) -> None:
         self._readable.discard(fd)
         self._poller.unregister(fd)
         if fd in self._writable:
             self._poller.register(fd, self._WRITE)
 
-    def unregister_writable(self, fd: int) -> None:
+    def unregister_writable(self, fd: Fd) -> None:
         self._writable.discard(fd)
         self._poller.unregister(fd)
         if fd in self._readable:
             self._poller.register(fd, self._READ)
 
-    def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
+    def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
         fds = self._poll_fds(timeout)  # type: ignore
         readable, writable = [], []
         for fd, eventmask in fds:
@@ -127,16 +130,16 @@ class PollPoller(Poller):
                 writable.append(fd)
         return readable, writable
 
-    def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[int, int]]:
+    def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[Fd, Fd]]:
         try:
-            return self._poller.poll(timeout * 1000)
-        except OSError as err:
-            if err.args[0] == errno.EINTR:
+            return self._poller.poll(timeout * 1000)  # type: ignore
+        except OSError as exc:
+            if exc.args[0] == errno.EINTR:
                 log.debug('EINTR encountered in poll')
                 return []
             raise
 
-    def _ignore_invalid(self, fd: int, eventmask: int) -> bool:
+    def _ignore_invalid(self, fd: Fd, eventmask: int) -> bool:
         if eventmask & select.POLLNVAL:
             # POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
             # more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
@@ -155,30 +158,30 @@ if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
             super().__init__()
 
             self._kqueue: ta.Optional[ta.Any] = select.kqueue()
-            self._readable: set[int] = set()
-            self._writable: set[int] = set()
+            self._readable: set[Fd] = set()
+            self._writable: set[Fd] = set()
 
-        def register_readable(self, fd: int) -> None:
+        def register_readable(self, fd: Fd) -> None:
             self._readable.add(fd)
             kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
             self._kqueue_control(fd, kevent)
 
-        def register_writable(self, fd: int) -> None:
+        def register_writable(self, fd: Fd) -> None:
             self._writable.add(fd)
             kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
             self._kqueue_control(fd, kevent)
 
-        def unregister_readable(self, fd: int) -> None:
+        def unregister_readable(self, fd: Fd) -> None:
             kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
             self._readable.discard(fd)
             self._kqueue_control(fd, kevent)
 
-        def unregister_writable(self, fd: int) -> None:
+        def unregister_writable(self, fd: Fd) -> None:
             kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
             self._writable.discard(fd)
             self._kqueue_control(fd, kevent)
 
-        def _kqueue_control(self, fd: int, kevent: 'select.kevent') -> None:
+        def _kqueue_control(self, fd: Fd, kevent: 'select.kevent') -> None:
             try:
                 self._kqueue.control([kevent], 0)  # type: ignore
             except OSError as error:
@@ -187,7 +190,7 @@ if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
                 else:
                     raise
 
-        def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[int], ta.List[int]]:
+        def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
             readable, writable = [], []  # type: ignore
 
             try:
@@ -226,8 +229,9 @@ else:
 
 def get_poller_impl() -> ta.Type[Poller]:
     if (
-            sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
-            hasattr(select, 'kqueue') and KqueuePoller is not None
+            (sys.platform == 'darwin' or sys.platform.startswith('freebsd')) and
+            hasattr(select, 'kqueue') and
+            KqueuePoller is not None
     ):
         return KqueuePoller
     elif hasattr(select, 'poll'):