ominfra 0.0.0.dev125__py3-none-any.whl → 0.0.0.dev127__py3-none-any.whl

ominfra/supervisor/groups.py

@@ -1,113 +1,16 @@
 # ruff: noqa: UP006 UP007
 import typing as ta
 
-from omlish.lite.typing import Func
-
+from .collections import KeyedCollectionAccessors
 from .configs import ProcessGroupConfig
-from .dispatchers import Dispatcher
 from .events import EventCallbacks
 from .events import ProcessGroupAddedEvent
 from .events import ProcessGroupRemovedEvent
-from .states import ProcessState
 from .types import Process
 from .types import ProcessGroup
-from .types import ServerContext
-
-
-##
 
 
-ProcessFactory = ta.NewType('ProcessFactory', Func[Process])  # (config: ProcessConfig, group: ProcessGroup)
-
-
-class ProcessGroupImpl(ProcessGroup):
-    def __init__(
-            self,
-            config: ProcessGroupConfig,
-            context: ServerContext,
-            *,
-            process_factory: ProcessFactory,
-    ):
-        super().__init__()
-
-        self._config = config
-        self._context = context
-        self._process_factory = process_factory
-
-        self._processes = {}
-        for pconfig in self._config.processes or []:
-            process = self._process_factory(pconfig, self)
-            self._processes[pconfig.name] = process
-
-    @property
-    def config(self) -> ProcessGroupConfig:
-        return self._config
-
-    @property
-    def name(self) -> str:
-        return self._config.name
-
-    @property
-    def context(self) -> ServerContext:
-        return self._context
-
-    def __repr__(self):
-        # repr can't return anything other than a native string, but the name might be unicode - a problem on Python 2.
-        name = self._config.name
-        return f'<{self.__class__.__name__} instance at {id(self)} named {name}>'
-
-    def remove_logs(self) -> None:
-        for process in self._processes.values():
-            process.remove_logs()
-
-    def reopen_logs(self) -> None:
-        for process in self._processes.values():
-            process.reopen_logs()
-
-    def stop_all(self) -> None:
-        processes = list(self._processes.values())
-        processes.sort()
-        processes.reverse()  # stop in desc priority order
-
-        for proc in processes:
-            state = proc.get_state()
-            if state == ProcessState.RUNNING:
-                # RUNNING -> STOPPING
-                proc.stop()
-
-            elif state == ProcessState.STARTING:
-                # STARTING -> STOPPING
-                proc.stop()
-
-            elif state == ProcessState.BACKOFF:
-                # BACKOFF -> FATAL
-                proc.give_up()
-
-    def get_unstopped_processes(self) -> ta.List[Process]:
-        return [x for x in self._processes.values() if not x.get_state().stopped]
-
-    def get_dispatchers(self) -> ta.Dict[int, Dispatcher]:
-        dispatchers: dict = {}
-        for process in self._processes.values():
-            dispatchers.update(process.get_dispatchers())
-        return dispatchers
-
-    def before_remove(self) -> None:
-        pass
-
-    def transition(self) -> None:
-        for proc in self._processes.values():
-            proc.transition()
-
-    def after_setuid(self) -> None:
-        for proc in self._processes.values():
-            proc.create_auto_child_logs()
-
-
-##
-
-
-class ProcessGroups:
+class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
     def __init__(
             self,
             *,
@@ -119,20 +22,17 @@ class ProcessGroups:
 
         self._by_name: ta.Dict[str, ProcessGroup] = {}
 
-    def get(self, name: str) -> ta.Optional[ProcessGroup]:
-        return self._by_name.get(name)
+    @property
+    def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
+        return self._by_name
 
-    def __getitem__(self, name: str) -> ProcessGroup:
-        return self._by_name[name]
+    #
 
-    def __len__(self) -> int:
-        return len(self._by_name)
+    def all_processes(self) -> ta.Iterator[Process]:
+        for g in self:
+            yield from g
 
-    def __iter__(self) -> ta.Iterator[ProcessGroup]:
-        return iter(self._by_name.values())
-
-    def all(self) -> ta.Mapping[str, ProcessGroup]:
-        return self._by_name
+    #
 
     def add(self, group: ProcessGroup) -> None:
         if (name := group.name) in self._by_name:
@@ -154,3 +54,26 @@ class ProcessGroups:
     def clear(self) -> None:
         # FIXME: events?
         self._by_name.clear()
+
+    #
+
+    class Diff(ta.NamedTuple):
+        added: ta.List[ProcessGroupConfig]
+        changed: ta.List[ProcessGroupConfig]
+        removed: ta.List[ProcessGroupConfig]
+
+    def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
+        cur = [group.config for group in self]
+
+        cur_by_name = {cfg.name: cfg for cfg in cur}
+        new_by_name = {cfg.name: cfg for cfg in new}
+
+        added = [cand for cand in new if cand.name not in cur_by_name]
+        removed = [cand for cand in cur if cand.name not in new_by_name]
+        changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
+
+        return ProcessGroupManager.Diff(
+            added,
+            changed,
+            removed,
+        )

ominfra/supervisor/groupsimpl.py

@@ -0,0 +1,86 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.typing import Func2
+
+from .configs import ProcessConfig
+from .configs import ProcessGroupConfig
+from .states import ProcessState
+from .types import Process
+from .types import ProcessGroup
+
+
+class ProcessFactory(Func2[ProcessConfig, ProcessGroup, Process]):
+    pass
+
+
+class ProcessGroupImpl(ProcessGroup):
+    def __init__(
+            self,
+            config: ProcessGroupConfig,
+            *,
+            process_factory: ProcessFactory,
+    ):
+        super().__init__()
+
+        self._config = config
+        self._process_factory = process_factory
+
+        by_name: ta.Dict[str, Process] = {}
+        for pconfig in self._config.processes or []:
+            p = check_isinstance(self._process_factory(pconfig, self), Process)
+            if p.name in by_name:
+                raise KeyError(f'name {p.name} of process {p} already registered by {by_name[p.name]}')
+            by_name[pconfig.name] = p
+        self._by_name = by_name
+
+    @property
+    def _by_key(self) -> ta.Mapping[str, Process]:
+        return self._by_name
+
+    #
+
+    def __repr__(self) -> str:
+        return f'<{self.__class__.__name__} instance at {id(self)} named {self._config.name}>'
+
+    #
+
+    @property
+    def name(self) -> str:
+        return self._config.name
+
+    @property
+    def config(self) -> ProcessGroupConfig:
+        return self._config
+
+    @property
+    def by_name(self) -> ta.Mapping[str, Process]:
+        return self._by_name
+
+    #
+
+    def get_unstopped_processes(self) -> ta.List[Process]:
+        return [x for x in self if not x.get_state().stopped]
+
+    def stop_all(self) -> None:
+        processes = list(self._by_name.values())
+        processes.sort()
+        processes.reverse()  # stop in desc priority order
+
+        for proc in processes:
+            state = proc.get_state()
+            if state == ProcessState.RUNNING:
+                # RUNNING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.STARTING:
+                # STARTING -> STOPPING
+                proc.stop()
+
+            elif state == ProcessState.BACKOFF:
+                # BACKOFF -> FATAL
+                proc.give_up()
+
+    def before_remove(self) -> None:
+        pass

ominfra/supervisor/inject.py

@@ -7,23 +7,33 @@ from omlish.lite.inject import inj
 
 from .configs import ServerConfig
 from .context import ServerContextImpl
-from .context import ServerEpoch
+from .dispatchersimpl import InputDispatcherImpl
+from .dispatchersimpl import OutputDispatcherImpl
 from .events import EventCallbacks
-from .groups import ProcessFactory
-from .groups import ProcessGroupImpl
+from .groups import ProcessGroupManager
+from .groupsimpl import ProcessFactory
+from .groupsimpl import ProcessGroupImpl
 from .poller import Poller
 from .poller import get_poller_impl
-from .process import InheritedFds
-from .process import ProcessImpl
+from .processes import PidHistory
+from .processesimpl import ProcessImpl
+from .processesimpl import ProcessSpawningFactory
+from .setup import DaemonizeListener
+from .setup import DaemonizeListeners
+from .setup import SupervisorUser
+from .setupimpl import SupervisorSetup
+from .setupimpl import SupervisorSetupImpl
 from .signals import SignalReceiver
+from .spawningimpl import InheritedFds
+from .spawningimpl import InputDispatcherFactory
+from .spawningimpl import OutputDispatcherFactory
+from .spawningimpl import ProcessSpawningImpl
 from .supervisor import ProcessGroupFactory
-from .supervisor import ProcessGroups
 from .supervisor import SignalHandler
 from .supervisor import Supervisor
 from .types import ServerContext
-
-
-##
+from .types import ServerEpoch
+from .users import get_user
 
 
 def bind_server(
@@ -35,7 +45,12 @@ def bind_server(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
-        inj.bind(get_poller_impl(), key=Poller, singleton=True),
+        inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
+
+        inj.bind(SupervisorSetupImpl, singleton=True),
+        inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
+
+        inj.bind(DaemonizeListener, array=True, to_key=Poller),
 
         inj.bind(ServerContextImpl, singleton=True),
         inj.bind(ServerContext, to_key=ServerContextImpl),
@@ -45,11 +60,18 @@ def bind_server(
         inj.bind(SignalReceiver, singleton=True),
 
         inj.bind(SignalHandler, singleton=True),
-        inj.bind(ProcessGroups, singleton=True),
+        inj.bind(ProcessGroupManager, singleton=True),
         inj.bind(Supervisor, singleton=True),
 
-        inj.bind_factory(ProcessGroupFactory, ProcessGroupImpl),
-        inj.bind_factory(ProcessFactory, ProcessImpl),
+        inj.bind(PidHistory()),
+
+        inj.bind_factory(ProcessGroupImpl, ProcessGroupFactory),
+        inj.bind_factory(ProcessImpl, ProcessFactory),
+
+        inj.bind_factory(ProcessSpawningImpl, ProcessSpawningFactory),
+
+        inj.bind_factory(OutputDispatcherImpl, OutputDispatcherFactory),
+        inj.bind_factory(InputDispatcherImpl, InputDispatcherFactory),
     ]
 
     #
@@ -61,4 +83,14 @@ def bind_server(
 
     #
 
+    if config.user is not None:
+        user = get_user(config.user)
+        lst.append(inj.bind(user, key=SupervisorUser))
+
+    #
+
+    lst.append(inj.bind(get_poller_impl(), key=Poller, singleton=True))
+
+    #
+
     return inj.as_bindings(*lst)

ominfra/supervisor/main.py

@@ -44,7 +44,7 @@ from .configs import prepare_server_config
 from .context import ServerContextImpl
 from .context import ServerEpoch
 from .inject import bind_server
-from .process import InheritedFds
+from .spawningimpl import InheritedFds
 from .states import SupervisorState
 from .supervisor import Supervisor
 from .utils import ExitNow

ominfra/supervisor/pipes.py

@@ -0,0 +1,83 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import fcntl
+import os
+import typing as ta
+
+from .utils import close_fd
+
+
+@dc.dataclass(frozen=True)
+class ProcessPipes:
+    child_stdin: ta.Optional[int] = None
+    stdin: ta.Optional[int] = None
+
+    stdout: ta.Optional[int] = None
+    child_stdout: ta.Optional[int] = None
+
+    stderr: ta.Optional[int] = None
+    child_stderr: ta.Optional[int] = None
+
+    def child_fds(self) -> ta.List[int]:
+        return [fd for fd in [self.child_stdin, self.child_stdout, self.child_stderr] if fd is not None]
+
+    def parent_fds(self) -> ta.List[int]:
+        return [fd for fd in [self.stdin, self.stdout, self.stderr] if fd is not None]
+
+
+def make_process_pipes(stderr=True) -> ProcessPipes:
+    """
+    Create pipes for parent to child stdin/stdout/stderr communications.  Open fd in non-blocking mode so we can
+    read them in the mainloop without blocking.  If stderr is False, don't create a pipe for stderr.
+    """
+
+    pipes: ta.Dict[str, ta.Optional[int]] = {
+        'child_stdin': None,
+        'stdin': None,
+
+        'stdout': None,
+        'child_stdout': None,
+
+        'stderr': None,
+        'child_stderr': None,
+    }
+
+    try:
+        pipes['child_stdin'], pipes['stdin'] = os.pipe()
+        pipes['stdout'], pipes['child_stdout'] = os.pipe()
+
+        if stderr:
+            pipes['stderr'], pipes['child_stderr'] = os.pipe()
+
+        for fd in (
+                pipes['stdout'],
+                pipes['stderr'],
+                pipes['stdin'],
+        ):
+            if fd is not None:
+                flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NDELAY
+                fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+        return ProcessPipes(**pipes)
+
+    except OSError:
+        for fd in pipes.values():
+            if fd is not None:
+                close_fd(fd)
+
+        raise
+
+
+def close_pipes(pipes: ProcessPipes) -> None:
+    close_parent_pipes(pipes)
+    close_child_pipes(pipes)
+
+
+def close_parent_pipes(pipes: ProcessPipes) -> None:
+    for fd in pipes.parent_fds():
+        close_fd(fd)
+
+
+def close_child_pipes(pipes: ProcessPipes) -> None:
+    for fd in pipes.child_fds():
+        close_fd(fd)

ominfra/supervisor/poller.py

@@ -7,8 +7,10 @@ import typing as ta
 
 from omlish.lite.logs import log
 
+from .setup import DaemonizeListener
 
-class Poller(abc.ABC):
+
+class Poller(DaemonizeListener, abc.ABC):
     def __init__(self) -> None:
         super().__init__()
 
@@ -226,8 +228,9 @@ else:
 
 def get_poller_impl() -> ta.Type[Poller]:
     if (
-            sys.platform == 'darwin' or sys.platform.startswith('freebsd') and
-            hasattr(select, 'kqueue') and KqueuePoller is not None
+            (sys.platform == 'darwin' or sys.platform.startswith('freebsd')) and
+            hasattr(select, 'kqueue') and
+            KqueuePoller is not None
     ):
         return KqueuePoller
     elif hasattr(select, 'poll'):

ominfra/supervisor/privileges.py

@@ -0,0 +1,65 @@
+# ruff: noqa: UP006 UP007
+import grp
+import os
+import pwd
+import typing as ta
+
+
+def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
+    """
+    Drop privileges to become the specified user, which may be a username or uid.  Called for supervisord startup
+    and when spawning subprocesses.  Returns None on success or a string error message if privileges could not be
+    dropped.
+    """
+
+    if user is None:
+        return 'No user specified to setuid to!'
+
+    # get uid for user, which can be a number or username
+    try:
+        uid = int(user)
+    except ValueError:
+        try:
+            pwrec = pwd.getpwnam(user)  # type: ignore
+        except KeyError:
+            return f"Can't find username {user!r}"
+        uid = pwrec[2]
+    else:
+        try:
+            pwrec = pwd.getpwuid(uid)
+        except KeyError:
+            return f"Can't find uid {uid!r}"
+
+    current_uid = os.getuid()
+
+    if current_uid == uid:
+        # do nothing and return successfully if the uid is already the current one.  this allows a supervisord
+        # running as an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in
+        # it.
+        return None
+
+    if current_uid != 0:
+        return "Can't drop privilege as nonroot user"
+
+    gid = pwrec[3]
+    if hasattr(os, 'setgroups'):
+        user = pwrec[0]
+        groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
+
+        # always put our primary gid first in this list, otherwise we can lose group info since sometimes the first
+        # group in the setgroups list gets overwritten on the subsequent setgid call (at least on freebsd 9 with
+        # python 2.7 - this will be safe though for all unix /python version combos)
+        groups.insert(0, gid)
+        try:
+            os.setgroups(groups)
+        except OSError:
+            return 'Could not set groups of effective user'
+
+    try:
+        os.setgid(gid)
+    except OSError:
+        return 'Could not set group id of effective user'
+
+    os.setuid(uid)
+
+    return None

ominfra/supervisor/processes.py

@@ -0,0 +1,18 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from .types import Process
+
+
+##
+
+
+class ProcessStateError(RuntimeError):
+    pass
+
+
+##
+
+
+class PidHistory(ta.Dict[int, Process]):
+    pass