ominfra 0.0.0.dev125__py3-none-any.whl → 0.0.0.dev127__py3-none-any.whl

ominfra/supervisor/supervisor.py

@@ -3,24 +3,27 @@ import signal
 import time
 import typing as ta
 
-from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check_isinstance
 from omlish.lite.check import check_not_none
 from omlish.lite.logs import log
-from omlish.lite.typing import Func
+from omlish.lite.typing import Func1
 
 from .configs import ProcessGroupConfig
 from .context import ServerContextImpl
-from .dispatchers import Dispatcher
+from .dispatchers import Dispatchers
 from .events import TICK_EVENTS
 from .events import EventCallbacks
 from .events import SupervisorRunningEvent
 from .events import SupervisorStoppingEvent
 from .groups import ProcessGroup
-from .groups import ProcessGroups
+from .groups import ProcessGroupManager
 from .poller import Poller
+from .processes import PidHistory
+from .setup import SupervisorSetup
 from .signals import SignalReceiver
 from .signals import sig_name
 from .states import SupervisorState
+from .types import OutputDispatcher
 from .types import Process
 from .utils import ExitNow
 from .utils import as_string
@@ -37,7 +40,7 @@ class SignalHandler:
             *,
             context: ServerContextImpl,
             signal_receiver: SignalReceiver,
-            process_groups: ProcessGroups,
+            process_groups: ProcessGroupManager,
     ) -> None:
         super().__init__()
 
@@ -77,8 +80,10 @@ class SignalHandler:
         elif sig == signal.SIGUSR2:
             log.info('received %s indicating log reopen request', sig_name(sig))
 
-            for group in self._process_groups:
-                group.reopen_logs()
+            for p in self._process_groups.all_processes():
+                for d in p.get_dispatchers():
+                    if isinstance(d, OutputDispatcher):
+                        d.reopen_logs()
 
         else:
             log.debug('received %s indicating nothing', sig_name(sig))
@@ -87,7 +92,8 @@ class SignalHandler:
 ##
 
 
-ProcessGroupFactory = ta.NewType('ProcessGroupFactory', Func[ProcessGroup])  # (config: ProcessGroupConfig)
+class ProcessGroupFactory(Func1[ProcessGroupConfig, ProcessGroup]):
+    pass
 
 
 class Supervisor:
@@ -96,10 +102,12 @@ class Supervisor:
             *,
             context: ServerContextImpl,
             poller: Poller,
-            process_groups: ProcessGroups,
+            process_groups: ProcessGroupManager,
             signal_handler: SignalHandler,
             event_callbacks: EventCallbacks,
             process_group_factory: ProcessGroupFactory,
+            pid_history: PidHistory,
+            setup: SupervisorSetup,
     ) -> None:
         super().__init__()
 
@@ -109,6 +117,8 @@ class Supervisor:
         self._signal_handler = signal_handler
         self._event_callbacks = event_callbacks
         self._process_group_factory = process_group_factory
+        self._pid_history = pid_history
+        self._setup = setup
 
         self._ticks: ta.Dict[int, float] = {}
         self._stop_groups: ta.Optional[ta.List[ProcessGroup]] = None  # list used for priority ordered shutdown
@@ -126,31 +136,13 @@ class Supervisor:
 
     #
 
-    class DiffToActive(ta.NamedTuple):
-        added: ta.List[ProcessGroupConfig]
-        changed: ta.List[ProcessGroupConfig]
-        removed: ta.List[ProcessGroupConfig]
-
-    def diff_to_active(self) -> DiffToActive:
-        new = self._context.config.groups or []
-        cur = [group.config for group in self._process_groups]
-
-        curdict = dict(zip([cfg.name for cfg in cur], cur))
-        newdict = dict(zip([cfg.name for cfg in new], new))
-
-        added = [cand for cand in new if cand.name not in curdict]
-        removed = [cand for cand in cur if cand.name not in newdict]
-
-        changed = [cand for cand in new if cand != curdict.get(cand.name, cand)]
-
-        return Supervisor.DiffToActive(added, changed, removed)
-
     def add_process_group(self, config: ProcessGroupConfig) -> bool:
         if self._process_groups.get(config.name) is not None:
             return False
 
-        group = self._process_group_factory(config)
-        group.after_setuid()
+        group = check_isinstance(self._process_group_factory(config), ProcessGroup)
+        for process in group:
+            process.after_setuid()
 
         self._process_groups.add(group)
 
@@ -164,11 +156,7 @@ class Supervisor:
 
         return True
 
-    def get_process_map(self) -> ta.Dict[int, Dispatcher]:
-        process_map: ta.Dict[int, Dispatcher] = {}
-        for group in self._process_groups:
-            process_map.update(group.get_dispatchers())
-        return process_map
+    #
 
     def shutdown_report(self) -> ta.List[Process]:
         unstopped: ta.List[Process] = []
@@ -191,25 +179,12 @@ class Supervisor:
 
     #
 
-    def main(self) -> None:
-        self.setup()
-        self.run()
-
-    @cached_nullary
-    def setup(self) -> None:
-        if not self._context.first:
-            # prevent crash on libdispatch-based systems, at least for the first request
-            self._context.cleanup_fds()
-
-        self._context.set_uid_or_exit()
-
-        if self._context.first:
-            self._context.set_rlimits_or_exit()
-
-        # this sets the options.logger object delay logger instantiation until after setuid
-        if not self._context.config.nocleanup:
-            # clean up old automatic logs
-            self._context.clear_auto_child_logdir()
+    def main(self, **kwargs: ta.Any) -> None:
+        self._setup.setup()
+        try:
+            self.run(**kwargs)
+        finally:
+            self._setup.cleanup()
 
     def run(
             self,
@@ -227,12 +202,6 @@ class Supervisor:
 
             self._signal_handler.set_signals()
 
-            if not self._context.config.nodaemon and self._context.first:
-                self._context.daemonize()
-
-            # writing pid file needs to come *after* daemonizing or pid will be wrong
-            self._context.write_pidfile()
-
             self._event_callbacks.notify(SupervisorRunningEvent())
 
             while True:
@@ -242,7 +211,7 @@ class Supervisor:
                 self._run_once()
 
         finally:
-            self._context.cleanup()
+            self._poller.close()
 
     #
 
@@ -271,18 +240,24 @@ class Supervisor:
                 # down, so push it back on to the end of the stop group queue
                 self._stop_groups.append(group)
 
+    def get_dispatchers(self) -> Dispatchers:
+        return Dispatchers(
+            d
+            for p in self._process_groups.all_processes()
+            for d in p.get_dispatchers()
+        )
+
     def _poll(self) -> None:
-        combined_map = {}
-        combined_map.update(self.get_process_map())
+        dispatchers = self.get_dispatchers()
 
-        pgroups = list(self._process_groups)
-        pgroups.sort()
+        sorted_groups = list(self._process_groups)
+        sorted_groups.sort()
 
         if self._context.state < SupervisorState.RUNNING:
             if not self._stopping:
                 # first time, set the stopping flag, do a notification and set stop_groups
                 self._stopping = True
-                self._stop_groups = pgroups[:]
+                self._stop_groups = sorted_groups[:]
                 self._event_callbacks.notify(SupervisorStoppingEvent())
 
             self._ordered_stop_groups_phase_1()
@@ -291,7 +266,7 @@ class Supervisor:
                 # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
                 raise ExitNow
 
-        for fd, dispatcher in combined_map.items():
+        for fd, dispatcher in dispatchers.items():
             if dispatcher.readable():
                 self._poller.register_readable(fd)
             if dispatcher.writable():
@@ -301,9 +276,9 @@ class Supervisor:
         r, w = self._poller.poll(timeout)
 
         for fd in r:
-            if fd in combined_map:
+            if fd in dispatchers:
                 try:
-                    dispatcher = combined_map[fd]
+                    dispatcher = dispatchers[fd]
                     log.debug('read event caused by %r', dispatcher)
                     dispatcher.handle_read_event()
                     if not dispatcher.readable():
@@ -311,9 +286,9 @@ class Supervisor:
                 except ExitNow:
                     raise
                 except Exception:  # noqa
-                    combined_map[fd].handle_error()
+                    dispatchers[fd].handle_error()
             else:
-                # if the fd is not in combined_map, we should unregister it. otherwise, it will be polled every
+                # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
                 # time, which may cause 100% cpu usage
                 log.debug('unexpected read event from fd %r', fd)
                 try:
@@ -322,9 +297,9 @@ class Supervisor:
                     pass
 
         for fd in w:
-            if fd in combined_map:
+            if fd in dispatchers:
                 try:
-                    dispatcher = combined_map[fd]
+                    dispatcher = dispatchers[fd]
                     log.debug('write event caused by %r', dispatcher)
                     dispatcher.handle_write_event()
                     if not dispatcher.writable():
@@ -332,7 +307,7 @@ class Supervisor:
                 except ExitNow:
                     raise
                 except Exception:  # noqa
-                    combined_map[fd].handle_error()
+                    dispatchers[fd].handle_error()
             else:
                 log.debug('unexpected write event from fd %r', fd)
                 try:
@@ -340,8 +315,9 @@ class Supervisor:
                 except Exception:  # noqa
                     pass
 
-        for group in pgroups:
-            group.transition()
+        for group in sorted_groups:
+            for process in group:
+                process.transition()
 
     def _reap(self, *, once: bool = False, depth: int = 0) -> None:
         if depth >= 100:
@@ -351,13 +327,13 @@ class Supervisor:
         if not pid:
             return
 
-        process = self._context.pid_history.get(pid, None)
+        process = self._pid_history.get(pid, None)
         if process is None:
             _, msg = decode_wait_status(check_not_none(sts))
             log.info('reaped unknown pid %s (%s)', pid, msg)
         else:
             process.finish(check_not_none(sts))
-            del self._context.pid_history[pid]
+            del self._pid_history[pid]
 
         if not once:
             # keep reaping until no more kids to reap, but don't recurse infinitely

ominfra/supervisor/types.py

@@ -3,6 +3,7 @@ import abc
 import functools
 import typing as ta
 
+from .collections import KeyedCollectionAccessors
 from .configs import ProcessConfig
 from .configs import ProcessGroupConfig
 from .configs import ServerConfig
@@ -10,6 +11,36 @@ from .states import ProcessState
 from .states import SupervisorState
 
 
+if ta.TYPE_CHECKING:
+    from .dispatchers import Dispatchers
+
+
+##
+
+
+ServerEpoch = ta.NewType('ServerEpoch', int)
+
+
+##
+
+
+@functools.total_ordering
+class ConfigPriorityOrdered(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def config(self) -> ta.Any:
+        raise NotImplementedError
+
+    def __lt__(self, other):
+        return self.config.priority < other.config.priority
+
+    def __eq__(self, other):
+        return self.config.priority == other.config.priority
+
+
+##
+
+
 class ServerContext(abc.ABC):
     @property
     @abc.abstractmethod
@@ -31,45 +62,60 @@ class ServerContext(abc.ABC):
         raise NotImplementedError
 
 
-# class Dispatcher(abc.ABC):
-#     pass
-#
-#
-# class OutputDispatcher(Dispatcher, abc.ABC):
-#     pass
-#
-#
-# class InputDispatcher(Dispatcher, abc.ABC):
-#     pass
+##
 
 
-@functools.total_ordering
-class Process(abc.ABC):
+class Dispatcher(abc.ABC):
     @property
     @abc.abstractmethod
-    def pid(self) -> int:
+    def process(self) -> 'Process':
         raise NotImplementedError
 
     @property
     @abc.abstractmethod
-    def config(self) -> ProcessConfig:
+    def channel(self) -> str:
         raise NotImplementedError
 
-    def __lt__(self, other):
-        return self.config.priority < other.config.priority
-
-    def __eq__(self, other):
-        return self.config.priority == other.config.priority
+    @property
+    @abc.abstractmethod
+    def fd(self) -> int:
+        raise NotImplementedError
 
     @property
     @abc.abstractmethod
-    def context(self) -> ServerContext:
+    def closed(self) -> bool:
         raise NotImplementedError
 
+    #
+
     @abc.abstractmethod
-    def finish(self, sts: int) -> None:
+    def close(self) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def handle_error(self) -> None:
+        raise NotImplementedError
+
+    #
+
+    @abc.abstractmethod
+    def readable(self) -> bool:
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def writable(self) -> bool:
+        raise NotImplementedError
+
+    #
+
+    def handle_read_event(self) -> None:
+        raise TypeError
+
+    def handle_write_event(self) -> None:
+        raise TypeError
+
+
+class OutputDispatcher(Dispatcher, abc.ABC):
     @abc.abstractmethod
     def remove_logs(self) -> None:
         raise NotImplementedError
@@ -78,67 +124,104 @@ class Process(abc.ABC):
     def reopen_logs(self) -> None:
         raise NotImplementedError
 
+
+class InputDispatcher(Dispatcher, abc.ABC):
     @abc.abstractmethod
-    def stop(self) -> ta.Optional[str]:
+    def write(self, chars: ta.Union[bytes, str]) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def give_up(self) -> None:
+    def flush(self) -> None:
         raise NotImplementedError
 
+
+##
+
+
+class Process(ConfigPriorityOrdered, abc.ABC):
+    @property
     @abc.abstractmethod
-    def transition(self) -> None:
+    def name(self) -> str:
         raise NotImplementedError
 
+    @property
     @abc.abstractmethod
-    def get_state(self) -> ProcessState:
+    def config(self) -> ProcessConfig:
         raise NotImplementedError
 
+    @property
     @abc.abstractmethod
-    def create_auto_child_logs(self) -> None:
+    def group(self) -> 'ProcessGroup':
         raise NotImplementedError
 
+    @property
     @abc.abstractmethod
-    def get_dispatchers(self) -> ta.Mapping[int, ta.Any]:  # Dispatcher]:
+    def pid(self) -> int:
         raise NotImplementedError
 
+    #
 
-@functools.total_ordering
-class ProcessGroup(abc.ABC):
     @property
     @abc.abstractmethod
-    def config(self) -> ProcessGroupConfig:
+    def context(self) -> ServerContext:
         raise NotImplementedError
 
-    def __lt__(self, other):
-        return self.config.priority < other.config.priority
+    @abc.abstractmethod
+    def finish(self, sts: int) -> None:
+        raise NotImplementedError
 
-    def __eq__(self, other):
-        return self.config.priority == other.config.priority
+    @abc.abstractmethod
+    def stop(self) -> ta.Optional[str]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def give_up(self) -> None:
+        raise NotImplementedError
 
     @abc.abstractmethod
     def transition(self) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def stop_all(self) -> None:
+    def get_state(self) -> ProcessState:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def after_setuid(self) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_dispatchers(self) -> 'Dispatchers':
         raise NotImplementedError
 
+
+##
+
+
+class ProcessGroup(
+    ConfigPriorityOrdered,
+    KeyedCollectionAccessors[str, Process],
+    abc.ABC,
+):
     @property
     @abc.abstractmethod
     def name(self) -> str:
         raise NotImplementedError
 
+    @property
     @abc.abstractmethod
-    def before_remove(self) -> None:
+    def config(self) -> ProcessGroupConfig:
         raise NotImplementedError
 
+    @property
     @abc.abstractmethod
-    def get_dispatchers(self) -> ta.Mapping[int, ta.Any]:  # Dispatcher]:
+    def by_name(self) -> ta.Mapping[str, Process]:
         raise NotImplementedError
 
+    #
+
     @abc.abstractmethod
-    def reopen_logs(self) -> None:
+    def stop_all(self) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod
@@ -146,5 +229,5 @@ class ProcessGroup(abc.ABC):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def after_setuid(self) -> None:
+    def before_remove(self) -> None:
         raise NotImplementedError

ominfra/supervisor/users.py

@@ -0,0 +1,64 @@
+# ruff: noqa: UP007
+import dataclasses as dc
+import grp
+import pwd
+
+
+##
+
+
+def name_to_uid(name: str) -> int:
+    try:
+        uid = int(name)
+    except ValueError:
+        try:
+            pwdrec = pwd.getpwnam(name)
+        except KeyError:
+            raise ValueError(f'Invalid user name {name}')  # noqa
+        uid = pwdrec[2]
+    else:
+        try:
+            pwd.getpwuid(uid)  # check if uid is valid
+        except KeyError:
+            raise ValueError(f'Invalid user id {name}')  # noqa
+    return uid
+
+
+def name_to_gid(name: str) -> int:
+    try:
+        gid = int(name)
+    except ValueError:
+        try:
+            grprec = grp.getgrnam(name)
+        except KeyError:
+            raise ValueError(f'Invalid group name {name}')  # noqa
+        gid = grprec[2]
+    else:
+        try:
+            grp.getgrgid(gid)  # check if gid is valid
+        except KeyError:
+            raise ValueError(f'Invalid group id {name}')  # noqa
+    return gid
+
+
+def gid_for_uid(uid: int) -> int:
+    pwrec = pwd.getpwuid(uid)
+    return pwrec[3]
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class User:
+    name: str
+    uid: int
+    gid: int
+
+
+def get_user(name: str) -> User:
+    return User(
+        name=name,
+        uid=(uid := name_to_uid(name)),
+        gid=gid_for_uid(uid),
+    )

{ominfra-0.0.0.dev125.dist-info → ominfra-0.0.0.dev127.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev125
+Version: 0.0.0.dev127
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev125
-Requires-Dist: omlish==0.0.0.dev125
+Requires-Dist: omdev==0.0.0.dev127
+Requires-Dist: omlish==0.0.0.dev127
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"