ominfra 0.0.0.dev131__py3-none-any.whl → 0.0.0.dev133__py3-none-any.whl

ominfra/scripts/supervisor.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omlish-amalg-output ../supervisor/main.py
-# ruff: noqa: N802 U006 UP006 UP007 UP012 UP036
+# ruff: noqa: N802 UP006 UP007 UP012 UP036
 # Supervisor is licensed under the following license:
 #
 #  A copyright notice accompanies this license document that identifies the copyright holders.
@@ -116,6 +116,10 @@ A2 = ta.TypeVar('A2')
 
 # events.py
 EventCallback = ta.Callable[['Event'], None]
+ProcessOutputChannel = ta.Literal['stdout', 'stderr']  # ta.TypeAlias
+
+# ../../omlish/lite/contextmanagers.py
+ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
 
 # ../../omlish/lite/http/parsing.py
 HttpHeaders = http.client.HTTPMessage  # ta.TypeAlias
@@ -1077,6 +1081,17 @@ class ProcessState(enum.IntEnum):
         return self in SIGNALABLE_STATES
 
 
+# http://supervisord.org/subprocess.html
+STATE_TRANSITIONS = {
+    ProcessState.STOPPED: (ProcessState.STARTING,),
+    ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
+    ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
+    ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
+    ProcessState.STOPPING: (ProcessState.STOPPED,),
+    ProcessState.EXITED: (ProcessState.STARTING,),
+    ProcessState.FATAL: (ProcessState.STARTING,),
+}
+
 STOPPED_STATES = (
     ProcessState.STOPPED,
     ProcessState.EXITED,
@@ -1576,16 +1591,16 @@ class FdIoPoller(abc.ABC):
     def register_readable(self, fd: int) -> bool:
         if fd in self._readable:
             return False
-        self._readable.add(fd)
         self._register_readable(fd)
+        self._readable.add(fd)
         return True
 
     @ta.final
     def register_writable(self, fd: int) -> bool:
         if fd in self._writable:
             return False
-        self._writable.add(fd)
         self._register_writable(fd)
+        self._writable.add(fd)
         return True
 
     @ta.final
@@ -1689,24 +1704,24 @@ if hasattr(select, 'poll'):
 
         #
 
-        _READ = select.POLLIN | select.POLLPRI | select.POLLHUP
-        _WRITE = select.POLLOUT
-
         def _register_readable(self, fd: int) -> None:
-            self._update_registration(fd)
+            self._update_registration(fd, r=True, w=fd in self._writable)
 
         def _register_writable(self, fd: int) -> None:
-            self._update_registration(fd)
+            self._update_registration(fd, r=fd in self._readable, w=True)
 
         def _unregister_readable(self, fd: int) -> None:
-            self._update_registration(fd)
+            self._update_registration(fd, r=False, w=False)
 
         def _unregister_writable(self, fd: int) -> None:
-            self._update_registration(fd)
+            self._update_registration(fd, r=fd in self._readable, w=False)
 
-        def _update_registration(self, fd: int) -> None:
-            r = fd in self._readable
-            w = fd in self._writable
+        #
+
+        _READ = select.POLLIN | select.POLLPRI | select.POLLHUP
+        _WRITE = select.POLLOUT
+
+        def _update_registration(self, fd: int, *, r: bool, w: bool) -> None:
             if r or w:
                 self._poller.register(fd, (self._READ if r else 0) | (self._WRITE if w else 0))
             else:
@@ -2119,7 +2134,7 @@ class EventCallbacks:
 
 
 class ProcessLogEvent(Event, abc.ABC):
-    channel: ta.Optional[str] = None
+    channel: ta.ClassVar[ProcessOutputChannel]
 
     def __init__(self, process, pid, data):
         super().__init__()
@@ -2144,7 +2159,7 @@ class ProcessCommunicationEvent(Event, abc.ABC):
     BEGIN_TOKEN = b'<!--XSUPERVISOR:BEGIN-->'
     END_TOKEN = b'<!--XSUPERVISOR:END-->'
 
-    channel: ta.ClassVar[str]
+    channel: ta.ClassVar[ProcessOutputChannel]
 
     def __init__(self, process, pid, data):
         super().__init__()
@@ -2516,6 +2531,64 @@ def get_user(name: str) -> User:
     )
 
 
+########################################
+# ../../../omlish/lite/contextmanagers.py
+
+
+##
+
+
+class ExitStacked:
+    _exit_stack: ta.Optional[contextlib.ExitStack] = None
+
+    def __enter__(self: ExitStackedT) -> ExitStackedT:
+        check_state(self._exit_stack is None)
+        es = self._exit_stack = contextlib.ExitStack()
+        es.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if (es := self._exit_stack) is None:
+            return None
+        self._exit_contexts()
+        return es.__exit__(exc_type, exc_val, exc_tb)
+
+    def _exit_contexts(self) -> None:
+        pass
+
+    def _enter_context(self, cm: ta.ContextManager[T]) -> T:
+        es = check_not_none(self._exit_stack)
+        return es.enter_context(cm)
+
+
+##
+
+
+@contextlib.contextmanager
+def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
+    try:
+        yield fn
+    finally:
+        fn()
+
+
+@contextlib.contextmanager
+def attr_setting(obj, attr, val, *, default=None):  # noqa
+    not_set = object()
+    orig = getattr(obj, attr, not_set)
+    try:
+        setattr(obj, attr, val)
+        if orig is not not_set:
+            yield orig
+        else:
+            yield default
+    finally:
+        if orig is not_set:
+            delattr(obj, attr)
+        else:
+            setattr(obj, attr, orig)
+
+
 ########################################
 # ../../../omlish/lite/fdio/handlers.py
 
@@ -2624,19 +2697,40 @@ if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
         #
 
         def _register_readable(self, fd: int) -> None:
-            self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD)
+            self._update_registration(fd, 'read', 'add')
 
         def _register_writable(self, fd: int) -> None:
-            self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
+            self._update_registration(fd, 'write', 'add')
 
         def _unregister_readable(self, fd: int) -> None:
-            self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
+            self._update_registration(fd, 'read', 'del')
 
         def _unregister_writable(self, fd: int) -> None:
-            self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
+            self._update_registration(fd, 'write', 'del')
+
+        #
 
-        def _control(self, fd: int, filter: int, flags: int) -> None:  # noqa
-            ke = select.kevent(fd, filter=filter, flags=flags)
+        _CONTROL_FILTER_BY_READ_OR_WRITE: ta.ClassVar[ta.Mapping[ta.Literal['read', 'write'], int]] = {
+            'read': select.KQ_FILTER_READ,
+            'write': select.KQ_FILTER_WRITE,
+        }
+
+        _CONTROL_FLAGS_BY_ADD_OR_DEL: ta.ClassVar[ta.Mapping[ta.Literal['add', 'del'], int]] = {
+            'add': select.KQ_EV_ADD,
+            'del': select.KQ_EV_DELETE,
+        }
+
+        def _update_registration(
+                self,
+                fd: int,
+                read_or_write: ta.Literal['read', 'write'],
+                add_or_del: ta.Literal['add', 'del'],
+        ) -> None:  # noqa
+            ke = select.kevent(
+                fd,
+                filter=self._CONTROL_FILTER_BY_READ_OR_WRITE[read_or_write],
+                flags=self._CONTROL_FLAGS_BY_ADD_OR_DEL[add_or_del],
+            )
             kq = self._get_kqueue()
             try:
                 kq.control([ke], 0)
@@ -2647,7 +2741,8 @@ if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
                     pass
                 elif exc.errno == errno.ENOENT:
                     # Can happen when trying to remove an already closed socket
-                    pass
+                    if add_or_del == 'add':
+                        raise
                 else:
                     raise
 
@@ -5951,7 +6046,7 @@ class HasDispatchers(abc.ABC):
 class ProcessDispatcher(FdIoHandler, abc.ABC):
     @property
     @abc.abstractmethod
-    def channel(self) -> str:
+    def channel(self) -> ProcessOutputChannel:
         raise NotImplementedError
 
     @property
@@ -6240,7 +6335,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
     def __init__(
             self,
             process: Process,
-            channel: str,
+            channel: ProcessOutputChannel,
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,
@@ -6268,7 +6363,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
         return self._process
 
     @property
-    def channel(self) -> str:
+    def channel(self) -> ProcessOutputChannel:
         return self._channel
 
     def fd(self) -> Fd:
@@ -6515,7 +6610,7 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
     def __init__(
             self,
             process: Process,
-            channel: str,
+            channel: ProcessOutputChannel,
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,
@@ -7024,9 +7119,9 @@ class IoManager(HasDispatchers):
         )
 
         timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
-        log.info(f'Polling: {timeout=}')  # noqa
+
         polled = self._poller.poll(timeout)
-        log.info(f'Polled: {polled=}')  # noqa
+
         if polled.msg is not None:
             log.error(polled.msg)
         if polled.exc is not None:
@@ -7151,6 +7246,8 @@ class HttpServer(HasDispatchers):
             self,
             handler: Handler,
             addr: Address = Address(('localhost', 8000)),
+            *,
+            exit_stack: contextlib.ExitStack,
     ) -> None:
         super().__init__()
 
@@ -7161,6 +7258,8 @@ class HttpServer(HasDispatchers):
 
         self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
 
+        exit_stack.enter_context(defer(self._server.close))  # noqa
+
     def get_dispatchers(self) -> Dispatchers:
         l = []
         for c in self._conns:
@@ -7205,6 +7304,7 @@ class SupervisorHttpHandler:
                     'processes': {
                         p.name: {
                             'pid': p.pid,
+                            'state': p.state.name,
                         }
                         for p in g
                     },
@@ -7572,7 +7672,6 @@ class ProcessImpl(Process):
         self._last_stop = now
 
         if now > self._last_start:
-            log.info(f'{now - self._last_start=}')  # noqa
             too_quickly = now - self._last_start < self._config.start_secs
         else:
             too_quickly = False
@@ -8227,11 +8326,29 @@ class Supervisor:
     #
 
     def _run_once(self) -> None:
-        now = time.time()
-        self._poll()
-        log.info(f'Poll took {time.time() - now}')  # noqa
+        if self._states.state < SupervisorState.RUNNING:
+            if not self._stopping:
+                # first time, set the stopping flag, do a notification and set stop_groups
+                self._stopping = True
+                self._stop_groups = sorted(self._process_groups)
+                self._event_callbacks.notify(SupervisorStoppingEvent())
+
+            self._ordered_stop_groups_phase_1()
+
+            if not self.shutdown_report():
+                # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
+                raise ExitNow
+
+        self._io.poll()
+
+        for group in sorted(self._process_groups):
+            for process in group:
+                process.transition()
+
         self._reap()
+
         self._signal_handler.handle_signals()
+
         self._tick()
 
         if self._states.state < SupervisorState.RUNNING:
@@ -8253,38 +8370,18 @@ class Supervisor:
                 # down, so push it back on to the end of the stop group queue
                 self._stop_groups.append(group)
 
-    def _poll(self) -> None:
-        sorted_groups = list(self._process_groups)
-        sorted_groups.sort()
-
-        if self._states.state < SupervisorState.RUNNING:
-            if not self._stopping:
-                # first time, set the stopping flag, do a notification and set stop_groups
-                self._stopping = True
-                self._stop_groups = sorted_groups[:]
-                self._event_callbacks.notify(SupervisorStoppingEvent())
-
-            self._ordered_stop_groups_phase_1()
-
-            if not self.shutdown_report():
-                # if there are no unstopped processes (we're done killing everything), it's OK to shutdown or reload
-                raise ExitNow
-
-        self._io.poll()
-
-        for group in sorted_groups:
-            for process in group:
-                process.transition()
+    #
 
     def _reap(self, *, once: bool = False, depth: int = 0) -> None:
         if depth >= 100:
             return
 
         wp = waitpid()
-        log.info(f'Waited pid: {wp}')  # noqa
+
         if wp is None or not wp.pid:
             return
 
+        log.info(f'Waited pid: {wp}')  # noqa
         process = self._pid_history.get(wp.pid, None)
         if process is None:
             _, msg = decode_wait_status(wp.sts)
@@ -8297,6 +8394,8 @@ class Supervisor:
             # keep reaping until no more kids to reap, but don't recurse infinitely
             self._reap(once=False, depth=depth + 1)
 
+    #
+
     def _tick(self, now: ta.Optional[float] = None) -> None:
         """Send one or more 'tick' events when the timeslice related to the period for the event type rolls over"""
 
@@ -8362,6 +8461,7 @@ class _FdIoPollerDaemonizeListener(DaemonizeListener):
 
 
 def bind_server(
+        exit_stack: contextlib.ExitStack,
         config: ServerConfig,
         *,
         server_epoch: ta.Optional[ServerEpoch] = None,
@@ -8370,6 +8470,8 @@ def bind_server(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
+        inj.bind(exit_stack),
+
         inj.bind_array(DaemonizeListener),
         inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
 
@@ -8425,8 +8527,10 @@ def bind_server(
         PollFdIoPoller,
         SelectFdIoPoller,
     ]))
-    lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
-    inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
+    lst.extend([
+        inj.bind(poller_impl, key=FdIoPoller, singleton=True),
+        inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True),
+    ])
 
     #
 
@@ -8495,18 +8599,20 @@ def main(
             prepare=prepare_server_config,
         )
 
-        injector = inj.create_injector(bind_server(
-            config,
-            server_epoch=ServerEpoch(epoch),
-            inherited_fds=inherited_fds,
-        ))
+        with contextlib.ExitStack() as es:
+            injector = inj.create_injector(bind_server(
+                es,
+                config,
+                server_epoch=ServerEpoch(epoch),
+                inherited_fds=inherited_fds,
+            ))
 
-        supervisor = injector[Supervisor]
+            supervisor = injector[Supervisor]
 
-        try:
-            supervisor.main()
-        except ExitNow:
-            pass
+            try:
+                supervisor.main()
+            except ExitNow:
+                pass
 
         if supervisor.state < SupervisorState.RESTARTING:
             break

ominfra/supervisor/dispatchersimpl.py

@@ -13,6 +13,7 @@ from .events import EventCallbacks
 from .events import ProcessCommunicationEvent
 from .events import ProcessLogStderrEvent
 from .events import ProcessLogStdoutEvent
+from .events import ProcessOutputChannel
 from .types import Process
 from .types import ProcessDispatcher
 from .types import ProcessInputDispatcher
@@ -29,7 +30,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
     def __init__(
             self,
             process: Process,
-            channel: str,
+            channel: ProcessOutputChannel,
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,
@@ -57,7 +58,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
         return self._process
 
     @property
-    def channel(self) -> str:
+    def channel(self) -> ProcessOutputChannel:
         return self._channel
 
     def fd(self) -> Fd:
@@ -304,7 +305,7 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
     def __init__(
             self,
             process: Process,
-            channel: str,
+            channel: ProcessOutputChannel,
             fd: Fd,
             *,
             event_callbacks: EventCallbacks,

ominfra/supervisor/events.py

@@ -8,6 +8,9 @@ from .states import ProcessState
 EventCallback = ta.Callable[['Event'], None]
 
 
+ProcessOutputChannel = ta.Literal['stdout', 'stderr']  # ta.TypeAlias
+
+
 ##
 
 
@@ -43,7 +46,7 @@ class EventCallbacks:
 
 
 class ProcessLogEvent(Event, abc.ABC):
-    channel: ta.Optional[str] = None
+    channel: ta.ClassVar[ProcessOutputChannel]
 
     def __init__(self, process, pid, data):
         super().__init__()
@@ -68,7 +71,7 @@ class ProcessCommunicationEvent(Event, abc.ABC):
     BEGIN_TOKEN = b'<!--XSUPERVISOR:BEGIN-->'
     END_TOKEN = b'<!--XSUPERVISOR:END-->'
 
-    channel: ta.ClassVar[str]
+    channel: ta.ClassVar[ProcessOutputChannel]
 
     def __init__(self, process, pid, data):
         super().__init__()

ominfra/supervisor/http.py

@@ -1,9 +1,11 @@
-# ruff: noqa: U006 UP007
+# ruff: noqa: UP006 UP007
+import contextlib
 import json
 import socket
 import typing as ta
 
 from omlish.lite.check import check_not_none
+from omlish.lite.contextmanagers import defer
 from omlish.lite.fdio.corohttp import CoroHttpServerConnectionFdIoHandler
 from omlish.lite.fdio.handlers import SocketFdIoHandler
 from omlish.lite.http.handlers import HttpHandler
@@ -59,6 +61,8 @@ class HttpServer(HasDispatchers):
             self,
             handler: Handler,
             addr: Address = Address(('localhost', 8000)),
+            *,
+            exit_stack: contextlib.ExitStack,
     ) -> None:
         super().__init__()
 
@@ -69,6 +73,8 @@ class HttpServer(HasDispatchers):
 
         self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
 
+        exit_stack.enter_context(defer(self._server.close))  # noqa
+
     def get_dispatchers(self) -> Dispatchers:
         l = []
         for c in self._conns:
@@ -113,6 +119,7 @@ class SupervisorHttpHandler:
                     'processes': {
                         p.name: {
                             'pid': p.pid,
+                            'state': p.state.name,
                         }
                         for p in g
                     },

ominfra/supervisor/inject.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007
+import contextlib
 import dataclasses as dc
 import typing as ta
 
@@ -56,6 +57,7 @@ class _FdIoPollerDaemonizeListener(DaemonizeListener):
 
 
 def bind_server(
+        exit_stack: contextlib.ExitStack,
         config: ServerConfig,
         *,
         server_epoch: ta.Optional[ServerEpoch] = None,
@@ -64,6 +66,8 @@ def bind_server(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
+        inj.bind(exit_stack),
+
         inj.bind_array(DaemonizeListener),
         inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
 
@@ -119,8 +123,10 @@ def bind_server(
         PollFdIoPoller,
         SelectFdIoPoller,
     ]))
-    lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
-    inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
+    lst.extend([
+        inj.bind(poller_impl, key=FdIoPoller, singleton=True),
+        inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True),
+    ])
 
     #
 

ominfra/supervisor/io.py

@@ -44,9 +44,9 @@ class IoManager(HasDispatchers):
         )
 
         timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
-        log.info(f'Polling: {timeout=}')  # noqa
+
         polled = self._poller.poll(timeout)
-        log.info(f'Polled: {polled=}')  # noqa
+
         if polled.msg is not None:
             log.error(polled.msg)
         if polled.exc is not None:

ominfra/supervisor/main.py

@@ -29,6 +29,7 @@
 #   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 #   STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 #   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import contextlib
 import itertools
 import os.path
 import typing as ta
@@ -96,18 +97,20 @@ def main(
             prepare=prepare_server_config,
         )
 
-        injector = inj.create_injector(bind_server(
-            config,
-            server_epoch=ServerEpoch(epoch),
-            inherited_fds=inherited_fds,
-        ))
+        with contextlib.ExitStack() as es:
+            injector = inj.create_injector(bind_server(
+                es,
+                config,
+                server_epoch=ServerEpoch(epoch),
+                inherited_fds=inherited_fds,
+            ))
 
-        supervisor = injector[Supervisor]
+            supervisor = injector[Supervisor]
 
-        try:
-            supervisor.main()
-        except ExitNow:
-            pass
+            try:
+                supervisor.main()
+            except ExitNow:
+                pass
 
         if supervisor.state < SupervisorState.RESTARTING:
             break

ominfra/supervisor/processimpl.py

@@ -378,7 +378,6 @@ class ProcessImpl(Process):
         self._last_stop = now
 
         if now > self._last_start:
-            log.info(f'{now - self._last_start=}')  # noqa
             too_quickly = now - self._last_start < self._config.start_secs
         else:
             too_quickly = False

ominfra/supervisor/states.py

@@ -27,6 +27,17 @@ class ProcessState(enum.IntEnum):
         return self in SIGNALABLE_STATES
 
 
+# http://supervisord.org/subprocess.html
+STATE_TRANSITIONS = {
+    ProcessState.STOPPED: (ProcessState.STARTING,),
+    ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
+    ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
+    ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
+    ProcessState.STOPPING: (ProcessState.STOPPED,),
+    ProcessState.EXITED: (ProcessState.STARTING,),
+    ProcessState.FATAL: (ProcessState.STARTING,),
+}
+
 STOPPED_STATES = (
     ProcessState.STOPPED,
     ProcessState.EXITED,