ominfra 0.0.0.dev129__py3-none-any.whl → 0.0.0.dev130__py3-none-any.whl

ominfra/supervisor/dispatchers.py

@@ -1,13 +1,14 @@
 # ruff: noqa: UP006 UP007
-from .types import Dispatcher
+from omlish.lite.fdio.handlers import FdIoHandler
+
 from .types import ProcessOutputDispatcher
 from .utils.collections import KeyedCollection
 from .utils.ostypes import Fd
 
 
-class Dispatchers(KeyedCollection[Fd, Dispatcher]):
-    def _key(self, v: Dispatcher) -> Fd:
-        return v.fd
+class Dispatchers(KeyedCollection[Fd, FdIoHandler]):
+    def _key(self, v: FdIoHandler) -> Fd:
+        return Fd(v.fd())
 
     #
 
@@ -16,9 +17,9 @@ class Dispatchers(KeyedCollection[Fd, Dispatcher]):
             # note that we *must* call readable() for every dispatcher, as it may have side effects for a given
             # dispatcher (eg. call handle_listener_state_change for event listener processes)
             if d.readable():
-                d.handle_read_event()
+                d.on_readable()
             if d.writable():
-                d.handle_write_event()
+                d.on_writable()
 
     #
 

ominfra/supervisor/dispatchersimpl.py

@@ -60,7 +60,6 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
     def channel(self) -> str:
         return self._channel
 
-    @property
     def fd(self) -> Fd:
         return self._fd
 
@@ -75,7 +74,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
             log.debug('fd %s closed, stopped monitoring %s', self._fd, self)
             self._closed = True
 
-    def handle_error(self) -> None:
+    def on_error(self, exc: ta.Optional[BaseException] = None) -> None:
         nil, t, v, tbinfo = compact_traceback()
 
         log.critical('uncaptured python exception, closing channel %s (%s:%s %s)', repr(self), t, v, tbinfo)
@@ -291,7 +290,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
             return False
         return True
 
-    def handle_read_event(self) -> None:
+    def on_readable(self) -> None:
         data = read_fd(self._fd)
         self._output_buffer += data
         self.record_output()
@@ -329,15 +328,12 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
             return True
         return False
 
-    def readable(self) -> bool:
-        return False
-
     def flush(self) -> None:
         # other code depends on this raising EPIPE if the pipe is closed
         sent = os.write(self._fd, as_bytes(self._input_buffer))
         self._input_buffer = self._input_buffer[sent:]
 
-    def handle_write_event(self) -> None:
+    def on_writable(self) -> None:
         if self._input_buffer:
             try:
                 self.flush()

ominfra/supervisor/groups.py

@@ -2,15 +2,20 @@
 import typing as ta
 
 from .configs import ProcessGroupConfig
+from .dispatchers import Dispatchers
 from .events import EventCallbacks
 from .events import ProcessGroupAddedEvent
 from .events import ProcessGroupRemovedEvent
+from .types import HasDispatchers
 from .types import Process
 from .types import ProcessGroup
 from .utils.collections import KeyedCollectionAccessors
 
 
-class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
+class ProcessGroupManager(
+    KeyedCollectionAccessors[str, ProcessGroup],
+    HasDispatchers,
+):
     def __init__(
             self,
             *,
@@ -34,6 +39,16 @@ class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
 
     #
 
+    def get_dispatchers(self) -> Dispatchers:
+        return Dispatchers(
+            d
+            for g in self
+            for p in g
+            for d in p.get_dispatchers()
+        )
+
+    #
+
     def add(self, group: ProcessGroup) -> None:
         if (name := group.name) in self._by_name:
             raise KeyError(f'Process group already exists: {name}')

ominfra/supervisor/http.py

@@ -0,0 +1,130 @@
+# ruff: noqa: U006 UP007
+import json
+import socket
+import typing as ta
+
+from omlish.lite.check import check_not_none
+from omlish.lite.fdio.corohttp import CoroHttpServerConnectionFdIoHandler
+from omlish.lite.fdio.handlers import SocketFdIoHandler
+from omlish.lite.http.handlers import HttpHandler
+from omlish.lite.http.handlers import HttpHandlerRequest
+from omlish.lite.http.handlers import HttpHandlerResponse
+from omlish.lite.json import JSON_PRETTY_KWARGS
+from omlish.lite.socket import SocketAddress
+
+from .dispatchers import Dispatchers
+from .groups import ProcessGroupManager
+from .types import HasDispatchers
+
+
+##
+
+
+class SocketServerFdIoHandler(SocketFdIoHandler):
+    def __init__(
+            self,
+            addr: SocketAddress,
+            on_connect: ta.Callable[[socket.socket, SocketAddress], None],
+    ) -> None:
+        sock = socket.create_server(addr)
+        sock.setblocking(False)
+
+        super().__init__(addr, sock)
+
+        self._on_connect = on_connect
+
+        sock.listen(1)
+
+    def readable(self) -> bool:
+        return True
+
+    def on_readable(self) -> None:
+        cli_sock, cli_addr = check_not_none(self._sock).accept()
+        cli_sock.setblocking(False)
+
+        self._on_connect(cli_sock, cli_addr)
+
+
+##
+
+
+class HttpServer(HasDispatchers):
+    class Address(ta.NamedTuple):
+        a: SocketAddress
+
+    class Handler(ta.NamedTuple):
+        h: HttpHandler
+
+    def __init__(
+            self,
+            handler: Handler,
+            addr: Address = Address(('localhost', 8000)),
+    ) -> None:
+        super().__init__()
+
+        self._handler = handler.h
+        self._addr = addr.a
+
+        self._server = SocketServerFdIoHandler(self._addr, self._on_connect)
+
+        self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
+
+    def get_dispatchers(self) -> Dispatchers:
+        l = []
+        for c in self._conns:
+            if not c.closed:
+                l.append(c)
+        self._conns = l
+        return Dispatchers([
+            self._server,
+            *l,
+        ])
+
+    def _on_connect(self, sock: socket.socket, addr: SocketAddress) -> None:
+        conn = CoroHttpServerConnectionFdIoHandler(
+            addr,
+            sock,
+            self._handler,
+        )
+
+        self._conns.append(conn)
+
+
+##
+
+
+class SupervisorHttpHandler:
+    def __init__(
+            self,
+            *,
+            groups: ProcessGroupManager,
+    ) -> None:
+        super().__init__()
+
+        self._groups = groups
+
+    def handle(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
+        dct = {
+            'method': req.method,
+            'path': req.path,
+            'data': len(req.data or b''),
+            'groups': {
+                g.name: {
+                    'processes': {
+                        p.name: {
+                            'pid': p.pid,
+                        }
+                        for p in g
+                    },
+                }
+                for g in self._groups
+            },
+        }
+
+        return HttpHandlerResponse(
+            200,
+            data=json.dumps(dct, **JSON_PRETTY_KWARGS).encode('utf-8') + b'\n',
+            headers={
+                'Content-Type': 'application/json',
+            },
+        )

ominfra/supervisor/inject.py

@@ -1,6 +1,11 @@
 # ruff: noqa: UP006 UP007
+import dataclasses as dc
 import typing as ta
 
+from omlish.lite.fdio.kqueue import KqueueFdIoPoller  # noqa
+from omlish.lite.fdio.pollers import FdIoPoller
+from omlish.lite.fdio.pollers import PollFdIoPoller  # noqa
+from omlish.lite.fdio.pollers import SelectFdIoPoller
 from omlish.lite.inject import InjectorBindingOrBindings
 from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
@@ -12,9 +17,10 @@ from .events import EventCallbacks
 from .groups import ProcessGroupManager
 from .groupsimpl import ProcessFactory
 from .groupsimpl import ProcessGroupImpl
+from .http import HttpServer
+from .http import SupervisorHttpHandler
+from .io import HasDispatchersList
 from .io import IoManager
-from .poller import Poller
-from .poller import get_poller_impl
 from .process import PidHistory
 from .processimpl import ProcessImpl
 from .processimpl import ProcessSpawningFactory
@@ -31,12 +37,24 @@ from .spawningimpl import ProcessSpawningImpl
 from .supervisor import ProcessGroupFactory
 from .supervisor import Supervisor
 from .supervisor import SupervisorStateManagerImpl
+from .types import HasDispatchers
 from .types import ServerEpoch
 from .types import SupervisorStateManager
 from .utils.signals import SignalReceiver
 from .utils.users import get_user
 
 
+@dc.dataclass(frozen=True)
+class _FdIoPollerDaemonizeListener(DaemonizeListener):
+    _poller: FdIoPoller
+
+    def before_daemonize(self) -> None:
+        self._poller.close()
+
+    def after_daemonize(self) -> None:
+        self._poller.reopen()
+
+
 def bind_server(
         config: ServerConfig,
         *,
@@ -46,22 +64,24 @@ def bind_server(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
+        inj.bind_array(DaemonizeListener),
         inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
 
         inj.bind(SupervisorSetupImpl, singleton=True),
         inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
 
-        inj.bind(DaemonizeListener, array=True, to_key=Poller),
-
         inj.bind(EventCallbacks, singleton=True),
 
         inj.bind(SignalReceiver, singleton=True),
 
         inj.bind(IoManager, singleton=True),
+        inj.bind_array(HasDispatchers),
+        inj.bind_array_type(HasDispatchers, HasDispatchersList),
 
         inj.bind(SignalHandler, singleton=True),
 
         inj.bind(ProcessGroupManager, singleton=True),
+        inj.bind(HasDispatchers, array=True, to_key=ProcessGroupManager),
 
         inj.bind(Supervisor, singleton=True),
 
@@ -94,7 +114,26 @@ def bind_server(
 
     #
 
-    lst.append(inj.bind(get_poller_impl(), key=Poller, singleton=True))
+    poller_impl = next(filter(None, [
+        KqueueFdIoPoller,
+        PollFdIoPoller,
+        SelectFdIoPoller,
+    ]))
+    lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
+    inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
+
+    #
+
+    def _provide_http_handler(s: SupervisorHttpHandler) -> HttpServer.Handler:
+        return HttpServer.Handler(s.handle)
+
+    lst.extend([
+        inj.bind(HttpServer, singleton=True, eager=True),
+        inj.bind(HasDispatchers, array=True, to_key=HttpServer),
+
+        inj.bind(SupervisorHttpHandler, singleton=True),
+        inj.bind(_provide_http_handler),
+    ])
 
     #
 

ominfra/supervisor/io.py

@@ -1,58 +1,71 @@
 # ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.fdio.pollers import FdIoPoller
 from omlish.lite.logs import log
 
 from .dispatchers import Dispatchers
-from .groups import ProcessGroupManager
-from .poller import Poller
 from .types import ExitNow
+from .types import HasDispatchers
+from .utils.ostypes import Fd
 
 
 ##
 
 
-class IoManager:
+HasDispatchersList = ta.NewType('HasDispatchersList', ta.Sequence[HasDispatchers])
+
+
+class IoManager(HasDispatchers):
     def __init__(
             self,
             *,
-            poller: Poller,
-            process_groups: ProcessGroupManager,
+            poller: FdIoPoller,
+            has_dispatchers_list: HasDispatchersList,
     ) -> None:
         super().__init__()
 
         self._poller = poller
-        self._process_groups = process_groups
+        self._has_dispatchers_list = has_dispatchers_list
 
     def get_dispatchers(self) -> Dispatchers:
         return Dispatchers(
             d
-            for p in self._process_groups.all_processes()
-            for d in p.get_dispatchers()
+            for hd in self._has_dispatchers_list
+            for d in hd.get_dispatchers()
         )
 
     def poll(self) -> None:
         dispatchers = self.get_dispatchers()
 
-        for fd, dispatcher in dispatchers.items():
-            if dispatcher.readable():
-                self._poller.register_readable(fd)
-            if dispatcher.writable():
-                self._poller.register_writable(fd)
+        self._poller.update(
+            {fd for fd, d in dispatchers.items() if d.readable()},
+            {fd for fd, d in dispatchers.items() if d.writable()},
+        )
 
         timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
-        r, w = self._poller.poll(timeout)
+        log.info(f'Polling: {timeout=}')  # noqa
+        polled = self._poller.poll(timeout)
+        log.info(f'Polled: {polled=}')  # noqa
+        if polled.msg is not None:
+            log.error(polled.msg)
+        if polled.exc is not None:
+            log.error('Poll exception: %r', polled.exc)
 
-        for fd in r:
+        for r in polled.r:
+            fd = Fd(r)
             if fd in dispatchers:
+                dispatcher = dispatchers[fd]
                 try:
-                    dispatcher = dispatchers[fd]
                     log.debug('read event caused by %r', dispatcher)
-                    dispatcher.handle_read_event()
+                    dispatcher.on_readable()
                     if not dispatcher.readable():
                         self._poller.unregister_readable(fd)
                 except ExitNow:
                     raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
+                except Exception as exc:  # noqa
+                    log.exception('Error in dispatcher: %r', dispatcher)
+                    dispatcher.on_error(exc)
             else:
                 # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
                 # time, which may cause 100% cpu usage
@@ -62,18 +75,20 @@ class IoManager:
                 except Exception:  # noqa
                     pass
 
-        for fd in w:
+        for w in polled.w:
+            fd = Fd(w)
             if fd in dispatchers:
+                dispatcher = dispatchers[fd]
                 try:
-                    dispatcher = dispatchers[fd]
                     log.debug('write event caused by %r', dispatcher)
-                    dispatcher.handle_write_event()
+                    dispatcher.on_writable()
                     if not dispatcher.writable():
                         self._poller.unregister_writable(fd)
                 except ExitNow:
                     raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
+                except Exception as exc:  # noqa
+                    log.exception('Error in dispatcher: %r', dispatcher)
+                    dispatcher.on_error(exc)
             else:
                 log.debug('unexpected write event from fd %r', fd)
                 try:

ominfra/supervisor/processimpl.py

@@ -378,6 +378,7 @@ class ProcessImpl(Process):
         self._last_stop = now
 
         if now > self._last_start:
+            log.info(f'{now - self._last_start=}')  # noqa
             too_quickly = now - self._last_start < self._config.startsecs
         else:
             too_quickly = False
@@ -447,8 +448,6 @@ class ProcessImpl(Process):
 
         self._check_and_adjust_for_system_clock_rollback(now)
 
-        logger = log
-
         if self._supervisor_states.state > SupervisorState.RESTARTING:
             # dont start any processes if supervisor is shutting down
             if state == ProcessState.EXITED:
@@ -480,14 +479,14 @@ class ProcessImpl(Process):
                 self.check_in_state(ProcessState.STARTING)
                 self.change_state(ProcessState.RUNNING)
                 msg = ('entered RUNNING state, process has stayed up for > than %s seconds (startsecs)' % self._config.startsecs)  # noqa
-                logger.info('success: %s %s', self.name, msg)
+                log.info('success: %s %s', self.name, msg)
 
         if state == ProcessState.BACKOFF:
             if self._backoff > self._config.startretries:
                 # BACKOFF -> FATAL if the proc has exceeded its number of retries
                 self.give_up()
                 msg = ('entered FATAL state, too many start retries too quickly')
-                logger.info('gave up: %s %s', self.name, msg)
+                log.info('gave up: %s %s', self.name, msg)
 
         elif state == ProcessState.STOPPING:
             time_left = self._delay - now

ominfra/supervisor/spawningimpl.py

@@ -7,6 +7,7 @@ import typing as ta
 
 from omlish.lite.check import check_isinstance
 from omlish.lite.check import check_not_none
+from omlish.lite.fdio.handlers import FdIoHandler
 from omlish.lite.typing import Func3
 
 from .configs import ProcessConfig
@@ -29,7 +30,6 @@ from .process import PidHistory
 from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .spawning import SpawnedProcess
-from .types import Dispatcher
 from .types import Process
 from .types import ProcessGroup
 from .types import ProcessInputDispatcher
@@ -200,7 +200,7 @@ class ProcessSpawningImpl(ProcessSpawning):
         return exe, args
 
     def _make_dispatchers(self, pipes: ProcessPipes) -> Dispatchers:
-        dispatchers: ta.List[Dispatcher] = []
+        dispatchers: ta.List[FdIoHandler] = []
 
         if pipes.stdout is not None:
             dispatchers.append(check_isinstance(self._output_dispatcher_factory(

ominfra/supervisor/supervisor.py

@@ -5,6 +5,7 @@ import time
 import typing as ta
 
 from omlish.lite.check import check_isinstance
+from omlish.lite.fdio.pollers import FdIoPoller
 from omlish.lite.logs import log
 from omlish.lite.typing import Func1
 
@@ -17,7 +18,6 @@ from .events import SupervisorStoppingEvent
 from .groups import ProcessGroup
 from .groups import ProcessGroupManager
 from .io import IoManager
-from .poller import Poller
 from .process import PidHistory
 from .setup import SupervisorSetup
 from .signals import SignalHandler
@@ -66,7 +66,7 @@ class Supervisor:
             self,
             *,
             config: ServerConfig,
-            poller: Poller,
+            poller: FdIoPoller,
             process_groups: ProcessGroupManager,
             signal_handler: SignalHandler,
             event_callbacks: EventCallbacks,
@@ -182,7 +182,9 @@ class Supervisor:
     #
 
     def _run_once(self) -> None:
+        now = time.time()
         self._poll()
+        log.info(f'Poll took {time.time() - now}')  # noqa
         self._reap()
         self._signal_handler.handle_signals()
         self._tick()
@@ -234,6 +236,7 @@ class Supervisor:
             return
 
         wp = waitpid()
+        log.info(f'Waited pid: {wp}')  # noqa
         if wp is None or not wp.pid:
             return
 

ominfra/supervisor/types.py

@@ -3,12 +3,13 @@ import abc
 import functools
 import typing as ta
 
+from omlish.lite.fdio.handlers import FdIoHandler
+
 from .configs import ProcessConfig
 from .configs import ProcessGroupConfig
 from .states import ProcessState
 from .states import SupervisorState
 from .utils.collections import KeyedCollectionAccessors
-from .utils.ostypes import Fd
 from .utils.ostypes import Pid
 from .utils.ostypes import Rc
 
@@ -61,69 +62,18 @@ class SupervisorStateManager(abc.ABC):
 ##
 
 
-class Dispatcher(abc.ABC):
-    @property
+class HasDispatchers(abc.ABC):
     @abc.abstractmethod
-    def channel(self) -> str:
+    def get_dispatchers(self) -> 'Dispatchers':
         raise NotImplementedError
 
-    @property
-    @abc.abstractmethod
-    def fd(self) -> Fd:
-        raise NotImplementedError
 
+class ProcessDispatcher(FdIoHandler, abc.ABC):
     @property
     @abc.abstractmethod
-    def closed(self) -> bool:
-        raise NotImplementedError
-
-    #
-
-    @abc.abstractmethod
-    def close(self) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def handle_error(self) -> None:
-        raise NotImplementedError
-
-    #
-
-    @abc.abstractmethod
-    def readable(self) -> bool:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def writable(self) -> bool:
-        raise NotImplementedError
-
-    #
-
-    def handle_read_event(self) -> None:
-        raise TypeError
-
-    def handle_write_event(self) -> None:
-        raise TypeError
-
-    #
-
-    def handle_connect(self) -> None:
-        raise TypeError
-
-    def handle_close(self) -> None:
-        raise TypeError
-
-    def handle_accepted(self, sock, addr) -> None:
-        raise TypeError
-
-
-class HasDispatchers(abc.ABC):
-    @abc.abstractmethod
-    def get_dispatchers(self) -> 'Dispatchers':
+    def channel(self) -> str:
         raise NotImplementedError
 
-
-class ProcessDispatcher(Dispatcher, abc.ABC):
     @property
     @abc.abstractmethod
     def process(self) -> 'Process':

{ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev129
+Version: 0.0.0.dev130
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev129
-Requires-Dist: omlish==0.0.0.dev129
+Requires-Dist: omdev==0.0.0.dev130
+Requires-Dist: omlish==0.0.0.dev130
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.18; extra == "all"