ominfra 0.0.0.dev129__py3-none-any.whl → 0.0.0.dev131__py3-none-any.whl

ominfra/supervisor/configs.py

@@ -37,32 +37,32 @@ class ProcessConfig:
     umask: ta.Optional[int] = None
     priority: int = 999
 
-    autostart: bool = True
-    autorestart: str = 'unexpected'
+    auto_start: bool = True
+    auto_restart: str = 'unexpected'
 
-    startsecs: int = 1
-    startretries: int = 3
+    start_secs: int = 1
+    start_retries: int = 3
 
-    numprocs: int = 1
-    numprocs_start: int = 0
+    num_procs: int = 1
+    num_procs_start: int = 0
 
     @dc.dataclass(frozen=True)
     class Log:
         file: ta.Optional[str] = None
-        capture_maxbytes: ta.Optional[int] = None
+        capture_max_bytes: ta.Optional[int] = None
         events_enabled: bool = False
         syslog: bool = False
         backups: ta.Optional[int] = None
-        maxbytes: ta.Optional[int] = None
+        max_bytes: ta.Optional[int] = None
 
     stdout: Log = Log()
     stderr: Log = Log()
 
-    stopsignal: int = signal.SIGTERM
-    stopwaitsecs: int = 10
-    stopasgroup: bool = False
+    stop_signal: int = signal.SIGTERM
+    stop_wait_secs: int = 10
+    stop_as_group: bool = False
 
-    killasgroup: bool = False
+    kill_as_group: bool = False
 
     exitcodes: ta.Sequence[int] = (0,)
 
@@ -87,14 +87,14 @@ class ServerConfig:
     umask: int = 0o22
     directory: ta.Optional[str] = None
     logfile: str = 'supervisord.log'
-    logfile_maxbytes: int = 50 * 1024 * 1024
+    logfile_max_bytes: int = 50 * 1024 * 1024
     logfile_backups: int = 10
     loglevel: int = logging.INFO
     pidfile: str = 'supervisord.pid'
     identifier: str = 'supervisor'
     child_logdir: str = '/dev/null'
-    minfds: int = 1024
-    minprocs: int = 200
+    min_fds: int = 1024
+    min_procs: int = 200
     nocleanup: bool = False
     strip_ansi: bool = False
     silent: bool = False
@@ -107,7 +107,7 @@ class ServerConfig:
             umask: ta.Union[int, str] = 0o22,
             directory: ta.Optional[str] = None,
             logfile: str = 'supervisord.log',
-            logfile_maxbytes: ta.Union[int, str] = 50 * 1024 * 1024,
+            logfile_max_bytes: ta.Union[int, str] = 50 * 1024 * 1024,
             loglevel: ta.Union[int, str] = logging.INFO,
             pidfile: str = 'supervisord.pid',
             child_logdir: ta.Optional[str] = None,
@@ -117,7 +117,7 @@ class ServerConfig:
             umask=parse_octal(umask),
             directory=check_existing_dir(directory) if directory is not None else None,
             logfile=check_path_with_existing_dir(logfile),
-            logfile_maxbytes=parse_bytes_size(logfile_maxbytes),
+            logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
             loglevel=parse_logging_level(loglevel),
             pidfile=check_path_with_existing_dir(pidfile),
             child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),

ominfra/supervisor/dispatchers.py

@@ -1,13 +1,14 @@
 # ruff: noqa: UP006 UP007
-from .types import Dispatcher
+from omlish.lite.fdio.handlers import FdIoHandler
+
 from .types import ProcessOutputDispatcher
 from .utils.collections import KeyedCollection
 from .utils.ostypes import Fd
 
 
-class Dispatchers(KeyedCollection[Fd, Dispatcher]):
-    def _key(self, v: Dispatcher) -> Fd:
-        return v.fd
+class Dispatchers(KeyedCollection[Fd, FdIoHandler]):
+    def _key(self, v: FdIoHandler) -> Fd:
+        return Fd(v.fd())
 
     #
 
@@ -16,9 +17,9 @@ class Dispatchers(KeyedCollection[Fd, Dispatcher]):
             # note that we *must* call readable() for every dispatcher, as it may have side effects for a given
             # dispatcher (eg. call handle_listener_state_change for event listener processes)
             if d.readable():
-                d.handle_read_event()
+                d.on_readable()
             if d.writable():
-                d.handle_write_event()
+                d.on_writable()
 
     #
 

ominfra/supervisor/dispatchersimpl.py

@@ -60,7 +60,6 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
     def channel(self) -> str:
         return self._channel
 
-    @property
     def fd(self) -> Fd:
         return self._fd
 
@@ -75,7 +74,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
             log.debug('fd %s closed, stopped monitoring %s', self._fd, self)
             self._closed = True
 
-    def handle_error(self) -> None:
+    def on_error(self, exc: ta.Optional[BaseException] = None) -> None:
         nil, t, v, tbinfo = compact_traceback()
 
         log.critical('uncaptured python exception, closing channel %s (%s:%s %s)', repr(self), t, v, tbinfo)
@@ -148,7 +147,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
         channel = self._channel  # noqa
 
         logfile = self._lc.file
-        maxbytes = self._lc.maxbytes  # noqa
+        max_bytes = self._lc.max_bytes  # noqa
         backups = self._lc.backups  # noqa
         to_syslog = self._lc.syslog
 
@@ -160,8 +159,8 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
         #         self.normal_log,
         #         filename=logfile,
         #         fmt='%(message)s',
-        #         rotating=bool(maxbytes),  # optimization
-        #         maxbytes=maxbytes,
+        #         rotating=bool(max_bytes),  # optimization
+        #         max_bytes=max_bytes,
         #         backups=backups,
         #     )
 
@@ -173,17 +172,17 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
 
     def _init_capture_log(self) -> None:
         """
-        Configure the capture log for this process.  This log is used to temporarily capture output when special output
+        Configure the capture log for this process. This log is used to temporarily capture output when special output
         is detected. Sets self.capture_log if capturing is enabled.
         """
 
-        capture_maxbytes = self._lc.capture_maxbytes
-        if capture_maxbytes:
+        capture_max_bytes = self._lc.capture_max_bytes
+        if capture_max_bytes:
             self._capture_log = logging.getLogger(__name__)
             # loggers.handle_boundIO(
             #     self._capture_log,
             #     fmt='%(message)s',
-            #     maxbytes=capture_maxbytes,
+            #     max_bytes=capture_max_bytes,
             # )
 
     def remove_logs(self) -> None:
@@ -291,12 +290,12 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
             return False
         return True
 
-    def handle_read_event(self) -> None:
+    def on_readable(self) -> None:
         data = read_fd(self._fd)
         self._output_buffer += data
         self.record_output()
         if not data:
-            # if we get no data back from the pipe, it means that the child process has ended.  See
+            # if we get no data back from the pipe, it means that the child process has ended. See
             # mail.python.org/pipermail/python-dev/2004-August/046850.html
             self.close()
 
@@ -329,15 +328,12 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
             return True
         return False
 
-    def readable(self) -> bool:
-        return False
-
     def flush(self) -> None:
         # other code depends on this raising EPIPE if the pipe is closed
         sent = os.write(self._fd, as_bytes(self._input_buffer))
         self._input_buffer = self._input_buffer[sent:]
 
-    def handle_write_event(self) -> None:
+    def on_writable(self) -> None:
         if self._input_buffer:
             try:
                 self.flush()

ominfra/supervisor/groups.py

@@ -2,15 +2,20 @@
 import typing as ta
 
 from .configs import ProcessGroupConfig
+from .dispatchers import Dispatchers
 from .events import EventCallbacks
 from .events import ProcessGroupAddedEvent
 from .events import ProcessGroupRemovedEvent
+from .types import HasDispatchers
 from .types import Process
 from .types import ProcessGroup
 from .utils.collections import KeyedCollectionAccessors
 
 
-class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
+class ProcessGroupManager(
+    KeyedCollectionAccessors[str, ProcessGroup],
+    HasDispatchers,
+):
     def __init__(
             self,
             *,
@@ -34,6 +39,16 @@ class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
 
     #
 
+    def get_dispatchers(self) -> Dispatchers:
+        return Dispatchers(
+            d
+            for g in self
+            for p in g
+            for d in p.get_dispatchers()
+        )
+
+    #
+
     def add(self, group: ProcessGroup) -> None:
         if (name := group.name) in self._by_name:
             raise KeyError(f'Process group already exists: {name}')

ominfra/supervisor/http.py

@@ -0,0 +1,130 @@
+# ruff: noqa: U006 UP007
+import json
+import socket
+import typing as ta
+
+from omlish.lite.check import check_not_none
+from omlish.lite.fdio.corohttp import CoroHttpServerConnectionFdIoHandler
+from omlish.lite.fdio.handlers import SocketFdIoHandler
+from omlish.lite.http.handlers import HttpHandler
+from omlish.lite.http.handlers import HttpHandlerRequest
+from omlish.lite.http.handlers import HttpHandlerResponse
+from omlish.lite.json import JSON_PRETTY_KWARGS
+from omlish.lite.socket import SocketAddress
+
+from .dispatchers import Dispatchers
+from .groups import ProcessGroupManager
+from .types import HasDispatchers
+
+
+##
+
+
+class SocketServerFdIoHandler(SocketFdIoHandler):
+    def __init__(
+            self,
+            addr: SocketAddress,
+            on_connect: ta.Callable[[socket.socket, SocketAddress], None],
+    ) -> None:
+        sock = socket.create_server(addr)
+        sock.setblocking(False)
+
+        super().__init__(addr, sock)
+
+        self._on_connect = on_connect
+
+        sock.listen(1)
+
+    def readable(self) -> bool:
+        return True
+
+    def on_readable(self) -> None:
+        cli_sock, cli_addr = check_not_none(self._sock).accept()
+        cli_sock.setblocking(False)
+
+        self._on_connect(cli_sock, cli_addr)
+
+
+##
+
+
+class HttpServer(HasDispatchers):
+    class Address(ta.NamedTuple):
+        a: SocketAddress
+
+    class Handler(ta.NamedTuple):
+        h: HttpHandler
+
+    def __init__(
+            self,
+            handler: Handler,
+            addr: Address = Address(('localhost', 8000)),
+    ) -> None:
+        super().__init__()
+
+        self._handler = handler.h
+        self._addr = addr.a
+
+        self._server = SocketServerFdIoHandler(self._addr, self._on_connect)
+
+        self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
+
+    def get_dispatchers(self) -> Dispatchers:
+        l = []
+        for c in self._conns:
+            if not c.closed:
+                l.append(c)
+        self._conns = l
+        return Dispatchers([
+            self._server,
+            *l,
+        ])
+
+    def _on_connect(self, sock: socket.socket, addr: SocketAddress) -> None:
+        conn = CoroHttpServerConnectionFdIoHandler(
+            addr,
+            sock,
+            self._handler,
+        )
+
+        self._conns.append(conn)
+
+
+##
+
+
+class SupervisorHttpHandler:
+    def __init__(
+            self,
+            *,
+            groups: ProcessGroupManager,
+    ) -> None:
+        super().__init__()
+
+        self._groups = groups
+
+    def handle(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
+        dct = {
+            'method': req.method,
+            'path': req.path,
+            'data': len(req.data or b''),
+            'groups': {
+                g.name: {
+                    'processes': {
+                        p.name: {
+                            'pid': p.pid,
+                        }
+                        for p in g
+                    },
+                }
+                for g in self._groups
+            },
+        }
+
+        return HttpHandlerResponse(
+            200,
+            data=json.dumps(dct, **JSON_PRETTY_KWARGS).encode('utf-8') + b'\n',
+            headers={
+                'Content-Type': 'application/json',
+            },
+        )

ominfra/supervisor/inject.py

@@ -1,6 +1,11 @@
 # ruff: noqa: UP006 UP007
+import dataclasses as dc
 import typing as ta
 
+from omlish.lite.fdio.kqueue import KqueueFdIoPoller  # noqa
+from omlish.lite.fdio.pollers import FdIoPoller
+from omlish.lite.fdio.pollers import PollFdIoPoller  # noqa
+from omlish.lite.fdio.pollers import SelectFdIoPoller
 from omlish.lite.inject import InjectorBindingOrBindings
 from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
@@ -12,9 +17,10 @@ from .events import EventCallbacks
 from .groups import ProcessGroupManager
 from .groupsimpl import ProcessFactory
 from .groupsimpl import ProcessGroupImpl
+from .http import HttpServer
+from .http import SupervisorHttpHandler
+from .io import HasDispatchersList
 from .io import IoManager
-from .poller import Poller
-from .poller import get_poller_impl
 from .process import PidHistory
 from .processimpl import ProcessImpl
 from .processimpl import ProcessSpawningFactory
@@ -31,12 +37,24 @@ from .spawningimpl import ProcessSpawningImpl
 from .supervisor import ProcessGroupFactory
 from .supervisor import Supervisor
 from .supervisor import SupervisorStateManagerImpl
+from .types import HasDispatchers
 from .types import ServerEpoch
 from .types import SupervisorStateManager
 from .utils.signals import SignalReceiver
 from .utils.users import get_user
 
 
+@dc.dataclass(frozen=True)
+class _FdIoPollerDaemonizeListener(DaemonizeListener):
+    _poller: FdIoPoller
+
+    def before_daemonize(self) -> None:
+        self._poller.close()
+
+    def after_daemonize(self) -> None:
+        self._poller.reopen()
+
+
 def bind_server(
         config: ServerConfig,
         *,
@@ -46,22 +64,24 @@ def bind_server(
     lst: ta.List[InjectorBindingOrBindings] = [
         inj.bind(config),
 
+        inj.bind_array(DaemonizeListener),
         inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
 
         inj.bind(SupervisorSetupImpl, singleton=True),
         inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
 
-        inj.bind(DaemonizeListener, array=True, to_key=Poller),
-
         inj.bind(EventCallbacks, singleton=True),
 
         inj.bind(SignalReceiver, singleton=True),
 
         inj.bind(IoManager, singleton=True),
+        inj.bind_array(HasDispatchers),
+        inj.bind_array_type(HasDispatchers, HasDispatchersList),
 
         inj.bind(SignalHandler, singleton=True),
 
         inj.bind(ProcessGroupManager, singleton=True),
+        inj.bind(HasDispatchers, array=True, to_key=ProcessGroupManager),
 
         inj.bind(Supervisor, singleton=True),
 
@@ -94,7 +114,26 @@ def bind_server(
 
     #
 
-    lst.append(inj.bind(get_poller_impl(), key=Poller, singleton=True))
+    poller_impl = next(filter(None, [
+        KqueueFdIoPoller,
+        PollFdIoPoller,
+        SelectFdIoPoller,
+    ]))
+    lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
+    inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
+
+    #
+
+    def _provide_http_handler(s: SupervisorHttpHandler) -> HttpServer.Handler:
+        return HttpServer.Handler(s.handle)
+
+    lst.extend([
+        inj.bind(HttpServer, singleton=True, eager=True),
+        inj.bind(HasDispatchers, array=True, to_key=HttpServer),
+
+        inj.bind(SupervisorHttpHandler, singleton=True),
+        inj.bind(_provide_http_handler),
+    ])
 
     #
 

ominfra/supervisor/io.py

@@ -1,58 +1,71 @@
 # ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.fdio.pollers import FdIoPoller
 from omlish.lite.logs import log
 
 from .dispatchers import Dispatchers
-from .groups import ProcessGroupManager
-from .poller import Poller
 from .types import ExitNow
+from .types import HasDispatchers
+from .utils.ostypes import Fd
 
 
 ##
 
 
-class IoManager:
+HasDispatchersList = ta.NewType('HasDispatchersList', ta.Sequence[HasDispatchers])
+
+
+class IoManager(HasDispatchers):
     def __init__(
             self,
             *,
-            poller: Poller,
-            process_groups: ProcessGroupManager,
+            poller: FdIoPoller,
+            has_dispatchers_list: HasDispatchersList,
     ) -> None:
         super().__init__()
 
         self._poller = poller
-        self._process_groups = process_groups
+        self._has_dispatchers_list = has_dispatchers_list
 
     def get_dispatchers(self) -> Dispatchers:
         return Dispatchers(
             d
-            for p in self._process_groups.all_processes()
-            for d in p.get_dispatchers()
+            for hd in self._has_dispatchers_list
+            for d in hd.get_dispatchers()
         )
 
     def poll(self) -> None:
         dispatchers = self.get_dispatchers()
 
-        for fd, dispatcher in dispatchers.items():
-            if dispatcher.readable():
-                self._poller.register_readable(fd)
-            if dispatcher.writable():
-                self._poller.register_writable(fd)
+        self._poller.update(
+            {fd for fd, d in dispatchers.items() if d.readable()},
+            {fd for fd, d in dispatchers.items() if d.writable()},
+        )
 
         timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
-        r, w = self._poller.poll(timeout)
+        log.info(f'Polling: {timeout=}')  # noqa
+        polled = self._poller.poll(timeout)
+        log.info(f'Polled: {polled=}')  # noqa
+        if polled.msg is not None:
+            log.error(polled.msg)
+        if polled.exc is not None:
+            log.error('Poll exception: %r', polled.exc)
 
-        for fd in r:
+        for r in polled.r:
+            fd = Fd(r)
             if fd in dispatchers:
+                dispatcher = dispatchers[fd]
                 try:
-                    dispatcher = dispatchers[fd]
                     log.debug('read event caused by %r', dispatcher)
-                    dispatcher.handle_read_event()
+                    dispatcher.on_readable()
                     if not dispatcher.readable():
                         self._poller.unregister_readable(fd)
                 except ExitNow:
                     raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
+                except Exception as exc:  # noqa
+                    log.exception('Error in dispatcher: %r', dispatcher)
+                    dispatcher.on_error(exc)
             else:
                 # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
                 # time, which may cause 100% cpu usage
@@ -62,18 +75,20 @@ class IoManager:
                 except Exception:  # noqa
                     pass
 
-        for fd in w:
+        for w in polled.w:
+            fd = Fd(w)
             if fd in dispatchers:
+                dispatcher = dispatchers[fd]
                 try:
-                    dispatcher = dispatchers[fd]
                     log.debug('write event caused by %r', dispatcher)
-                    dispatcher.handle_write_event()
+                    dispatcher.on_writable()
                     if not dispatcher.writable():
                         self._poller.unregister_writable(fd)
                 except ExitNow:
                     raise
-                except Exception:  # noqa
-                    dispatchers[fd].handle_error()
+                except Exception as exc:  # noqa
+                    log.exception('Error in dispatcher: %r', dispatcher)
+                    dispatcher.on_error(exc)
             else:
                 log.debug('unexpected write event from fd %r', fd)
                 try:

ominfra/supervisor/pipes.py

@@ -29,8 +29,8 @@ class ProcessPipes:
 
 def make_process_pipes(stderr=True) -> ProcessPipes:
     """
-    Create pipes for parent to child stdin/stdout/stderr communications.  Open fd in non-blocking mode so we can
-    read them in the mainloop without blocking.  If stderr is False, don't create a pipe for stderr.
+    Create pipes for parent to child stdin/stdout/stderr communications. Open fd in non-blocking mode so we can read
+    them in the mainloop without blocking. If stderr is False, don't create a pipe for stderr.
     """
 
     pipes: ta.Dict[str, ta.Optional[Fd]] = {

ominfra/supervisor/privileges.py

@@ -7,9 +7,8 @@ import typing as ta
 
 def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
     """
-    Drop privileges to become the specified user, which may be a username or uid.  Called for supervisord startup
-    and when spawning subprocesses.  Returns None on success or a string error message if privileges could not be
-    dropped.
+    Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
+    when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
     """
 
     if user is None:
@@ -33,9 +32,8 @@ def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
     current_uid = os.getuid()
 
     if current_uid == uid:
-        # do nothing and return successfully if the uid is already the current one.  this allows a supervisord
-        # running as an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in
-        # it.
+        # do nothing and return successfully if the uid is already the current one. this allows a supervisord running as
+        # an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in it.
         return None
 
     if current_uid != 0: