ominfra 0.0.0.dev127__py3-none-any.whl → 0.0.0.dev129__py3-none-any.whl

ominfra/supervisor/configs.py

@@ -7,11 +7,21 @@ import typing as ta
 
 from ..configs import ConfigMapping
 from ..configs import build_config_named_children
-from .datatypes import byte_size
-from .datatypes import existing_directory
-from .datatypes import existing_dirpath
-from .datatypes import logging_level
-from .datatypes import octal_type
+from .utils.fs import check_existing_dir
+from .utils.fs import check_path_with_existing_dir
+from .utils.strings import parse_bytes_size
+from .utils.strings import parse_octal
+
+
+##
+
+
+class RestartWhenExitUnexpected:
+    pass
+
+
+class RestartUnconditionally:
+    pass
 
 
 ##
@@ -104,12 +114,12 @@ class ServerConfig:
             **kwargs: ta.Any,
     ) -> 'ServerConfig':
         return cls(
-            umask=octal_type(umask),
-            directory=existing_directory(directory) if directory is not None else None,
-            logfile=existing_dirpath(logfile),
-            logfile_maxbytes=byte_size(logfile_maxbytes),
-            loglevel=logging_level(loglevel),
-            pidfile=existing_dirpath(pidfile),
+            umask=parse_octal(umask),
+            directory=check_existing_dir(directory) if directory is not None else None,
+            logfile=check_path_with_existing_dir(logfile),
+            logfile_maxbytes=parse_bytes_size(logfile_maxbytes),
+            loglevel=parse_logging_level(loglevel),
+            pidfile=check_path_with_existing_dir(pidfile),
             child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
             **kwargs,
         )
@@ -129,3 +139,16 @@ def prepare_server_config(dct: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.An
     group_dcts = build_config_named_children(out.get('groups'))
     out['groups'] = [prepare_process_group_config(group_dct) for group_dct in group_dcts or []]
     return out
+
+
+##
+
+
+def parse_logging_level(value: ta.Union[str, int]) -> int:
+    if isinstance(value, int):
+        return value
+    s = str(value).lower()
+    level = logging.getLevelNamesMapping().get(s.upper())
+    if level is None:
+        raise ValueError(f'bad logging level name {value!r}')
+    return level

ominfra/supervisor/dispatchers.py

@@ -1,11 +1,12 @@
 # ruff: noqa: UP006 UP007
-from .collections import KeyedCollection
 from .types import Dispatcher
-from .types import OutputDispatcher
+from .types import ProcessOutputDispatcher
+from .utils.collections import KeyedCollection
+from .utils.ostypes import Fd
 
 
-class Dispatchers(KeyedCollection[int, Dispatcher]):
-    def _key(self, v: Dispatcher) -> int:
+class Dispatchers(KeyedCollection[Fd, Dispatcher]):
+    def _key(self, v: Dispatcher) -> Fd:
         return v.fd
 
     #
@@ -23,10 +24,10 @@ class Dispatchers(KeyedCollection[int, Dispatcher]):
 
     def remove_logs(self) -> None:
         for d in self:
-            if isinstance(d, OutputDispatcher):
+            if isinstance(d, ProcessOutputDispatcher):
                 d.remove_logs()
 
     def reopen_logs(self) -> None:
         for d in self:
-            if isinstance(d, OutputDispatcher):
+            if isinstance(d, ProcessOutputDispatcher):
                 d.reopen_logs()

ominfra/supervisor/dispatchersimpl.py

@@ -8,29 +8,32 @@ import typing as ta
 from omlish.lite.logs import log
 
 from .configs import ProcessConfig
+from .configs import ServerConfig
 from .events import EventCallbacks
 from .events import ProcessCommunicationEvent
 from .events import ProcessLogStderrEvent
 from .events import ProcessLogStdoutEvent
-from .types import Dispatcher
-from .types import InputDispatcher
-from .types import OutputDispatcher
 from .types import Process
-from .utils import as_bytes
-from .utils import compact_traceback
-from .utils import find_prefix_at_end
-from .utils import read_fd
-from .utils import strip_escapes
-
-
-class BaseDispatcherImpl(Dispatcher, abc.ABC):
+from .types import ProcessDispatcher
+from .types import ProcessInputDispatcher
+from .types import ProcessOutputDispatcher
+from .utils.diag import compact_traceback
+from .utils.fds import read_fd
+from .utils.ostypes import Fd
+from .utils.strings import as_bytes
+from .utils.strings import find_prefix_at_end
+from .utils.strings import strip_escapes
+
+
+class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
     def __init__(
             self,
             process: Process,
             channel: str,
-            fd: int,
+            fd: Fd,
             *,
             event_callbacks: EventCallbacks,
+            server_config: ServerConfig,
     ) -> None:
         super().__init__()
 
@@ -38,6 +41,7 @@ class BaseDispatcherImpl(Dispatcher, abc.ABC):
         self._channel = channel  # 'stderr' or 'stdout'
         self._fd = fd
         self._event_callbacks = event_callbacks
+        self._server_config = server_config
 
         self._closed = False  # True if close() has been called
 
@@ -57,7 +61,7 @@ class BaseDispatcherImpl(Dispatcher, abc.ABC):
         return self._channel
 
     @property
-    def fd(self) -> int:
+    def fd(self) -> Fd:
         return self._fd
 
     @property
@@ -78,7 +82,7 @@ class BaseDispatcherImpl(Dispatcher, abc.ABC):
         self.close()
 
 
-class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
+class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispatcher):
     """
     Dispatcher for one channel (stdout or stderr) of one process. Serves several purposes:
 
@@ -91,15 +95,17 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
             self,
             process: Process,
             event_type: ta.Type[ProcessCommunicationEvent],
-            fd: int,
+            fd: Fd,
             *,
             event_callbacks: EventCallbacks,
+            server_config: ServerConfig,
     ) -> None:
         super().__init__(
             process,
             event_type.channel,
             fd,
             event_callbacks=event_callbacks,
+            server_config=server_config,
         )
 
         self._event_type = event_type
@@ -123,11 +129,10 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
 
         self._main_log_level = logging.DEBUG
 
-        self._log_to_main_log = process.context.config.loglevel <= self._main_log_level
+        self._log_to_main_log = self._server_config.loglevel <= self._main_log_level
 
-        config = self._process.config
-        self._stdout_events_enabled = config.stdout.events_enabled
-        self._stderr_events_enabled = config.stderr.events_enabled
+        self._stdout_events_enabled = self._process.config.stdout.events_enabled
+        self._stderr_events_enabled = self._process.config.stderr.events_enabled
 
     _child_log: ta.Optional[logging.Logger] = None  # the current logger (normal_log or capture_log)
     _normal_log: ta.Optional[logging.Logger] = None  # the "normal" (non-capture) logger
@@ -198,7 +203,7 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
         if not data:
             return
 
-        if self._process.context.config.strip_ansi:
+        if self._server_config.strip_ansi:
             data = strip_escapes(as_bytes(data))
 
         if self._child_log:
@@ -296,20 +301,22 @@ class OutputDispatcherImpl(BaseDispatcherImpl, OutputDispatcher):
             self.close()
 
 
-class InputDispatcherImpl(BaseDispatcherImpl, InputDispatcher):
+class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatcher):
     def __init__(
             self,
             process: Process,
             channel: str,
-            fd: int,
+            fd: Fd,
             *,
             event_callbacks: EventCallbacks,
+            server_config: ServerConfig,
     ) -> None:
         super().__init__(
             process,
             channel,
             fd,
             event_callbacks=event_callbacks,
+            server_config=server_config,
         )
 
         self._input_buffer = b''

ominfra/supervisor/groups.py

@@ -1,13 +1,13 @@
 # ruff: noqa: UP006 UP007
 import typing as ta
 
-from .collections import KeyedCollectionAccessors
 from .configs import ProcessGroupConfig
 from .events import EventCallbacks
 from .events import ProcessGroupAddedEvent
 from .events import ProcessGroupRemovedEvent
 from .types import Process
 from .types import ProcessGroup
+from .utils.collections import KeyedCollectionAccessors
 
 
 class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):

ominfra/supervisor/groupsimpl.py

@@ -61,7 +61,7 @@ class ProcessGroupImpl(ProcessGroup):
     #
 
     def get_unstopped_processes(self) -> ta.List[Process]:
-        return [x for x in self if not x.get_state().stopped]
+        return [x for x in self if not x.state.stopped]
 
     def stop_all(self) -> None:
         processes = list(self._by_name.values())
@@ -69,7 +69,7 @@ class ProcessGroupImpl(ProcessGroup):
         processes.reverse()  # stop in desc priority order
 
         for proc in processes:
-            state = proc.get_state()
+            state = proc.state
             if state == ProcessState.RUNNING:
                 # RUNNING -> STOPPING
                 proc.stop()

ominfra/supervisor/inject.py

@@ -6,34 +6,35 @@ from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
 
 from .configs import ServerConfig
-from .context import ServerContextImpl
-from .dispatchersimpl import InputDispatcherImpl
-from .dispatchersimpl import OutputDispatcherImpl
+from .dispatchersimpl import ProcessInputDispatcherImpl
+from .dispatchersimpl import ProcessOutputDispatcherImpl
 from .events import EventCallbacks
 from .groups import ProcessGroupManager
 from .groupsimpl import ProcessFactory
 from .groupsimpl import ProcessGroupImpl
+from .io import IoManager
 from .poller import Poller
 from .poller import get_poller_impl
-from .processes import PidHistory
-from .processesimpl import ProcessImpl
-from .processesimpl import ProcessSpawningFactory
+from .process import PidHistory
+from .processimpl import ProcessImpl
+from .processimpl import ProcessSpawningFactory
 from .setup import DaemonizeListener
 from .setup import DaemonizeListeners
 from .setup import SupervisorUser
 from .setupimpl import SupervisorSetup
 from .setupimpl import SupervisorSetupImpl
-from .signals import SignalReceiver
+from .signals import SignalHandler
 from .spawningimpl import InheritedFds
-from .spawningimpl import InputDispatcherFactory
-from .spawningimpl import OutputDispatcherFactory
+from .spawningimpl import ProcessInputDispatcherFactory
+from .spawningimpl import ProcessOutputDispatcherFactory
 from .spawningimpl import ProcessSpawningImpl
 from .supervisor import ProcessGroupFactory
-from .supervisor import SignalHandler
 from .supervisor import Supervisor
-from .types import ServerContext
+from .supervisor import SupervisorStateManagerImpl
 from .types import ServerEpoch
-from .users import get_user
+from .types import SupervisorStateManager
+from .utils.signals import SignalReceiver
+from .utils.users import get_user
 
 
 def bind_server(
@@ -52,17 +53,21 @@ def bind_server(
 
         inj.bind(DaemonizeListener, array=True, to_key=Poller),
 
-        inj.bind(ServerContextImpl, singleton=True),
-        inj.bind(ServerContext, to_key=ServerContextImpl),
-
         inj.bind(EventCallbacks, singleton=True),
 
         inj.bind(SignalReceiver, singleton=True),
 
+        inj.bind(IoManager, singleton=True),
+
         inj.bind(SignalHandler, singleton=True),
+
         inj.bind(ProcessGroupManager, singleton=True),
+
         inj.bind(Supervisor, singleton=True),
 
+        inj.bind(SupervisorStateManagerImpl, singleton=True),
+        inj.bind(SupervisorStateManager, to_key=SupervisorStateManagerImpl),
+
         inj.bind(PidHistory()),
 
         inj.bind_factory(ProcessGroupImpl, ProcessGroupFactory),
@@ -70,8 +75,8 @@ def bind_server(
 
         inj.bind_factory(ProcessSpawningImpl, ProcessSpawningFactory),
 
-        inj.bind_factory(OutputDispatcherImpl, OutputDispatcherFactory),
-        inj.bind_factory(InputDispatcherImpl, InputDispatcherFactory),
+        inj.bind_factory(ProcessOutputDispatcherImpl, ProcessOutputDispatcherFactory),
+        inj.bind_factory(ProcessInputDispatcherImpl, ProcessInputDispatcherFactory),
     ]
 
     #

ominfra/supervisor/io.py

@@ -0,0 +1,82 @@
+# ruff: noqa: UP006 UP007
+from omlish.lite.logs import log
+
+from .dispatchers import Dispatchers
+from .groups import ProcessGroupManager
+from .poller import Poller
+from .types import ExitNow
+
+
+##
+
+
+class IoManager:
+    def __init__(
+            self,
+            *,
+            poller: Poller,
+            process_groups: ProcessGroupManager,
+    ) -> None:
+        super().__init__()
+
+        self._poller = poller
+        self._process_groups = process_groups
+
+    def get_dispatchers(self) -> Dispatchers:
+        return Dispatchers(
+            d
+            for p in self._process_groups.all_processes()
+            for d in p.get_dispatchers()
+        )
+
+    def poll(self) -> None:
+        dispatchers = self.get_dispatchers()
+
+        for fd, dispatcher in dispatchers.items():
+            if dispatcher.readable():
+                self._poller.register_readable(fd)
+            if dispatcher.writable():
+                self._poller.register_writable(fd)
+
+        timeout = 1  # this cannot be fewer than the smallest TickEvent (5)
+        r, w = self._poller.poll(timeout)
+
+        for fd in r:
+            if fd in dispatchers:
+                try:
+                    dispatcher = dispatchers[fd]
+                    log.debug('read event caused by %r', dispatcher)
+                    dispatcher.handle_read_event()
+                    if not dispatcher.readable():
+                        self._poller.unregister_readable(fd)
+                except ExitNow:
+                    raise
+                except Exception:  # noqa
+                    dispatchers[fd].handle_error()
+            else:
+                # if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
+                # time, which may cause 100% cpu usage
+                log.debug('unexpected read event from fd %r', fd)
+                try:
+                    self._poller.unregister_readable(fd)
+                except Exception:  # noqa
+                    pass
+
+        for fd in w:
+            if fd in dispatchers:
+                try:
+                    dispatcher = dispatchers[fd]
+                    log.debug('write event caused by %r', dispatcher)
+                    dispatcher.handle_write_event()
+                    if not dispatcher.writable():
+                        self._poller.unregister_writable(fd)
+                except ExitNow:
+                    raise
+                except Exception:  # noqa
+                    dispatchers[fd].handle_error()
+            else:
+                log.debug('unexpected write event from fd %r', fd)
+                try:
+                    self._poller.unregister_writable(fd)
+                except Exception:  # noqa
+                    pass

ominfra/supervisor/main.py

@@ -37,18 +37,18 @@ from omlish.lite.http.coroserver import CoroHttpServer
 from omlish.lite.inject import inj
 from omlish.lite.journald import journald_log_handler_factory
 from omlish.lite.logs import configure_standard_logging
+from omlish.lite.runtime import is_debugger_attached
 
 from ..configs import read_config_file
 from .configs import ServerConfig
 from .configs import prepare_server_config
-from .context import ServerContextImpl
-from .context import ServerEpoch
 from .inject import bind_server
 from .spawningimpl import InheritedFds
 from .states import SupervisorState
 from .supervisor import Supervisor
-from .utils import ExitNow
-from .utils import get_open_fds
+from .types import ExitNow
+from .types import ServerEpoch
+from .utils.fds import get_open_fds
 
 
 ##
@@ -79,7 +79,7 @@ def main(
     if not no_logging:
         configure_standard_logging(
             'INFO',
-            handler_factory=journald_log_handler_factory if not args.no_journald else None,
+            handler_factory=journald_log_handler_factory if not (args.no_journald or is_debugger_attached()) else None,
         )
 
     #
@@ -102,7 +102,6 @@ def main(
             inherited_fds=inherited_fds,
         ))
 
-        context = injector[ServerContextImpl]
         supervisor = injector[Supervisor]
 
         try:
@@ -110,7 +109,7 @@ def main(
         except ExitNow:
             pass
 
-        if context.state < SupervisorState.RESTARTING:
+        if supervisor.state < SupervisorState.RESTARTING:
             break
 
 

ominfra/supervisor/pipes.py

@@ -4,24 +4,26 @@ import fcntl
 import os
 import typing as ta
 
-from .utils import close_fd
+from .utils.fds import close_fd
+from .utils.fds import make_pipe
+from .utils.ostypes import Fd
 
 
 @dc.dataclass(frozen=True)
 class ProcessPipes:
-    child_stdin: ta.Optional[int] = None
-    stdin: ta.Optional[int] = None
+    child_stdin: ta.Optional[Fd] = None
+    stdin: ta.Optional[Fd] = None
 
-    stdout: ta.Optional[int] = None
-    child_stdout: ta.Optional[int] = None
+    stdout: ta.Optional[Fd] = None
+    child_stdout: ta.Optional[Fd] = None
 
-    stderr: ta.Optional[int] = None
-    child_stderr: ta.Optional[int] = None
+    stderr: ta.Optional[Fd] = None
+    child_stderr: ta.Optional[Fd] = None
 
-    def child_fds(self) -> ta.List[int]:
+    def child_fds(self) -> ta.List[Fd]:
         return [fd for fd in [self.child_stdin, self.child_stdout, self.child_stderr] if fd is not None]
 
-    def parent_fds(self) -> ta.List[int]:
+    def parent_fds(self) -> ta.List[Fd]:
         return [fd for fd in [self.stdin, self.stdout, self.stderr] if fd is not None]
 
 
@@ -31,7 +33,7 @@ def make_process_pipes(stderr=True) -> ProcessPipes:
     read them in the mainloop without blocking.  If stderr is False, don't create a pipe for stderr.
     """
 
-    pipes: ta.Dict[str, ta.Optional[int]] = {
+    pipes: ta.Dict[str, ta.Optional[Fd]] = {
         'child_stdin': None,
         'stdin': None,
 
@@ -43,11 +45,11 @@ def make_process_pipes(stderr=True) -> ProcessPipes:
     }
 
     try:
-        pipes['child_stdin'], pipes['stdin'] = os.pipe()
-        pipes['stdout'], pipes['child_stdout'] = os.pipe()
+        pipes['child_stdin'], pipes['stdin'] = make_pipe()
+        pipes['stdout'], pipes['child_stdout'] = make_pipe()
 
         if stderr:
-            pipes['stderr'], pipes['child_stderr'] = os.pipe()
+            pipes['stderr'], pipes['child_stderr'] = make_pipe()
 
         for fd in (
                 pipes['stdout'],