ominfra 0.0.0.dev127__py3-none-any.whl → 0.0.0.dev128__py3-none-any.whl

ominfra/supervisor/{processesimpl.py → processimpl.py}

@@ -11,14 +11,13 @@ from omlish.lite.logs import log
 from omlish.lite.typing import Func1
 
 from .configs import ProcessConfig
-from .datatypes import RestartUnconditionally
+from .configs import RestartUnconditionally
 from .dispatchers import Dispatchers
 from .events import PROCESS_STATE_EVENT_MAP
 from .events import EventCallbacks
 from .pipes import ProcessPipes
 from .pipes import close_parent_pipes
-from .processes import ProcessStateError
-from .signals import sig_name
+from .process import ProcessStateError
 from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .states import ProcessState
@@ -27,8 +26,10 @@ from .types import InputDispatcher
 from .types import Process
 from .types import ProcessGroup
 from .types import ServerContext
-from .utils import as_string
-from .utils import decode_wait_status
+from .utils.os import decode_wait_status
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
+from .utils.signals import sig_name
 
 
 class ProcessSpawningFactory(Func1[Process, ProcessSpawning]):
@@ -66,7 +67,7 @@ class ProcessImpl(Process):
         self._pipes = ProcessPipes()
 
         self._state = ProcessState.STOPPED
-        self._pid = 0  # 0 when not running
+        self._pid = Pid(0)  # 0 when not running
 
         self._last_start = 0.  # Last time the subprocess was started; 0 if never
         self._last_stop = 0.  # Last time the subprocess was stopped; 0 if never
@@ -80,7 +81,7 @@ class ProcessImpl(Process):
 
         self._backoff = 0  # backoff counter (to startretries)
 
-        self._exitstatus: ta.Optional[int] = None  # status attached to dead process by finish()
+        self._exitstatus: ta.Optional[Rc] = None  # status attached to dead process by finish()
         self._spawn_err: ta.Optional[str] = None  # error message attached by spawn() if any
 
     #
@@ -103,7 +104,7 @@ class ProcessImpl(Process):
         return self._group
 
     @property
-    def pid(self) -> int:
+    def pid(self) -> Pid:
         return self._pid
 
     #
@@ -122,11 +123,9 @@ class ProcessImpl(Process):
 
     #
 
-    def spawn(self) -> ta.Optional[int]:
-        process_name = as_string(self._config.name)
-
+    def spawn(self) -> ta.Optional[Pid]:
         if self.pid:
-            log.warning('process \'%s\' already running', process_name)
+            log.warning('process \'%s\' already running', self.name)
             return None
 
         self.check_in_state(
@@ -249,7 +248,7 @@ class ProcessImpl(Process):
             self._check_and_adjust_for_system_clock_rollback(now)
 
             if now > (self._last_stop_report + 2):  # every 2 seconds
-                log.info('waiting for %s to stop', as_string(self._config.name))
+                log.info('waiting for %s to stop', self.name)
                 self._last_stop_report = now
 
     def give_up(self) -> None:
@@ -269,18 +268,17 @@ class ProcessImpl(Process):
         """
         now = time.time()
 
-        process_name = as_string(self._config.name)
         # If the process is in BACKOFF and we want to stop or kill it, then BACKOFF -> STOPPED.  This is needed because
         # if startretries is a large number and the process isn't starting successfully, the stop request would be
         # blocked for a long time waiting for the retries.
         if self._state == ProcessState.BACKOFF:
-            log.debug('Attempted to kill %s, which is in BACKOFF state.', process_name)
+            log.debug('Attempted to kill %s, which is in BACKOFF state.', self.name)
             self.change_state(ProcessState.STOPPED)
             return None
 
         args: tuple
         if not self.pid:
-            fmt, args = "attempted to kill %s with sig %s but it wasn't running", (process_name, sig_name(sig))
+            fmt, args = "attempted to kill %s with sig %s but it wasn't running", (self.name, sig_name(sig))
             log.debug(fmt, *args)
             return fmt % args
 
@@ -294,7 +292,7 @@ class ProcessImpl(Process):
         if killasgroup:
             as_group = 'process group '
 
-        log.debug('killing %s (pid %s) %s with signal %s', process_name, self.pid, as_group, sig_name(sig))
+        log.debug('killing %s (pid %s) %s with signal %s', self.name, self.pid, as_group, sig_name(sig))
 
         # RUNNING/STARTING/STOPPING -> STOPPING
         self._killing = True
@@ -303,24 +301,24 @@ class ProcessImpl(Process):
         self.check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
         self.change_state(ProcessState.STOPPING)
 
-        pid = self.pid
+        kpid = int(self.pid)
         if killasgroup:
             # send to the whole process group instead
-            pid = -self.pid
+            kpid = -kpid
 
         try:
             try:
-                os.kill(pid, sig)
+                os.kill(kpid, sig)
             except OSError as exc:
                 if exc.errno == errno.ESRCH:
-                    log.debug('unable to signal %s (pid %s), it probably just exited on its own: %s', process_name, self.pid, str(exc))  # noqa
+                    log.debug('unable to signal %s (pid %s), it probably just exited on its own: %s', self.name, self.pid, str(exc))  # noqa
                     # we could change the state here but we intentionally do not.  we will do it during normal SIGCHLD
                     # processing.
                     return None
                 raise
         except Exception:  # noqa
             tb = traceback.format_exc()
-            fmt, args = 'unknown problem killing %s (%s):%s', (process_name, self.pid, tb)
+            fmt, args = 'unknown problem killing %s (%s):%s', (self.name, self.pid, tb)
             log.critical(fmt, *args)
             self.change_state(ProcessState.UNKNOWN)
             self._killing = False
@@ -336,14 +334,13 @@ class ProcessImpl(Process):
         Return None if the signal was sent, or an error message string if an error occurred or if the subprocess is not
         running.
         """
-        process_name = as_string(self._config.name)
         args: tuple
         if not self.pid:
-            fmt, args = "attempted to send %s sig %s but it wasn't running", (process_name, sig_name(sig))
+            fmt, args = "Attempted to send %s sig %s but it wasn't running", (self.name, sig_name(sig))
             log.debug(fmt, *args)
             return fmt % args
 
-        log.debug('sending %s (pid %s) sig %s', process_name, self.pid, sig_name(sig))
+        log.debug('sending %s (pid %s) sig %s', self.name, self.pid, sig_name(sig))
 
         self.check_in_state(ProcessState.RUNNING, ProcessState.STARTING, ProcessState.STOPPING)
 
@@ -354,7 +351,7 @@ class ProcessImpl(Process):
                 if exc.errno == errno.ESRCH:
                     log.debug(
                         'unable to signal %s (pid %s), it probably just now exited on its own: %s',
-                        process_name,
+                        self.name,
                         self.pid,
                         str(exc),
                     )
@@ -364,14 +361,14 @@ class ProcessImpl(Process):
                 raise
         except Exception:  # noqa
             tb = traceback.format_exc()
-            fmt, args = 'unknown problem sending sig %s (%s):%s', (process_name, self.pid, tb)
+            fmt, args = 'unknown problem sending sig %s (%s):%s', (self.name, self.pid, tb)
             log.critical(fmt, *args)
             self.change_state(ProcessState.UNKNOWN)
             return fmt % args
 
         return None
 
-    def finish(self, sts: int) -> None:
+    def finish(self, sts: Rc) -> None:
         """The process was reaped and we need to report and manage its state."""
 
         self._dispatchers.drain()
@@ -383,7 +380,6 @@ class ProcessImpl(Process):
         self._check_and_adjust_for_system_clock_rollback(now)
 
         self._last_stop = now
-        process_name = as_string(self._config.name)
 
         if now > self._last_start:
             too_quickly = now - self._last_start < self._config.startsecs
@@ -392,7 +388,7 @@ class ProcessImpl(Process):
             log.warning(
                 "process '%s' (%s) last_start time is in the future, don't know how long process was running so "
                 "assuming it did not exit too quickly",
-                process_name,
+                self.name,
                 self.pid,
             )
 
@@ -402,9 +398,9 @@ class ProcessImpl(Process):
             # likely the result of a stop request implies STOPPING -> STOPPED
             self._killing = False
             self._delay = 0
-            self._exitstatus = es
+            self._exitstatus = Rc(es)
 
-            fmt, args = 'stopped: %s (%s)', (process_name, msg)
+            fmt, args = 'stopped: %s (%s)', (self.name, msg)
             self.check_in_state(ProcessState.STOPPING)
             self.change_state(ProcessState.STOPPED)
             if exit_expected:
@@ -418,7 +414,7 @@ class ProcessImpl(Process):
             self._spawn_err = 'Exited too quickly (process log may have details)'
             self.check_in_state(ProcessState.STARTING)
             self.change_state(ProcessState.BACKOFF)
-            log.warning('exited: %s (%s)', process_name, msg + '; not expected')
+            log.warning('exited: %s (%s)', self.name, msg + '; not expected')
 
         else:
             # this finish was not the result of a stop request, the program was in the RUNNING state but exited implies
@@ -437,14 +433,14 @@ class ProcessImpl(Process):
             if exit_expected:
                 # expected exit code
                 self.change_state(ProcessState.EXITED, expected=True)
-                log.info('exited: %s (%s)', process_name, msg + '; expected')
+                log.info('exited: %s (%s)', self.name, msg + '; expected')
             else:
                 # unexpected exit code
                 self._spawn_err = f'Bad exit code {es}'
                 self.change_state(ProcessState.EXITED, expected=False)
-                log.warning('exited: %s (%s)', process_name, msg + '; not expected')
+                log.warning('exited: %s (%s)', self.name, msg + '; not expected')
 
-        self._pid = 0
+        self._pid = Pid(0)
         close_parent_pipes(self._pipes)
         self._pipes = ProcessPipes()
         self._dispatchers = Dispatchers([])
@@ -482,7 +478,6 @@ class ProcessImpl(Process):
                         # BACKOFF -> STARTING
                         self.spawn()
 
-        process_name = as_string(self._config.name)
         if state == ProcessState.STARTING:
             if now - self._last_start > self._config.startsecs:
                 # STARTING -> RUNNING if the proc has started successfully and it has stayed up for at least
@@ -492,21 +487,21 @@ class ProcessImpl(Process):
                 self.check_in_state(ProcessState.STARTING)
                 self.change_state(ProcessState.RUNNING)
                 msg = ('entered RUNNING state, process has stayed up for > than %s seconds (startsecs)' % self._config.startsecs)  # noqa
-                logger.info('success: %s %s', process_name, msg)
+                logger.info('success: %s %s', self.name, msg)
 
         if state == ProcessState.BACKOFF:
             if self._backoff > self._config.startretries:
                 # BACKOFF -> FATAL if the proc has exceeded its number of retries
                 self.give_up()
                 msg = ('entered FATAL state, too many start retries too quickly')
-                logger.info('gave up: %s %s', process_name, msg)
+                logger.info('gave up: %s %s', self.name, msg)
 
         elif state == ProcessState.STOPPING:
             time_left = self._delay - now
             if time_left <= 0:
                 # kill processes which are taking too long to stop with a final sigkill.  if this doesn't kill it, the
                 # process will be stuck in the STOPPING state forever.
-                log.warning('killing \'%s\' (%s) with SIGKILL', process_name, self.pid)
+                log.warning('killing \'%s\' (%s) with SIGKILL', self.name, self.pid)
                 self.kill(signal.SIGKILL)
 
     def after_setuid(self) -> None:

ominfra/supervisor/setup.py

@@ -2,7 +2,7 @@
 import abc
 import typing as ta
 
-from .users import User
+from .utils.users import User
 
 
 ##

ominfra/supervisor/setupimpl.py

@@ -14,8 +14,9 @@ from .setup import DaemonizeListeners
 from .setup import SupervisorSetup
 from .setup import SupervisorUser
 from .types import ServerEpoch
-from .utils import real_exit
-from .utils import try_unlink
+from .utils.fs import try_unlink
+from .utils.os import real_exit
+from .utils.ostypes import Rc
 
 
 ##
@@ -238,7 +239,7 @@ class SupervisorSetupImpl(SupervisorSetup):
         if pid != 0:
             # Parent
             log.debug('supervisord forked; parent exiting')
-            real_exit(0)
+            real_exit(Rc(0))
 
         # Child
         log.info('daemonizing the supervisord process')

ominfra/supervisor/spawning.py

@@ -5,11 +5,12 @@ import dataclasses as dc
 from .dispatchers import Dispatchers
 from .pipes import ProcessPipes
 from .types import Process
+from .utils.ostypes import Pid
 
 
 @dc.dataclass(frozen=True)
 class SpawnedProcess:
-    pid: int
+    pid: Pid
     pipes: ProcessPipes
     dispatchers: Dispatchers
 

ominfra/supervisor/spawningimpl.py

@@ -25,7 +25,7 @@ from .pipes import close_child_pipes
 from .pipes import close_pipes
 from .pipes import make_process_pipes
 from .privileges import drop_privileges
-from .processes import PidHistory
+from .process import PidHistory
 from .spawning import ProcessSpawnError
 from .spawning import ProcessSpawning
 from .spawning import SpawnedProcess
@@ -34,22 +34,25 @@ from .types import InputDispatcher
 from .types import OutputDispatcher
 from .types import Process
 from .types import ProcessGroup
-from .utils import as_bytes
-from .utils import close_fd
-from .utils import compact_traceback
-from .utils import get_path
-from .utils import real_exit
+from .utils.diag import compact_traceback
+from .utils.fds import close_fd
+from .utils.fs import get_path
+from .utils.os import real_exit
+from .utils.ostypes import Fd
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
+from .utils.strings import as_bytes
 
 
-class OutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], int, OutputDispatcher]):
+class OutputDispatcherFactory(Func3[Process, ta.Type[ProcessCommunicationEvent], Fd, OutputDispatcher]):
     pass
 
 
-class InputDispatcherFactory(Func3[Process, str, int, InputDispatcher]):
+class InputDispatcherFactory(Func3[Process, str, Fd, InputDispatcher]):
     pass
 
 
-InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[int])
+InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[Fd])
 
 
 ##
@@ -120,7 +123,7 @@ class ProcessSpawningImpl(ProcessSpawning):
             raise ProcessSpawnError(f"Unknown error making dispatchers for '{self.process.name}': {exc}") from exc
 
         try:
-            pid = os.fork()
+            pid = Pid(os.fork())
         except OSError as exc:
             code = exc.args[0]
             if code == errno.EAGAIN:
@@ -299,7 +302,7 @@ class ProcessSpawningImpl(ProcessSpawning):
 
         finally:
             os.write(2, as_bytes('supervisor: child process was not spawned\n'))
-            real_exit(127)  # exit process with code for spawn failure
+            real_exit(Rc(127))  # exit process with code for spawn failure
 
         raise RuntimeError('Unreachable')
 
@@ -316,7 +319,7 @@ class ProcessSpawningImpl(ProcessSpawning):
         for i in range(3, self._server_config.minfds):
             if i in self._inherited_fds:
                 continue
-            close_fd(i)
+            close_fd(Fd(i))
 
     def _set_uid(self) -> ta.Optional[str]:
         if self.config.uid is None:

ominfra/supervisor/supervisor.py

@@ -18,17 +18,25 @@ from .events import SupervisorStoppingEvent
 from .groups import ProcessGroup
 from .groups import ProcessGroupManager
 from .poller import Poller
-from .processes import PidHistory
+from .process import PidHistory
 from .setup import SupervisorSetup
-from .signals import SignalReceiver
-from .signals import sig_name
 from .states import SupervisorState
 from .types import OutputDispatcher
 from .types import Process
-from .utils import ExitNow
-from .utils import as_string
-from .utils import decode_wait_status
-from .utils import timeslice
+from .utils.os import decode_wait_status
+from .utils.signals import SignalReceiver
+from .utils.signals import sig_name
+
+
+##
+
+
+class ExitNow(Exception):  # noqa
+    pass
+
+
+def timeslice(period: int, when: float) -> int:
+    return int(when - (when % period))
 
 
 ##
@@ -168,7 +176,7 @@ class Supervisor:
             # throttle 'waiting for x to die' reports
             now = time.time()
             if now > (self._last_shutdown_report + 3):  # every 3 secs
-                names = [as_string(p.config.name) for p in unstopped]
+                names = [p.config.name for p in unstopped]
                 namestr = ', '.join(names)
                 log.info('waiting for %s to die', namestr)
                 self._last_shutdown_report = now

ominfra/supervisor/types.py

@@ -3,12 +3,15 @@ import abc
 import functools
 import typing as ta
 
-from .collections import KeyedCollectionAccessors
 from .configs import ProcessConfig
 from .configs import ProcessGroupConfig
 from .configs import ServerConfig
 from .states import ProcessState
 from .states import SupervisorState
+from .utils.collections import KeyedCollectionAccessors
+from .utils.ostypes import Fd
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
 
 
 if ta.TYPE_CHECKING:
@@ -56,11 +59,6 @@ class ServerContext(abc.ABC):
     def set_state(self, state: SupervisorState) -> None:
         raise NotImplementedError
 
-    @property
-    @abc.abstractmethod
-    def pid_history(self) -> ta.Dict[int, 'Process']:
-        raise NotImplementedError
-
 
 ##
 
@@ -78,7 +76,7 @@ class Dispatcher(abc.ABC):
 
     @property
     @abc.abstractmethod
-    def fd(self) -> int:
+    def fd(self) -> Fd:
         raise NotImplementedError
 
     @property
@@ -156,7 +154,7 @@ class Process(ConfigPriorityOrdered, abc.ABC):
 
     @property
     @abc.abstractmethod
-    def pid(self) -> int:
+    def pid(self) -> Pid:
         raise NotImplementedError
 
     #
@@ -167,7 +165,7 @@ class Process(ConfigPriorityOrdered, abc.ABC):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def finish(self, sts: int) -> None:
+    def finish(self, sts: Rc) -> None:
         raise NotImplementedError
 
     @abc.abstractmethod

ominfra/supervisor/utils/diag.py

@@ -0,0 +1,31 @@
+# ruff: noqa: UP006 UP007
+import sys
+import types
+import typing as ta
+
+
+def compact_traceback() -> ta.Tuple[
+    ta.Tuple[str, str, int],
+    ta.Type[BaseException],
+    BaseException,
+    types.TracebackType,
+]:
+    t, v, tb = sys.exc_info()
+    if not tb:
+        raise RuntimeError('No traceback')
+
+    tbinfo = []
+    while tb:
+        tbinfo.append((
+            tb.tb_frame.f_code.co_filename,
+            tb.tb_frame.f_code.co_name,
+            str(tb.tb_lineno),
+        ))
+        tb = tb.tb_next
+
+    # just to be safe
+    del tb
+
+    file, function, line = tbinfo[-1]
+    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])  # noqa
+    return (file, function, line), t, v, info  # type: ignore

ominfra/supervisor/utils/fds.py

@@ -0,0 +1,46 @@
+# ruff: noqa: UP006 UP007
+import errno
+import os
+import typing as ta
+
+from .ostypes import Fd
+
+
+class PipeFds(ta.NamedTuple):
+    r: Fd
+    w: Fd
+
+
+def make_pipe() -> PipeFds:
+    return PipeFds(*os.pipe())  # type: ignore
+
+
+def read_fd(fd: Fd) -> bytes:
+    try:
+        data = os.read(fd, 2 << 16)  # 128K
+    except OSError as why:
+        if why.args[0] not in (errno.EWOULDBLOCK, errno.EBADF, errno.EINTR):
+            raise
+        data = b''
+    return data
+
+
+def close_fd(fd: Fd) -> bool:
+    try:
+        os.close(fd)
+    except OSError:
+        return False
+    return True
+
+
+def is_fd_open(fd: Fd) -> bool:
+    try:
+        n = os.dup(fd)
+    except OSError:
+        return False
+    os.close(n)
+    return True
+
+
+def get_open_fds(limit: int) -> ta.FrozenSet[Fd]:
+    return frozenset(fd for i in range(limit) if is_fd_open(fd := Fd(i)))

ominfra/supervisor/utils/fs.py

@@ -0,0 +1,47 @@
+# ruff: noqa: UP006 UP007
+import os
+import tempfile
+import typing as ta
+
+
+def try_unlink(path: str) -> bool:
+    try:
+        os.unlink(path)
+    except OSError:
+        return False
+    return True
+
+
+def mktempfile(suffix: str, prefix: str, dir: str) -> str:  # noqa
+    fd, filename = tempfile.mkstemp(suffix, prefix, dir)
+    os.close(fd)
+    return filename
+
+
+def get_path() -> ta.Sequence[str]:
+    """Return a list corresponding to $PATH, or a default."""
+
+    path = ['/bin', '/usr/bin', '/usr/local/bin']
+    if 'PATH' in os.environ:
+        p = os.environ['PATH']
+        if p:
+            path = p.split(os.pathsep)
+    return path
+
+
+def check_existing_dir(v: str) -> str:
+    nv = os.path.expanduser(v)
+    if os.path.isdir(nv):
+        return nv
+    raise ValueError(f'{v} is not an existing directory')
+
+
+def check_path_with_existing_dir(v: str) -> str:
+    nv = os.path.expanduser(v)
+    dir = os.path.dirname(nv)  # noqa
+    if not dir:
+        # relative pathname with no directory component
+        return nv
+    if os.path.isdir(dir):
+        return nv
+    raise ValueError(f'The directory named as part of the path {v} does not exist')

ominfra/supervisor/utils/os.py

@@ -0,0 +1,45 @@
+# ruff: noqa: UP006 UP007
+import os
+import typing as ta
+
+from .ostypes import Rc
+from .signals import sig_name
+
+
+##
+
+
+def real_exit(code: Rc) -> None:
+    os._exit(code)  # noqa
+
+
+##
+
+
+def decode_wait_status(sts: int) -> ta.Tuple[Rc, str]:
+    """
+    Decode the status returned by wait() or waitpid().
+
+    Return a tuple (exitstatus, message) where exitstatus is the exit status, or -1 if the process was killed by a
+    signal; and message is a message telling what happened.  It is the caller's responsibility to display the message.
+    """
+
+    if os.WIFEXITED(sts):
+        es = os.WEXITSTATUS(sts) & 0xffff
+        msg = f'exit status {es}'
+        return Rc(es), msg
+
+    elif os.WIFSIGNALED(sts):
+        sig = os.WTERMSIG(sts)
+        msg = f'terminated by {sig_name(sig)}'
+        if hasattr(os, 'WCOREDUMP'):
+            iscore = os.WCOREDUMP(sts)
+        else:
+            iscore = bool(sts & 0x80)
+        if iscore:
+            msg += ' (core dumped)'
+        return Rc(-1), msg
+
+    else:
+        msg = 'unknown termination cause 0x%04x' % sts  # noqa
+        return Rc(-1), msg

ominfra/supervisor/utils/ostypes.py

@@ -0,0 +1,9 @@
+import typing as ta
+
+
+Fd = ta.NewType('Fd', int)
+Pid = ta.NewType('Pid', int)
+Rc = ta.NewType('Rc', int)
+
+Uid = ta.NewType('Uid', int)
+Gid = ta.NewType('Gid', int)