ominfra 0.0.0.dev126__py3-none-any.whl → 0.0.0.dev128__py3-none-any.whl

ominfra/supervisor/configs.py

@@ -7,11 +7,21 @@ import typing as ta
 
 from ..configs import ConfigMapping
 from ..configs import build_config_named_children
-from .datatypes import byte_size
-from .datatypes import existing_directory
-from .datatypes import existing_dirpath
-from .datatypes import logging_level
-from .datatypes import octal_type
+from .utils.fs import check_existing_dir
+from .utils.fs import check_path_with_existing_dir
+from .utils.strings import parse_bytes_size
+from .utils.strings import parse_octal
+
+
+##
+
+
+class RestartWhenExitUnexpected:
+    pass
+
+
+class RestartUnconditionally:
+    pass
 
 
 ##
@@ -104,12 +114,12 @@ class ServerConfig:
             **kwargs: ta.Any,
     ) -> 'ServerConfig':
         return cls(
-            umask=octal_type(umask),
-            directory=existing_directory(directory) if directory is not None else None,
-            logfile=existing_dirpath(logfile),
-            logfile_maxbytes=byte_size(logfile_maxbytes),
-            loglevel=logging_level(loglevel),
-            pidfile=existing_dirpath(pidfile),
+            umask=parse_octal(umask),
+            directory=check_existing_dir(directory) if directory is not None else None,
+            logfile=check_path_with_existing_dir(logfile),
+            logfile_maxbytes=parse_bytes_size(logfile_maxbytes),
+            loglevel=parse_logging_level(loglevel),
+            pidfile=check_path_with_existing_dir(pidfile),
             child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
             **kwargs,
         )
@@ -129,3 +139,16 @@ def prepare_server_config(dct: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.An
     group_dcts = build_config_named_children(out.get('groups'))
     out['groups'] = [prepare_process_group_config(group_dct) for group_dct in group_dcts or []]
     return out
+
+
+##
+
+
+def parse_logging_level(value: ta.Union[str, int]) -> int:
+    if isinstance(value, int):
+        return value
+    s = str(value).lower()
+    level = logging.getLevelNamesMapping().get(s.upper())
+    if level is None:
+        raise ValueError(f'bad logging level name {value!r}')
+    return level

ominfra/supervisor/context.py

@@ -1,34 +1,18 @@
 # ruff: noqa: UP006 UP007
 import errno
-import fcntl
-import grp
 import os
-import pwd
-import re
-import resource
-import stat
 import typing as ta
-import warnings
 
 from omlish.lite.logs import log
 
 from .configs import ServerConfig
-from .datatypes import gid_for_uid
-from .datatypes import name_to_uid
-from .exceptions import NoPermissionError
-from .exceptions import NotExecutableError
-from .exceptions import NotFoundError
 from .poller import Poller
 from .states import SupervisorState
-from .types import Process
 from .types import ServerContext
-from .utils import close_fd
-from .utils import mktempfile
-from .utils import real_exit
-from .utils import try_unlink
-
-
-ServerEpoch = ta.NewType('ServerEpoch', int)
+from .types import ServerEpoch
+from .utils.fs import mktempfile
+from .utils.ostypes import Pid
+from .utils.ostypes import Rc
 
 
 class ServerContextImpl(ServerContext):
@@ -45,19 +29,8 @@ class ServerContextImpl(ServerContext):
         self._poller = poller
         self._epoch = epoch
 
-        self._pid_history: ta.Dict[int, Process] = {}
         self._state: SupervisorState = SupervisorState.RUNNING
 
-        if config.user is not None:
-            uid = name_to_uid(config.user)
-            self._uid: ta.Optional[int] = uid
-            self._gid: ta.Optional[int] = gid_for_uid(uid)
-        else:
-            self._uid = None
-            self._gid = None
-
-        self._unlink_pidfile = False
-
     @property
     def config(self) -> ServerConfig:
         return self._config
@@ -77,21 +50,9 @@ class ServerContextImpl(ServerContext):
     def set_state(self, state: SupervisorState) -> None:
         self._state = state
 
-    @property
-    def pid_history(self) -> ta.Dict[int, Process]:
-        return self._pid_history
-
-    @property
-    def uid(self) -> ta.Optional[int]:
-        return self._uid
-
-    @property
-    def gid(self) -> ta.Optional[int]:
-        return self._gid
+    #
 
-    ##
-
-    def waitpid(self) -> ta.Tuple[ta.Optional[int], ta.Optional[int]]:
+    def waitpid(self) -> ta.Tuple[ta.Optional[Pid], ta.Optional[Rc]]:
         # Need pthread_sigmask here to avoid concurrent sigchld, but Python doesn't offer in Python < 3.4.  There is
         # still a race condition here; we can get a sigchld while we're sitting in the waitpid call. However, AFAICT, if
         # waitpid is interrupted by SIGCHLD, as long as we call waitpid again (which happens every so often during the
@@ -107,167 +68,7 @@ class ServerContextImpl(ServerContext):
             if code == errno.EINTR:
                 log.debug('EINTR during reap')
             pid, sts = None, None
-        return pid, sts
-
-    def set_uid_or_exit(self) -> None:
-        """
-        Set the uid of the supervisord process.  Called during supervisord startup only.  No return value.  Exits the
-        process via usage() if privileges could not be dropped.
-        """
-
-        if self.uid is None:
-            if os.getuid() == 0:
-                warnings.warn(
-                    'Supervisor is running as root.  Privileges were not dropped because no user is specified in the '
-                    'config file.  If you intend to run as root, you can set user=root in the config file to avoid '
-                    'this message.',
-                )
-        else:
-            msg = drop_privileges(self.uid)
-            if msg is None:
-                log.info('Set uid to user %s succeeded', self.uid)
-            else:  # failed to drop privileges
-                raise RuntimeError(msg)
-
-    def set_rlimits_or_exit(self) -> None:
-        """
-        Set the rlimits of the supervisord process.  Called during supervisord startup only.  No return value.  Exits
-        the process via usage() if any rlimits could not be set.
-        """
-
-        limits = []
-
-        if hasattr(resource, 'RLIMIT_NOFILE'):
-            limits.append({
-                'msg': (
-                    'The minimum number of file descriptors required to run this process is %(min_limit)s as per the '
-                    '"minfds" command-line argument or config file setting. The current environment will only allow '
-                    'you to open %(hard)s file descriptors.  Either raise the number of usable file descriptors in '
-                    'your environment (see README.rst) or lower the minfds setting in the config file to allow the '
-                    'process to start.'
-                ),
-                'min': self.config.minfds,
-                'resource': resource.RLIMIT_NOFILE,
-                'name': 'RLIMIT_NOFILE',
-            })
-
-        if hasattr(resource, 'RLIMIT_NPROC'):
-            limits.append({
-                'msg': (
-                    'The minimum number of available processes required to run this program is %(min_limit)s as per '
-                    'the "minprocs" command-line argument or config file setting. The current environment will only '
-                    'allow you to open %(hard)s processes.  Either raise the number of usable processes in your '
-                    'environment (see README.rst) or lower the minprocs setting in the config file to allow the '
-                    'program to start.'
-                ),
-                'min': self.config.minprocs,
-                'resource': resource.RLIMIT_NPROC,
-                'name': 'RLIMIT_NPROC',
-            })
-
-        for limit in limits:
-            min_limit = limit['min']
-            res = limit['resource']
-            msg = limit['msg']
-            name = limit['name']
-
-            soft, hard = resource.getrlimit(res)  # type: ignore
-
-            # -1 means unlimited
-            if soft < min_limit and soft != -1:  # type: ignore
-                if hard < min_limit and hard != -1:  # type: ignore
-                    # setrlimit should increase the hard limit if we are root, if not then setrlimit raises and we print
-                    # usage
-                    hard = min_limit  # type: ignore
-
-                try:
-                    resource.setrlimit(res, (min_limit, hard))  # type: ignore
-                    log.info('Increased %s limit to %s', name, min_limit)
-                except (resource.error, ValueError):
-                    raise RuntimeError(msg % dict(  # type: ignore  # noqa
-                        min_limit=min_limit,
-                        res=res,
-                        name=name,
-                        soft=soft,
-                        hard=hard,
-                    ))
-
-    def cleanup(self) -> None:
-        if self._unlink_pidfile:
-            try_unlink(self.config.pidfile)
-        self._poller.close()
-
-    def cleanup_fds(self) -> None:
-        # try to close any leaked file descriptors (for reload)
-        start = 5
-        os.closerange(start, self.config.minfds)
-
-    def clear_auto_child_logdir(self) -> None:
-        # must be called after realize()
-        child_logdir = self.config.child_logdir
-        fnre = re.compile(rf'.+?---{self.config.identifier}-\S+\.log\.?\d{{0,4}}')
-        try:
-            filenames = os.listdir(child_logdir)
-        except OSError:
-            log.warning('Could not clear child_log dir')
-            return
-
-        for filename in filenames:
-            if fnre.match(filename):
-                pathname = os.path.join(child_logdir, filename)
-                try:
-                    os.remove(pathname)
-                except OSError:
-                    log.warning('Failed to clean up %r', pathname)
-
-    def daemonize(self) -> None:
-        self._poller.before_daemonize()
-        self._daemonize()
-        self._poller.after_daemonize()
-
-    def _daemonize(self) -> None:
-        # To daemonize, we need to become the leader of our own session (process) group.  If we do not, signals sent to
-        # our parent process will also be sent to us.   This might be bad because signals such as SIGINT can be sent to
-        # our parent process during normal (uninteresting) operations such as when we press Ctrl-C in the parent
-        # terminal window to escape from a logtail command. To disassociate ourselves from our parent's session group we
-        # use os.setsid.  It means "set session id", which has the effect of disassociating a process from is current
-        # session and process group and setting itself up as a new session leader.
-        #
-        # Unfortunately we cannot call setsid if we're already a session group leader, so we use "fork" to make a copy
-        # of ourselves that is guaranteed to not be a session group leader.
-        #
-        # We also change directories, set stderr and stdout to null, and change our umask.
-        #
-        # This explanation was (gratefully) garnered from
-        # http://www.cems.uwe.ac.uk/~irjohnso/coursenotes/lrc/system/daemons/d3.htm
-
-        pid = os.fork()
-        if pid != 0:
-            # Parent
-            log.debug('supervisord forked; parent exiting')
-            real_exit(0)
-
-        # Child
-        log.info('daemonizing the supervisord process')
-        if self.config.directory:
-            try:
-                os.chdir(self.config.directory)
-            except OSError as err:
-                log.critical("can't chdir into %r: %s", self.config.directory, err)
-            else:
-                log.info('set current directory: %r', self.config.directory)
-
-        os.dup2(0, os.open('/dev/null', os.O_RDONLY))
-        os.dup2(1, os.open('/dev/null', os.O_WRONLY))
-        os.dup2(2, os.open('/dev/null', os.O_WRONLY))
-
-        os.setsid()
-
-        os.umask(self.config.umask)
-
-        # XXX Stevens, in his Advanced Unix book, section 13.3 (page 417) recommends calling umask(0) and closing unused
-        # file descriptors.  In his Network Programming book, he additionally recommends ignoring SIGHUP and forking
-        # again after the setsid() call, for obscure SVR4 reasons.
+        return pid, sts  # type: ignore
 
     def get_auto_child_log_name(self, name: str, identifier: str, channel: str) -> str:
         prefix = f'{name}-{channel}---{identifier}-'
@@ -277,142 +78,3 @@ class ServerContextImpl(ServerContext):
             dir=self.config.child_logdir,
         )
         return logfile
-
-    def write_pidfile(self) -> None:
-        pid = os.getpid()
-        try:
-            with open(self.config.pidfile, 'w') as f:
-                f.write(f'{pid}\n')
-        except OSError:
-            log.critical('could not write pidfile %s', self.config.pidfile)
-        else:
-            self._unlink_pidfile = True
-            log.info('supervisord started with pid %s', pid)
-
-
-def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
-    """
-    Drop privileges to become the specified user, which may be a username or uid.  Called for supervisord startup
-    and when spawning subprocesses.  Returns None on success or a string error message if privileges could not be
-    dropped.
-    """
-
-    if user is None:
-        return 'No user specified to setuid to!'
-
-    # get uid for user, which can be a number or username
-    try:
-        uid = int(user)
-    except ValueError:
-        try:
-            pwrec = pwd.getpwnam(user)  # type: ignore
-        except KeyError:
-            return f"Can't find username {user!r}"
-        uid = pwrec[2]
-    else:
-        try:
-            pwrec = pwd.getpwuid(uid)
-        except KeyError:
-            return f"Can't find uid {uid!r}"
-
-    current_uid = os.getuid()
-
-    if current_uid == uid:
-        # do nothing and return successfully if the uid is already the current one.  this allows a supervisord
-        # running as an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in
-        # it.
-        return None
-
-    if current_uid != 0:
-        return "Can't drop privilege as nonroot user"
-
-    gid = pwrec[3]
-    if hasattr(os, 'setgroups'):
-        user = pwrec[0]
-        groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
-
-        # always put our primary gid first in this list, otherwise we can lose group info since sometimes the first
-        # group in the setgroups list gets overwritten on the subsequent setgid call (at least on freebsd 9 with
-        # python 2.7 - this will be safe though for all unix /python version combos)
-        groups.insert(0, gid)
-        try:
-            os.setgroups(groups)
-        except OSError:
-            return 'Could not set groups of effective user'
-
-    try:
-        os.setgid(gid)
-    except OSError:
-        return 'Could not set group id of effective user'
-
-    os.setuid(uid)
-
-    return None
-
-
-def make_pipes(stderr=True) -> ta.Mapping[str, int]:
-    """
-    Create pipes for parent to child stdin/stdout/stderr communications.  Open fd in non-blocking mode so we can
-    read them in the mainloop without blocking.  If stderr is False, don't create a pipe for stderr.
-    """
-
-    pipes: ta.Dict[str, ta.Optional[int]] = {
-        'child_stdin': None,
-        'stdin': None,
-        'stdout': None,
-        'child_stdout': None,
-        'stderr': None,
-        'child_stderr': None,
-    }
-
-    try:
-        stdin, child_stdin = os.pipe()
-        pipes['child_stdin'], pipes['stdin'] = stdin, child_stdin
-
-        stdout, child_stdout = os.pipe()
-        pipes['stdout'], pipes['child_stdout'] = stdout, child_stdout
-
-        if stderr:
-            stderr, child_stderr = os.pipe()
-            pipes['stderr'], pipes['child_stderr'] = stderr, child_stderr
-
-        for fd in (pipes['stdout'], pipes['stderr'], pipes['stdin']):
-            if fd is not None:
-                flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NDELAY
-                fcntl.fcntl(fd, fcntl.F_SETFL, flags)
-
-        return pipes  # type: ignore
-
-    except OSError:
-        for fd in pipes.values():
-            if fd is not None:
-                close_fd(fd)
-        raise
-
-
-def close_parent_pipes(pipes: ta.Mapping[str, int]) -> None:
-    for fdname in ('stdin', 'stdout', 'stderr'):
-        fd = pipes.get(fdname)
-        if fd is not None:
-            close_fd(fd)
-
-
-def close_child_pipes(pipes: ta.Mapping[str, int]) -> None:
-    for fdname in ('child_stdin', 'child_stdout', 'child_stderr'):
-        fd = pipes.get(fdname)
-        if fd is not None:
-            close_fd(fd)
-
-
-def check_execv_args(filename, argv, st) -> None:
-    if st is None:
-        raise NotFoundError(f"can't find command {filename!r}")
-
-    elif stat.S_ISDIR(st[stat.ST_MODE]):
-        raise NotExecutableError(f'command at {filename!r} is a directory')
-
-    elif not (stat.S_IMODE(st[stat.ST_MODE]) & 0o111):
-        raise NotExecutableError(f'command at {filename!r} is not executable')
-
-    elif not os.access(filename, os.X_OK):
-        raise NoPermissionError(f'no permission to run command {filename!r}')