ominfra 0.0.0.dev223__py3-none-any.whl → 0.0.0.dev225__py3-none-any.whl

ominfra/clouds/aws/journald2aws/driver.py

@@ -42,7 +42,7 @@ from omlish.lite.check import check
 from omlish.lite.contextmanagers import ExitStacked
 from omlish.lite.logs import log
 from omlish.lite.runtime import is_debugger_attached
-from omlish.os.pidfile import Pidfile
+from omlish.os.pidfiles.pidfile import Pidfile
 
 from ....journald.messages import JournalctlMessage  # noqa
 from ....journald.tailer import JournalctlTailerWorker

ominfra/journald/tailer.py

@@ -410,8 +410,8 @@ import typing as ta
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.logs import log
-from omlish.subprocesses import subprocess_close
-from omlish.subprocesses import subprocess_shell_wrap_exec
+from omlish.subprocesses.utils import subprocess_close
+from omlish.subprocesses.wrap import subprocess_shell_wrap_exec
 
 from ..threadworkers import ThreadWorker
 from .messages import JournalctlMessage  # noqa

ominfra/manage/commands/subprocess.py

@@ -8,9 +8,9 @@ import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
-from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
-from omlish.subprocesses import SubprocessChannelOption
-from omlish.subprocesses import subprocess_maybe_shell_wrap_exec
+from omlish.subprocesses.base import SUBPROCESS_CHANNEL_OPTION_VALUES
+from omlish.subprocesses.base import SubprocessChannelOption
+from omlish.subprocesses.wrap import subprocess_maybe_shell_wrap_exec
 
 from .base import Command
 from .base import CommandExecutor

ominfra/manage/remote/spawning.py

@@ -10,8 +10,8 @@ import typing as ta
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.shlex import shlex_maybe_quote
-from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
-from omlish.subprocesses import SubprocessChannelOption
+from omlish.subprocesses.base import SUBPROCESS_CHANNEL_OPTION_VALUES
+from omlish.subprocesses.base import SubprocessChannelOption
 
 
 ##

ominfra/scripts/journald2aws.py

@@ -86,9 +86,6 @@ AsyncExitStackedT = ta.TypeVar('AsyncExitStackedT', bound='AsyncExitStacked')
 # ../../../threadworkers.py
 ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
 
-# ../../../../omlish/subprocesses.py
-SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
-
 
 ########################################
 # ../../../../../omlish/configs/types.py
@@ -1945,42 +1942,107 @@ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
 
 
 ########################################
-# ../../../../../omlish/os/pidfile.py
+# ../../../../../omlish/os/pidfiles/pidfile.py
+"""
+TODO:
+ - reliable pid retrieval
+  - contents are *ignored*, just advisory
+  - check double-check:
+   - 1) get pid of flock holder
+   - 2) get pidfd to that
+   - 3) recheck current pid of flock holder == that pid
+  - racy as to if it's a different actual process as initial check, just with same pid, but due to 'identity' / semantic
+    meaning of the named pidfile the processes are considered equivalent
+"""
+
+
+##
 
 
 class Pidfile:
-    def __init__(self, path: str) -> None:
+    def __init__(
+            self,
+            path: str,
+            *,
+            inheritable: bool = True,
+    ) -> None:
         super().__init__()
-        self._path = path
 
-    _f: ta.TextIO
+        self._path = path
+        self._inheritable = inheritable
 
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}({self._path!r})'
 
+    #
+
+    _f: ta.TextIO
+
+    def fileno(self) -> ta.Optional[int]:
+        if hasattr(self, '_f'):
+            return self._f.fileno()
+        else:
+            return None
+
+    #
+
     def __enter__(self) -> 'Pidfile':
         fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
+
         try:
-            os.set_inheritable(fd, True)
+            if self._inheritable:
+                os.set_inheritable(fd, True)
+
             f = os.fdopen(fd, 'r+')
+
         except Exception:
             try:
                 os.close(fd)
             except Exception:  # noqa
                 pass
             raise
+
         self._f = f
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if hasattr(self, '_f'):
-            self._f.close()
-            del self._f
+        self.close()
+
+    #
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+
+        if '_f' in state:
+            if os.get_inheritable(fd := state.pop('_f').fileno()):
+                state['__fd'] = fd
+
+        return state
+
+    def __setstate__(self, state):
+        if '_f' in state:
+            raise RuntimeError
+
+        if '__fd' in state:
+            state['_f'] = os.fdopen(state.pop('__fd'), 'r+')
+
+        self.__dict__.update(state)
+
+    #
+
+    def close(self) -> bool:
+        if not hasattr(self, '_f'):
+            return False
+
+        self._f.close()
+        del self._f
+        return True
 
     def try_lock(self) -> bool:
         try:
             fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
             return True
+
         except OSError:
             return False
 
@@ -1988,27 +2050,57 @@ class Pidfile:
         if not self.try_lock():
             raise RuntimeError('Could not get lock')
 
+    #
+
     def write(self, pid: ta.Optional[int] = None) -> None:
         self.ensure_locked()
+
         if pid is None:
             pid = os.getpid()
+
+        self._f.seek(0)
+        self._f.truncate()
         self._f.write(f'{pid}\n')
         self._f.flush()
 
     def clear(self) -> None:
         self.ensure_locked()
+
         self._f.seek(0)
         self._f.truncate()
 
     def read(self) -> int:
         if self.try_lock():
             raise RuntimeError('Got lock')
+
         self._f.seek(0)
-        return int(self._f.read())
+        return int(self._f.read())  # FIXME: could be empty or hold old value, race w proc start
 
     def kill(self, sig: int = signal.SIGTERM) -> None:
         pid = self.read()
-        os.kill(pid, sig)  # FIXME: Still racy
+        os.kill(pid, sig)  # FIXME: Still racy - pidfd_send_signal?
+
+
+########################################
+# ../../../../../omlish/subprocesses/utils.py
+
+
+##
+
+
+def subprocess_close(
+        proc: subprocess.Popen,
+        timeout: ta.Optional[float] = None,
+) -> None:
+    # TODO: terminate, sleep, kill
+    if proc.stdout:
+        proc.stdout.close()
+    if proc.stderr:
+        proc.stderr.close()
+    if proc.stdin:
+        proc.stdin.close()
+
+    proc.wait(timeout)
 
 
 ########################################
@@ -4245,23 +4337,7 @@ def configure_standard_logging(
 
 
 ########################################
-# ../../../../../omlish/subprocesses.py
-
-
-##
-
-
-# Valid channel type kwarg values:
-#  - A special flag negative int
-#  - A positive fd int
-#  - A file-like object
-#  - None
-
-SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
-    'pipe': subprocess.PIPE,
-    'stdout': subprocess.STDOUT,
-    'devnull': subprocess.DEVNULL,
-}
+# ../../../../../omlish/subprocesses/wrap.py
 
 
 ##
@@ -4281,446 +4357,6 @@ def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
         return cmd
 
 
-##
-
-
-def subprocess_close(
-        proc: subprocess.Popen,
-        timeout: ta.Optional[float] = None,
-) -> None:
-    # TODO: terminate, sleep, kill
-    if proc.stdout:
-        proc.stdout.close()
-    if proc.stderr:
-        proc.stderr.close()
-    if proc.stdin:
-        proc.stdin.close()
-
-    proc.wait(timeout)
-
-
-##
-
-
-class VerboseCalledProcessError(subprocess.CalledProcessError):
-    @classmethod
-    def from_std(cls, e: subprocess.CalledProcessError) -> 'VerboseCalledProcessError':
-        return cls(
-            e.returncode,
-            e.cmd,
-            output=e.output,
-            stderr=e.stderr,
-        )
-
-    def __str__(self) -> str:
-        msg = super().__str__()
-        if self.output is not None:
-            msg += f' Output: {self.output!r}'
-        if self.stderr is not None:
-            msg += f' Stderr: {self.stderr!r}'
-        return msg
-
-
-class BaseSubprocesses(abc.ABC):  # noqa
-    DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
-
-    def __init__(
-            self,
-            *,
-            log: ta.Optional[logging.Logger] = None,
-            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
-    ) -> None:
-        super().__init__()
-
-        self._log = log if log is not None else self.DEFAULT_LOGGER
-        self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
-
-    def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
-        self._log = log
-
-    #
-
-    def prepare_args(
-            self,
-            *cmd: str,
-            env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
-            extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
-            quiet: bool = False,
-            shell: bool = False,
-            **kwargs: ta.Any,
-    ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
-        if self._log:
-            self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
-            if extra_env:
-                self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
-
-        #
-
-        if extra_env:
-            env = {**(env if env is not None else os.environ), **extra_env}
-
-        #
-
-        if quiet and 'stderr' not in kwargs:
-            if self._log and not self._log.isEnabledFor(logging.DEBUG):
-                kwargs['stderr'] = subprocess.DEVNULL
-
-        for chk in ('stdout', 'stderr'):
-            try:
-                chv = kwargs[chk]
-            except KeyError:
-                continue
-            kwargs[chk] = SUBPROCESS_CHANNEL_OPTION_VALUES.get(chv, chv)
-
-        #
-
-        if not shell:
-            cmd = subprocess_maybe_shell_wrap_exec(*cmd)
-
-        #
-
-        return cmd, dict(
-            env=env,
-            shell=shell,
-            **kwargs,
-        )
-
-    @contextlib.contextmanager
-    def wrap_call(
-            self,
-            *cmd: ta.Any,
-            raise_verbose: bool = False,
-            **kwargs: ta.Any,
-    ) -> ta.Iterator[None]:
-        start_time = time.time()
-        try:
-            if self._log:
-                self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
-
-            yield
-
-        except Exception as exc:  # noqa
-            if self._log:
-                self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
-
-            if (
-                    raise_verbose and
-                    isinstance(exc, subprocess.CalledProcessError) and
-                    not isinstance(exc, VerboseCalledProcessError) and
-                    (exc.output is not None or exc.stderr is not None)
-            ):
-                raise VerboseCalledProcessError.from_std(exc) from exc
-
-            raise
-
-        finally:
-            end_time = time.time()
-            elapsed_s = end_time - start_time
-
-            if self._log:
-                self._log.debug('Subprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
-
-    @contextlib.contextmanager
-    def prepare_and_wrap(
-            self,
-            *cmd: ta.Any,
-            raise_verbose: bool = False,
-            **kwargs: ta.Any,
-    ) -> ta.Iterator[ta.Tuple[
-        ta.Tuple[ta.Any, ...],
-        ta.Dict[str, ta.Any],
-    ]]:
-        cmd, kwargs = self.prepare_args(*cmd, **kwargs)
-
-        with self.wrap_call(
-                *cmd,
-                raise_verbose=raise_verbose,
-                **kwargs,
-        ):
-            yield cmd, kwargs
-
-    #
-
-    DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
-        FileNotFoundError,
-        subprocess.CalledProcessError,
-    )
-
-    def try_fn(
-            self,
-            fn: ta.Callable[..., T],
-            *cmd: str,
-            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
-            **kwargs: ta.Any,
-    ) -> ta.Union[T, Exception]:
-        if try_exceptions is None:
-            try_exceptions = self._try_exceptions
-
-        try:
-            return fn(*cmd, **kwargs)
-
-        except try_exceptions as e:  # noqa
-            if self._log and self._log.isEnabledFor(logging.DEBUG):
-                self._log.exception('command failed')
-            return e
-
-    async def async_try_fn(
-            self,
-            fn: ta.Callable[..., ta.Awaitable[T]],
-            *cmd: ta.Any,
-            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
-            **kwargs: ta.Any,
-    ) -> ta.Union[T, Exception]:
-        if try_exceptions is None:
-            try_exceptions = self._try_exceptions
-
-        try:
-            return await fn(*cmd, **kwargs)
-
-        except try_exceptions as e:  # noqa
-            if self._log and self._log.isEnabledFor(logging.DEBUG):
-                self._log.exception('command failed')
-            return e
-
-
-##
-
-
-@dc.dataclass(frozen=True)
-class SubprocessRun:
-    cmd: ta.Sequence[str]
-    input: ta.Any = None
-    timeout: ta.Optional[float] = None
-    check: bool = False
-    capture_output: ta.Optional[bool] = None
-    kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None
-
-
-@dc.dataclass(frozen=True)
-class SubprocessRunOutput(ta.Generic[T]):
-    proc: T
-
-    returncode: int  # noqa
-
-    stdout: ta.Optional[bytes] = None
-    stderr: ta.Optional[bytes] = None
-
-
-class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
-    @abc.abstractmethod
-    def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
-        raise NotImplementedError
-
-    def run(
-            self,
-            *cmd: str,
-            input: ta.Any = None,  # noqa
-            timeout: ta.Optional[float] = None,
-            check: bool = False,
-            capture_output: ta.Optional[bool] = None,
-            **kwargs: ta.Any,
-    ) -> SubprocessRunOutput:
-        return self.run_(SubprocessRun(
-            cmd=cmd,
-            input=input,
-            timeout=timeout,
-            check=check,
-            capture_output=capture_output,
-            kwargs=kwargs,
-        ))
-
-    #
-
-    @abc.abstractmethod
-    def check_call(
-            self,
-            *cmd: str,
-            stdout: ta.Any = sys.stderr,
-            **kwargs: ta.Any,
-    ) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def check_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bytes:
-        raise NotImplementedError
-
-    #
-
-    def check_output_str(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> str:
-        return self.check_output(*cmd, **kwargs).decode().strip()
-
-    #
-
-    def try_call(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bool:
-        if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
-            return False
-        else:
-            return True
-
-    def try_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> ta.Optional[bytes]:
-        if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
-            return None
-        else:
-            return ret
-
-    def try_output_str(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> ta.Optional[str]:
-        if (ret := self.try_output(*cmd, **kwargs)) is None:
-            return None
-        else:
-            return ret.decode().strip()
-
-
-##
-
-
-class Subprocesses(AbstractSubprocesses):
-    def run_(self, run: SubprocessRun) -> SubprocessRunOutput[subprocess.CompletedProcess]:
-        proc = subprocess.run(
-            run.cmd,
-            input=run.input,
-            timeout=run.timeout,
-            check=run.check,
-            capture_output=run.capture_output or False,
-            **(run.kwargs or {}),
-        )
-
-        return SubprocessRunOutput(
-            proc=proc,
-
-            returncode=proc.returncode,
-
-            stdout=proc.stdout,  # noqa
-            stderr=proc.stderr,  # noqa
-        )
-
-    def check_call(
-            self,
-            *cmd: str,
-            stdout: ta.Any = sys.stderr,
-            **kwargs: ta.Any,
-    ) -> None:
-        with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs):  # noqa
-            subprocess.check_call(cmd, **kwargs)
-
-    def check_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bytes:
-        with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs):  # noqa
-            return subprocess.check_output(cmd, **kwargs)
-
-
-subprocesses = Subprocesses()
-
-
-##
-
-
-class AbstractAsyncSubprocesses(BaseSubprocesses):
-    @abc.abstractmethod
-    async def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
-        raise NotImplementedError
-
-    def run(
-            self,
-            *cmd: str,
-            input: ta.Any = None,  # noqa
-            timeout: ta.Optional[float] = None,
-            check: bool = False,
-            capture_output: ta.Optional[bool] = None,
-            **kwargs: ta.Any,
-    ) -> ta.Awaitable[SubprocessRunOutput]:
-        return self.run_(SubprocessRun(
-            cmd=cmd,
-            input=input,
-            timeout=timeout,
-            check=check,
-            capture_output=capture_output,
-            kwargs=kwargs,
-        ))
-
-    #
-
-    @abc.abstractmethod
-    async def check_call(
-            self,
-            *cmd: str,
-            stdout: ta.Any = sys.stderr,
-            **kwargs: ta.Any,
-    ) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    async def check_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bytes:
-        raise NotImplementedError
-
-    #
-
-    async def check_output_str(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> str:
-        return (await self.check_output(*cmd, **kwargs)).decode().strip()
-
-    #
-
-    async def try_call(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> bool:
-        if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
-            return False
-        else:
-            return True
-
-    async def try_output(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> ta.Optional[bytes]:
-        if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
-            return None
-        else:
-            return ret
-
-    async def try_output_str(
-            self,
-            *cmd: str,
-            **kwargs: ta.Any,
-    ) -> ta.Optional[str]:
-        if (ret := await self.try_output(*cmd, **kwargs)) is None:
-            return None
-        else:
-            return ret.decode().strip()
-
-
 ########################################
 # ../poster.py
 """