omlish 0.0.0.dev224__py3-none-any.whl → 0.0.0.dev226__py3-none-any.whl

omlish/os/pidfiles/pidfile.py

@@ -0,0 +1,152 @@
+# ruff: noqa: UP007
+# @omlish-lite
+"""
+TODO:
+ - reliable pid retrieval
+  - contents are *ignored*, just advisory
+  - check double-check:
+   - 1) get pid of flock holder
+   - 2) get pidfd to that
+   - 3) recheck current pid of flock holder == that pid
+  - racy as to if it's a different actual process as initial check, just with same pid, but due to 'identity' / semantic
+    meaning of the named pidfile the processes are considered equivalent
+  - read_checked(), contextmanager
+"""
+import fcntl
+import os
+import signal
+import typing as ta
+
+
+##
+
+
+class Pidfile:
+    def __init__(
+            self,
+            path: str,
+            *,
+            inheritable: bool = True,
+    ) -> None:
+        super().__init__()
+
+        self._path = path
+        self._inheritable = inheritable
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    @property
+    def inheritable(self) -> bool:
+        return self._inheritable
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({self._path!r})'
+
+    #
+
+    _f: ta.TextIO
+
+    def fileno(self) -> ta.Optional[int]:
+        if hasattr(self, '_f'):
+            return self._f.fileno()
+        else:
+            return None
+
+    #
+
+    def __enter__(self) -> 'Pidfile':
+        fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
+
+        try:
+            if self._inheritable:
+                os.set_inheritable(fd, True)
+
+            f = os.fdopen(fd, 'r+')
+
+        except Exception:
+            try:
+                os.close(fd)
+            except Exception:  # noqa
+                pass
+            raise
+
+        self._f = f
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    #
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+
+        if '_f' in state:
+            # self._inheritable may be decoupled from actual file inheritability - for example when using the manager.
+            if os.get_inheritable(fd := state.pop('_f').fileno()):
+                state['__fd'] = fd
+
+        return state
+
+    def __setstate__(self, state):
+        if '_f' in state:
+            raise RuntimeError
+
+        if '__fd' in state:
+            state['_f'] = os.fdopen(state.pop('__fd'), 'r+')
+
+        self.__dict__.update(state)
+
+    #
+
+    def close(self) -> bool:
+        if not hasattr(self, '_f'):
+            return False
+
+        self._f.close()
+        del self._f
+        return True
+
+    def try_lock(self) -> bool:
+        try:
+            fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+
+        except OSError:
+            return False
+
+    def ensure_locked(self) -> None:
+        if not self.try_lock():
+            raise RuntimeError('Could not get lock')
+
+    #
+
+    def write(self, pid: ta.Optional[int] = None) -> None:
+        self.ensure_locked()
+
+        if pid is None:
+            pid = os.getpid()
+
+        self._f.seek(0)
+        self._f.truncate()
+        self._f.write(f'{pid}\n')
+        self._f.flush()
+
+    def clear(self) -> None:
+        self.ensure_locked()
+
+        self._f.seek(0)
+        self._f.truncate()
+
+    def read(self) -> int:
+        if self.try_lock():
+            raise RuntimeError('Got lock')
+
+        self._f.seek(0)
+        return int(self._f.read())
+
+    def kill(self, sig: int = signal.SIGTERM) -> None:
+        pid = self.read()
+        os.kill(pid, sig)

omlish/secrets/crypto.py

@@ -76,7 +76,6 @@ class Crypto(abc.ABC):
 
 
 class FernetCrypto(Crypto):
-
     def generate_key(self) -> bytes:
         return cry_fernet.Fernet.generate_key()
 
@@ -98,7 +97,7 @@ class FernetCrypto(Crypto):
             raise DecryptionError from e
 
 
-class AesgsmCrypto(Crypto):
+class AesgcmCrypto(Crypto):
     """https://stackoverflow.com/a/59835994"""
 
     def generate_key(self) -> bytes:

omlish/secrets/openssl.py

@@ -33,7 +33,7 @@ else:
 DEFAULT_KEY_SIZE = 64
 
 
-def generate_key(self, sz: int = DEFAULT_KEY_SIZE) -> bytes:
+def generate_key(sz: int = DEFAULT_KEY_SIZE) -> bytes:
     # !! https://docs.openssl.org/3.0/man7/passphrase-encoding/
     # Must not contain null bytes!
     return secrets.token_hex(sz).encode('ascii')

omlish/secrets/tempssl.py

@@ -6,9 +6,9 @@ import tempfile
 import typing as ta
 
 from ..lite.cached import cached_nullary
-from ..subprocesses import SubprocessRun
-from ..subprocesses import SubprocessRunOutput
-from ..subprocesses import subprocesses
+from ..subprocesses.run import SubprocessRun
+from ..subprocesses.run import SubprocessRunnable
+from ..subprocesses.run import SubprocessRunOutput
 from .ssl import SslCert
 
 
@@ -18,7 +18,7 @@ class TempSslCert(ta.NamedTuple):
 
 
 @dc.dataclass(frozen=True)
-class TempSslCertGenerator:
+class TempSslCertGenerator(SubprocessRunnable[TempSslCert]):
     @cached_nullary
     def temp_dir(self) -> str:
         return tempfile.mkdtemp()
@@ -64,9 +64,6 @@ class TempSslCertGenerator:
             temp_dir=self.temp_dir(),
         )
 
-    def run(self) -> TempSslCert:
-        return self.handle_run_output(subprocesses.run_(self.make_run()))
-
 
 def generate_temp_localhost_ssl_cert() -> TempSslCert:
     return TempSslCertGenerator().run()

omlish/sockets/handlers.py

@@ -17,3 +17,7 @@ class SocketHandler_(abc.ABC):  # noqa
     @abc.abstractmethod
     def __call__(self, addr: SocketAddress, f: SocketIoPair) -> None:
         raise NotImplementedError
+
+
+class SocketHandlerClose(Exception):  # noqa
+    pass

omlish/sockets/server/handlers.py

@@ -3,6 +3,7 @@
 import abc
 import concurrent.futures as cf
 import dataclasses as dc
+import logging
 import socket
 import typing as ta
 
@@ -132,3 +133,24 @@ class ExecutorSocketServerHandler(SocketServerHandler_):
 
     def __call__(self, conn: SocketAndAddress) -> None:
         self.executor.submit(self.handler, conn)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class ExceptionLoggingSocketServerHandler(SocketServerHandler_):
+    handler: SocketServerHandler
+    log: logging.Logger
+
+    ignored: ta.Optional[ta.Container[ta.Type[Exception]]] = None
+
+    def __call__(self, conn: SocketAndAddress) -> None:
+        try:
+            return self.handler(conn)
+
+        except Exception as e:  # noqa
+            if (ignored := self.ignored) is None or type(e) not in ignored:
+                self.log.exception('Error in handler %r for conn %r', self.handler, conn)
+
+            raise

omlish/subprocesses/async_.py

@@ -0,0 +1,97 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import sys
+import typing as ta
+
+from ..lite.timeouts import TimeoutLike
+from .base import BaseSubprocesses
+from .run import SubprocessRun
+from .run import SubprocessRunOutput
+
+
+##
+
+
+class AbstractAsyncSubprocesses(BaseSubprocesses):
+    @abc.abstractmethod
+    async def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
+        raise NotImplementedError
+
+    def run(
+            self,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[TimeoutLike] = None,
+            check: bool = False,
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Awaitable[SubprocessRunOutput]:
+        return self.run_(SubprocessRun(
+            cmd=cmd,
+            input=input,
+            timeout=timeout,
+            check=check,
+            capture_output=capture_output,
+            kwargs=kwargs,
+        ))
+
+    #
+
+    @abc.abstractmethod
+    async def check_call(
+            self,
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    async def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        raise NotImplementedError
+
+    #
+
+    async def check_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> str:
+        return (await self.check_output(*cmd, **kwargs)).decode().strip()
+
+    #
+
+    async def try_call(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bool:
+        if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
+            return False
+        else:
+            return True
+
+    async def try_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[bytes]:
+        if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
+            return None
+        else:
+            return ret
+
+    async def try_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[str]:
+        if (ret := await self.try_output(*cmd, **kwargs)) is None:
+            return None
+        else:
+            return ret.decode().strip()

omlish/subprocesses/base.py

@@ -0,0 +1,221 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import contextlib
+import logging
+import os
+import subprocess
+import time
+import typing as ta
+
+from ..lite.timeouts import Timeout
+from .wrap import subprocess_maybe_shell_wrap_exec
+
+
+T = ta.TypeVar('T')
+SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
+
+
+##
+
+
+# Valid channel type kwarg values:
+#  - A special flag negative int
+#  - A positive fd int
+#  - A file-like object
+#  - None
+
+SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
+    'pipe': subprocess.PIPE,
+    'stdout': subprocess.STDOUT,
+    'devnull': subprocess.DEVNULL,
+}
+
+
+##
+
+
+class VerboseCalledProcessError(subprocess.CalledProcessError):
+    @classmethod
+    def from_std(cls, e: subprocess.CalledProcessError) -> 'VerboseCalledProcessError':
+        return cls(
+            e.returncode,
+            e.cmd,
+            output=e.output,
+            stderr=e.stderr,
+        )
+
+    def __str__(self) -> str:
+        msg = super().__str__()
+        if self.output is not None:
+            msg += f' Output: {self.output!r}'
+        if self.stderr is not None:
+            msg += f' Stderr: {self.stderr!r}'
+        return msg
+
+
+class BaseSubprocesses(abc.ABC):  # noqa
+    DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
+
+    def __init__(
+            self,
+            *,
+            log: ta.Optional[logging.Logger] = None,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._log = log if log is not None else self.DEFAULT_LOGGER
+        self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
+
+    def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
+        self._log = log
+
+    #
+
+    def prepare_args(
+            self,
+            *cmd: str,
+            env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            quiet: bool = False,
+            shell: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
+        if self._log:
+            self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
+            if extra_env:
+                self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
+
+        #
+
+        if extra_env:
+            env = {**(env if env is not None else os.environ), **extra_env}
+
+        #
+
+        if quiet and 'stderr' not in kwargs:
+            if self._log and not self._log.isEnabledFor(logging.DEBUG):
+                kwargs['stderr'] = subprocess.DEVNULL
+
+        for chk in ('stdout', 'stderr'):
+            try:
+                chv = kwargs[chk]
+            except KeyError:
+                continue
+            kwargs[chk] = SUBPROCESS_CHANNEL_OPTION_VALUES.get(chv, chv)
+
+        #
+
+        if not shell:
+            cmd = subprocess_maybe_shell_wrap_exec(*cmd)
+
+        #
+
+        if 'timeout' in kwargs:
+            kwargs['timeout'] = Timeout.of(kwargs['timeout']).or_(None)
+
+        #
+
+        return cmd, dict(
+            env=env,
+            shell=shell,
+            **kwargs,
+        )
+
+    @contextlib.contextmanager
+    def wrap_call(
+            self,
+            *cmd: ta.Any,
+            raise_verbose: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[None]:
+        start_time = time.time()
+        try:
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
+
+            yield
+
+        except Exception as exc:  # noqa
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
+
+            if (
+                    raise_verbose and
+                    isinstance(exc, subprocess.CalledProcessError) and
+                    not isinstance(exc, VerboseCalledProcessError) and
+                    (exc.output is not None or exc.stderr is not None)
+            ):
+                raise VerboseCalledProcessError.from_std(exc) from exc
+
+            raise
+
+        finally:
+            end_time = time.time()
+            elapsed_s = end_time - start_time
+
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
+
+    @contextlib.contextmanager
+    def prepare_and_wrap(
+            self,
+            *cmd: ta.Any,
+            raise_verbose: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[ta.Tuple[
+        ta.Tuple[ta.Any, ...],
+        ta.Dict[str, ta.Any],
+    ]]:
+        cmd, kwargs = self.prepare_args(*cmd, **kwargs)
+
+        with self.wrap_call(
+                *cmd,
+                raise_verbose=raise_verbose,
+                **kwargs,
+        ):
+            yield cmd, kwargs
+
+    #
+
+    DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
+        FileNotFoundError,
+        subprocess.CalledProcessError,
+    )
+
+    def try_fn(
+            self,
+            fn: ta.Callable[..., T],
+            *cmd: str,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Union[T, Exception]:
+        if try_exceptions is None:
+            try_exceptions = self._try_exceptions
+
+        try:
+            return fn(*cmd, **kwargs)
+
+        except try_exceptions as e:  # noqa
+            if self._log and self._log.isEnabledFor(logging.DEBUG):
+                self._log.exception('command failed')
+            return e
+
+    async def async_try_fn(
+            self,
+            fn: ta.Callable[..., ta.Awaitable[T]],
+            *cmd: ta.Any,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Union[T, Exception]:
+        if try_exceptions is None:
+            try_exceptions = self._try_exceptions
+
+        try:
+            return await fn(*cmd, **kwargs)
+
+        except try_exceptions as e:  # noqa
+            if self._log and self._log.isEnabledFor(logging.DEBUG):
+                self._log.exception('command failed')
+            return e

omlish/subprocesses/run.py

@@ -0,0 +1,138 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import typing as ta
+
+from ..lite.check import check
+from ..lite.timeouts import TimeoutLike
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SubprocessRunOutput(ta.Generic[T]):
+    proc: T
+
+    returncode: int  # noqa
+
+    stdout: ta.Optional[bytes] = None
+    stderr: ta.Optional[bytes] = None
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SubprocessRun:
+    cmd: ta.Sequence[str]
+    input: ta.Any = None
+    timeout: ta.Optional[TimeoutLike] = None
+    check: bool = False
+    capture_output: ta.Optional[bool] = None
+    kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None
+
+    #
+
+    _FIELD_NAMES: ta.ClassVar[ta.FrozenSet[str]]
+
+    def replace(self, **kwargs: ta.Any) -> 'SubprocessRun':
+        if not kwargs:
+            return self
+
+        field_kws = {}
+        extra_kws = {}
+        for k, v in kwargs.items():
+            if k in self._FIELD_NAMES:
+                field_kws[k] = v
+            else:
+                extra_kws[k] = v
+
+        return dc.replace(self, **{
+            **dict(kwargs={
+                **(self.kwargs or {}),
+                **extra_kws,
+            }),
+            **field_kws,  # passing a kwarg named 'kwargs' intentionally clobbers
+        })
+
+    #
+
+    @classmethod
+    def of(
+            cls,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[TimeoutLike] = None,
+            check: bool = False,  # noqa
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> 'SubprocessRun':
+        return cls(
+            cmd=cmd,
+            input=input,
+            timeout=timeout,
+            check=check,
+            capture_output=capture_output,
+            kwargs=kwargs,
+        )
+
+    #
+
+    _DEFAULT_SUBPROCESSES: ta.ClassVar[ta.Optional[ta.Any]] = None  # AbstractSubprocesses
+
+    def run(
+            self,
+            subprocesses: ta.Optional[ta.Any] = None,  # AbstractSubprocesses
+            **kwargs: ta.Any,
+    ) -> SubprocessRunOutput:
+        if subprocesses is None:
+            subprocesses = self._DEFAULT_SUBPROCESSES
+        return check.not_none(subprocesses).run_(self.replace(**kwargs))  # type: ignore[attr-defined]
+
+    _DEFAULT_ASYNC_SUBPROCESSES: ta.ClassVar[ta.Optional[ta.Any]] = None  # AbstractAsyncSubprocesses
+
+    async def async_run(
+            self,
+            async_subprocesses: ta.Optional[ta.Any] = None,  # AbstractAsyncSubprocesses
+            **kwargs: ta.Any,
+    ) -> SubprocessRunOutput:
+        if async_subprocesses is None:
+            async_subprocesses = self._DEFAULT_ASYNC_SUBPROCESSES
+        return await check.not_none(async_subprocesses).run_(self.replace(**kwargs))  # type: ignore[attr-defined]
+
+
+SubprocessRun._FIELD_NAMES = frozenset(fld.name for fld in dc.fields(SubprocessRun))  # noqa
+
+
+##
+
+
+class SubprocessRunnable(abc.ABC, ta.Generic[T]):
+    @abc.abstractmethod
+    def make_run(self) -> SubprocessRun:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def handle_run_output(self, output: SubprocessRunOutput) -> T:
+        raise NotImplementedError
+
+    #
+
+    def run(
+            self,
+            subprocesses: ta.Optional[ta.Any] = None,  # AbstractSubprocesses
+            **kwargs: ta.Any,
+    ) -> T:
+        return self.handle_run_output(self.make_run().run(subprocesses, **kwargs))
+
+    async def async_run(
+            self,
+            async_subprocesses: ta.Optional[ta.Any] = None,  # AbstractAsyncSubprocesses
+            **kwargs: ta.Any,
+    ) -> T:
+        return self.handle_run_output(await self.make_run().async_run(async_subprocesses, **kwargs))