omlish 0.0.0.dev154__tar.gz → 0.0.0.dev156__tar.gz

{omlish-0.0.0.dev154/omlish.egg-info → omlish-0.0.0.dev156}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev154
+Version: 0.0.0.dev156
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev154 → omlish-0.0.0.dev156}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev154'
-__revision__ = '148f19432b8d3a70543b9466386aa540fe95b6be'
+__version__ = '0.0.0.dev156'
+__revision__ = '594c80ed50a0c5515f35cac4b31a4ed482993a4b'
 
 
 #

omlish-0.0.0.dev156/omlish/lite/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite

omlish-0.0.0.dev156/omlish/lite/asyncio/subprocesses.py

@@ -0,0 +1,273 @@
+# ruff: noqa: UP006 UP007
+import asyncio.base_subprocess
+import asyncio.subprocess
+import contextlib
+import dataclasses as dc
+import functools
+import subprocess
+import sys
+import typing as ta
+
+from ...asyncs.asyncio.timeouts import asyncio_maybe_timeout
+from ..check import check
+from ..logs import log
+from ..subprocesses import AbstractSubprocesses
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class AsyncioProcessCommunicator:
+    def __init__(
+            self,
+            proc: asyncio.subprocess.Process,
+            loop: ta.Optional[ta.Any] = None,
+    ) -> None:
+        super().__init__()
+
+        if loop is None:
+            loop = asyncio.get_running_loop()
+
+        self._proc = proc
+        self._loop = loop
+
+        self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
+            proc._transport,  # type: ignore  # noqa
+            asyncio.base_subprocess.BaseSubprocessTransport,
+        )
+
+    @property
+    def _debug(self) -> bool:
+        return self._loop.get_debug()
+
+    async def _feed_stdin(self, input: bytes) -> None:  # noqa
+        stdin = check.not_none(self._proc.stdin)
+        try:
+            if input is not None:
+                stdin.write(input)
+                if self._debug:
+                    log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
+
+            await stdin.drain()
+
+        except (BrokenPipeError, ConnectionResetError) as exc:
+            # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
+            # exceptions.
+            if self._debug:
+                log.debug('%r communicate: stdin got %r', self, exc)
+
+        if self._debug:
+            log.debug('%r communicate: close stdin', self)
+
+        stdin.close()
+
+    async def _noop(self) -> None:
+        return None
+
+    async def _read_stream(self, fd: int) -> bytes:
+        transport: ta.Any = check.not_none(self._transport.get_pipe_transport(fd))
+
+        if fd == 2:
+            stream = check.not_none(self._proc.stderr)
+        else:
+            check.equal(fd, 1)
+            stream = check.not_none(self._proc.stdout)
+
+        if self._debug:
+            name = 'stdout' if fd == 1 else 'stderr'
+            log.debug('%r communicate: read %s', self, name)
+
+        output = await stream.read()
+
+        if self._debug:
+            name = 'stdout' if fd == 1 else 'stderr'
+            log.debug('%r communicate: close %s', self, name)
+
+        transport.close()
+
+        return output
+
+    class Communication(ta.NamedTuple):
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def _communicate(
+            self,
+            input: ta.Any = None,  # noqa
+    ) -> Communication:
+        stdin_fut: ta.Any
+        if self._proc.stdin is not None:
+            stdin_fut = self._feed_stdin(input)
+        else:
+            stdin_fut = self._noop()
+
+        stdout_fut: ta.Any
+        if self._proc.stdout is not None:
+            stdout_fut = self._read_stream(1)
+        else:
+            stdout_fut = self._noop()
+
+        stderr_fut: ta.Any
+        if self._proc.stderr is not None:
+            stderr_fut = self._read_stream(2)
+        else:
+            stderr_fut = self._noop()
+
+        stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
+
+        await self._proc.wait()
+
+        return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
+
+    async def communicate(
+            self,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> Communication:
+        return await asyncio_maybe_timeout(self._communicate(input), timeout)
+
+
+##
+
+
+class AsyncioSubprocesses(AbstractSubprocesses):
+    async def communicate(
+            self,
+            proc: asyncio.subprocess.Process,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
+        return await AsyncioProcessCommunicator(proc).communicate(input, timeout)  # noqa
+
+    #
+
+    @contextlib.asynccontextmanager
+    async def popen(
+            self,
+            *cmd: str,
+            shell: bool = False,
+            timeout: ta.Optional[float] = None,
+            **kwargs: ta.Any,
+    ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
+        fac: ta.Any
+        if shell:
+            fac = functools.partial(
+                asyncio.create_subprocess_shell,
+                check.single(cmd),
+            )
+        else:
+            fac = functools.partial(
+                asyncio.create_subprocess_exec,
+                *cmd,
+            )
+
+        with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs):  # noqa
+            proc: asyncio.subprocess.Process = await fac(**kwargs)
+            try:
+                yield proc
+
+            finally:
+                await asyncio_maybe_timeout(proc.wait(), timeout)
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class RunOutput:
+        proc: asyncio.subprocess.Process
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def run(
+            self,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+            check: bool = False,  # noqa
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> RunOutput:
+        if capture_output:
+            kwargs.setdefault('stdout', subprocess.PIPE)
+            kwargs.setdefault('stderr', subprocess.PIPE)
+
+        proc: asyncio.subprocess.Process
+        async with self.popen(*cmd, **kwargs) as proc:
+            stdout, stderr = await self.communicate(proc, input, timeout)
+
+        if check and proc.returncode:
+            raise subprocess.CalledProcessError(
+                proc.returncode,
+                cmd,
+                output=stdout,
+                stderr=stderr,
+            )
+
+        return self.RunOutput(
+            proc,
+            stdout,
+            stderr,
+        )
+
+    #
+
+    async def check_call(
+            self,
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        with self.prepare_and_wrap(*cmd, stdout=stdout, check=True, **kwargs) as (cmd, kwargs):  # noqa
+            await self.run(*cmd, **kwargs)
+
+    async def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs):  # noqa
+            return check.not_none((await self.run(*cmd, **kwargs)).stdout)
+
+    async def check_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> str:
+        return (await self.check_output(*cmd, **kwargs)).decode().strip()
+
+    #
+
+    async def try_call(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bool:
+        if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
+            return False
+        else:
+            return True
+
+    async def try_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[bytes]:
+        if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
+            return None
+        else:
+            return ret
+
+    async def try_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[str]:
+        if (ret := await self.try_output(*cmd, **kwargs)) is None:
+            return None
+        else:
+            return ret.decode().strip()
+
+
+asyncio_subprocesses = AsyncioSubprocesses()

{omlish-0.0.0.dev154 → omlish-0.0.0.dev156}/omlish/lite/check.py

@@ -2,8 +2,6 @@
 """
 TODO:
  - def maybe(v: lang.Maybe[T])
- - patch / override lite.check ?
-  - checker interface?
 """
 import collections
 import threading

{omlish-0.0.0.dev154 → omlish-0.0.0.dev156}/omlish/lite/contextmanagers.py

@@ -61,3 +61,17 @@ def attr_setting(obj, attr, val, *, default=None):  # noqa
             delattr(obj, attr)
         else:
             setattr(obj, attr, orig)
+
+
+##
+
+
+class aclosing(contextlib.AbstractAsyncContextManager):  # noqa
+    def __init__(self, thing):
+        self.thing = thing
+
+    async def __aenter__(self):
+        return self.thing
+
+    async def __aexit__(self, *exc_info):
+        await self.thing.aclose()

{omlish-0.0.0.dev154 → omlish-0.0.0.dev156}/omlish/lite/marshal.py

@@ -1,7 +1,8 @@
 """
 TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
- - nonstrict toggle
+ - namedtuple
+ - literals
 """
 # ruff: noqa: UP006 UP007
 import abc
@@ -23,6 +24,7 @@ from .reflect import deep_subclasses
 from .reflect import get_optional_alias_arg
 from .reflect import is_generic_alias
 from .reflect import is_union_alias
+from .strings import snake_case
 
 
 T = ta.TypeVar('T')
@@ -314,14 +316,18 @@ class ObjMarshalerManager:
     ) -> ObjMarshaler:
         if isinstance(ty, type):
             if abc.ABC in ty.__bases__:
-                return PolymorphicObjMarshaler.of([  # type: ignore
+                impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__]  # type: ignore
+                if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
+                    ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
+                else:
+                    ins = {ity: ity.__qualname__ for ity in impls}
+                return PolymorphicObjMarshaler.of([
                     PolymorphicObjMarshaler.Impl(
                         ity,
-                        ity.__qualname__,
+                        itn,
                         rec(ity),
                     )
-                    for ity in deep_subclasses(ty)
-                    if abc.ABC not in ity.__bases__
+                    for ity, itn in ins.items()
                 ])
 
             if issubclass(ty, enum.Enum):

omlish-0.0.0.dev156/omlish/lite/subprocesses.py

@@ -0,0 +1,251 @@
+# ruff: noqa: UP006 UP007
+import abc
+import contextlib
+import logging
+import os
+import shlex
+import subprocess
+import sys
+import time
+import typing as ta
+
+from omlish.lite.logs import log
+from omlish.lite.runtime import is_debugger_attached
+
+
+T = ta.TypeVar('T')
+SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
+
+
+##
+
+
+SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
+    'pipe': subprocess.PIPE,
+    'stdout': subprocess.STDOUT,
+    'devnull': subprocess.DEVNULL,
+}
+
+
+##
+
+
+_SUBPROCESS_SHELL_WRAP_EXECS = False
+
+
+def subprocess_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
+    return ('sh', '-c', ' '.join(map(shlex.quote, cmd)))
+
+
+def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
+    if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
+        return subprocess_shell_wrap_exec(*cmd)
+    else:
+        return cmd
+
+
+##
+
+
+def subprocess_close(
+        proc: subprocess.Popen,
+        timeout: ta.Optional[float] = None,
+) -> None:
+    # TODO: terminate, sleep, kill
+    if proc.stdout:
+        proc.stdout.close()
+    if proc.stderr:
+        proc.stderr.close()
+    if proc.stdin:
+        proc.stdin.close()
+
+    proc.wait(timeout)
+
+
+##
+
+
+class AbstractSubprocesses(abc.ABC):  # noqa
+    DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = log
+
+    def __init__(
+            self,
+            *,
+            log: ta.Optional[logging.Logger] = None,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._log = log if log is not None else self.DEFAULT_LOGGER
+        self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
+
+    #
+
+    def prepare_args(
+            self,
+            *cmd: str,
+            env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            quiet: bool = False,
+            shell: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
+        if self._log:
+            self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
+            if extra_env:
+                self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
+
+        if extra_env:
+            env = {**(env if env is not None else os.environ), **extra_env}
+
+        if quiet and 'stderr' not in kwargs:
+            if self._log and not self._log.isEnabledFor(logging.DEBUG):
+                kwargs['stderr'] = subprocess.DEVNULL
+
+        if not shell:
+            cmd = subprocess_maybe_shell_wrap_exec(*cmd)
+
+        return cmd, dict(
+            env=env,
+            shell=shell,
+            **kwargs,
+        )
+
+    @contextlib.contextmanager
+    def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
+        start_time = time.time()
+        try:
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
+            yield
+
+        except Exception as exc:  # noqa
+            if self._log:
+                self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
+            raise
+
+        finally:
+            end_time = time.time()
+            elapsed_s = end_time - start_time
+            if self._log:
+                self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
+
+    @contextlib.contextmanager
+    def prepare_and_wrap(
+            self,
+            *cmd: ta.Any,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[ta.Tuple[
+        ta.Tuple[ta.Any, ...],
+        ta.Dict[str, ta.Any],
+    ]]:
+        cmd, kwargs = self.prepare_args(*cmd, **kwargs)
+        with self.wrap_call(*cmd, **kwargs):
+            yield cmd, kwargs
+
+    #
+
+    DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
+        FileNotFoundError,
+        subprocess.CalledProcessError,
+    )
+
+    def try_fn(
+            self,
+            fn: ta.Callable[..., T],
+            *cmd: str,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Union[T, Exception]:
+        if try_exceptions is None:
+            try_exceptions = self._try_exceptions
+
+        try:
+            return fn(*cmd, **kwargs)
+
+        except try_exceptions as e:  # noqa
+            if self._log and self._log.isEnabledFor(logging.DEBUG):
+                self._log.exception('command failed')
+            return e
+
+    async def async_try_fn(
+            self,
+            fn: ta.Callable[..., ta.Awaitable[T]],
+            *cmd: ta.Any,
+            try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
+            **kwargs: ta.Any,
+    ) -> ta.Union[T, Exception]:
+        if try_exceptions is None:
+            try_exceptions = self._try_exceptions
+
+        try:
+            return await fn(*cmd, **kwargs)
+
+        except try_exceptions as e:  # noqa
+            if self._log and self._log.isEnabledFor(logging.DEBUG):
+                self._log.exception('command failed')
+            return e
+
+
+##
+
+
+class Subprocesses(AbstractSubprocesses):
+    def check_call(
+            self,
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs):  # noqa
+            subprocess.check_call(cmd, **kwargs)
+
+    def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs):  # noqa
+            return subprocess.check_output(cmd, **kwargs)
+
+    def check_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> str:
+        return self.check_output(*cmd, **kwargs).decode().strip()
+
+    #
+
+    def try_call(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bool:
+        if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
+            return False
+        else:
+            return True
+
+    def try_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[bytes]:
+        if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
+            return None
+        else:
+            return ret
+
+    def try_output_str(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> ta.Optional[str]:
+        if (ret := self.try_output(*cmd, **kwargs)) is None:
+            return None
+        else:
+            return ret.decode().strip()
+
+
+subprocesses = Subprocesses()