omlish 0.0.0.dev146__tar.gz → 0.0.0.dev148__tar.gz

{omlish-0.0.0.dev146/omlish.egg-info → omlish-0.0.0.dev148}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev146
+Version: 0.0.0.dev148
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev146'
-__revision__ = '3e5441b65738d0b3c80a43abb47b62f9ef5a6bee'
+__version__ = '0.0.0.dev148'
+__revision__ = '01263e3c2d1b82ca19c3d1d6916041ca4276737b'
 
 
 #

omlish-0.0.0.dev148/omlish/lite/asyncio/asyncio.py

@@ -0,0 +1,63 @@
+# ruff: noqa: UP006 UP007
+import asyncio.base_subprocess
+import asyncio.subprocess
+import typing as ta
+
+
+AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
+
+
+##
+
+
+ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
+
+
+async def asyncio_open_stream_reader(
+        f: ta.IO,
+        loop: ta.Any = None,
+        *,
+        limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
+) -> asyncio.StreamReader:
+    if loop is None:
+        loop = asyncio.get_running_loop()
+
+    reader = asyncio.StreamReader(limit=limit, loop=loop)
+    await loop.connect_read_pipe(
+        lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
+        f,
+    )
+
+    return reader
+
+
+async def asyncio_open_stream_writer(
+        f: ta.IO,
+        loop: ta.Any = None,
+) -> asyncio.StreamWriter:
+    if loop is None:
+        loop = asyncio.get_running_loop()
+
+    writer_transport, writer_protocol = await loop.connect_write_pipe(
+        lambda: asyncio.streams.FlowControlMixin(loop=loop),
+        f,
+    )
+
+    return asyncio.streams.StreamWriter(
+        writer_transport,
+        writer_protocol,
+        None,
+        loop,
+    )
+
+
+##
+
+
+def asyncio_maybe_timeout(
+        fut: AwaitableT,
+        timeout: ta.Optional[float] = None,
+) -> AwaitableT:
+    if timeout is not None:
+        fut = asyncio.wait_for(fut, timeout)  # type: ignore
+    return fut

omlish-0.0.0.dev148/omlish/lite/asyncio/subprocesses.py

@@ -0,0 +1,297 @@
+# ruff: noqa: UP006 UP007
+import asyncio.base_subprocess
+import asyncio.subprocess
+import contextlib
+import functools
+import logging
+import subprocess
+import sys
+import typing as ta
+
+from ..check import check_equal
+from ..check import check_isinstance
+from ..check import check_not_none
+from ..check import check_single
+from ..logs import log
+from ..subprocesses import DEFAULT_SUBPROCESS_TRY_EXCEPTIONS
+from ..subprocesses import prepare_subprocess_invocation
+from ..subprocesses import subprocess_common_context
+from .asyncio import asyncio_maybe_timeout
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+@contextlib.asynccontextmanager
+async def asyncio_subprocess_popen(
+        *cmd: str,
+        shell: bool = False,
+        timeout: ta.Optional[float] = None,
+        **kwargs: ta.Any,
+) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
+    fac: ta.Any
+    if shell:
+        fac = functools.partial(
+            asyncio.create_subprocess_shell,
+            check_single(cmd),
+        )
+    else:
+        fac = functools.partial(
+            asyncio.create_subprocess_exec,
+            *cmd,
+        )
+
+    with subprocess_common_context(
+            *cmd,
+            shell=shell,
+            timeout=timeout,
+            **kwargs,
+    ):
+        proc: asyncio.subprocess.Process
+        proc = await fac(**kwargs)
+        try:
+            yield proc
+
+        finally:
+            await asyncio_maybe_timeout(proc.wait(), timeout)
+
+
+##
+
+
+class AsyncioProcessCommunicator:
+    def __init__(
+            self,
+            proc: asyncio.subprocess.Process,
+            loop: ta.Optional[ta.Any] = None,
+    ) -> None:
+        super().__init__()
+
+        if loop is None:
+            loop = asyncio.get_running_loop()
+
+        self._proc = proc
+        self._loop = loop
+
+        self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check_isinstance(
+            proc._transport,  # type: ignore  # noqa
+            asyncio.base_subprocess.BaseSubprocessTransport,
+        )
+
+    @property
+    def _debug(self) -> bool:
+        return self._loop.get_debug()
+
+    async def _feed_stdin(self, input: bytes) -> None:  # noqa
+        stdin = check_not_none(self._proc.stdin)
+        try:
+            if input is not None:
+                stdin.write(input)
+                if self._debug:
+                    log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
+
+            await stdin.drain()
+
+        except (BrokenPipeError, ConnectionResetError) as exc:
+            # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
+            # exceptions.
+            if self._debug:
+                log.debug('%r communicate: stdin got %r', self, exc)
+
+        if self._debug:
+            log.debug('%r communicate: close stdin', self)
+
+        stdin.close()
+
+    async def _noop(self) -> None:
+        return None
+
+    async def _read_stream(self, fd: int) -> bytes:
+        transport: ta.Any = check_not_none(self._transport.get_pipe_transport(fd))
+
+        if fd == 2:
+            stream = check_not_none(self._proc.stderr)
+        else:
+            check_equal(fd, 1)
+            stream = check_not_none(self._proc.stdout)
+
+        if self._debug:
+            name = 'stdout' if fd == 1 else 'stderr'
+            log.debug('%r communicate: read %s', self, name)
+
+        output = await stream.read()
+
+        if self._debug:
+            name = 'stdout' if fd == 1 else 'stderr'
+            log.debug('%r communicate: close %s', self, name)
+
+        transport.close()
+
+        return output
+
+    class Communication(ta.NamedTuple):
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def _communicate(
+            self,
+            input: ta.Any = None,  # noqa
+    ) -> Communication:
+        stdin_fut: ta.Any
+        if self._proc.stdin is not None:
+            stdin_fut = self._feed_stdin(input)
+        else:
+            stdin_fut = self._noop()
+
+        stdout_fut: ta.Any
+        if self._proc.stdout is not None:
+            stdout_fut = self._read_stream(1)
+        else:
+            stdout_fut = self._noop()
+
+        stderr_fut: ta.Any
+        if self._proc.stderr is not None:
+            stderr_fut = self._read_stream(2)
+        else:
+            stderr_fut = self._noop()
+
+        stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
+
+        await self._proc.wait()
+
+        return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
+
+    async def communicate(
+            self,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> Communication:
+        return await asyncio_maybe_timeout(self._communicate(input), timeout)
+
+
+async def asyncio_subprocess_communicate(
+        proc: asyncio.subprocess.Process,
+        input: ta.Any = None,  # noqa
+        timeout: ta.Optional[float] = None,
+) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
+    return await AsyncioProcessCommunicator(proc).communicate(input, timeout)  # noqa
+
+
+##
+
+
+async def _asyncio_subprocess_check_run(
+        *args: str,
+        input: ta.Any = None,  # noqa
+        timeout: ta.Optional[float] = None,
+        **kwargs: ta.Any,
+) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
+    args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
+
+    proc: asyncio.subprocess.Process
+    async with asyncio_subprocess_popen(*args, **kwargs) as proc:
+        stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
+
+    if proc.returncode:
+        raise subprocess.CalledProcessError(
+            proc.returncode,
+            args,
+            output=stdout,
+            stderr=stderr,
+        )
+
+    return stdout, stderr
+
+
+async def asyncio_subprocess_check_call(
+        *args: str,
+        stdout: ta.Any = sys.stderr,
+        input: ta.Any = None,  # noqa
+        timeout: ta.Optional[float] = None,
+        **kwargs: ta.Any,
+) -> None:
+    _, _ = await _asyncio_subprocess_check_run(
+        *args,
+        stdout=stdout,
+        input=input,
+        timeout=timeout,
+        **kwargs,
+    )
+
+
+async def asyncio_subprocess_check_output(
+        *args: str,
+        input: ta.Any = None,  # noqa
+        timeout: ta.Optional[float] = None,
+        **kwargs: ta.Any,
+) -> bytes:
+    stdout, stderr = await _asyncio_subprocess_check_run(
+        *args,
+        stdout=asyncio.subprocess.PIPE,
+        input=input,
+        timeout=timeout,
+        **kwargs,
+    )
+
+    return check_not_none(stdout)
+
+
+async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
+    return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
+
+
+##
+
+
+async def _asyncio_subprocess_try_run(
+        fn: ta.Callable[..., ta.Awaitable[T]],
+        *args: ta.Any,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> ta.Union[T, Exception]:
+    try:
+        return await fn(*args, **kwargs)
+    except try_exceptions as e:  # noqa
+        if log.isEnabledFor(logging.DEBUG):
+            log.exception('command failed')
+        return e
+
+
+async def asyncio_subprocess_try_call(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> bool:
+    if isinstance(await _asyncio_subprocess_try_run(
+            asyncio_subprocess_check_call,
+            *args,
+            try_exceptions=try_exceptions,
+            **kwargs,
+    ), Exception):
+        return False
+    else:
+        return True
+
+
+async def asyncio_subprocess_try_output(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> ta.Optional[bytes]:
+    if isinstance(ret := await _asyncio_subprocess_try_run(
+            asyncio_subprocess_check_output,
+            *args,
+            try_exceptions=try_exceptions,
+            **kwargs,
+    ), Exception):
+        return None
+    else:
+        return ret
+
+
+async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
+    out = await asyncio_subprocess_try_output(*args, **kwargs)
+    return out.decode().strip() if out is not None else None

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish/lite/cached.py

@@ -7,7 +7,10 @@ T = ta.TypeVar('T')
 CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
 
 
-class _cached_nullary:  # noqa
+##
+
+
+class _AbstractCachedNullary:
     def __init__(self, fn):
         super().__init__()
         self._fn = fn
@@ -15,20 +18,42 @@ class _cached_nullary:  # noqa
         functools.update_wrapper(self, fn)
 
     def __call__(self, *args, **kwargs):  # noqa
-        if self._value is self._missing:
-            self._value = self._fn()
-        return self._value
+        raise TypeError
 
     def __get__(self, instance, owner):  # noqa
         bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
         return bound
 
 
+##
+
+
+class _CachedNullary(_AbstractCachedNullary):
+    def __call__(self, *args, **kwargs):  # noqa
+        if self._value is self._missing:
+            self._value = self._fn()
+        return self._value
+
+
 def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
-    return _cached_nullary(fn)
+    return _CachedNullary(fn)
 
 
 def static_init(fn: CallableT) -> CallableT:
     fn = cached_nullary(fn)
     fn()
     return fn
+
+
+##
+
+
+class _AsyncCachedNullary(_AbstractCachedNullary):
+    async def __call__(self, *args, **kwargs):
+        if self._value is self._missing:
+            self._value = await self._fn()
+        return self._value
+
+
+def async_cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
+    return _AsyncCachedNullary(fn)

omlish-0.0.0.dev148/omlish/lite/deathsig.py

@@ -0,0 +1,22 @@
+import ctypes as ct
+import sys
+
+
+LINUX_PR_SET_PDEATHSIG = 1  # Second arg is a signal
+LINUX_PR_GET_PDEATHSIG = 2  # Second arg is a ptr to return the signal
+
+
+def set_process_deathsig(sig: int) -> bool:
+    if sys.platform == 'linux':
+        libc = ct.CDLL('libc.so.6')
+
+        # int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5);
+        libc.prctl.restype = ct.c_int
+        libc.prctl.argtypes = [ct.c_int, ct.c_ulong, ct.c_ulong, ct.c_ulong, ct.c_ulong]
+
+        libc.prctl(LINUX_PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
+
+        return True
+
+    else:
+        return False

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish/lite/pycharm.py

@@ -36,7 +36,6 @@ def pycharm_debug_connect(prd: PycharmRemoteDebug) -> None:
 def pycharm_debug_preamble(prd: PycharmRemoteDebug) -> str:
     import inspect
     import textwrap
-
     return textwrap.dedent(f"""
         {inspect.getsource(pycharm_debug_connect)}
 

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish/lite/subprocesses.py

@@ -1,15 +1,18 @@
 # ruff: noqa: UP006 UP007
+import contextlib
 import logging
 import os
 import shlex
 import subprocess
 import sys
+import time
 import typing as ta
 
 from .logs import log
 from .runtime import is_debugger_attached
 
 
+T = ta.TypeVar('T')
 SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']
 
 
@@ -40,7 +43,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
         return args
 
 
-def _prepare_subprocess_invocation(
+def prepare_subprocess_invocation(
         *args: str,
         env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
         extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
@@ -48,9 +51,9 @@ def _prepare_subprocess_invocation(
         shell: bool = False,
         **kwargs: ta.Any,
 ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
-    log.debug(args)
+    log.debug('prepare_subprocess_invocation: args=%r', args)
     if extra_env:
-        log.debug(extra_env)
+        log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
 
     if extra_env:
         env = {**(env if env is not None else os.environ), **extra_env}
@@ -69,14 +72,46 @@ def _prepare_subprocess_invocation(
     )
 
 
-def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
-    args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
-    return subprocess.check_call(args, **kwargs)  # type: ignore
+##
+
+
+@contextlib.contextmanager
+def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
+    start_time = time.time()
+    try:
+        log.debug('subprocess_common_context.try: args=%r', args)
+        yield
+
+    except Exception as exc:  # noqa
+        log.debug('subprocess_common_context.except: exc=%r', exc)
+        raise
+
+    finally:
+        end_time = time.time()
+        elapsed_s = end_time - start_time
+        log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
+
+
+##
+
+
+def subprocess_check_call(
+        *args: str,
+        stdout: ta.Any = sys.stderr,
+        **kwargs: ta.Any,
+) -> None:
+    args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
+    with subprocess_common_context(*args, **kwargs):
+        return subprocess.check_call(args, **kwargs)  # type: ignore
 
 
-def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
-    args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
-    return subprocess.check_output(args, **kwargs)
+def subprocess_check_output(
+        *args: str,
+        **kwargs: ta.Any,
+) -> bytes:
+    args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
+    with subprocess_common_context(*args, **kwargs):
+        return subprocess.check_output(args, **kwargs)
 
 
 def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
@@ -92,16 +127,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
 )
 
 
-def subprocess_try_call(
-        *args: str,
+def _subprocess_try_run(
+        fn: ta.Callable[..., T],
+        *args: ta.Any,
         try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
         **kwargs: ta.Any,
-) -> bool:
+) -> ta.Union[T, Exception]:
     try:
-        subprocess_check_call(*args, **kwargs)
+        return fn(*args, **kwargs)
     except try_exceptions as e:  # noqa
         if log.isEnabledFor(logging.DEBUG):
             log.exception('command failed')
+        return e
+
+
+def subprocess_try_call(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> bool:
+    if isinstance(_subprocess_try_run(
+            subprocess_check_call,
+            *args,
+            try_exceptions=try_exceptions,
+            **kwargs,
+    ), Exception):
         return False
     else:
         return True
@@ -112,12 +162,15 @@ def subprocess_try_output(
         try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
         **kwargs: ta.Any,
 ) -> ta.Optional[bytes]:
-    try:
-        return subprocess_check_output(*args, **kwargs)
-    except try_exceptions as e:  # noqa
-        if log.isEnabledFor(logging.DEBUG):
-            log.exception('command failed')
+    if isinstance(ret := _subprocess_try_run(
+            subprocess_check_output,
+            *args,
+            try_exceptions=try_exceptions,
+            **kwargs,
+    ), Exception):
         return None
+    else:
+        return ret
 
 
 def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish/manifests/load.py

@@ -31,6 +31,8 @@ class ManifestLoader:
         self._cls_cache: ta.Dict[str, type] = {}
         self._raw_cache: ta.Dict[str, ta.Optional[ta.Sequence[Manifest]]] = {}
 
+    #
+
     @classmethod
     def from_entry_point(
             cls,
@@ -52,6 +54,8 @@ class ManifestLoader:
 
         return cls(module_remap=rm, **kwargs)
 
+    #
+
     def load_cls(self, key: str) -> type:
         try:
             return self._cls_cache[key]
@@ -153,6 +157,8 @@ class ManifestLoader:
 
         return lst
 
+    #
+
     ENTRY_POINT_GROUP = 'omlish.manifests'
 
     def discover(self) -> ta.Sequence[str]:

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish/testing/pytest/plugins/asyncs.py

@@ -142,7 +142,12 @@ class AsyncsPlugin:
         if len(bes) > 1 and set(bes) != {'trio', 'trio_asyncio'}:
             raise Exception(f'{item.nodeid}: multiple async backends specified: {bes}')
         elif is_async_function(item.obj) and not bes:
-            raise Exception(f'{item.nodeid}: async def function and no async plugin specified')
+            from _pytest.unittest import UnitTestCase  # noqa
+            if isinstance(item.parent, UnitTestCase):
+                # unittest handles these itself.
+                pass
+            else:
+                raise Exception(f'{item.nodeid}: async def function and no async plugin specified')
 
         if 'trio_asyncio' in bes:
             obj = item.obj

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev146
+Version: 0.0.0.dev148
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/omlish.egg-info/SOURCES.txt

@@ -324,6 +324,7 @@ omlish/lite/__init__.py
 omlish/lite/cached.py
 omlish/lite/check.py
 omlish/lite/contextmanagers.py
+omlish/lite/deathsig.py
 omlish/lite/docker.py
 omlish/lite/inject.py
 omlish/lite/io.py
@@ -343,6 +344,9 @@ omlish/lite/socketserver.py
 omlish/lite/strings.py
 omlish/lite/subprocesses.py
 omlish/lite/typing.py
+omlish/lite/asyncio/__init__.py
+omlish/lite/asyncio/asyncio.py
+omlish/lite/asyncio/subprocesses.py
 omlish/lite/fdio/__init__.py
 omlish/lite/fdio/corohttp.py
 omlish/lite/fdio/handlers.py

{omlish-0.0.0.dev146 → omlish-0.0.0.dev148}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '>=3.12'
-version = '0.0.0.dev146'
+version = '0.0.0.dev148'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',