omlish 0.0.0.dev239__tar.gz → 0.0.0.dev241__tar.gz

{omlish-0.0.0.dev239/omlish.egg-info → omlish-0.0.0.dev241}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev239
+Version: 0.0.0.dev241
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev239'
-__revision__ = 'f241f36977bf74d1ea680d415782228917ec3669'
+__version__ = '0.0.0.dev241'
+__revision__ = '97c9969eb271b44d0d57f2dae10a680aa62c02e8'
 
 
 #

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/asyncs/asyncio/all.py

@@ -1,7 +1,12 @@
 # ruff: noqa: I001
 from .asyncio import (  # noqa
+    asyncio_ensure_task as ensure_task,
+
     asyncio_once as once,
-    asyncio_wait_concurrent as wait_concurrent,
+
     drain_asyncio_tasks as drain_tasks,
     draining_asyncio_tasks as draining_tasks,
+
+    asyncio_wait_concurrent as wait_concurrent,
+    asyncio_wait_maybe_concurrent as wait_maybe_concurrent,
 )

omlish-0.0.0.dev241/omlish/asyncs/asyncio/asyncio.py

@@ -0,0 +1,109 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+TODO:
+ - split module
+"""
+import asyncio
+import contextlib
+import functools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
+
+
+##
+
+
+def asyncio_ensure_task(obj: ta.Awaitable) -> asyncio.Task:
+    if isinstance(obj, asyncio.Task):
+        return obj
+    elif isinstance(obj, ta.Coroutine):
+        return asyncio.create_task(obj)
+    else:
+        raise TypeError(obj)
+
+
+##
+
+
+def asyncio_once(fn: CallableT) -> CallableT:
+    task = None
+
+    @functools.wraps(fn)
+    async def inner(*args, **kwargs):
+        nonlocal task
+        if not task:
+            task = asyncio.create_task(fn(*args, **kwargs))
+        return await task
+
+    return ta.cast(CallableT, inner)
+
+
+##
+
+
+def drain_asyncio_tasks(loop=None):
+    if loop is None:
+        loop = asyncio.get_running_loop()
+
+    while loop._ready or loop._scheduled:  # noqa
+        loop._run_once()  # noqa
+
+
+@contextlib.contextmanager
+def draining_asyncio_tasks() -> ta.Iterator[None]:
+    loop = asyncio.get_running_loop()
+    try:
+        yield
+    finally:
+        if loop is not None:
+            drain_asyncio_tasks(loop)  # noqa
+
+
+##
+
+
+async def asyncio_wait_concurrent(
+        awaitables: ta.Iterable[ta.Awaitable[T]],
+        concurrency: ta.Union[int, asyncio.Semaphore],
+        *,
+        return_when: ta.Any = asyncio.FIRST_EXCEPTION,
+) -> ta.List[T]:
+    if isinstance(concurrency, asyncio.Semaphore):
+        semaphore = concurrency
+    elif isinstance(concurrency, int):
+        semaphore = asyncio.Semaphore(concurrency)
+    else:
+        raise TypeError(concurrency)
+
+    async def limited_task(a):
+        async with semaphore:
+            return await a
+
+    futs = [asyncio.create_task(limited_task(a)) for a in awaitables]
+    done, pending = await asyncio.wait(futs, return_when=return_when)
+
+    for fut in pending:
+        fut.cancel()
+
+    for fut in done:
+        if fut.exception():
+            raise fut.exception()  # type: ignore
+
+    return [fut.result() for fut in done]
+
+
+async def asyncio_wait_maybe_concurrent(
+        awaitables: ta.Iterable[ta.Awaitable[T]],
+        concurrency: ta.Union[int, asyncio.Semaphore, None],
+) -> ta.List[T]:
+    # Note: Only supports return_when=asyncio.FIRST_EXCEPTION
+    if concurrency is None:
+        return [await a for a in awaitables]
+
+    else:
+        return await asyncio_wait_concurrent(awaitables, concurrency)

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/docker/all.py

@@ -27,5 +27,5 @@ from .hub import (  # noqa
 )
 
 from .timebomb import (  # noqa
-    timebomb_payload,
+    docker_timebomb_payload as timebomb_payload,
 )

omlish-0.0.0.dev241/omlish/docker/ns1.py

@@ -0,0 +1,98 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import re
+import typing as ta
+
+from ..lite.check import check
+from ..subprocesses.run import SubprocessRun
+from ..subprocesses.run import SubprocessRunnable
+from ..subprocesses.run import SubprocessRunOutput
+
+
+##
+
+
+DEFAULT_DOCKER_NS1_RUN_IMAGE: str = 'debian'
+
+
+def build_docker_ns1_run_args(
+        *cmd: str,
+        image: ta.Optional[str] = None,
+        nsenter: str = 'nsenter',
+) -> ta.List[str]:
+    """
+    - https://gist.github.com/BretFisher/5e1a0c7bcca4c735e716abf62afad389
+    - https://github.com/justincormack/nsenter1/blob/8d3ba504b2c14d73c70cf34f1ec6943c093f1b02/nsenter1.c
+
+    alt:
+     - nc -U ~/Library/Containers/com.docker.docker/Data/debug-shell.sock
+    """
+
+    return [
+        '--privileged',
+        '--pid=host',
+        (image if image is not None else DEFAULT_DOCKER_NS1_RUN_IMAGE),
+
+        nsenter,
+        '-t', '1',
+
+        '-m',  # mount
+        '-u',  # uts
+        '-i',  # ipc
+        '-n',  # net
+        '-p',  # pid
+        '-C',  # cgroup
+        # '-U',  # user
+        '-T',  # time
+
+        *cmd,
+    ]
+
+
+def build_docker_ns1_run_cmd(
+        *cmd: str,
+        exe: str = 'docker',
+        run_args: ta.Optional[ta.Sequence[str]] = None,
+        **kwargs: ta.Any,
+) -> ta.List[str]:
+    if run_args is not None:
+        check.not_isinstance(run_args, str)
+
+    return [
+        exe,
+        'run',
+        '--rm',
+        *(run_args if run_args is not None else []),
+        '-i',
+        *build_docker_ns1_run_args(*cmd, **kwargs),
+    ]
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DockerNs1ListUsedTcpPortsCommand(SubprocessRunnable[ta.List[int]]):
+    kwargs: ta.Optional[ta.Mapping[str, str]] = None
+
+    def make_run(self) -> SubprocessRun:
+        return SubprocessRun.of(
+            *build_docker_ns1_run_cmd(
+                'netstat', '-tan',
+            ),
+            stdout='pipe',
+            stderr='devnull',
+            check=True,
+        )
+
+    _NETSTAT_LINE_PAT: ta.ClassVar[re.Pattern] = re.compile(r'\d{1,3}(.(\d{1,3})){3}:(?P<port>\d+)')
+
+    def handle_run_output(self, output: SubprocessRunOutput) -> ta.List[int]:
+        lines = [s for l in check.not_none(output.stdout).decode().splitlines() if (s := l.strip())]
+        return [
+            int(m.groupdict()['port'])
+            for l in lines
+            if len(ps := l.split(maxsplit=4)) > 3
+            if (m := self._NETSTAT_LINE_PAT.fullmatch(ps[3])) is not None
+        ]

omlish-0.0.0.dev239/omlish/docker/portrelay.py → omlish-0.0.0.dev241/omlish/docker/ports.py

@@ -1,13 +1,33 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
+"""
+TODO:
+ - docstring
+ - timebomb
+ - auto-discover available ports
+"""
 import dataclasses as dc
 import os
 import typing as ta
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class DockerPortRelay:
-    docker_port: int
+    """
+    Uses roughly the following command to forward connections from inside docker-for-mac's vm to the mac host:
+
+      docker run --rm -i -p 5001:5000 alpine/socat -d -d TCP-LISTEN:5000,fork,reuseaddr TCP:host.docker.internal:5021
+
+    This allows requests made by the docker daemon running inside the vm to `host.docker.internal:5001` to be forwarded
+    to the mac host on port 5021. The reason for this is to be able to use a docker registry running locally directly on
+    the host mac - specifically to be able to do so with ssl certificate checking disabled (which docker will only do on
+    localhost, which on a mac in the vm isn't actually the mac host - hence the necessity of the relay).
+    """
+
+    docker_port: int  # port
     host_port: int
 
     name: ta.Optional[str] = None
@@ -30,7 +50,7 @@ class DockerPortRelay:
 
     def run_args(self) -> ta.List[str]:
         if (name := self.name) is None:
-            name = f'docker_port_relay-{os.getpid()}'
+            name = f'docker_port_relay-{os.getpid()}-{self.docker_port}-{self.intermediate_port}-{self.host_port}'
 
         return [
             '--name', name,

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/docker/timebomb.py

@@ -1,10 +1,14 @@
+# @omlish-lite
 import shlex
 
 
-_DEFAULT_TIMEBOMB_NAME = '-'.join([*__name__.split('.'), 'timebomb'])
+##
 
 
-def timebomb_payload(delay_s: float, name: str = _DEFAULT_TIMEBOMB_NAME) -> str:
+_DEFAULT_DOCKER_TIMEBOMB_NAME = 'omlish-timebomb'
+
+
+def docker_timebomb_payload(delay_s: float, name: str = _DEFAULT_DOCKER_TIMEBOMB_NAME) -> str:
     return (
         '('
         f'echo {shlex.quote(name)} && '

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/http/coro/simple.py

@@ -1,5 +1,9 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
+"""
+TODO:
+ - logging
+"""
 import concurrent.futures as cf
 import contextlib
 import functools
@@ -15,6 +19,7 @@ from ...sockets.server.handlers import SocketServerHandler
 from ...sockets.server.handlers import SocketWrappingSocketServerHandler
 from ...sockets.server.handlers import StandardSocketServerHandler
 from ...sockets.server.server import SocketServer
+from ...sockets.server.ssl import SslErrorHandlingSocketServerHandler
 from ...sockets.server.threading import ThreadingSocketServerHandler
 from ..handlers import HttpHandler
 from ..parsing import HttpRequestParser
@@ -35,6 +40,7 @@ def make_simple_http_server(
         *,
         server_version: HttpProtocolVersion = HttpProtocolVersions.HTTP_1_1,
         ssl_context: ta.Optional['ssl.SSLContext'] = None,
+        ignore_ssl_errors: bool = False,
         executor: ta.Optional[cf.Executor] = None,
         use_threads: bool = False,
 ) -> ta.Iterator[SocketServer]:
@@ -72,6 +78,11 @@ def make_simple_http_server(
                 )),
             )
 
+        if ignore_ssl_errors:
+            server_handler = SslErrorHandlingSocketServerHandler(
+                server_handler,
+            )
+
         #
 
         server_handler = StandardSocketServerHandler(

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/lite/inject.py

@@ -510,14 +510,6 @@ _INJECTION_INSPECTION_CACHE: ta.MutableMapping[ta.Any, _InjectionInspection] = w
 
 def _do_injection_inspect(obj: ta.Any) -> _InjectionInspection:
     tgt = obj
-    if isinstance(tgt, type) and tgt.__init__ is not object.__init__:  # type: ignore[misc]
-        # Python 3.8's inspect.signature can't handle subclasses overriding __new__, always generating *args/**kwargs.
-        #  - https://bugs.python.org/issue40897
-        #  - https://github.com/python/cpython/commit/df7c62980d15acd3125dfbd81546dad359f7add7
-        tgt = tgt.__init__  # type: ignore[misc]
-        has_generic_base = True
-    else:
-        has_generic_base = False
 
     # inspect.signature(eval_str=True) was added in 3.10 and we have to support 3.8, so we have to get_type_hints to
     # eval str annotations *in addition to* getting the signature for parameter information.
@@ -525,23 +517,40 @@ def _do_injection_inspect(obj: ta.Any) -> _InjectionInspection:
     has_partial = False
     while True:
         if isinstance(uw, functools.partial):
-            has_partial = True
             uw = uw.func
+            has_partial = True
         else:
             if (uw2 := inspect.unwrap(uw)) is uw:
                 break
             uw = uw2
 
-    if has_generic_base and has_partial:
+    has_args_offset = False
+
+    if isinstance(tgt, type) and tgt.__new__ is not object.__new__:
+        # Python 3.8's inspect.signature can't handle subclasses overriding __new__, always generating *args/**kwargs.
+        #  - https://bugs.python.org/issue40897
+        #  - https://github.com/python/cpython/commit/df7c62980d15acd3125dfbd81546dad359f7add7
+        tgt = tgt.__init__  # type: ignore[misc]
+        has_args_offset = True
+
+    if tgt in (object.__init__, object.__new__):
+        # inspect strips self for types but not the underlying methods.
+        def dummy(self):
+            pass
+        tgt = dummy
+        has_args_offset = True
+
+    if has_partial and has_args_offset:
+        # TODO: unwrap partials masking parameters like modern python
         raise InjectorError(
-            'Injector inspection does not currently support both a typing.Generic base and a functools.partial: '
+            'Injector inspection does not currently support both an args offset and a functools.partial: '
             f'{obj}',
         )
 
     return _InjectionInspection(
         inspect.signature(tgt),
         ta.get_type_hints(uw),
-        1 if has_generic_base else 0,
+        1 if has_args_offset else 0,
     )
 
 

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/marshal/base.py

@@ -13,6 +13,7 @@ See:
  - https://github.com/python-attrs/cattrs
  - https://github.com/jcrist/msgspec
  - https://github.com/Fatal1ty/mashumaro
+ - https://github.com/Kotlin/kotlinx.serialization/blob/master/docs/serializers.md#custom-serializers
 
 cattrs:
  *

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/metadata.py

@@ -8,6 +8,7 @@ TODO:
   - merge mro?
   - are these better left up to callers? too usecase-specific to favor either way?
 """
+import threading
 import types
 import typing as ta
 
@@ -60,6 +61,8 @@ def _unwrap_object_metadata_target(obj: ta.Any) -> ta.Any:
 ##
 
 
+_OBJECT_METADATA_LOCK = threading.RLock()
+
 _OBJECT_METADATA_ATTR = '__' + __name__.replace('.', '_') + '__metadata__'
 
 
@@ -71,15 +74,14 @@ def append_object_metadata(obj: T, *mds: ObjectMetadata) -> T:
     dct = tgt.__dict__
 
     if isinstance(dct, types.MappingProxyType):
-        for _ in range(2):
-            try:
-                lst = dct[_OBJECT_METADATA_ATTR]
-            except KeyError:
-                setattr(tgt, _OBJECT_METADATA_ATTR, [])
-            else:
-                break
-        else:
-            raise RuntimeError
+        try:
+            lst = dct[_OBJECT_METADATA_ATTR]
+        except KeyError:
+            with _OBJECT_METADATA_LOCK:
+                try:
+                    lst = dct[_OBJECT_METADATA_ATTR]
+                except KeyError:
+                    setattr(tgt, _OBJECT_METADATA_ATTR, lst := [])
 
     else:
         lst = dct.setdefault(_OBJECT_METADATA_ATTR, [])

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/os/mangle.py

@@ -1,5 +1,9 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
+"""
+TODO:
+ - use ..text.mangle
+"""
 
 
 #

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/sockets/bind.py

@@ -175,11 +175,11 @@ class SocketBinder(abc.ABC, ta.Generic[SocketBinderConfigT]):
         if hasattr(self, '_socket'):
             raise self.AlreadyBoundError
 
-        socket = socket_.socket(self.address_family, socket_.SOCK_STREAM)
-        self._socket = socket
+        sock = socket_.socket(self.address_family, socket_.SOCK_STREAM)
+        self._socket = sock
 
         if self._config.allow_reuse_address and hasattr(socket_, 'SO_REUSEADDR'):
-            socket.setsockopt(socket_.SOL_SOCKET, socket_.SO_REUSEADDR, 1)
+            sock.setsockopt(socket_.SOL_SOCKET, socket_.SO_REUSEADDR, 1)
 
         # Since Linux 6.12.9, SO_REUSEPORT is not allowed on other address families than AF_INET/AF_INET6.
         if (
@@ -187,13 +187,13 @@ class SocketBinder(abc.ABC, ta.Generic[SocketBinderConfigT]):
                 self.address_family in (socket_.AF_INET, socket_.AF_INET6)
         ):
             try:
-                socket.setsockopt(socket_.SOL_SOCKET, socket_.SO_REUSEPORT, 1)
+                sock.setsockopt(socket_.SOL_SOCKET, socket_.SO_REUSEPORT, 1)
             except OSError as err:
                 if err.errno not in (errno.ENOPROTOOPT, errno.EINVAL):
                     raise
 
-        if self._config.set_inheritable and hasattr(socket, 'set_inheritable'):
-            socket.set_inheritable(True)
+        if self._config.set_inheritable and hasattr(sock, 'set_inheritable'):
+            sock.set_inheritable(True)
 
     def _pre_bind(self) -> None:
         pass
@@ -224,7 +224,7 @@ class SocketBinder(abc.ABC, ta.Generic[SocketBinderConfigT]):
         self.socket.listen(self._config.listen_backlog)
 
     @abc.abstractmethod
-    def accept(self, socket: ta.Optional[socket_.socket] = None) -> SocketAndAddress:
+    def accept(self, sock: ta.Optional[socket_.socket] = None) -> SocketAndAddress:
         raise NotImplementedError
 
 
@@ -270,11 +270,11 @@ class TcpSocketBinder(SocketBinder):
 
     #
 
-    def accept(self, socket: ta.Optional[socket_.socket] = None) -> SocketAndAddress:
-        if socket is None:
-            socket = self.socket
+    def accept(self, sock: ta.Optional[socket_.socket] = None) -> SocketAndAddress:
+        if sock is None:
+            sock = self.socket
 
-        conn, client_address = socket.accept()
+        conn, client_address = sock.accept()
         return SocketAndAddress(conn, client_address)
 
 

omlish-0.0.0.dev241/omlish/sockets/ports.py

@@ -0,0 +1,62 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import contextlib
+import socket
+import typing as ta
+
+from ..lite.check import check
+from ..lite.timeouts import Timeout
+from ..lite.timeouts import TimeoutLike
+
+
+##
+
+
+DEFAULT_AVAILABLE_PORT_HOST: str = '127.0.0.1'
+
+
+@contextlib.contextmanager
+def get_available_port_context(host: ta.Optional[str] = None) -> ta.Iterator[int]:
+    if host is None:
+        host = DEFAULT_AVAILABLE_PORT_HOST
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind((host, 0))
+        sock.listen(1)
+        port = sock.getsockname()[1]
+        yield port
+
+
+def get_available_port(host: ta.Optional[str] = None) -> int:
+    with get_available_port_context(host) as port:
+        pass
+    return port
+
+
+def get_available_ports(
+        n: int,
+        *,
+        host: ta.Optional[str] = None,
+        exclude: ta.Optional[ta.Iterable[int]] = None,
+        timeout: ta.Optional[TimeoutLike] = None,
+) -> ta.List[int]:
+    exclude = set(exclude or [])
+
+    seen: ta.Set[int] = set()
+    ret: ta.List[int] = []
+
+    timeout = Timeout.of(timeout)
+
+    with contextlib.ExitStack() as es:
+        while len(ret) < n:
+            timeout()
+
+            cur = es.enter_context(get_available_port_context(host))
+
+            check.not_in(cur, seen)
+            seen.add(cur)
+
+            if cur not in exclude:
+                ret.append(cur)
+
+    return ret

{omlish-0.0.0.dev239 → omlish-0.0.0.dev241}/omlish/sockets/server/handlers.py

@@ -13,7 +13,7 @@ from ..io import SocketIoPair
 from ..io import close_socket_immediately
 
 
-SocketServerHandler = ta.Callable[[SocketAndAddress], None]  # ta.TypeAlias
+SocketServerHandler = ta.Callable[['SocketAndAddress'], None]  # ta.TypeAlias
 
 
 ##

omlish-0.0.0.dev241/omlish/sockets/server/ssl.py

@@ -0,0 +1,39 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import logging
+import typing as ta
+
+from ..addresses import SocketAndAddress
+from ..io import close_socket_immediately
+from .handlers import SocketServerHandler
+from .handlers import SocketServerHandler_
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SslErrorHandlingSocketServerHandler(SocketServerHandler_):
+    handler: SocketServerHandler
+
+    log: ta.Optional[logging.Logger] = None
+
+    #
+
+    _error_cls: ta.ClassVar[ta.Optional[ta.Type[BaseException]]] = None
+
+    @classmethod
+    def _get_error_cls(cls) -> ta.Type[BaseException]:
+        if (error_cls := cls._error_cls) is None:
+            import ssl
+            error_cls = cls._error_cls = ssl.SSLError
+        return error_cls
+
+    def __call__(self, conn: SocketAndAddress) -> None:
+        try:
+            self.handler(conn)
+        except self._get_error_cls():  # noqa
+            if (log := self.log) is not None:
+                log.exception('SSL Error in connection %r', conn)
+            close_socket_immediately(conn.socket)