omdev 0.0.0.dev241__tar.gz → 0.0.0.dev243__tar.gz

{omdev-0.0.0.dev241/omdev.egg-info → omdev-0.0.0.dev243}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev241
+Version: 0.0.0.dev243
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev241
+Requires-Dist: omlish==0.0.0.dev243
 Provides-Extra: all
 Requires-Dist: black~=25.1; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/ci/ci.py

@@ -17,6 +17,7 @@ from .compose import get_compose_service_dependencies
 from .docker.buildcaching import DockerBuildCaching
 from .docker.cache import DockerCacheKey
 from .docker.cmds import build_docker_image
+from .docker.cmds import ensure_docker_image_setup
 from .docker.imagepulling import DockerImagePulling
 from .docker.utils import build_docker_file_hash
 from .requirements import build_requirements_hash
@@ -51,6 +52,8 @@ class Ci(AsyncExitStacked):
 
         no_dependencies: bool = False
 
+        setup_only: bool = False
+
         run_options: ta.Optional[ta.Sequence[str]] = None
 
         #
@@ -174,18 +177,21 @@ class Ci(AsyncExitStacked):
     #
 
     @cached_nullary
-    def pull_dependencies_funcs(self) -> ta.Sequence[ta.Callable[[], ta.Awaitable]]:
+    def get_dependency_images(self) -> ta.Sequence[str]:
         deps = get_compose_service_dependencies(
             self._config.compose_file,
             self._config.service,
         )
+        return sorted(deps.values())
 
+    @cached_nullary
+    def pull_dependencies_funcs(self) -> ta.Sequence[ta.Callable[[], ta.Awaitable]]:
         return [
             async_cached_nullary(functools.partial(
                 self._docker_image_pulling.pull_docker_image,
                 dep_image,
             ))
-            for dep_image in deps.values()
+            for dep_image in self.get_dependency_images()
         ]
 
     #
@@ -233,7 +239,26 @@ class Ci(AsyncExitStacked):
 
     #
 
+    async def _run_setup_only(self) -> None:
+        image_ids = [
+            await self.resolve_ci_image(),
+
+            *(self.get_dependency_images() if not self._config.no_dependencies else []),
+        ]
+
+        for image_id in image_ids:
+            with log_timing_context(f'Run setup only: {image_id}'):
+                await ensure_docker_image_setup(
+                    image_id,
+                    cwd=self._config.project_dir,
+                )
+
+    #
+
     async def run(self) -> None:
         await self.setup()
 
-        await self._run_compose()
+        if self._config.setup_only:
+            await self._run_setup_only()
+        else:
+            await self._run_compose()

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/ci/cli.py

@@ -97,6 +97,8 @@ class CiCli(ArgparseCli):
 
         argparse_arg('--no-dependencies', action='store_true'),
 
+        argparse_arg('--setup-only', action='store_true'),
+
         argparse_arg('-e', '--env', action='append'),
         argparse_arg('-v', '--volume', action='append'),
 
@@ -211,6 +213,8 @@ class CiCli(ArgparseCli):
 
             no_dependencies=self.args.no_dependencies,
 
+            setup_only=self.args.setup_only,
+
             run_options=run_options,
         )
 

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/ci/docker/cacheserved/cache.py

@@ -6,6 +6,7 @@ import json
 import os.path
 import typing as ta
 
+from omlish.asyncs.asyncio.sockets import asyncio_wait_until_can_connect
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.json import json_dumps_compact
@@ -126,17 +127,14 @@ class CacheServedDockerCache(DockerCache):
             dds_run_task = asyncio.create_task(dds.run())
             try:
                 timeout = Timeout.of(self._config.server_start_timeout)
-                while True:
-                    timeout()
-                    try:
-                        reader, writer = await asyncio.open_connection('localhost', self._config.port)
-                    except Exception as e:  # noqa
-                        log.exception('Failed to connect to cache server - will try again')
-                    else:
-                        writer.close()
-                        await asyncio.wait_for(writer.wait_closed(), timeout=timeout.remaining())
-                        break
-                    await asyncio.sleep(self._config.server_start_sleep)
+
+                await asyncio_wait_until_can_connect(
+                    'localhost',
+                    self._config.port,
+                    timeout=timeout,
+                    on_fail=lambda _: log.exception('Failed to connect to cache server - will try again'),
+                    sleep_s=self._config.server_start_sleep,
+                )
 
                 if (prc := self._config.pull_run_cmd) is not None:
                     pull_cmd = [

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/ci/docker/cmds.py

@@ -123,3 +123,21 @@ async def load_docker_tar(
         tar_file: str,
 ) -> str:
     return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
+
+
+##
+
+
+async def ensure_docker_image_setup(
+        image: str,
+        *,
+        cwd: ta.Optional[str] = None,
+) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'run',
+        '--rm',
+        '--entrypoint', '/bin/true',  # FIXME: lol
+        image,
+        **(dict(cwd=cwd) if cwd is not None else {}),
+    )

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/oci/tars.py

@@ -79,11 +79,7 @@ class OciDataTarWriter(ExitStacked):
             tar_sha256=self._tw.sha256(),
         )
 
-    def __enter__(self) -> 'OciDataTarWriter':
-        super().__enter__()
-
-        #
-
+    def _enter_contexts(self) -> None:
         self._cw = self._FileWrapper(self._f)
 
         if self._compression is OciCompression.GZIP:
@@ -112,16 +108,12 @@ class OciDataTarWriter(ExitStacked):
         self._tw = self._FileWrapper(self._cf)
 
         self._tf = self._enter_context(
-            tarfile.open(  # type: ignore
+            tarfile.open(  # type: ignore  # noqa
                 fileobj=self._tw,
                 mode='w',
             ),
         )
 
-        #
-
-        return self
-
     def tar_file(self) -> tarfile.TarFile:
         return self._tf
 

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/scripts/ci.py

@@ -2607,6 +2607,48 @@ class ArgparseCli:
             return fn()
 
 
+########################################
+# ../../../omlish/asyncs/asyncio/sockets.py
+
+
+async def asyncio_wait_until_can_connect(
+        host: ta.Any = None,
+        port: ta.Any = None,
+        *,
+        timeout: ta.Optional[TimeoutLike] = None,
+        on_fail: ta.Optional[ta.Callable[[BaseException], None]] = None,
+        sleep_s: float = .1,
+        exception: ta.Union[ta.Type[BaseException], ta.Tuple[ta.Type[BaseException], ...]] = (Exception,),
+) -> None:
+    timeout = Timeout.of(timeout)
+
+    async def inner():
+        while True:
+            timeout()
+
+            try:
+                reader, writer = await asyncio.open_connection(host, port)
+
+            except asyncio.CancelledError:
+                raise
+
+            except exception as e:  # noqa
+                if on_fail is not None:
+                    on_fail(e)
+
+            else:
+                writer.close()
+                await asyncio.wait_for(writer.wait_closed(), timeout=timeout.or_(None))
+                break
+
+            await asyncio.sleep(min(sleep_s, timeout.remaining()))
+
+    if timeout() != float('inf'):
+        await asyncio.wait_for(inner(), timeout=timeout())
+    else:
+        await inner()
+
+
 ########################################
 # ../../../omlish/asyncs/asyncio/timeouts.py
 
@@ -3041,20 +3083,64 @@ class HttpRequestParser:
 
 
 class ExitStacked:
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        for a in ('__enter__', '__exit__'):
+            for b in cls.__bases__:
+                if b is ExitStacked:
+                    continue
+                try:
+                    fn = getattr(b, a)
+                except AttributeError:
+                    pass
+                else:
+                    if fn is not getattr(ExitStacked, a):
+                        raise TypeError(f'ExitStacked subclass {cls} must not not override {a} via {b}')
+
     _exit_stack: ta.Optional[contextlib.ExitStack] = None
 
+    @contextlib.contextmanager
+    def _exit_stacked_init_wrapper(self) -> ta.Iterator[None]:
+        """
+        Overridable wrapper around __enter__ which deliberately does not have access to an _exit_stack yet. Intended for
+        things like wrapping __enter__ in a lock.
+        """
+
+        yield
+
+    @ta.final
     def __enter__(self: ExitStackedT) -> ExitStackedT:
-        check.state(self._exit_stack is None)
-        es = self._exit_stack = contextlib.ExitStack()
-        es.__enter__()
-        return self
+        """
+        Final because any contexts entered during this init must be exited if any exception is thrown, and user
+        overriding would likely interfere with that. Override `_enter_contexts` for such init.
+        """
+
+        with self._exit_stacked_init_wrapper():
+            check.state(self._exit_stack is None)
+            es = self._exit_stack = contextlib.ExitStack()
+            es.__enter__()
+            try:
+                self._enter_contexts()
+            except Exception:  # noqa
+                es.__exit__(*sys.exc_info())
+                raise
+            return self
 
+    @ta.final
     def __exit__(self, exc_type, exc_val, exc_tb):
         if (es := self._exit_stack) is None:
             return None
-        self._exit_contexts()
+        try:
+            self._exit_contexts()
+        except Exception:  # noqa
+            es.__exit__(*sys.exc_info())
+            raise
         return es.__exit__(exc_type, exc_val, exc_tb)
 
+    def _enter_contexts(self) -> None:
+        pass
+
     def _exit_contexts(self) -> None:
         pass
 
@@ -3064,20 +3150,54 @@ class ExitStacked:
 
 
 class AsyncExitStacked:
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        for a in ('__aenter__', '__aexit__'):
+            for b in cls.__bases__:
+                if b is AsyncExitStacked:
+                    continue
+                try:
+                    fn = getattr(b, a)
+                except AttributeError:
+                    pass
+                else:
+                    if fn is not getattr(AsyncExitStacked, a):
+                        raise TypeError(f'AsyncExitStacked subclass {cls} must not not override {a} via {b}')
+
     _exit_stack: ta.Optional[contextlib.AsyncExitStack] = None
 
+    @contextlib.asynccontextmanager
+    async def _async_exit_stacked_init_wrapper(self) -> ta.AsyncGenerator[None, None]:
+        yield
+
+    @ta.final
     async def __aenter__(self: AsyncExitStackedT) -> AsyncExitStackedT:
-        check.state(self._exit_stack is None)
-        es = self._exit_stack = contextlib.AsyncExitStack()
-        await es.__aenter__()
-        return self
+        async with self._async_exit_stacked_init_wrapper():
+            check.state(self._exit_stack is None)
+            es = self._exit_stack = contextlib.AsyncExitStack()
+            await es.__aenter__()
+            try:
+                await self._async_enter_contexts()
+            except Exception:  # noqa
+                await es.__aexit__(*sys.exc_info())
+                raise
+            return self
 
+    @ta.final
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         if (es := self._exit_stack) is None:
             return None
-        await self._async_exit_contexts()
+        try:
+            await self._async_exit_contexts()
+        except Exception:  # noqa
+            await es.__aexit__(*sys.exc_info())
+            raise
         return await es.__aexit__(exc_type, exc_val, exc_tb)
 
+    async def _async_enter_contexts(self) -> None:
+        pass
+
     async def _async_exit_contexts(self) -> None:
         pass
 
@@ -6952,11 +7072,7 @@ class OciDataTarWriter(ExitStacked):
             tar_sha256=self._tw.sha256(),
         )
 
-    def __enter__(self) -> 'OciDataTarWriter':
-        super().__enter__()
-
-        #
-
+    def _enter_contexts(self) -> None:
         self._cw = self._FileWrapper(self._f)
 
         if self._compression is OciCompression.GZIP:
@@ -6985,16 +7101,12 @@ class OciDataTarWriter(ExitStacked):
         self._tw = self._FileWrapper(self._cf)
 
         self._tf = self._enter_context(
-            tarfile.open(  # type: ignore
+            tarfile.open(  # type: ignore  # noqa
                 fileobj=self._tw,
                 mode='w',
             ),
         )
 
-        #
-
-        return self
-
     def tar_file(self) -> tarfile.TarFile:
         return self._tf
 
@@ -8800,58 +8912,88 @@ class SocketServer(abc.ABC):
 
     #
 
-    @contextlib.contextmanager
-    def _listen_context(self) -> ta.Iterator[SelectorProtocol]:
-        with contextlib.ExitStack() as es:
-            es.enter_context(self._lock)
-            es.enter_context(self._binder)
+    class PollResult(enum.Enum):
+        TIMEOUT = enum.auto()
+        CONNECTION = enum.auto()
+        ERROR = enum.auto()
+        SHUTDOWN = enum.auto()
 
-            self._binder.listen()
+    class PollContext(ExitStacked, abc.ABC):
+        @abc.abstractmethod
+        def poll(self, timeout: ta.Optional[float] = None) -> 'SocketServer.PollResult':
+            raise NotImplementedError
 
-            self._is_shutdown.clear()
-            try:
-                # XXX: Consider using another file descriptor or connecting to the socket to wake this up instead of
-                # polling. Polling reduces our responsiveness to a shutdown request and wastes cpu at all other times.
-                with self.Selector() as selector:
-                    selector.register(self._binder.fileno(), selectors.EVENT_READ)
+    class _PollContext(PollContext):
+        def __init__(self, server: 'SocketServer') -> None:
+            super().__init__()
 
-                    yield selector
+            self._server = server
 
-            finally:
-                self._is_shutdown.set()
+        _selector: ta.Any = None
 
-    @contextlib.contextmanager
-    def loop_context(self, poll_interval: ta.Optional[float] = None) -> ta.Iterator[ta.Iterator[bool]]:
-        if poll_interval is None:
-            poll_interval = self._poll_interval
+        def _enter_contexts(self) -> None:
+            self._enter_context(self._server._lock)  # noqa: SLF001
+            self._enter_context(self._server._binder)  # noqa: SLF001
 
-        with self._listen_context() as selector:
-            def loop():
-                while not self._should_shutdown:
-                    ready = selector.select(poll_interval)
+            self._server._binder.listen()  # noqa: SLF001
+
+            self._server._is_shutdown.clear()  # noqa: SLF001
+            self._enter_context(defer(self._server._is_shutdown.set))  # noqa
+
+            # XXX: Consider using another file descriptor or connecting to the socket to wake this up instead of
+            # polling. Polling reduces our responsiveness to a shutdown request and wastes cpu at all other times.
+            self._selector = self._enter_context(self._server.Selector())
+            self._selector.register(self._server._binder.fileno(), selectors.EVENT_READ)  # noqa: SLF001
+
+        def poll(self, timeout: ta.Optional[float] = None) -> 'SocketServer.PollResult':
+            if self._server._should_shutdown:  # noqa: SLF001
+                return SocketServer.PollResult.SHUTDOWN
 
-                    # bpo-35017: shutdown() called during select(), exit immediately.
-                    if self._should_shutdown:
-                        break  # type: ignore[unreachable]
+            ready = self._selector.select(timeout)
 
-                    if ready:
-                        try:
-                            conn = self._binder.accept()
+            # bpo-35017: shutdown() called during select(), exit immediately.
+            if self._server._should_shutdown:  # noqa: SLF001
+                return SocketServer.PollResult.SHUTDOWN  # type: ignore[unreachable]
 
-                        except OSError as exc:
-                            self._handle_error(exc)
+            if not ready:
+                return SocketServer.PollResult.TIMEOUT
 
-                            return
+            try:
+                conn = self._server._binder.accept()  # noqa: SLF001
+
+            except OSError as exc:
+                self._server._handle_error(exc)  # noqa: SLF001
+
+                return SocketServer.PollResult.ERROR
+
+            try:
+                self._server._handler(conn)  # noqa: SLF001
 
-                        try:
-                            self._handler(conn)
+            except Exception as exc:  # noqa
+                self._server._handle_error(exc, conn)  # noqa: SLF001
 
-                        except Exception as exc:  # noqa
-                            self._handle_error(exc, conn)
+                close_socket_immediately(conn.socket)
 
-                            close_socket_immediately(conn.socket)
+            return SocketServer.PollResult.CONNECTION
 
-                    yield bool(ready)
+    def poll_context(self) -> PollContext:
+        return self._PollContext(self)
+
+    #
+
+    @contextlib.contextmanager
+    def loop_context(self, poll_interval: ta.Optional[float] = None) -> ta.Iterator[ta.Iterator[bool]]:
+        if poll_interval is None:
+            poll_interval = self._poll_interval
+
+        with self.poll_context() as pc:
+            def loop():
+                while True:
+                    res = pc.poll(poll_interval)
+                    if res in (SocketServer.PollResult.ERROR, SocketServer.PollResult.SHUTDOWN):
+                        return
+                    else:
+                        yield res == SocketServer.PollResult.CONNECTION
 
             yield loop()
 
@@ -10983,6 +11125,24 @@ async def load_docker_tar(
     return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
 
 
+##
+
+
+async def ensure_docker_image_setup(
+        image: str,
+        *,
+        cwd: ta.Optional[str] = None,
+) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'run',
+        '--rm',
+        '--entrypoint', '/bin/true',  # FIXME: lol
+        image,
+        **(dict(cwd=cwd) if cwd is not None else {}),
+    )
+
+
 ########################################
 # ../docker/dataserver.py
 
@@ -11442,17 +11602,14 @@ class CacheServedDockerCache(DockerCache):
             dds_run_task = asyncio.create_task(dds.run())
             try:
                 timeout = Timeout.of(self._config.server_start_timeout)
-                while True:
-                    timeout()
-                    try:
-                        reader, writer = await asyncio.open_connection('localhost', self._config.port)
-                    except Exception as e:  # noqa
-                        log.exception('Failed to connect to cache server - will try again')
-                    else:
-                        writer.close()
-                        await asyncio.wait_for(writer.wait_closed(), timeout=timeout.remaining())
-                        break
-                    await asyncio.sleep(self._config.server_start_sleep)
+
+                await asyncio_wait_until_can_connect(
+                    'localhost',
+                    self._config.port,
+                    timeout=timeout,
+                    on_fail=lambda _: log.exception('Failed to connect to cache server - will try again'),
+                    sleep_s=self._config.server_start_sleep,
+                )
 
                 if (prc := self._config.pull_run_cmd) is not None:
                     pull_cmd = [
@@ -11613,6 +11770,8 @@ class Ci(AsyncExitStacked):
 
         no_dependencies: bool = False
 
+        setup_only: bool = False
+
         run_options: ta.Optional[ta.Sequence[str]] = None
 
         #
@@ -11736,18 +11895,21 @@ class Ci(AsyncExitStacked):
     #
 
     @cached_nullary
-    def pull_dependencies_funcs(self) -> ta.Sequence[ta.Callable[[], ta.Awaitable]]:
+    def get_dependency_images(self) -> ta.Sequence[str]:
         deps = get_compose_service_dependencies(
             self._config.compose_file,
             self._config.service,
         )
+        return sorted(deps.values())
 
+    @cached_nullary
+    def pull_dependencies_funcs(self) -> ta.Sequence[ta.Callable[[], ta.Awaitable]]:
         return [
             async_cached_nullary(functools.partial(
                 self._docker_image_pulling.pull_docker_image,
                 dep_image,
             ))
-            for dep_image in deps.values()
+            for dep_image in self.get_dependency_images()
         ]
 
     #
@@ -11795,10 +11957,29 @@ class Ci(AsyncExitStacked):
 
     #
 
+    async def _run_setup_only(self) -> None:
+        image_ids = [
+            await self.resolve_ci_image(),
+
+            *(self.get_dependency_images() if not self._config.no_dependencies else []),
+        ]
+
+        for image_id in image_ids:
+            with log_timing_context(f'Run setup only: {image_id}'):
+                await ensure_docker_image_setup(
+                    image_id,
+                    cwd=self._config.project_dir,
+                )
+
+    #
+
     async def run(self) -> None:
         await self.setup()
 
-        await self._run_compose()
+        if self._config.setup_only:
+            await self._run_setup_only()
+        else:
+            await self._run_compose()
 
 
 ########################################
@@ -11979,6 +12160,8 @@ class CiCli(ArgparseCli):
 
         argparse_arg('--no-dependencies', action='store_true'),
 
+        argparse_arg('--setup-only', action='store_true'),
+
         argparse_arg('-e', '--env', action='append'),
         argparse_arg('-v', '--volume', action='append'),
 
@@ -12093,6 +12276,8 @@ class CiCli(ArgparseCli):
 
             no_dependencies=self.args.no_dependencies,
 
+            setup_only=self.args.setup_only,
+
             run_options=run_options,
         )
 

{omdev-0.0.0.dev241 → omdev-0.0.0.dev243}/omdev/tools/json/parsing.py

@@ -55,10 +55,8 @@ class DelimitingParser:
 class StreamBuilder(lang.ExitStacked):
     _builder: JsonObjectBuilder | None = None
 
-    def __enter__(self) -> ta.Self:
-        super().__enter__()
+    def _enter_contexts(self) -> None:
         self._builder = self._enter_context(JsonObjectBuilder())
-        return self
 
     def build(self, e: JsonStreamParserEvent) -> ta.Generator[ta.Any, None, None]:
         yield from check.not_none(self._builder)(e)
@@ -69,12 +67,10 @@ class StreamParser(lang.ExitStacked):
     _lex: JsonStreamLexer
     _parse: JsonStreamParser
 
-    def __enter__(self) -> ta.Self:
-        super().__enter__()
+    def _enter_contexts(self) -> None:
         self._decoder = codecs.getincrementaldecoder('utf-8')()
         self._lex = self._enter_context(JsonStreamLexer())
         self._parse = self._enter_context(JsonStreamParser())
-        return self
 
     def parse(self, b: bytes) -> ta.Generator[JsonStreamParserEvent, None, None]:
         for s in self._decoder.decode(b, not b):