omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev214__py3-none-any.whl

omdev/scripts/ci.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omlish-amalg-output ../ci/cli.py
-# ruff: noqa: N802 UP006 UP007 UP036
+# ruff: noqa: N802 TC003 UP006 UP007 UP036
 """
 Inputs:
  - requirements.txt
@@ -12,17 +12,20 @@ Inputs:
 
 ==
 
-./python -m ci run --cache-dir ci/cache ci/project omlish-ci
+./python -m omdev.ci run --cache-dir omdev/ci/tests/cache omdev/ci/tests/project omlish-ci
 """
 import abc
 import argparse
 import asyncio
+import asyncio.base_subprocess
+import asyncio.subprocess
 import collections
 import contextlib
 import dataclasses as dc
 import datetime
 import functools
 import hashlib
+import http.client
 import inspect
 import itertools
 import json
@@ -39,6 +42,8 @@ import threading
 import time
 import types
 import typing as ta
+import urllib.parse
+import urllib.request
 
 
 ########################################
@@ -54,9 +59,12 @@ if sys.version_info < (3, 8):
 # shell.py
 T = ta.TypeVar('T')
 
-# ../../omlish/lite/cached.py
+# ../../omlish/asyncs/asyncio/asyncio.py
 CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
 
+# ../../omlish/asyncs/asyncio/timeouts.py
+AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
+
 # ../../omlish/lite/check.py
 SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
 CheckMessage = ta.Union[str, ta.Callable[..., ta.Optional[str]], None]  # ta.TypeAlias
@@ -70,11 +78,40 @@ ArgparseCmdFn = ta.Callable[[], ta.Optional[int]]  # ta.TypeAlias
 
 # ../../omlish/lite/contextmanagers.py
 ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
+AsyncExitStackedT = ta.TypeVar('AsyncExitStackedT', bound='AsyncExitStacked')
 
 # ../../omlish/subprocesses.py
 SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
 
 
+########################################
+# ../consts.py
+
+
+CI_CACHE_VERSION = 1
+
+
+########################################
+# ../github/env.py
+
+
+@dc.dataclass(frozen=True)
+class GithubEnvVar:
+    k: str
+
+    def __call__(self) -> ta.Optional[str]:
+        return os.environ.get(self.k)
+
+
+GITHUB_ENV_VARS: ta.Set[GithubEnvVar] = set()
+
+
+def register_github_env_var(k: str) -> GithubEnvVar:
+    ev = GithubEnvVar(k)
+    GITHUB_ENV_VARS.add(ev)
+    return ev
+
+
 ########################################
 # ../shell.py
 
@@ -113,6 +150,84 @@ class ShellCmd:
         )
 
 
+########################################
+# ../../../omlish/asyncs/asyncio/asyncio.py
+
+
+def asyncio_once(fn: CallableT) -> CallableT:
+    future = None
+
+    @functools.wraps(fn)
+    async def inner(*args, **kwargs):
+        nonlocal future
+        if not future:
+            future = asyncio.create_task(fn(*args, **kwargs))
+        return await future
+
+    return ta.cast(CallableT, inner)
+
+
+def drain_tasks(loop=None):
+    if loop is None:
+        loop = asyncio.get_running_loop()
+
+    while loop._ready or loop._scheduled:  # noqa
+        loop._run_once()  # noqa
+
+
+@contextlib.contextmanager
+def draining_asyncio_tasks() -> ta.Iterator[None]:
+    loop = asyncio.get_running_loop()
+    try:
+        yield
+    finally:
+        if loop is not None:
+            drain_tasks(loop)  # noqa
+
+
+async def asyncio_wait_concurrent(
+        coros: ta.Iterable[ta.Awaitable[T]],
+        concurrency: ta.Union[int, asyncio.Semaphore],
+        *,
+        return_when: ta.Any = asyncio.FIRST_EXCEPTION,
+) -> ta.List[T]:
+    if isinstance(concurrency, asyncio.Semaphore):
+        semaphore = concurrency
+    elif isinstance(concurrency, int):
+        semaphore = asyncio.Semaphore(concurrency)
+    else:
+        raise TypeError(concurrency)
+
+    async def limited_task(coro):
+        async with semaphore:
+            return await coro
+
+    tasks = [asyncio.create_task(limited_task(coro)) for coro in coros]
+    done, pending = await asyncio.wait(tasks, return_when=return_when)
+
+    for task in pending:
+        task.cancel()
+
+    for task in done:
+        if task.exception():
+            raise task.exception()  # type: ignore
+
+    return [task.result() for task in done]
+
+
+########################################
+# ../../../omlish/asyncs/asyncio/timeouts.py
+
+
+def asyncio_maybe_timeout(
+        fut: AwaitableT,
+        timeout: ta.Optional[float] = None,
+) -> AwaitableT:
+    if timeout is not None:
+        fut = asyncio.wait_for(fut, timeout)  # type: ignore
+    return fut
+
+
 ########################################
 # ../../../omlish/lite/cached.py
 
@@ -208,6 +323,17 @@ class Checks:
 
     #
 
+    def register_on_raise_breakpoint_if_env_var_set(self, key: str) -> None:
+        import os
+
+        def on_raise(exc: Exception) -> None:  # noqa
+            if key in os.environ:
+                breakpoint()  # noqa
+
+        self.register_on_raise(on_raise)
+
+    #
+
     def set_exception_factory(self, factory: CheckExceptionFactory) -> None:
         self._exception_factory = factory
 
@@ -523,6 +649,18 @@ class Checks:
 
         return v
 
+    def not_equal(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
+        if o == v:
+            self._raise(
+                ValueError,
+                'Must not be equal',
+                msg,
+                Checks._ArgsKwargs(v, o),
+                render_fmt='%s == %s',
+            )
+
+        return v
+
     def is_(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
         if o is not v:
             self._raise(
@@ -956,170 +1094,183 @@ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
 
 
 ########################################
-# ../cache.py
+# ../../../omlish/os/files.py
 
 
-##
+def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
+    if exist_ok:
+        # First try to bump modification time
+        # Implementation note: GNU touch uses the UTIME_NOW option of the utimensat() / futimens() functions.
+        try:
+            os.utime(self, None)
+        except OSError:
+            pass
+        else:
+            return
+    flags = os.O_CREAT | os.O_WRONLY
+    if not exist_ok:
+        flags |= os.O_EXCL
+    fd = os.open(self, flags, mode)
+    os.close(fd)
 
 
-@abc.abstractmethod
-class FileCache(abc.ABC):
-    @abc.abstractmethod
-    def get_file(self, key: str) -> ta.Optional[str]:
-        raise NotImplementedError
+def unlink_if_exists(path: str) -> None:
+    try:
+        os.unlink(path)
+    except FileNotFoundError:
+        pass
 
-    @abc.abstractmethod
-    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
-        raise NotImplementedError
 
+@contextlib.contextmanager
+def unlinking_if_exists(path: str) -> ta.Iterator[None]:
+    try:
+        yield
+    finally:
+        unlink_if_exists(path)
 
-#
 
+########################################
+# ../cache.py
 
-class DirectoryFileCache(FileCache):
-    def __init__(self, dir: str) -> None:  # noqa
-        super().__init__()
 
-        self._dir = dir
+##
 
-    #
 
-    def get_cache_file_path(
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    def __init__(
             self,
-            key: str,
             *,
-            make_dirs: bool = False,
-    ) -> str:
-        if make_dirs:
-            os.makedirs(self._dir, exist_ok=True)
-        return os.path.join(self._dir, key)
-
-    def format_incomplete_file(self, f: str) -> str:
-        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
-
-    #
-
-    def get_file(self, key: str) -> ta.Optional[str]:
-        cache_file_path = self.get_cache_file_path(key)
-        if not os.path.exists(cache_file_path):
-            return None
-        return cache_file_path
-
-    def put_file(self, key: str, file_path: str) -> None:
-        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
-        shutil.copyfile(file_path, cache_file_path)
+            version: int = CI_CACHE_VERSION,
+    ) -> None:
+        super().__init__()
 
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version = version
 
-##
+    @property
+    def version(self) -> int:
+        return self._version
 
+    #
 
-class ShellCache(abc.ABC):
     @abc.abstractmethod
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
         raise NotImplementedError
 
-    class PutFileCmdContext(abc.ABC):
-        def __init__(self) -> None:
-            super().__init__()
-
-            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
-
-        @property
-        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
-            return self._state
-
-        #
+    @abc.abstractmethod
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
 
-        @property
-        @abc.abstractmethod
-        def cmd(self) -> ShellCmd:
-            raise NotImplementedError
 
-        #
+#
 
-        def __enter__(self):
-            return self
 
-        def __exit__(self, exc_type, exc_val, exc_tb):
-            if exc_val is None:
-                self.commit()
-            else:
-                self.abort()
+class DirectoryFileCache(FileCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            no_create: bool = False,
+            no_purge: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:  # noqa
+        super().__init__(**kwargs)
 
-        #
+        self._dir = dir
+        self._no_create = no_create
+        self._no_purge = no_purge
 
-        @abc.abstractmethod
-        def _commit(self) -> None:
-            raise NotImplementedError
+    #
 
-        def commit(self) -> None:
-            if self._state == 'committed':
-                return
-            elif self._state == 'open':
-                self._commit()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
+    VERSION_FILE_NAME = '.ci-cache-version'
 
-        #
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
 
-        @abc.abstractmethod
-        def _abort(self) -> None:
-            raise NotImplementedError
+        if self._no_create:
+            check.state(os.path.isdir(self._dir))
 
-        def abort(self) -> None:
-            if self._state == 'aborted':
-                return
-            elif self._state == 'open':
-                self._abort()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
+        elif not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
 
-    @abc.abstractmethod
-    def put_file_cmd(self, key: str) -> PutFileCmdContext:
-        raise NotImplementedError
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
 
+        if dir_version == self._version:
+            return
 
-#
+        if self._no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
 
+        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
 
-class DirectoryShellCache(ShellCache):
-    def __init__(self, dfc: DirectoryFileCache) -> None:
-        super().__init__()
+        for n in sorted(os.listdir(self._dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self._dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
 
-        self._dfc = dfc
+        os.unlink(version_file)
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        f = self._dfc.get_file(key)
-        if f is None:
-            return None
-        return ShellCmd(f'cat {shlex.quote(f)}')
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
 
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(self, tf: str, f: str) -> None:
-            super().__init__()
+    #
 
-            self._tf = tf
-            self._f = f
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self._dir, key)
 
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
 
-        def _commit(self) -> None:
-            os.replace(self._tf, self._f)
+    #
 
-        def _abort(self) -> None:
-            os.unlink(self._tf)
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
 
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        f = self._dfc.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path
 
 
 ########################################
-# ../github/cacheapi.py
+# ../github/api.py
 """
 export FILE_SIZE=$(stat --format="%s" $FILE)
 
@@ -1205,7 +1356,7 @@ class GithubCacheServiceV1:
     @dc.dataclass(frozen=True)
     class ReserveCacheRequest:
         key: str
-        cache_size: ta.Optional[int]
+        cache_size: ta.Optional[int] = None
         version: ta.Optional[str] = None
 
     @dc.dataclass(frozen=True)
@@ -1320,16 +1471,27 @@ class GithubCacheServiceV2:
 
 
 ########################################
-# ../utils.py
+# ../github/bootstrap.py
+"""
+sudo rm -rf \
+    /usr/local/.ghcup \
+    /opt/hostedtoolcache \
 
+/usr/local/.ghcup       6.4G, 3391250 files
+/opt/hostedtoolcache    8.0G, 14843980 files
+/usr/local/lib/android  6.4G, 17251667 files
+"""
 
-##
 
+GITHUB_ACTIONS_ENV_VAR = register_github_env_var('GITHUB_ACTIONS')
 
-def make_temp_file() -> str:
-    file_fd, file = tempfile.mkstemp()
-    os.close(file_fd)
-    return file
+
+def is_in_github_actions() -> bool:
+    return GITHUB_ACTIONS_ENV_VAR() is not None
+
+
+########################################
+# ../utils.py
 
 
 ##
@@ -1378,7 +1540,7 @@ class LogTimingContext:
     def __enter__(self) -> 'LogTimingContext':
         self._begin_time = time.time()
 
-        self._log.log(self._level, f'Begin {self._description}')  # noqa
+        self._log.log(self._level, f'Begin : {self._description}')  # noqa
 
         return self
 
@@ -1387,7 +1549,7 @@ class LogTimingContext:
 
         self._log.log(
             self._level,
-            f'End {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
+            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
         )
 
 
@@ -1713,6 +1875,33 @@ class ExitStacked:
         return es.enter_context(cm)
 
 
+class AsyncExitStacked:
+    _exit_stack: ta.Optional[contextlib.AsyncExitStack] = None
+
+    async def __aenter__(self: AsyncExitStackedT) -> AsyncExitStackedT:
+        check.state(self._exit_stack is None)
+        es = self._exit_stack = contextlib.AsyncExitStack()
+        await es.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if (es := self._exit_stack) is None:
+            return None
+        await self._async_exit_contexts()
+        return await es.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def _async_exit_contexts(self) -> None:
+        pass
+
+    def _enter_context(self, cm: ta.ContextManager[T]) -> T:
+        es = check.not_none(self._exit_stack)
+        return es.enter_context(cm)
+
+    async def _enter_async_context(self, cm: ta.AsyncContextManager[T]) -> T:
+        es = check.not_none(self._exit_stack)
+        return await es.enter_async_context(cm)
+
+
 ##
 
 
@@ -1724,6 +1913,17 @@ def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
         fn()
 
 
+@contextlib.asynccontextmanager
+async def adefer(fn: ta.Callable) -> ta.AsyncGenerator[ta.Callable, None]:
+    try:
+        yield fn
+    finally:
+        await fn()
+
+
+##
+
+
 @contextlib.contextmanager
 def attr_setting(obj, attr, val, *, default=None):  # noqa
     not_set = object()
@@ -1827,55 +2027,576 @@ class JsonLogFormatter(logging.Formatter):
 
 
 ########################################
-# ../../../omlish/logs/standard.py
-"""
-TODO:
- - structured
- - prefixed
- - debug
- - optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
-"""
+# ../../../omlish/os/temp.py
 
 
-##
+def make_temp_file(**kwargs: ta.Any) -> str:
+    file_fd, file = tempfile.mkstemp(**kwargs)
+    os.close(file_fd)
+    return file
 
 
-STANDARD_LOG_FORMAT_PARTS = [
-    ('asctime', '%(asctime)-15s'),
-    ('process', 'pid=%(process)-6s'),
-    ('thread', 'tid=%(thread)x'),
-    ('levelname', '%(levelname)s'),
-    ('name', '%(name)s'),
-    ('separator', '::'),
-    ('message', '%(message)s'),
-]
+@contextlib.contextmanager
+def temp_file_context(**kwargs: ta.Any) -> ta.Iterator[str]:
+    path = make_temp_file(**kwargs)
+    try:
+        yield path
+    finally:
+        unlink_if_exists(path)
 
 
-class StandardLogFormatter(logging.Formatter):
-    @staticmethod
-    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
-        return ' '.join(v for k, v in parts)
+@contextlib.contextmanager
+def temp_dir_context(
+        root_dir: ta.Optional[str] = None,
+        **kwargs: ta.Any,
+) -> ta.Iterator[str]:
+    path = tempfile.mkdtemp(dir=root_dir, **kwargs)
+    try:
+        yield path
+    finally:
+        shutil.rmtree(path, ignore_errors=True)
 
-    converter = datetime.datetime.fromtimestamp  # type: ignore
 
-    def formatTime(self, record, datefmt=None):
-        ct = self.converter(record.created)  # type: ignore
-        if datefmt:
-            return ct.strftime(datefmt)  # noqa
-        else:
-            t = ct.strftime('%Y-%m-%d %H:%M:%S')
-            return '%s.%03d' % (t, record.msecs)  # noqa
+@contextlib.contextmanager
+def temp_named_file_context(
+        root_dir: ta.Optional[str] = None,
+        cleanup: bool = True,
+        **kwargs: ta.Any,
+) -> ta.Iterator[tempfile._TemporaryFileWrapper]:  # noqa
+    with tempfile.NamedTemporaryFile(dir=root_dir, delete=False, **kwargs) as f:
+        try:
+            yield f
+        finally:
+            if cleanup:
+                shutil.rmtree(f.name, ignore_errors=True)
+
+
+########################################
+# ../github/client.py
 
 
 ##
 
 
-class StandardConfiguredLogHandler(ProxyLogHandler):
-    def __init_subclass__(cls, **kwargs):
-        raise TypeError('This class serves only as a marker and should not be subclassed.')
+class GithubCacheClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
 
+    @abc.abstractmethod
+    def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
+        raise NotImplementedError
 
-##
+    @abc.abstractmethod
+    def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def upload_file(self, key: str, in_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+##
+
+
+class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+    AUTH_TOKEN_ENV_VAR = register_github_env_var('ACTIONS_RUNTIME_TOKEN')  # noqa
+
+    KEY_SUFFIX_ENV_VAR = register_github_env_var('GITHUB_RUN_ID')
+
+    #
+
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
+
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
+
+            cache_version: int = CI_CACHE_VERSION,
+
+            loop: ta.Optional[asyncio.AbstractEventLoop] = None,
+    ) -> None:
+        super().__init__()
+
+        #
+
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        self._service_url = GithubCacheServiceV1.get_service_url(base_url)
+
+        if auth_token is None:
+            auth_token = self.AUTH_TOKEN_ENV_VAR()
+        self._auth_token = auth_token
+
+        #
+
+        self._key_prefix = key_prefix
+
+        if key_suffix is None:
+            key_suffix = self.KEY_SUFFIX_ENV_VAR()
+        self._key_suffix = check.non_empty_str(key_suffix)
+
+        #
+
+        self._cache_version = check.isinstance(cache_version, int)
+
+        #
+
+        self._given_loop = loop
+
+    #
+
+    def _get_loop(self) -> asyncio.AbstractEventLoop:
+        if (loop := self._given_loop) is not None:
+            return loop
+        return asyncio.get_event_loop()
+
+    #
+
+    def build_request_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            *,
+            content_type: ta.Optional[str] = None,
+            json_content: bool = False,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': ';'.join([
+                'application/json',
+                f'api-version={GithubCacheServiceV1.API_VERSION}',
+            ]),
+        }
+
+        if (auth_token := self._auth_token):
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is None and json_content:
+            content_type = 'application/json'
+        if content_type is not None:
+            dct['Content-Type'] = content_type
+
+        if headers:
+            dct.update(headers)
+
+        return dct
+
+    #
+
+    def load_json_bytes(self, b: ta.Optional[bytes]) -> ta.Optional[ta.Any]:
+        if not b:
+            return None
+        return json.loads(b.decode('utf-8-sig'))
+
+    #
+
+    async def send_url_request(
+            self,
+            req: urllib.request.Request,
+    ) -> ta.Tuple[http.client.HTTPResponse, ta.Optional[bytes]]:
+        def run_sync():
+            with urllib.request.urlopen(req) as resp:  # noqa
+                body = resp.read()
+            return (resp, body)
+
+        return await self._get_loop().run_in_executor(None, run_sync)  # noqa
+
+    #
+
+    @dc.dataclass()
+    class ServiceRequestError(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def __str__(self) -> str:
+            return repr(self)
+
+    async def send_service_request(
+            self,
+            path: str,
+            *,
+            method: ta.Optional[str] = None,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            content_type: ta.Optional[str] = None,
+            content: ta.Optional[bytes] = None,
+            json_content: ta.Optional[ta.Any] = None,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        url = f'{self._service_url}/{path}'
+
+        if content is not None and json_content is not None:
+            raise RuntimeError('Must not pass both content and json_content')
+        elif json_content is not None:
+            content = json_dumps_compact(json_content).encode('utf-8')
+            header_json_content = True
+        else:
+            header_json_content = False
+
+        if method is None:
+            method = 'POST' if content is not None else 'GET'
+
+        #
+
+        req = urllib.request.Request(  # noqa
+            url,
+            method=method,
+            headers=self.build_request_headers(
+                headers,
+                content_type=content_type,
+                json_content=header_json_content,
+            ),
+            data=content,
+        )
+
+        resp, body = await self.send_url_request(req)
+
+        #
+
+        if success_status_codes is not None:
+            is_success = resp.status in success_status_codes
+        else:
+            is_success = (200 <= resp.status <= 300)
+        if not is_success:
+            raise self.ServiceRequestError(resp.status, body)
+
+        return self.load_json_bytes(body)
+
+    #
+
+    KEY_PART_SEPARATOR = '--'
+
+    def fix_key(self, s: str, partial_suffix: bool = False) -> str:
+        return self.KEY_PART_SEPARATOR.join([
+            *([self._key_prefix] if self._key_prefix else []),
+            s,
+            ('' if partial_suffix else self._key_suffix),
+        ])
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
+
+    #
+
+    def build_get_entry_url_path(self, *keys: str) -> str:
+        qp = dict(
+            keys=','.join(urllib.parse.quote_plus(k) for k in keys),
+            version=str(self._cache_version),
+        )
+
+        return '?'.join([
+            'cache',
+            '&'.join([
+                f'{k}={v}'
+                for k, v in qp.items()
+            ]),
+        ])
+
+    GET_ENTRY_SUCCESS_STATUS_CODES = (200, 204)
+
+
+##
+
+
+class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
+    DEFAULT_CONCURRENCY = 4
+
+    DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024
+
+    def __init__(
+            self,
+            *,
+            concurrency: int = DEFAULT_CONCURRENCY,
+            chunk_size: int = DEFAULT_CHUNK_SIZE,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        check.arg(concurrency > 0)
+        self._concurrency = concurrency
+
+        check.arg(chunk_size > 0)
+        self._chunk_size = chunk_size
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1BaseClient.Entry]:
+        obj = await self.send_service_request(
+            self.build_get_entry_url_path(self.fix_key(key, partial_suffix=True)),
+        )
+        if obj is None:
+            return None
+
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        ))
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class _DownloadChunk:
+        key: str
+        url: str
+        out_file: str
+        offset: int
+        size: int
+
+    async def _download_file_chunk_urllib(self, chunk: _DownloadChunk) -> None:
+        req = urllib.request.Request(  # noqa
+            chunk.url,
+            headers={
+                'Range': f'bytes={chunk.offset}-{chunk.offset + chunk.size - 1}',
+            },
+        )
+
+        _, buf_ = await self.send_url_request(req)
+
+        buf = check.not_none(buf_)
+        check.equal(len(buf), chunk.size)
+
+        #
+
+        def write_sync():
+            with open(chunk.out_file, 'r+b') as f:  # noqa
+                f.seek(chunk.offset, os.SEEK_SET)
+                f.write(buf)
+
+        await self._get_loop().run_in_executor(None, write_sync)  # noqa
+
+    # async def _download_file_chunk_curl(self, chunk: _DownloadChunk) -> None:
+    #     async with contextlib.AsyncExitStack() as es:
+    #         f = open(chunk.out_file, 'r+b')
+    #         f.seek(chunk.offset, os.SEEK_SET)
+    #
+    #         tmp_file = es.enter_context(temp_file_context())  # noqa
+    #
+    #         proc = await es.enter_async_context(asyncio_subprocesses.popen(
+    #             'curl',
+    #             '-s',
+    #             '-w', '%{json}',
+    #             '-H', f'Range: bytes={chunk.offset}-{chunk.offset + chunk.size - 1}',
+    #             chunk.url,
+    #             output=subprocess.PIPE,
+    #         ))
+    #
+    #         futs = asyncio.gather(
+    #
+    #         )
+    #
+    #         await proc.wait()
+    #
+    #         with open(tmp_file, 'r') as f:  # noqa
+    #             curl_json = tmp_file.read()
+    #
+    #     curl_res = json.loads(curl_json.decode().strip())
+    #
+    #     status_code = check.isinstance(curl_res['response_code'], int)
+    #
+    #     if not (200 <= status_code <= 300):
+    #         raise RuntimeError(f'Curl chunk download {chunk} failed: {curl_res}')
+
+    async def _download_file_chunk(self, chunk: _DownloadChunk) -> None:
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {chunk.key} '
+                f'file {chunk.out_file} '
+                f'chunk {chunk.offset} - {chunk.offset + chunk.size}',
+        ):
+            await self._download_file_chunk_urllib(chunk)
+
+    async def _download_file(self, entry: GithubCacheServiceV1BaseClient.Entry, out_file: str) -> None:
+        key = check.non_empty_str(entry.artifact.cache_key)
+        url = check.non_empty_str(entry.artifact.archive_location)
+
+        head_resp, _ = await self.send_url_request(urllib.request.Request(  # noqa
+            url,
+            method='HEAD',
+        ))
+        file_size = int(head_resp.headers['Content-Length'])
+
+        #
+
+        with open(out_file, 'xb') as f:  # noqa
+            f.truncate(file_size)
+
+        #
+
+        download_tasks = []
+        chunk_size = self._chunk_size
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            offset = i * chunk_size
+            size = min(chunk_size, file_size - offset)
+            chunk = self._DownloadChunk(
+                key,
+                url,
+                out_file,
+                offset,
+                size,
+            )
+            download_tasks.append(self._download_file_chunk(chunk))
+
+        await asyncio_wait_concurrent(download_tasks, self._concurrency)
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry1 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry1.artifact.cache_key} '
+                f'version {entry1.artifact.cache_version} '
+                f'to {out_file}',
+        ):
+            await self._download_file(entry1, out_file)
+
+    #
+
+    async def _upload_file_chunk(
+            self,
+            key: str,
+            cache_id: int,
+            in_file: str,
+            offset: int,
+            size: int,
+    ) -> None:
+        with log_timing_context(
+                f'Uploading github cache {key} '
+                f'file {in_file} '
+                f'chunk {offset} - {offset + size}',
+        ):
+            with open(in_file, 'rb') as f:  # noqa
+                f.seek(offset)
+                buf = f.read(size)
+
+            check.equal(len(buf), size)
+
+            await self.send_service_request(
+                f'caches/{cache_id}',
+                method='PATCH',
+                content_type='application/octet-stream',
+                headers={
+                    'Content-Range': f'bytes {offset}-{offset + size - 1}/*',
+                },
+                content=buf,
+                success_status_codes=[204],
+            )
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=fixed_key,
+            cache_size=file_size,
+            version=str(self._cache_version),
+        )
+        reserve_resp_obj = await self.send_service_request(
+            'caches',
+            json_content=GithubCacheServiceV1.dataclass_to_json(reserve_req),
+            success_status_codes=[201],
+        )
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} got id {cache_id}')  # noqa
+
+        #
+
+        upload_tasks = []
+        chunk_size = self._chunk_size
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            offset = i * chunk_size
+            size = min(chunk_size, file_size - offset)
+            upload_tasks.append(self._upload_file_chunk(
+                fixed_key,
+                cache_id,
+                in_file,
+                offset,
+                size,
+            ))
+
+        await asyncio_wait_concurrent(upload_tasks, self._concurrency)
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        await self.send_service_request(
+            f'caches/{cache_id}',
+            json_content=GithubCacheServiceV1.dataclass_to_json(commit_req),
+            success_status_codes=[204],
+        )
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)
+
+
+########################################
+# ../../../omlish/logs/standard.py
+"""
+TODO:
+ - structured
+ - prefixed
+ - debug
+ - optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
+"""
+
+
+##
+
+
+STANDARD_LOG_FORMAT_PARTS = [
+    ('asctime', '%(asctime)-15s'),
+    ('process', 'pid=%(process)-6s'),
+    ('thread', 'tid=%(thread)x'),
+    ('levelname', '%(levelname)s'),
+    ('name', '%(name)s'),
+    ('separator', '::'),
+    ('message', '%(message)s'),
+]
+
+
+class StandardLogFormatter(logging.Formatter):
+    @staticmethod
+    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
+        return ' '.join(v for k, v in parts)
+
+    converter = datetime.datetime.fromtimestamp  # type: ignore
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            return ct.strftime(datefmt)  # noqa
+        else:
+            t = ct.strftime('%Y-%m-%d %H:%M:%S')
+            return '%s.%03d' % (t, record.msecs)  # noqa
+
+
+##
+
+
+class StandardConfiguredLogHandler(ProxyLogHandler):
+    def __init_subclass__(cls, **kwargs):
+        raise TypeError('This class serves only as a marker and should not be subclassed.')
+
+
+##
 
 
 @contextlib.contextmanager
@@ -2276,6 +2997,395 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
             return ret.decode().strip()
 
 
+########################################
+# ../github/cache.py
+
+
+##
+
+
+class GithubFileCache(FileCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            client: ta.Optional[GithubCacheClient] = None,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        self._dir = check.not_none(dir)
+
+        if client is None:
+            client = GithubCacheServiceV1Client(
+                cache_version=self._version,
+            )
+        self._client: GithubCacheClient = client
+
+        self._local = DirectoryFileCache(
+            self._dir,
+            version=self._version,
+        )
+
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return local_file
+
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
+
+        tmp_file = self._local.format_incomplete_file(local_file)
+        with unlinking_if_exists(tmp_file):
+            await self._client.download_file(entry, tmp_file)
+
+            os.replace(tmp_file, local_file)
+
+        return local_file
+
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = await self._local.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )
+
+        await self._client.upload_file(key, cache_file_path)
+
+        return cache_file_path
+
+
+########################################
+# ../github/cli.py
+"""
+See:
+ - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
+"""
+
+
+class GithubCli(ArgparseCli):
+    @argparse_cmd()
+    def list_referenced_env_vars(self) -> None:
+        print('\n'.join(sorted(ev.k for ev in GITHUB_ENV_VARS)))
+
+    @argparse_cmd(
+        argparse_arg('key'),
+    )
+    async def get_cache_entry(self) -> None:
+        client = GithubCacheServiceV1Client()
+        entry = await client.get_entry(self.args.key)
+        if entry is None:
+            return
+        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
+
+    @argparse_cmd(
+        argparse_arg('repository-id'),
+    )
+    def list_cache_entries(self) -> None:
+        raise NotImplementedError
+
+
+########################################
+# ../requirements.py
+"""
+TODO:
+ - pip compile lol
+  - but still support git+ stuff
+ - req.txt format aware hash
+  - more than just whitespace
+ - pyproject req rewriting
+ - download_requirements bootstrap off prev? not worth the dl?
+  - big deps (torch) change less, probably worth it
+ - follow embedded -r automatically like pyp
+"""
+
+
+##
+
+
+def build_requirements_hash(
+        requirements_txts: ta.Sequence[str],
+) -> str:
+    txt_file_contents: dict = {}
+
+    for txt_file in requirements_txts:
+        txt_file_name = os.path.basename(txt_file)
+        check.not_in(txt_file_name, txt_file_contents)
+        with open(txt_file) as f:
+            txt_contents = f.read()
+        txt_file_contents[txt_file_name] = txt_contents
+
+    #
+
+    lines = []
+    for txt_file, txt_contents in sorted(txt_file_contents.items()):
+        txt_hash = sha256_str(txt_contents)
+        lines.append(f'{txt_file}={txt_hash}')
+
+    return sha256_str('\n'.join(lines))
+
+
+##
+
+
+def download_requirements(
+        image: str,
+        requirements_dir: str,
+        requirements_txts: ta.Sequence[str],
+) -> None:
+    requirements_txt_dir = tempfile.mkdtemp()
+    with defer(lambda: shutil.rmtree(requirements_txt_dir)):
+        for rt in requirements_txts:
+            shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
+
+        subprocesses.check_call(
+            'docker',
+            'run',
+            '--rm',
+            '-i',
+            '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
+            '-v', f'{requirements_txt_dir}:/requirements_txt',
+            image,
+            'pip',
+            'download',
+            '-d', '/requirements',
+            *itertools.chain.from_iterable(
+                ['-r', f'/requirements_txt/{os.path.basename(rt)}']
+                for rt in requirements_txts
+            ),
+        )
+
+
+########################################
+# ../../../omlish/asyncs/asyncio/subprocesses.py
+
+
+##
+
+
+class AsyncioProcessCommunicator:
+    def __init__(
+            self,
+            proc: asyncio.subprocess.Process,
+            loop: ta.Optional[ta.Any] = None,
+            *,
+            log: ta.Optional[logging.Logger] = None,
+    ) -> None:
+        super().__init__()
+
+        if loop is None:
+            loop = asyncio.get_running_loop()
+
+        self._proc = proc
+        self._loop = loop
+        self._log = log
+
+        self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
+            proc._transport,  # type: ignore  # noqa
+            asyncio.base_subprocess.BaseSubprocessTransport,
+        )
+
+    @property
+    def _debug(self) -> bool:
+        return self._loop.get_debug()
+
+    async def _feed_stdin(self, input: bytes) -> None:  # noqa
+        stdin = check.not_none(self._proc.stdin)
+        try:
+            if input is not None:
+                stdin.write(input)
+                if self._debug and self._log is not None:
+                    self._log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
+
+            await stdin.drain()
+
+        except (BrokenPipeError, ConnectionResetError) as exc:
+            # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
+            # exceptions.
+            if self._debug and self._log is not None:
+                self._log.debug('%r communicate: stdin got %r', self, exc)
+
+        if self._debug and self._log is not None:
+            self._log.debug('%r communicate: close stdin', self)
+
+        stdin.close()
+
+    async def _noop(self) -> None:
+        return None
+
+    async def _read_stream(self, fd: int) -> bytes:
+        transport: ta.Any = check.not_none(self._transport.get_pipe_transport(fd))
+
+        if fd == 2:
+            stream = check.not_none(self._proc.stderr)
+        else:
+            check.equal(fd, 1)
+            stream = check.not_none(self._proc.stdout)
+
+        if self._debug and self._log is not None:
+            name = 'stdout' if fd == 1 else 'stderr'
+            self._log.debug('%r communicate: read %s', self, name)
+
+        output = await stream.read()
+
+        if self._debug and self._log is not None:
+            name = 'stdout' if fd == 1 else 'stderr'
+            self._log.debug('%r communicate: close %s', self, name)
+
+        transport.close()
+
+        return output
+
+    class Communication(ta.NamedTuple):
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def _communicate(
+            self,
+            input: ta.Any = None,  # noqa
+    ) -> Communication:
+        stdin_fut: ta.Any
+        if self._proc.stdin is not None:
+            stdin_fut = self._feed_stdin(input)
+        else:
+            stdin_fut = self._noop()
+
+        stdout_fut: ta.Any
+        if self._proc.stdout is not None:
+            stdout_fut = self._read_stream(1)
+        else:
+            stdout_fut = self._noop()
+
+        stderr_fut: ta.Any
+        if self._proc.stderr is not None:
+            stderr_fut = self._read_stream(2)
+        else:
+            stderr_fut = self._noop()
+
+        stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
+
+        await self._proc.wait()
+
+        return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
+
+    async def communicate(
+            self,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> Communication:
+        return await asyncio_maybe_timeout(self._communicate(input), timeout)
+
+
+##
+
+
+class AsyncioSubprocesses(AbstractAsyncSubprocesses):
+    async def communicate(
+            self,
+            proc: asyncio.subprocess.Process,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
+        return await AsyncioProcessCommunicator(proc).communicate(input, timeout)  # noqa
+
+    #
+
+    @contextlib.asynccontextmanager
+    async def popen(
+            self,
+            *cmd: str,
+            shell: bool = False,
+            timeout: ta.Optional[float] = None,
+            **kwargs: ta.Any,
+    ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
+        fac: ta.Any
+        if shell:
+            fac = functools.partial(
+                asyncio.create_subprocess_shell,
+                check.single(cmd),
+            )
+        else:
+            fac = functools.partial(
+                asyncio.create_subprocess_exec,
+                *cmd,
+            )
+
+        with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs):  # noqa
+            proc: asyncio.subprocess.Process = await fac(**kwargs)
+            try:
+                yield proc
+
+            finally:
+                await asyncio_maybe_timeout(proc.wait(), timeout)
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class RunOutput:
+        proc: asyncio.subprocess.Process
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def run(
+            self,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+            check: bool = False,  # noqa
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> RunOutput:
+        if capture_output:
+            kwargs.setdefault('stdout', subprocess.PIPE)
+            kwargs.setdefault('stderr', subprocess.PIPE)
+
+        proc: asyncio.subprocess.Process
+        async with self.popen(*cmd, **kwargs) as proc:
+            stdout, stderr = await self.communicate(proc, input, timeout)
+
+        if check and proc.returncode:
+            raise subprocess.CalledProcessError(
+                proc.returncode,
+                cmd,
+                output=stdout,
+                stderr=stderr,
+            )
+
+        return self.RunOutput(
+            proc,
+            stdout,
+            stderr,
+        )
+
+    #
+
+    async def check_call(
+            self,
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        with self.prepare_and_wrap(*cmd, stdout=stdout, check=True, **kwargs) as (cmd, kwargs):  # noqa
+            await self.run(*cmd, **kwargs)
+
+    async def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs):  # noqa
+            return check.not_none((await self.run(*cmd, **kwargs)).stdout)
+
+
+asyncio_subprocesses = AsyncioSubprocesses()
+
+
 ########################################
 # ../compose.py
 """
@@ -2307,7 +3417,7 @@ def get_compose_service_dependencies(
 ##
 
 
-class DockerComposeRun(ExitStacked):
+class DockerComposeRun(AsyncExitStacked):
     @dc.dataclass(frozen=True)
     class Config:
         compose_file: str
@@ -2325,6 +3435,7 @@ class DockerComposeRun(ExitStacked):
 
         #
 
+        no_dependencies: bool = False
         no_dependency_cleanup: bool = False
 
         #
@@ -2343,40 +3454,6 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    @property
-    def image_tag(self) -> str:
-        pfx = 'sha256:'
-        if (image := self._cfg.image).startswith(pfx):
-            image = image[len(pfx):]
-
-        return f'{self._cfg.service}:{image}'
-
-    @cached_nullary
-    def tag_image(self) -> str:
-        image_tag = self.image_tag
-
-        subprocesses.check_call(
-            'docker',
-            'tag',
-            self._cfg.image,
-            image_tag,
-            **self._subprocess_kwargs,
-        )
-
-        def delete_tag() -> None:
-            subprocesses.check_call(
-                'docker',
-                'rmi',
-                image_tag,
-                **self._subprocess_kwargs,
-            )
-
-        self._enter_context(defer(delete_tag))  # noqa
-
-        return image_tag
-
-    #
-
     def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
         out = dict(in_dct)
 
@@ -2390,29 +3467,28 @@ class DockerComposeRun(ExitStacked):
         in_service: dict = in_services[self._cfg.service]
         out_services[self._cfg.service] = out_service = dict(in_service)
 
-        out_service['image'] = self.image_tag
+        out_service['image'] = self._cfg.image
 
         for k in ['build', 'platform']:
             if k in out_service:
-                del out_service[k]
-
-        out_service['links'] = [
-            f'{l}:{l}' if ':' not in l else l
-            for l in out_service.get('links', [])
-        ]
+                del out_service[k]
 
         #
 
-        depends_on = in_service.get('depends_on', [])
+        if not self._cfg.no_dependencies:
+            depends_on = in_service.get('depends_on', [])
 
-        for dep_service, in_dep_service_dct in list(in_services.items()):
-            if dep_service not in depends_on:
-                continue
+            for dep_service, in_dep_service_dct in list(in_services.items()):
+                if dep_service not in depends_on:
+                    continue
 
-            out_dep_service: dict = dict(in_dep_service_dct)
-            out_services[dep_service] = out_dep_service
+                out_dep_service: dict = dict(in_dep_service_dct)
+                out_services[dep_service] = out_dep_service
 
-            out_dep_service['ports'] = []
+                out_dep_service['ports'] = []
+
+        else:
+            out_service['depends_on'] = []
 
         #
 
@@ -2438,22 +3514,20 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    def _cleanup_dependencies(self) -> None:
-        subprocesses.check_call(
+    async def _cleanup_dependencies(self) -> None:
+        await asyncio_subprocesses.check_call(
             'docker',
             'compose',
             '-f', self.rewrite_compose_file(),
             'down',
         )
 
-    def run(self) -> None:
-        self.tag_image()
-
+    async def run(self) -> None:
         compose_file = self.rewrite_compose_file()
 
-        with contextlib.ExitStack() as es:
-            if not self._cfg.no_dependency_cleanup:
-                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+        async with contextlib.AsyncExitStack() as es:
+            if not (self._cfg.no_dependencies or self._cfg.no_dependency_cleanup):
+                await es.enter_async_context(adefer(self._cleanup_dependencies))  # noqa
 
             sh_cmd = ' '.join([
                 'docker',
@@ -2472,8 +3546,8 @@ class DockerComposeRun(ExitStacked):
 
             run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
 
-            run_cmd.run(
-                subprocesses.check_call,
+            await run_cmd.run(
+                asyncio_subprocesses.check_call,
                 **self._subprocess_kwargs,
             )
 
@@ -2505,535 +3579,142 @@ def read_docker_tar_image_tag(tar_file: str) -> str:
         with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
             m = mf.read()
 
-    manifests = json.loads(m.decode('utf-8'))
-    manifest = check.single(manifests)
-    tag = check.non_empty_str(check.single(manifest['RepoTags']))
-    return tag
-
-
-def read_docker_tar_image_id(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
-            i = mf.read()
-
-    index = json.loads(i.decode('utf-8'))
-    manifest = check.single(index['manifests'])
-    image_id = check.non_empty_str(manifest['digest'])
-    return image_id
-
-
-##
-
-
-def is_docker_image_present(image: str) -> bool:
-    out = subprocesses.check_output(
-        'docker',
-        'images',
-        '--format', 'json',
-        image,
-    )
-
-    out_s = out.decode('utf-8').strip()
-    if not out_s:
-        return False
-
-    json.loads(out_s)  # noqa
-    return True
-
-
-def pull_docker_image(
-        image: str,
-) -> None:
-    subprocesses.check_call(
-        'docker',
-        'pull',
-        image,
-    )
-
-
-def build_docker_image(
-        docker_file: str,
-        *,
-        cwd: ta.Optional[str] = None,
-) -> str:
-    id_file = make_temp_file()
-    with defer(lambda: os.unlink(id_file)):
-        subprocesses.check_call(
-            'docker',
-            'build',
-            '-f', os.path.abspath(docker_file),
-            '--iidfile', id_file,
-            '--squash',
-            '.',
-            **(dict(cwd=cwd) if cwd is not None else {}),
-        )
-
-        with open(id_file) as f:
-            image_id = check.single(f.read().strip().splitlines()).strip()
-
-    return image_id
-
-
-##
-
-
-def save_docker_tar_cmd(
-        image: str,
-        output_cmd: ShellCmd,
-) -> None:
-    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
-    cmd.run(subprocesses.check_call)
-
-
-def save_docker_tar(
-        image: str,
-        tar_file: str,
-) -> None:
-    return save_docker_tar_cmd(
-        image,
-        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
-    )
-
-
-#
-
-
-def load_docker_tar_cmd(
-        input_cmd: ShellCmd,
-) -> str:
-    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
-
-    out = cmd.run(subprocesses.check_output).decode()
-
-    line = check.single(out.strip().splitlines())
-    loaded = line.partition(':')[2].strip()
-    return loaded
-
-
-def load_docker_tar(
-        tar_file: str,
-) -> str:
-    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
-
-
-########################################
-# ../github/cache.py
-
-
-##
-
-
-class GithubV1CacheShellClient:
-    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
-    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
-
-    def __init__(
-            self,
-            *,
-            base_url: ta.Optional[str] = None,
-            auth_token: ta.Optional[str] = None,
-    ) -> None:
-        super().__init__()
-
-        if base_url is None:
-            base_url = os.environ[self.BASE_URL_ENV_KEY]
-        self._base_url = check.non_empty_str(base_url)
-
-        if auth_token is None:
-            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
-        self._auth_token = auth_token
-
-        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
-
-    #
-
-    _MISSING = object()
-
-    def build_headers(
-            self,
-            *,
-            auth_token: ta.Any = _MISSING,
-            content_type: ta.Optional[str] = None,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
-        }
-
-        if auth_token is self._MISSING:
-            auth_token = self._auth_token
-        if auth_token:
-            dct['Authorization'] = f'Bearer {auth_token}'
-
-        if content_type is not None:
-            dct['Content-Type'] = content_type
-
-        return dct
-
-    #
-
-    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
-
-    def build_curl_cmd(
-            self,
-            method: str,
-            url: str,
-            *,
-            json_content: bool = False,
-            content_type: ta.Optional[str] = None,
-    ) -> ShellCmd:
-        if content_type is None and json_content:
-            content_type = 'application/json'
-
-        env = {}
-
-        header_auth_token: ta.Optional[str]
-        if self._auth_token:
-            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
-            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
-        else:
-            header_auth_token = None
-
-        hdrs = self.build_headers(
-            auth_token=header_auth_token,
-            content_type=content_type,
-        )
-
-        url = f'{self._service_url}/{url}'
-
-        cmd = ' '.join([
-            'curl',
-            '-s',
-            '-X', method,
-            url,
-            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
-        ])
-
-        return ShellCmd(
-            cmd,
-            env=env,
-        )
-
-    def build_post_json_curl_cmd(
-            self,
-            url: str,
-            obj: ta.Any,
-            **kwargs: ta.Any,
-    ) -> ShellCmd:
-        curl_cmd = self.build_curl_cmd(
-            'POST',
-            url,
-            json_content=True,
-            **kwargs,
-        )
-
-        obj_json = json_dumps_compact(obj)
-
-        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
-
-    #
-
-    @dc.dataclass()
-    class CurlError(RuntimeError):
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def __str__(self) -> str:
-            return repr(self)
-
-    @dc.dataclass(frozen=True)
-    class CurlResult:
-        status_code: int
-        body: ta.Optional[bytes]
-
-        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
-            return GithubV1CacheShellClient.CurlError(
-                status_code=self.status_code,
-                body=self.body,
-            )
-
-    def run_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            raise_: bool = False,
-    ) -> CurlResult:
-        out_file = make_temp_file()
-        with defer(lambda: os.unlink(out_file)):
-            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
-
-            out_json_bytes = run_cmd.run(subprocesses.check_output)
-
-            out_json = json.loads(out_json_bytes.decode())
-            status_code = check.isinstance(out_json['response_code'], int)
-
-            with open(out_file, 'rb') as f:
-                body = f.read()
-
-            result = self.CurlResult(
-                status_code=status_code,
-                body=body,
-            )
-
-        if raise_ and (500 <= status_code <= 600):
-            raise result.as_error()
-
-        return result
-
-    def run_json_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            success_status_codes: ta.Optional[ta.Container[int]] = None,
-    ) -> ta.Optional[ta.Any]:
-        result = self.run_curl_cmd(cmd, raise_=True)
-
-        if success_status_codes is not None:
-            is_success = result.status_code in success_status_codes
-        else:
-            is_success = 200 <= result.status_code < 300
-
-        if is_success:
-            if not (body := result.body):
-                return None
-            return json.loads(body.decode('utf-8-sig'))
-
-        elif result.status_code == 404:
-            return None
-
-        else:
-            raise result.as_error()
-
-    #
-
-    def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
-        return self.build_curl_cmd(
-            'GET',
-            f'cache?keys={key}',
-        )
-
-    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
-        curl_cmd = self.build_get_entry_curl_cmd(key)
-
-        obj = self.run_json_curl_cmd(
-            curl_cmd,
-            success_status_codes=[200, 204],
-        )
-        if obj is None:
-            return None
-
-        return GithubCacheServiceV1.dataclass_from_json(
-            GithubCacheServiceV1.ArtifactCacheEntry,
-            obj,
-        )
-
-    #
-
-    def build_download_get_entry_cmd(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> ShellCmd:
-        return ShellCmd(' '.join([
-            'aria2c',
-            '-x', '4',
-            '-o', out_file,
-            check.non_empty_str(entry.archive_location),
-        ]))
-
-    def download_get_entry(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> None:
-        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
-        dl_cmd.run(subprocesses.check_call)
-
-    #
-
-    def upload_cache_entry(
-            self,
-            key: str,
-            in_file: str,
-    ) -> None:
-        check.state(os.path.isfile(in_file))
-
-        file_size = os.stat(in_file).st_size
-
-        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=key,
-            cache_size=file_size,
-        )
-        reserve_cmd = self.build_post_json_curl_cmd(
-            'caches',
-            GithubCacheServiceV1.dataclass_to_json(reserve_req),
-        )
-        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
-            reserve_cmd,
-            success_status_codes=[201],
-        ))
-        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
-            GithubCacheServiceV1.ReserveCacheResponse,
-            reserve_resp_obj,
-        )
-
-        raise NotImplementedError
-
-
-##
-
-
-class GithubShellCache(ShellCache):
-    def __init__(
-            self,
-            dir: str,  # noqa
-            *,
-            client: ta.Optional[GithubV1CacheShellClient] = None,
-    ) -> None:
-        super().__init__()
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
 
-        self._dir = check.not_none(dir)
 
-        if client is None:
-            client = GithubV1CacheShellClient()
-        self._client = client
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
 
-        self._local = DirectoryFileCache(self._dir)
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        local_file = self._local.get_cache_file_path(key)
-        if os.path.exists(local_file):
-            return ShellCmd(f'cat {shlex.quote(local_file)}')
 
-        if (entry := self._client.run_get_entry(key)) is None:
-            return None
+##
 
-        tmp_file = self._local.format_incomplete_file(local_file)
-        try:
-            self._client.download_get_entry(entry, tmp_file)
 
-            os.replace(tmp_file, local_file)
+async def is_docker_image_present(image: str) -> bool:
+    out = await asyncio_subprocesses.check_output(
+        'docker',
+        'images',
+        '--format', 'json',
+        image,
+    )
 
-        except BaseException:  # noqa
-            os.unlink(tmp_file)
+    out_s = out.decode('utf-8').strip()
+    if not out_s:
+        return False
 
-            raise
+    json.loads(out_s)  # noqa
+    return True
 
-        return ShellCmd(f'cat {shlex.quote(local_file)}')
 
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(
-                self,
-                owner: 'GithubShellCache',
-                key: str,
-                tmp_file: str,
-                local_file: str,
-        ) -> None:
-            super().__init__()
+async def pull_docker_image(
+        image: str,
+) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'pull',
+        image,
+    )
 
-            self._owner = owner
-            self._key = key
-            self._tmp_file = tmp_file
-            self._local_file = local_file
 
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tmp_file)}')
+async def build_docker_image(
+        docker_file: str,
+        *,
+        tag: ta.Optional[str] = None,
+        cwd: ta.Optional[str] = None,
+        run_options: ta.Optional[ta.Sequence[str]] = None,
+) -> str:
+    with temp_file_context() as id_file:
+        await asyncio_subprocesses.check_call(
+            'docker',
+            'build',
+            '-f', os.path.abspath(docker_file),
+            '--iidfile', id_file,
+            *(['--tag', tag] if tag is not None else []),
+            *(run_options or []),
+            '.',
+            **(dict(cwd=cwd) if cwd is not None else {}),
+        )
 
-        def _commit(self) -> None:
-            os.replace(self._tmp_file, self._local_file)
+        with open(id_file) as f:  # noqa
+            image_id = check.single(f.read().strip().splitlines()).strip()
 
-            self._owner._client.upload_cache_entry(self._key, self._local_file)  # noqa
+    return image_id
 
-        def _abort(self) -> None:
-            os.unlink(self._tmp_file)
 
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        local_file = self._local.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(
-            self,
-            key,
-            self._local.format_incomplete_file(local_file),
-            local_file,
-        )
+async def tag_docker_image(image: str, tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'tag',
+        image,
+        tag,
+    )
 
 
-########################################
-# ../requirements.py
-"""
-TODO:
- - pip compile lol
-  - but still support git+ stuff
- - req.txt format aware hash
-  - more than just whitespace
- - pyproject req rewriting
- - download_requirements bootstrap off prev? not worth the dl?
-  - big deps (torch) change less, probably worth it
- - follow embedded -r automatically like pyp
-"""
+async def delete_docker_tag(tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'rmi',
+        tag,
+    )
 
 
 ##
 
 
-def build_requirements_hash(
-        requirements_txts: ta.Sequence[str],
-) -> str:
-    txt_file_contents: dict = {}
+async def save_docker_tar_cmd(
+        image: str,
+        output_cmd: ShellCmd,
+) -> None:
+    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
+    await cmd.run(asyncio_subprocesses.check_call)
 
-    for txt_file in requirements_txts:
-        txt_file_name = os.path.basename(txt_file)
-        check.not_in(txt_file_name, txt_file_contents)
-        with open(txt_file) as f:
-            txt_contents = f.read()
-        txt_file_contents[txt_file_name] = txt_contents
 
-    #
+async def save_docker_tar(
+        image: str,
+        tar_file: str,
+) -> None:
+    return await save_docker_tar_cmd(
+        image,
+        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
+    )
 
-    lines = []
-    for txt_file, txt_contents in sorted(txt_file_contents.items()):
-        txt_hash = sha256_str(txt_contents)
-        lines.append(f'{txt_file}={txt_hash}')
 
-    return sha256_str('\n'.join(lines))
+#
 
 
-##
+async def load_docker_tar_cmd(
+        input_cmd: ShellCmd,
+) -> str:
+    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
 
+    out = (await cmd.run(asyncio_subprocesses.check_output)).decode()
 
-def download_requirements(
-        image: str,
-        requirements_dir: str,
-        requirements_txts: ta.Sequence[str],
-) -> None:
-    requirements_txt_dir = tempfile.mkdtemp()
-    with defer(lambda: shutil.rmtree(requirements_txt_dir)):
-        for rt in requirements_txts:
-            shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
+    line = check.single(out.strip().splitlines())
+    loaded = line.partition(':')[2].strip()
+    return loaded
 
-        subprocesses.check_call(
-            'docker',
-            'run',
-            '--rm',
-            '-i',
-            '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
-            '-v', f'{requirements_txt_dir}:/requirements_txt',
-            image,
-            'pip',
-            'download',
-            '-d', '/requirements',
-            *itertools.chain.from_iterable(
-                ['-r', f'/requirements_txt/{os.path.basename(rt)}']
-                for rt in requirements_txts
-            ),
-        )
+
+async def load_docker_tar(
+        tar_file: str,
+) -> str:
+    return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
 
 
 ########################################
 # ../ci.py
 
 
-class Ci(ExitStacked):
-    FILE_NAME_HASH_LEN = 16
+class Ci(AsyncExitStacked):
+    KEY_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
     class Config:
@@ -3046,9 +3727,18 @@ class Ci(ExitStacked):
 
         cmd: ShellCmd
 
+        #
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
+        always_build: bool = False
+
+        no_dependencies: bool = False
+
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        #
 
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
@@ -3057,44 +3747,17 @@ class Ci(ExitStacked):
             self,
             cfg: Config,
             *,
-            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
-        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
-        if self._shell_cache is None:
-            return None
-
-        get_cache_cmd = self._shell_cache.get_file_cmd(key)
-        if get_cache_cmd is None:
-            return None
-
-        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
-
-        return load_docker_tar_cmd(get_cache_cmd)
-
-    def _save_cache_docker_image(self, key: str, image: str) -> None:
-        if self._shell_cache is None:
-            return
-
-        with self._shell_cache.put_file_cmd(key) as put_cache:
-            put_cache_cmd = put_cache.cmd
-
-            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
-
-            save_docker_tar_cmd(image, put_cache_cmd)
-
-    #
-
-    def _load_docker_image(self, image: str) -> None:
-        if not self._cfg.always_pull and is_docker_image_present(image):
+    async def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
 
         dep_suffix = image
@@ -3102,181 +3765,202 @@ class Ci(ExitStacked):
             dep_suffix = dep_suffix.replace(c, '-')
 
         cache_key = f'docker-{dep_suffix}'
-        if self._load_cache_docker_image(cache_key) is not None:
+        if (await self._load_cache_docker_image(cache_key)) is not None:
             return
 
-        pull_docker_image(image)
+        await pull_docker_image(image)
 
-        self._save_cache_docker_image(cache_key, image)
+        await self._save_cache_docker_image(cache_key, image)
 
-    def load_docker_image(self, image: str) -> None:
+    async def load_docker_image(self, image: str) -> None:
         with log_timing_context(f'Load docker image: {image}'):
-            self._load_docker_image(image)
+            await self._load_docker_image(image)
 
-    @cached_nullary
-    def load_compose_service_dependencies(self) -> None:
-        deps = get_compose_service_dependencies(
-            self._cfg.compose_file,
-            self._cfg.service,
-        )
+    #
 
-        for dep_image in deps.values():
-            self.load_docker_image(dep_image)
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
 
-    #
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
 
-    def _resolve_ci_image(self) -> str:
-        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
 
-        cache_key = f'ci-{docker_file_hash}'
-        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
-            return cache_image_id
+        return await load_docker_tar_cmd(get_cache_cmd)
 
-        image_id = build_docker_image(
-            self._cfg.docker_file,
-            cwd=self._cfg.project_dir,
-        )
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
 
-        self._save_cache_docker_image(cache_key, image_id)
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
 
-        return image_id
+            await save_docker_tar_cmd(image, write_tmp_cmd)
 
-    @cached_nullary
-    def resolve_ci_image(self) -> str:
-        with log_timing_context('Resolve ci image') as ltc:
-            image_id = self._resolve_ci_image()
-            ltc.set_description(f'Resolve ci image: {image_id}')
-            return image_id
+            await self._file_cache.put_file(key, tmp_file, steal=True)
 
     #
 
-    def _resolve_requirements_dir(self) -> str:
-        requirements_txts = [
-            os.path.join(self._cfg.project_dir, rf)
-            for rf in check.not_none(self._cfg.requirements_txts)
-        ]
+    async def _resolve_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
+        image_tag = f'{self._cfg.service}:{cache_key}'
 
-        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
+
+        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
 
-        tar_file_key = f'requirements-{requirements_hash}'
-        tar_file_name = f'{tar_file_key}.tar'
+        image_id = await build_and_tag(image_tag)
 
-        temp_dir = tempfile.mkdtemp()
-        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+        await self._save_cache_docker_image(cache_key, image_id)
 
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
-            with tarfile.open(cache_tar_file) as tar:
-                tar.extractall(path=temp_dir)  # noqa
+        return image_tag
 
-            return temp_dir
+    #
 
-        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
-        os.makedirs(temp_requirements_dir)
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_base_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            return await build_docker_image(
+                self._cfg.docker_file,
+                tag=image_tag,
+                cwd=self._cfg.project_dir,
+            )
 
-        download_requirements(
-            self.resolve_ci_image(),
-            temp_requirements_dir,
-            requirements_txts,
-        )
+        cache_key = f'ci-base-{self.docker_file_hash()}'
 
-        if self._file_cache is not None:
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+        return await self._resolve_docker_image(cache_key, build_and_tag)
 
-            with tarfile.open(temp_tar_file, 'w') as tar:
-                for requirement_file in os.listdir(temp_requirements_dir):
-                    tar.add(
-                        os.path.join(temp_requirements_dir, requirement_file),
-                        arcname=requirement_file,
-                    )
+    @async_cached_nullary
+    async def resolve_ci_base_image(self) -> str:
+        with log_timing_context('Resolve ci base image') as ltc:
+            image_id = await self._resolve_ci_base_image()
+            ltc.set_description(f'Resolve ci base image: {image_id}')
+            return image_id
 
-            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
+    #
 
-        return temp_requirements_dir
+    @cached_nullary
+    def requirements_txts(self) -> ta.Sequence[str]:
+        return [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
 
     @cached_nullary
-    def resolve_requirements_dir(self) -> str:
-        with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = self._resolve_requirements_dir()
-            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
-            return requirements_dir
+    def requirements_hash(self) -> str:
+        return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            base_image = await self.resolve_ci_base_image()
+
+            setup_cmds = [
+                ' '.join([
+                    'pip install',
+                    '--no-cache-dir',
+                    '--root-user-action ignore',
+                    'uv',
+                ]),
+                ' '.join([
+                    'uv pip install',
+                    '--no-cache',
+                    '--index-strategy unsafe-best-match',
+                    '--system',
+                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                ]),
+            ]
+            setup_cmd = ' && '.join(setup_cmds)
+
+            docker_file_lines = [
+                f'FROM {base_image}',
+                'RUN mkdir /project',
+                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                f'RUN {setup_cmd}',
+                'RUN rm /project/*',
+                'WORKDIR /project',
+            ]
+
+            with temp_file_context() as docker_file:
+                with open(docker_file, 'w') as f:  # noqa
+                    f.write('\n'.join(docker_file_lines))
+
+                return await build_docker_image(
+                    docker_file,
+                    tag=image_tag,
+                    cwd=self._cfg.project_dir,
+                )
+
+        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = await self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
 
     #
 
-    def _run_compose_(self) -> None:
-        setup_cmds = [
-            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
-            (
-                'uv pip install --system --find-links /requirements ' +
-                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
-            ),
-        ]
-
-        #
+    @async_cached_nullary
+    async def load_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
 
-        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
-            *setup_cmds,
-            f'({self._cfg.cmd.s})',
-        ]))
+        for dep_image in deps.values():
+            await self.load_docker_image(dep_image)
 
-        #
+    #
 
-        with DockerComposeRun(DockerComposeRun.Config(
+    async def _run_compose_(self) -> None:
+        async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
-            image=self.resolve_ci_image(),
+            image=await self.resolve_ci_image(),
 
-            cmd=ci_cmd,
+            cmd=self._cfg.cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+                *(self._cfg.run_options or []),
             ],
 
             cwd=self._cfg.project_dir,
+
+            no_dependencies=self._cfg.no_dependencies,
         )) as ci_compose_run:
-            ci_compose_run.run()
+            await ci_compose_run.run()
 
-    def _run_compose(self) -> None:
+    async def _run_compose(self) -> None:
         with log_timing_context('Run compose'):
-            self._run_compose_()
+            await self._run_compose_()
 
     #
 
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
-
-        self.resolve_ci_image()
-
-        self.resolve_requirements_dir()
-
-        self._run_compose()
-
-
-########################################
-# ../github/cli.py
-"""
-See:
- - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
-"""
-
+    async def run(self) -> None:
+        await self.resolve_ci_image()
 
-class GithubCli(ArgparseCli):
-    @argparse_cmd(
-        argparse_arg('key'),
-    )
-    def get_cache_entry(self) -> None:
-        shell_client = GithubV1CacheShellClient()
-        entry = shell_client.run_get_entry(self.args.key)
-        if entry is None:
-            return
-        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
+        await self.load_dependencies()
 
-    @argparse_cmd(
-        argparse_arg('repository-id'),
-    )
-    def list_cache_entries(self) -> None:
-        raise NotImplementedError
+        await self._run_compose()
 
 
 ########################################
@@ -3314,8 +3998,8 @@ class CiCli(ArgparseCli):
     @argparse_cmd(
         accepts_unknown=True,
     )
-    def github(self) -> ta.Optional[int]:
-        return GithubCli(self.unknown_args).cli_run()
+    async def github(self) -> ta.Optional[int]:
+        return await GithubCli(self.unknown_args).async_cli_run()
 
     #
 
@@ -3325,18 +4009,33 @@ class CiCli(ArgparseCli):
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
-        argparse_arg('--github-cache', action='store_true'),
+
         argparse_arg('--cache-dir'),
+
+        argparse_arg('--github', action='store_true'),
+        argparse_arg('--github-detect', action='store_true'),
+
         argparse_arg('--always-pull', action='store_true'),
+        argparse_arg('--always-build', action='store_true'),
+
+        argparse_arg('--no-dependencies', action='store_true'),
+
+        argparse_arg('-e', '--env', action='append'),
+        argparse_arg('-v', '--volume', action='append'),
+
+        argparse_arg('cmd', nargs=argparse.REMAINDER),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
-        service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
-        always_pull = self.args.always_pull
+
+        #
+
+        cmd = ' '.join(self.args.cmd)
+        check.non_empty_str(cmd)
 
         #
 
@@ -3348,6 +4047,7 @@ class CiCli(ArgparseCli):
             for alt in alts:
                 alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
+                    log.debug('Using %s', alt_file)
                     return alt_file
             return None
 
@@ -3381,6 +4081,7 @@ class CiCli(ArgparseCli):
                 'requirements-ci.txt',
             ]:
                 if os.path.exists(os.path.join(project_dir, rf)):
+                    log.debug('Using %s', rf)
                     requirements_txts.append(rf)
         else:
             for rf in requirements_txts:
@@ -3388,46 +4089,60 @@ class CiCli(ArgparseCli):
 
         #
 
-        shell_cache: ta.Optional[ShellCache] = None
+        github = self.args.github
+        if not github and self.args.github_detect:
+            github = is_in_github_actions()
+            if github:
+                log.debug('Github detected')
+
+        #
+
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
-            if not os.path.exists(cache_dir):
-                os.makedirs(cache_dir)
-            check.state(os.path.isdir(cache_dir))
-
-            directory_file_cache = DirectoryFileCache(cache_dir)
+            cache_dir = os.path.abspath(cache_dir)
+            log.debug('Using cache dir %s', cache_dir)
+            if github:
+                file_cache = GithubFileCache(cache_dir)
+            else:
+                file_cache = DirectoryFileCache(cache_dir)
 
-            file_cache = directory_file_cache
+        #
 
-            if self.args.github_cache:
-                shell_cache = GithubShellCache(cache_dir)
-            else:
-                shell_cache = DirectoryShellCache(directory_file_cache)
+        run_options: ta.List[str] = []
+        for run_arg, run_arg_vals in [
+            ('-e', self.args.env or []),
+            ('-v', self.args.volume or []),
+        ]:
+            run_options.extend(itertools.chain.from_iterable(
+                [run_arg, run_arg_val]
+                for run_arg_val in run_arg_vals
+            ))
 
         #
 
-        with Ci(
+        async with Ci(
                 Ci.Config(
                     project_dir=project_dir,
 
                     docker_file=docker_file,
 
                     compose_file=compose_file,
-                    service=service,
+                    service=self.args.service,
 
                     requirements_txts=requirements_txts,
 
-                    cmd=ShellCmd(' && '.join([
-                        'cd /project',
-                        'python3 -m pytest -svv test.py',
-                    ])),
+                    cmd=ShellCmd(cmd),
+
+                    always_pull=self.args.always_pull,
+                    always_build=self.args.always_build,
+
+                    no_dependencies=self.args.no_dependencies,
 
-                    always_pull=always_pull,
+                    run_options=run_options,
                 ),
                 file_cache=file_cache,
-                shell_cache=shell_cache,
         ) as ci:
-            ci.run()
+            await ci.run()
 
 
 async def _async_main() -> ta.Optional[int]: