omdev 0.0.0.dev211__py3-none-any.whl → 0.0.0.dev213__py3-none-any.whl

omdev/scripts/ci.py

@@ -17,6 +17,8 @@ Inputs:
 import abc
 import argparse
 import asyncio
+import asyncio.base_subprocess
+import asyncio.subprocess
 import collections
 import contextlib
 import dataclasses as dc
@@ -39,6 +41,7 @@ import threading
 import time
 import types
 import typing as ta
+import urllib.parse
 
 
 ########################################
@@ -54,6 +57,9 @@ if sys.version_info < (3, 8):
 # shell.py
 T = ta.TypeVar('T')
 
+# ../../omlish/asyncs/asyncio/timeouts.py
+AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
+
 # ../../omlish/lite/cached.py
 CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
 
@@ -70,6 +76,7 @@ ArgparseCmdFn = ta.Callable[[], ta.Optional[int]]  # ta.TypeAlias
 
 # ../../omlish/lite/contextmanagers.py
 ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
+AsyncExitStackedT = ta.TypeVar('AsyncExitStackedT', bound='AsyncExitStacked')
 
 # ../../omlish/subprocesses.py
 SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
@@ -113,6 +120,19 @@ class ShellCmd:
         )
 
 
+########################################
+# ../../../omlish/asyncs/asyncio/timeouts.py
+
+
+def asyncio_maybe_timeout(
+        fut: AwaitableT,
+        timeout: ta.Optional[float] = None,
+) -> AwaitableT:
+    if timeout is not None:
+        fut = asyncio.wait_for(fut, timeout)  # type: ignore
+    return fut
+
+
 ########################################
 # ../../../omlish/lite/cached.py
 
@@ -208,6 +228,17 @@ class Checks:
 
     #
 
+    def register_on_raise_breakpoint_if_env_var_set(self, key: str) -> None:
+        import os
+
+        def on_raise(exc: Exception) -> None:  # noqa
+            if key in os.environ:
+                breakpoint()  # noqa
+
+        self.register_on_raise(on_raise)
+
+    #
+
     def set_exception_factory(self, factory: CheckExceptionFactory) -> None:
         self._exception_factory = factory
 
@@ -523,6 +554,18 @@ class Checks:
 
         return v
 
+    def not_equal(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
+        if o == v:
+            self._raise(
+                ValueError,
+                'Must not be equal',
+                msg,
+                Checks._ArgsKwargs(v, o),
+                render_fmt='%s == %s',
+            )
+
+        return v
+
     def is_(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
         if o is not v:
             self._raise(
@@ -1205,7 +1248,7 @@ class GithubCacheServiceV1:
     @dc.dataclass(frozen=True)
     class ReserveCacheRequest:
         key: str
-        cache_size: ta.Optional[int]
+        cache_size: ta.Optional[int] = None
         version: ta.Optional[str] = None
 
     @dc.dataclass(frozen=True)
@@ -1713,6 +1756,33 @@ class ExitStacked:
         return es.enter_context(cm)
 
 
+class AsyncExitStacked:
+    _exit_stack: ta.Optional[contextlib.AsyncExitStack] = None
+
+    async def __aenter__(self: AsyncExitStackedT) -> AsyncExitStackedT:
+        check.state(self._exit_stack is None)
+        es = self._exit_stack = contextlib.AsyncExitStack()
+        await es.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if (es := self._exit_stack) is None:
+            return None
+        await self._async_exit_contexts()
+        return await es.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def _async_exit_contexts(self) -> None:
+        pass
+
+    def _enter_context(self, cm: ta.ContextManager[T]) -> T:
+        es = check.not_none(self._exit_stack)
+        return es.enter_context(cm)
+
+    async def _enter_async_context(self, cm: ta.AsyncContextManager[T]) -> T:
+        es = check.not_none(self._exit_stack)
+        return await es.enter_async_context(cm)
+
+
 ##
 
 
@@ -1724,6 +1794,17 @@ def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
         fn()
 
 
+@contextlib.asynccontextmanager
+async def adefer(fn: ta.Callable) -> ta.AsyncGenerator[ta.Callable, None]:
+    try:
+        yield fn
+    finally:
+        await fn()
+
+
+##
+
+
 @contextlib.contextmanager
 def attr_setting(obj, attr, val, *, default=None):  # noqa
     not_set = object()
@@ -2277,154 +2358,619 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
 
 
 ########################################
-# ../compose.py
-"""
-TODO:
- - fix rmi - only when not referenced anymore
-"""
+# ../github/curl.py
 
 
 ##
 
 
-def get_compose_service_dependencies(
-        compose_file: str,
-        service: str,
-) -> ta.Dict[str, str]:
-    compose_dct = read_yaml_file(compose_file)
+class GithubServiceCurlClient:
+    def __init__(
+            self,
+            service_url: str,
+            auth_token: ta.Optional[str] = None,
+            *,
+            api_version: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
 
-    services = compose_dct['services']
-    service_dct = services[service]
+        self._service_url = check.non_empty_str(service_url)
+        self._auth_token = auth_token
+        self._api_version = api_version
 
-    out = {}
-    for dep_service in service_dct.get('depends_on', []):
-        dep_service_dct = services[dep_service]
-        out[dep_service] = dep_service_dct['image']
+    #
 
-    return out
+    _MISSING = object()
 
+    def build_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            *,
+            auth_token: ta.Any = _MISSING,
+            content_type: ta.Optional[str] = None,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': ';'.join([
+                'application/json',
+                *([f'api-version={self._api_version}'] if self._api_version else []),
+            ]),
+        }
 
-##
+        if auth_token is self._MISSING:
+            auth_token = self._auth_token
+        if auth_token:
+            dct['Authorization'] = f'Bearer {auth_token}'
 
+        if content_type is not None:
+            dct['Content-Type'] = content_type
 
-class DockerComposeRun(ExitStacked):
-    @dc.dataclass(frozen=True)
-    class Config:
-        compose_file: str
-        service: str
+        if headers:
+            dct.update(headers)
 
-        image: str
+        return dct
 
-        cmd: ShellCmd
+    #
 
-        #
+    HEADER_AUTH_TOKEN_ENV_KEY_PREFIX = '_GITHUB_SERVICE_AUTH_TOKEN'  # noqa
 
-        run_options: ta.Optional[ta.Sequence[str]] = None
+    @property
+    def header_auth_token_env_key(self) -> str:
+        return f'{self.HEADER_AUTH_TOKEN_ENV_KEY_PREFIX}_{id(self)}'
 
-        cwd: ta.Optional[str] = None
+    def build_cmd(
+            self,
+            method: str,
+            url: str,
+            *,
+            json_content: bool = False,
+            content_type: ta.Optional[str] = None,
+            headers: ta.Optional[ta.Dict[str, str]] = None,
+    ) -> ShellCmd:
+        if content_type is None and json_content:
+            content_type = 'application/json'
 
-        #
+        env = {}
 
-        no_dependency_cleanup: bool = False
+        header_auth_token: ta.Optional[str]
+        if self._auth_token:
+            header_env_key = self.header_auth_token_env_key
+            env[header_env_key] = self._auth_token
+            header_auth_token = f'${header_env_key}'
+        else:
+            header_auth_token = None
 
-        #
+        built_hdrs = self.build_headers(
+            headers,
+            auth_token=header_auth_token,
+            content_type=content_type,
+        )
 
-        def __post_init__(self) -> None:
-            check.not_isinstance(self.run_options, str)
+        url = f'{self._service_url}/{url}'
 
-    def __init__(self, cfg: Config) -> None:
-        super().__init__()
+        cmd = ' '.join([
+            'curl',
+            '-s',
+            '-X', method,
+            url,
+            *[f'-H "{k}: {v}"' for k, v in built_hdrs.items()],
+        ])
 
-        self._cfg = cfg
+        return ShellCmd(
+            cmd,
+            env=env,
+        )
 
-        self._subprocess_kwargs = {
-            **(dict(cwd=self._cfg.cwd) if self._cfg.cwd is not None else {}),
-        }
+    def build_post_json_cmd(
+            self,
+            url: str,
+            obj: ta.Any,
+            **kwargs: ta.Any,
+    ) -> ShellCmd:
+        curl_cmd = self.build_cmd(
+            'POST',
+            url,
+            json_content=True,
+            **kwargs,
+        )
 
-    #
+        obj_json = json_dumps_compact(obj)
 
-    @property
-    def image_tag(self) -> str:
-        pfx = 'sha256:'
-        if (image := self._cfg.image).startswith(pfx):
-            image = image[len(pfx):]
+        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
 
-        return f'{self._cfg.service}:{image}'
+    #
 
-    @cached_nullary
-    def tag_image(self) -> str:
-        image_tag = self.image_tag
+    @dc.dataclass()
+    class Error(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
 
-        subprocesses.check_call(
-            'docker',
-            'tag',
-            self._cfg.image,
-            image_tag,
-            **self._subprocess_kwargs,
-        )
+        def __str__(self) -> str:
+            return repr(self)
 
-        def delete_tag() -> None:
-            subprocesses.check_call(
-                'docker',
-                'rmi',
-                image_tag,
-                **self._subprocess_kwargs,
+    @dc.dataclass(frozen=True)
+    class Result:
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def as_error(self) -> 'GithubServiceCurlClient.Error':
+            return GithubServiceCurlClient.Error(
+                status_code=self.status_code,
+                body=self.body,
             )
 
-        self._enter_context(defer(delete_tag))  # noqa
+    def run_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            raise_: bool = False,
+            **subprocess_kwargs: ta.Any,
+    ) -> Result:
+        out_file = make_temp_file()
+        with defer(lambda: os.unlink(out_file)):
+            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
 
-        return image_tag
+            out_json_bytes = run_cmd.run(
+                subprocesses.check_output,
+                **subprocess_kwargs,
+            )
 
-    #
+            out_json = json.loads(out_json_bytes.decode())
+            status_code = check.isinstance(out_json['response_code'], int)
 
-    def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
-        out = dict(in_dct)
+            with open(out_file, 'rb') as f:
+                body = f.read()
 
-        #
+            result = self.Result(
+                status_code=status_code,
+                body=body,
+            )
 
-        in_services = in_dct['services']
-        out['services'] = out_services = {}
+        if raise_ and (500 <= status_code <= 600):
+            raise result.as_error()
 
-        #
+        return result
 
-        in_service: dict = in_services[self._cfg.service]
-        out_services[self._cfg.service] = out_service = dict(in_service)
+    def run_json_cmd(
+            self,
+            cmd: ShellCmd,
+            *,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        result = self.run_cmd(cmd, raise_=True)
 
-        out_service['image'] = self.image_tag
+        if success_status_codes is not None:
+            is_success = result.status_code in success_status_codes
+        else:
+            is_success = 200 <= result.status_code < 300
 
-        for k in ['build', 'platform']:
-            if k in out_service:
-                del out_service[k]
+        if is_success:
+            if not (body := result.body):
+                return None
+            return json.loads(body.decode('utf-8-sig'))
 
-        out_service['links'] = [
-            f'{l}:{l}' if ':' not in l else l
-            for l in out_service.get('links', [])
-        ]
+        elif result.status_code == 404:
+            return None
 
-        #
+        else:
+            raise result.as_error()
 
-        depends_on = in_service.get('depends_on', [])
 
-        for dep_service, in_dep_service_dct in list(in_services.items()):
-            if dep_service not in depends_on:
-                continue
+########################################
+# ../requirements.py
+"""
+TODO:
+ - pip compile lol
+  - but still support git+ stuff
+ - req.txt format aware hash
+  - more than just whitespace
+ - pyproject req rewriting
+ - download_requirements bootstrap off prev? not worth the dl?
+  - big deps (torch) change less, probably worth it
+ - follow embedded -r automatically like pyp
+"""
 
-            out_dep_service: dict = dict(in_dep_service_dct)
-            out_services[dep_service] = out_dep_service
 
-            out_dep_service['ports'] = []
+##
 
-        #
 
-        return out
+def build_requirements_hash(
+        requirements_txts: ta.Sequence[str],
+) -> str:
+    txt_file_contents: dict = {}
 
-    @cached_nullary
-    def rewrite_compose_file(self) -> str:
-        in_dct = read_yaml_file(self._cfg.compose_file)
+    for txt_file in requirements_txts:
+        txt_file_name = os.path.basename(txt_file)
+        check.not_in(txt_file_name, txt_file_contents)
+        with open(txt_file) as f:
+            txt_contents = f.read()
+        txt_file_contents[txt_file_name] = txt_contents
 
-        out_dct = self._rewrite_compose_dct(in_dct)
+    #
 
-        #
+    lines = []
+    for txt_file, txt_contents in sorted(txt_file_contents.items()):
+        txt_hash = sha256_str(txt_contents)
+        lines.append(f'{txt_file}={txt_hash}')
+
+    return sha256_str('\n'.join(lines))
+
+
+##
+
+
+def download_requirements(
+        image: str,
+        requirements_dir: str,
+        requirements_txts: ta.Sequence[str],
+) -> None:
+    requirements_txt_dir = tempfile.mkdtemp()
+    with defer(lambda: shutil.rmtree(requirements_txt_dir)):
+        for rt in requirements_txts:
+            shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
+
+        subprocesses.check_call(
+            'docker',
+            'run',
+            '--rm',
+            '-i',
+            '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
+            '-v', f'{requirements_txt_dir}:/requirements_txt',
+            image,
+            'pip',
+            'download',
+            '-d', '/requirements',
+            *itertools.chain.from_iterable(
+                ['-r', f'/requirements_txt/{os.path.basename(rt)}']
+                for rt in requirements_txts
+            ),
+        )
+
+
+########################################
+# ../../../omlish/asyncs/asyncio/subprocesses.py
+
+
+##
+
+
+class AsyncioProcessCommunicator:
+    def __init__(
+            self,
+            proc: asyncio.subprocess.Process,
+            loop: ta.Optional[ta.Any] = None,
+            *,
+            log: ta.Optional[logging.Logger] = None,
+    ) -> None:
+        super().__init__()
+
+        if loop is None:
+            loop = asyncio.get_running_loop()
+
+        self._proc = proc
+        self._loop = loop
+        self._log = log
+
+        self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
+            proc._transport,  # type: ignore  # noqa
+            asyncio.base_subprocess.BaseSubprocessTransport,
+        )
+
+    @property
+    def _debug(self) -> bool:
+        return self._loop.get_debug()
+
+    async def _feed_stdin(self, input: bytes) -> None:  # noqa
+        stdin = check.not_none(self._proc.stdin)
+        try:
+            if input is not None:
+                stdin.write(input)
+                if self._debug and self._log is not None:
+                    self._log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
+
+            await stdin.drain()
+
+        except (BrokenPipeError, ConnectionResetError) as exc:
+            # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
+            # exceptions.
+            if self._debug and self._log is not None:
+                self._log.debug('%r communicate: stdin got %r', self, exc)
+
+        if self._debug and self._log is not None:
+            self._log.debug('%r communicate: close stdin', self)
+
+        stdin.close()
+
+    async def _noop(self) -> None:
+        return None
+
+    async def _read_stream(self, fd: int) -> bytes:
+        transport: ta.Any = check.not_none(self._transport.get_pipe_transport(fd))
+
+        if fd == 2:
+            stream = check.not_none(self._proc.stderr)
+        else:
+            check.equal(fd, 1)
+            stream = check.not_none(self._proc.stdout)
+
+        if self._debug and self._log is not None:
+            name = 'stdout' if fd == 1 else 'stderr'
+            self._log.debug('%r communicate: read %s', self, name)
+
+        output = await stream.read()
+
+        if self._debug and self._log is not None:
+            name = 'stdout' if fd == 1 else 'stderr'
+            self._log.debug('%r communicate: close %s', self, name)
+
+        transport.close()
+
+        return output
+
+    class Communication(ta.NamedTuple):
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def _communicate(
+            self,
+            input: ta.Any = None,  # noqa
+    ) -> Communication:
+        stdin_fut: ta.Any
+        if self._proc.stdin is not None:
+            stdin_fut = self._feed_stdin(input)
+        else:
+            stdin_fut = self._noop()
+
+        stdout_fut: ta.Any
+        if self._proc.stdout is not None:
+            stdout_fut = self._read_stream(1)
+        else:
+            stdout_fut = self._noop()
+
+        stderr_fut: ta.Any
+        if self._proc.stderr is not None:
+            stderr_fut = self._read_stream(2)
+        else:
+            stderr_fut = self._noop()
+
+        stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
+
+        await self._proc.wait()
+
+        return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
+
+    async def communicate(
+            self,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> Communication:
+        return await asyncio_maybe_timeout(self._communicate(input), timeout)
+
+
+##
+
+
+class AsyncioSubprocesses(AbstractAsyncSubprocesses):
+    async def communicate(
+            self,
+            proc: asyncio.subprocess.Process,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+    ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
+        return await AsyncioProcessCommunicator(proc).communicate(input, timeout)  # noqa
+
+    #
+
+    @contextlib.asynccontextmanager
+    async def popen(
+            self,
+            *cmd: str,
+            shell: bool = False,
+            timeout: ta.Optional[float] = None,
+            **kwargs: ta.Any,
+    ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
+        fac: ta.Any
+        if shell:
+            fac = functools.partial(
+                asyncio.create_subprocess_shell,
+                check.single(cmd),
+            )
+        else:
+            fac = functools.partial(
+                asyncio.create_subprocess_exec,
+                *cmd,
+            )
+
+        with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs):  # noqa
+            proc: asyncio.subprocess.Process = await fac(**kwargs)
+            try:
+                yield proc
+
+            finally:
+                await asyncio_maybe_timeout(proc.wait(), timeout)
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class RunOutput:
+        proc: asyncio.subprocess.Process
+        stdout: ta.Optional[bytes]
+        stderr: ta.Optional[bytes]
+
+    async def run(
+            self,
+            *cmd: str,
+            input: ta.Any = None,  # noqa
+            timeout: ta.Optional[float] = None,
+            check: bool = False,  # noqa
+            capture_output: ta.Optional[bool] = None,
+            **kwargs: ta.Any,
+    ) -> RunOutput:
+        if capture_output:
+            kwargs.setdefault('stdout', subprocess.PIPE)
+            kwargs.setdefault('stderr', subprocess.PIPE)
+
+        proc: asyncio.subprocess.Process
+        async with self.popen(*cmd, **kwargs) as proc:
+            stdout, stderr = await self.communicate(proc, input, timeout)
+
+        if check and proc.returncode:
+            raise subprocess.CalledProcessError(
+                proc.returncode,
+                cmd,
+                output=stdout,
+                stderr=stderr,
+            )
+
+        return self.RunOutput(
+            proc,
+            stdout,
+            stderr,
+        )
+
+    #
+
+    async def check_call(
+            self,
+            *cmd: str,
+            stdout: ta.Any = sys.stderr,
+            **kwargs: ta.Any,
+    ) -> None:
+        with self.prepare_and_wrap(*cmd, stdout=stdout, check=True, **kwargs) as (cmd, kwargs):  # noqa
+            await self.run(*cmd, **kwargs)
+
+    async def check_output(
+            self,
+            *cmd: str,
+            **kwargs: ta.Any,
+    ) -> bytes:
+        with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs):  # noqa
+            return check.not_none((await self.run(*cmd, **kwargs)).stdout)
+
+
+asyncio_subprocesses = AsyncioSubprocesses()
+
+
+########################################
+# ../compose.py
+"""
+TODO:
+ - fix rmi - only when not referenced anymore
+"""
+
+
+##
+
+
+def get_compose_service_dependencies(
+        compose_file: str,
+        service: str,
+) -> ta.Dict[str, str]:
+    compose_dct = read_yaml_file(compose_file)
+
+    services = compose_dct['services']
+    service_dct = services[service]
+
+    out = {}
+    for dep_service in service_dct.get('depends_on', []):
+        dep_service_dct = services[dep_service]
+        out[dep_service] = dep_service_dct['image']
+
+    return out
+
+
+##
+
+
+class DockerComposeRun(AsyncExitStacked):
+    @dc.dataclass(frozen=True)
+    class Config:
+        compose_file: str
+        service: str
+
+        image: str
+
+        cmd: ShellCmd
+
+        #
+
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        cwd: ta.Optional[str] = None
+
+        #
+
+        no_dependencies: bool = False
+        no_dependency_cleanup: bool = False
+
+        #
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.run_options, str)
+
+    def __init__(self, cfg: Config) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+
+        self._subprocess_kwargs = {
+            **(dict(cwd=self._cfg.cwd) if self._cfg.cwd is not None else {}),
+        }
+
+    #
+
+    def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
+        out = dict(in_dct)
+
+        #
+
+        in_services = in_dct['services']
+        out['services'] = out_services = {}
+
+        #
+
+        in_service: dict = in_services[self._cfg.service]
+        out_services[self._cfg.service] = out_service = dict(in_service)
+
+        out_service['image'] = self._cfg.image
+
+        for k in ['build', 'platform']:
+            if k in out_service:
+                del out_service[k]
+
+        out_service['links'] = [
+            f'{l}:{l}' if ':' not in l else l
+            for l in out_service.get('links', [])
+        ]
+
+        #
+
+        if not self._cfg.no_dependencies:
+            depends_on = in_service.get('depends_on', [])
+
+            for dep_service, in_dep_service_dct in list(in_services.items()):
+                if dep_service not in depends_on:
+                    continue
+
+                out_dep_service: dict = dict(in_dep_service_dct)
+                out_services[dep_service] = out_dep_service
+
+                out_dep_service['ports'] = []
+
+        else:
+            out_service['depends_on'] = []
+            out_service['links'] = []
+
+        #
+
+        return out
+
+    @cached_nullary
+    def rewrite_compose_file(self) -> str:
+        in_dct = read_yaml_file(self._cfg.compose_file)
+
+        out_dct = self._rewrite_compose_dct(in_dct)
+
+        #
 
         out_compose_file = make_temp_file()
         self._enter_context(defer(lambda: os.unlink(out_compose_file)))  # noqa
@@ -2438,22 +2984,20 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    def _cleanup_dependencies(self) -> None:
-        subprocesses.check_call(
+    async def _cleanup_dependencies(self) -> None:
+        await asyncio_subprocesses.check_call(
             'docker',
             'compose',
             '-f', self.rewrite_compose_file(),
             'down',
         )
 
-    def run(self) -> None:
-        self.tag_image()
-
+    async def run(self) -> None:
         compose_file = self.rewrite_compose_file()
 
-        with contextlib.ExitStack() as es:
-            if not self._cfg.no_dependency_cleanup:
-                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+        async with contextlib.AsyncExitStack() as es:
+            if not (self._cfg.no_dependencies or self._cfg.no_dependency_cleanup):
+                await es.enter_async_context(adefer(self._cleanup_dependencies))  # noqa
 
             sh_cmd = ' '.join([
                 'docker',
@@ -2461,7 +3005,10 @@ class DockerComposeRun(ExitStacked):
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
                 *(self._cfg.run_options or []),
                 self._cfg.service,
                 'sh', '-c', shlex.quote(self._cfg.cmd.s),
@@ -2469,8 +3016,8 @@ class DockerComposeRun(ExitStacked):
 
             run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
 
-            run_cmd.run(
-                subprocesses.check_call,
+            await run_cmd.run(
+                asyncio_subprocesses.check_call,
                 **self._subprocess_kwargs,
             )
 
@@ -2522,8 +3069,8 @@ def read_docker_tar_image_id(tar_file: str) -> str:
 ##
 
 
-def is_docker_image_present(image: str) -> bool:
-    out = subprocesses.check_output(
+async def is_docker_image_present(image: str) -> bool:
+    out = await asyncio_subprocesses.check_output(
         'docker',
         'images',
         '--format', 'json',
@@ -2538,55 +3085,74 @@ def is_docker_image_present(image: str) -> bool:
     return True
 
 
-def pull_docker_image(
+async def pull_docker_image(
         image: str,
 ) -> None:
-    subprocesses.check_call(
+    await asyncio_subprocesses.check_call(
         'docker',
         'pull',
         image,
     )
 
 
-def build_docker_image(
+async def build_docker_image(
         docker_file: str,
         *,
+        tag: ta.Optional[str] = None,
         cwd: ta.Optional[str] = None,
 ) -> str:
     id_file = make_temp_file()
     with defer(lambda: os.unlink(id_file)):
-        subprocesses.check_call(
+        await asyncio_subprocesses.check_call(
             'docker',
             'build',
             '-f', os.path.abspath(docker_file),
             '--iidfile', id_file,
             '--squash',
+            *(['--tag', tag] if tag is not None else []),
             '.',
             **(dict(cwd=cwd) if cwd is not None else {}),
         )
 
-        with open(id_file) as f:
+        with open(id_file) as f:  # noqa
             image_id = check.single(f.read().strip().splitlines()).strip()
 
     return image_id
 
 
+async def tag_docker_image(image: str, tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'tag',
+        image,
+        tag,
+    )
+
+
+async def delete_docker_tag(tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'rmi',
+        tag,
+    )
+
+
 ##
 
 
-def save_docker_tar_cmd(
+async def save_docker_tar_cmd(
         image: str,
         output_cmd: ShellCmd,
 ) -> None:
     cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
-    cmd.run(subprocesses.check_call)
+    await cmd.run(asyncio_subprocesses.check_call)
 
 
-def save_docker_tar(
+async def save_docker_tar(
         image: str,
         tar_file: str,
 ) -> None:
-    return save_docker_tar_cmd(
+    return await save_docker_tar_cmd(
         image,
         ShellCmd(f'cat > {shlex.quote(tar_file)}'),
     )
@@ -2595,276 +3161,187 @@ def save_docker_tar(
 #
 
 
-def load_docker_tar_cmd(
-        input_cmd: ShellCmd,
-) -> str:
-    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
-
-    out = cmd.run(subprocesses.check_output).decode()
-
-    line = check.single(out.strip().splitlines())
-    loaded = line.partition(':')[2].strip()
-    return loaded
-
-
-def load_docker_tar(
-        tar_file: str,
-) -> str:
-    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
-
-
-########################################
-# ../github/cache.py
-
-
-##
-
-
-class GithubV1CacheShellClient:
-    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
-    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
-
-    def __init__(
-            self,
-            *,
-            base_url: ta.Optional[str] = None,
-            auth_token: ta.Optional[str] = None,
-    ) -> None:
-        super().__init__()
-
-        if base_url is None:
-            base_url = os.environ[self.BASE_URL_ENV_KEY]
-        self._base_url = check.non_empty_str(base_url)
-
-        if auth_token is None:
-            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
-        self._auth_token = auth_token
-
-        self._service_url = GithubCacheServiceV1.get_service_url(self._base_url)
-
-    #
-
-    _MISSING = object()
-
-    def build_headers(
-            self,
-            *,
-            auth_token: ta.Any = _MISSING,
-            content_type: ta.Optional[str] = None,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': f'application/json;api-version={GithubCacheServiceV1.API_VERSION}',
-        }
-
-        if auth_token is self._MISSING:
-            auth_token = self._auth_token
-        if auth_token:
-            dct['Authorization'] = f'Bearer {auth_token}'
-
-        if content_type is not None:
-            dct['Content-Type'] = content_type
+async def load_docker_tar_cmd(
+        input_cmd: ShellCmd,
+) -> str:
+    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
 
-        return dct
+    out = (await cmd.run(asyncio_subprocesses.check_output)).decode()
 
-    #
+    line = check.single(out.strip().splitlines())
+    loaded = line.partition(':')[2].strip()
+    return loaded
 
-    HEADER_AUTH_TOKEN_ENV_KEY = '_GITHUB_CACHE_AUTH_TOKEN'  # noqa
 
-    def build_curl_cmd(
-            self,
-            method: str,
-            url: str,
-            *,
-            json_content: bool = False,
-            content_type: ta.Optional[str] = None,
-    ) -> ShellCmd:
-        if content_type is None and json_content:
-            content_type = 'application/json'
+async def load_docker_tar(
+        tar_file: str,
+) -> str:
+    return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
 
-        env = {}
 
-        header_auth_token: ta.Optional[str]
-        if self._auth_token:
-            env[self.HEADER_AUTH_TOKEN_ENV_KEY] = self._auth_token
-            header_auth_token = f'${self.HEADER_AUTH_TOKEN_ENV_KEY}'
-        else:
-            header_auth_token = None
+########################################
+# ../github/cache.py
 
-        hdrs = self.build_headers(
-            auth_token=header_auth_token,
-            content_type=content_type,
-        )
 
-        url = f'{self._service_url}/{url}'
+##
 
-        cmd = ' '.join([
-            'curl',
-            '-s',
-            '-X', method,
-            url,
-            *[f'-H "{k}: {v}"' for k, v in hdrs.items()],
-        ])
 
-        return ShellCmd(
-            cmd,
-            env=env,
-        )
+class GithubCacheShellClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
 
-    def build_post_json_curl_cmd(
-            self,
-            url: str,
-            obj: ta.Any,
-            **kwargs: ta.Any,
-    ) -> ShellCmd:
-        curl_cmd = self.build_curl_cmd(
-            'POST',
-            url,
-            json_content=True,
-            **kwargs,
-        )
+    @abc.abstractmethod
+    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
+        raise NotImplementedError
 
-        obj_json = json_dumps_compact(obj)
+    @abc.abstractmethod
+    def download_get_entry(self, entry: Entry, out_file: str) -> None:
+        raise NotImplementedError
 
-        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
+    @abc.abstractmethod
+    def upload_cache_entry(self, key: str, in_file: str) -> None:
+        raise NotImplementedError
 
-    #
 
-    @dc.dataclass()
-    class CurlError(RuntimeError):
-        status_code: int
-        body: ta.Optional[bytes]
+#
 
-        def __str__(self) -> str:
-            return repr(self)
 
-    @dc.dataclass(frozen=True)
-    class CurlResult:
-        status_code: int
-        body: ta.Optional[bytes]
+class GithubCacheServiceV1ShellClient(GithubCacheShellClient):
+    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
+    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
 
-        def as_error(self) -> 'GithubV1CacheShellClient.CurlError':
-            return GithubV1CacheShellClient.CurlError(
-                status_code=self.status_code,
-                body=self.body,
-            )
+    KEY_SUFFIX_ENV_KEY = 'GITHUB_RUN_ID'
+
+    CACHE_VERSION: ta.ClassVar[int] = 1
+
+    #
 
-    def run_curl_cmd(
+    def __init__(
             self,
-            cmd: ShellCmd,
             *,
-            raise_: bool = False,
-    ) -> CurlResult:
-        out_file = make_temp_file()
-        with defer(lambda: os.unlink(out_file)):
-            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
 
-            out_json_bytes = run_cmd.run(subprocesses.check_output)
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
 
-            out_json = json.loads(out_json_bytes.decode())
-            status_code = check.isinstance(out_json['response_code'], int)
+        #
 
-            with open(out_file, 'rb') as f:
-                body = f.read()
+        if base_url is None:
+            base_url = os.environ[self.BASE_URL_ENV_KEY]
+        service_url = GithubCacheServiceV1.get_service_url(base_url)
 
-            result = self.CurlResult(
-                status_code=status_code,
-                body=body,
-            )
+        if auth_token is None:
+            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
 
-        if raise_ and (500 <= status_code <= 600):
-            raise result.as_error()
+        self._curl = GithubServiceCurlClient(
+            service_url,
+            auth_token,
+            api_version=GithubCacheServiceV1.API_VERSION,
+        )
 
-        return result
+        #
 
-    def run_json_curl_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            success_status_codes: ta.Optional[ta.Container[int]] = None,
-    ) -> ta.Optional[ta.Any]:
-        result = self.run_curl_cmd(cmd, raise_=True)
+        self._key_prefix = key_prefix
 
-        if success_status_codes is not None:
-            is_success = result.status_code in success_status_codes
-        else:
-            is_success = 200 <= result.status_code < 300
+        if key_suffix is None:
+            key_suffix = os.environ[self.KEY_SUFFIX_ENV_KEY]
+        self._key_suffix = check.non_empty_str(key_suffix)
 
-        if is_success:
-            if not (body := result.body):
-                return None
-            return json.loads(body.decode('utf-8-sig'))
+    #
 
-        elif result.status_code == 404:
-            return None
+    KEY_PART_SEPARATOR = '--'
 
-        else:
-            raise result.as_error()
+    def fix_key(self, s: str) -> str:
+        return self.KEY_PART_SEPARATOR.join([
+            *([self._key_prefix] if self._key_prefix else []),
+            s,
+            self._key_suffix,
+        ])
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheShellClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
     #
 
     def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
-        return self.build_curl_cmd(
+        fixed_key = self.fix_key(key)
+
+        qp = dict(
+            keys=fixed_key,
+            version=str(self.CACHE_VERSION),
+        )
+
+        return self._curl.build_cmd(
             'GET',
-            f'cache?keys={key}',
+            shlex.quote('?'.join([
+                'cache',
+                '&'.join([
+                    f'{k}={urllib.parse.quote_plus(v)}'
+                    for k, v in qp.items()
+                ]),
+            ])),
         )
 
-    def run_get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1.ArtifactCacheEntry]:
-        curl_cmd = self.build_get_entry_curl_cmd(key)
+    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
+        fixed_key = self.fix_key(key)
+        curl_cmd = self.build_get_entry_curl_cmd(fixed_key)
 
-        obj = self.run_json_curl_cmd(
+        obj = self._curl.run_json_cmd(
             curl_cmd,
             success_status_codes=[200, 204],
         )
         if obj is None:
             return None
 
-        return GithubCacheServiceV1.dataclass_from_json(
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
             GithubCacheServiceV1.ArtifactCacheEntry,
             obj,
-        )
+        ))
 
     #
 
-    def build_download_get_entry_cmd(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> ShellCmd:
+    def build_download_get_entry_cmd(self, entry: Entry, out_file: str) -> ShellCmd:
         return ShellCmd(' '.join([
             'aria2c',
             '-x', '4',
             '-o', out_file,
-            check.non_empty_str(entry.archive_location),
+            check.non_empty_str(entry.artifact.archive_location),
         ]))
 
-    def download_get_entry(
-            self,
-            entry: GithubCacheServiceV1.ArtifactCacheEntry,
-            out_file: str,
-    ) -> None:
-        dl_cmd = self.build_download_get_entry_cmd(entry, out_file)
+    def download_get_entry(self, entry: GithubCacheShellClient.Entry, out_file: str) -> None:
+        dl_cmd = self.build_download_get_entry_cmd(
+            check.isinstance(entry, GithubCacheServiceV1ShellClient.Entry),
+            out_file,
+        )
         dl_cmd.run(subprocesses.check_call)
 
     #
 
-    def upload_cache_entry(
-            self,
-            key: str,
-            in_file: str,
-    ) -> None:
+    def upload_cache_entry(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
         check.state(os.path.isfile(in_file))
 
         file_size = os.stat(in_file).st_size
 
+        #
+
         reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=key,
+            key=fixed_key,
             cache_size=file_size,
+            version=str(self.CACHE_VERSION),
         )
-        reserve_cmd = self.build_post_json_curl_cmd(
+        reserve_cmd = self._curl.build_post_json_cmd(
             'caches',
             GithubCacheServiceV1.dataclass_to_json(reserve_req),
         )
-        reserve_resp_obj: ta.Any = check.not_none(self.run_json_curl_cmd(
+        reserve_resp_obj: ta.Any = check.not_none(self._curl.run_json_cmd(
             reserve_cmd,
             success_status_codes=[201],
         ))
@@ -2872,8 +3349,66 @@ class GithubV1CacheShellClient:
             GithubCacheServiceV1.ReserveCacheResponse,
             reserve_resp_obj,
         )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
 
-        raise NotImplementedError
+        #
+
+        tmp_file = make_temp_file()
+
+        print(f'{file_size=}')
+        num_written = 0
+        chunk_size = 32 * 1024 * 1024
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            ofs = i * chunk_size
+            sz = min(chunk_size, file_size - ofs)
+
+            patch_cmd = self._curl.build_cmd(
+                'PATCH',
+                f'caches/{cache_id}',
+                content_type='application/octet-stream',
+                headers={
+                    'Content-Range': f'bytes {ofs}-{ofs + sz - 1}/*',
+                },
+            )
+
+            #
+
+            # patch_data_cmd = dc.replace(patch_cmd, s=' | '.join([
+            #     f'dd if={in_file} bs={chunk_size} skip={i} count=1 status=none',
+            #     f'{patch_cmd.s} --data-binary -',
+            # ]))
+            # print(f'{patch_data_cmd.s=}')
+            # patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+
+            #
+
+            with open(in_file, 'rb') as f:
+                f.seek(ofs)
+                buf = f.read(sz)
+            with open(tmp_file, 'wb') as f:
+                f.write(buf)
+            num_written += len(buf)
+            print(f'{num_written=}')
+            patch_data_cmd = dc.replace(patch_cmd, s=f'{patch_cmd.s} --data-binary @{tmp_file}')
+            print(f'{patch_data_cmd.s=}')
+            patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+
+            #
+
+            check.equal(patch_result.status_code, 204)
+            ofs += sz
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        commit_cmd = self._curl.build_post_json_cmd(
+            f'caches/{cache_id}',
+            GithubCacheServiceV1.dataclass_to_json(commit_req),
+        )
+        commit_result = self._curl.run_cmd(commit_cmd, raise_=True)
+        check.equal(commit_result.status_code, 204)
 
 
 ##
@@ -2884,15 +3419,15 @@ class GithubShellCache(ShellCache):
             self,
             dir: str,  # noqa
             *,
-            client: ta.Optional[GithubV1CacheShellClient] = None,
+            client: ta.Optional[GithubCacheShellClient] = None,
     ) -> None:
         super().__init__()
 
         self._dir = check.not_none(dir)
 
         if client is None:
-            client = GithubV1CacheShellClient()
-        self._client = client
+            client = GithubCacheServiceV1ShellClient()
+        self._client: GithubCacheShellClient = client
 
         self._local = DirectoryFileCache(self._dir)
 
@@ -2954,82 +3489,11 @@ class GithubShellCache(ShellCache):
         )
 
 
-########################################
-# ../requirements.py
-"""
-TODO:
- - pip compile lol
-  - but still support git+ stuff
- - req.txt format aware hash
-  - more than just whitespace
- - pyproject req rewriting
- - download_requirements bootstrap off prev? not worth the dl?
-  - big deps (torch) change less, probably worth it
- - follow embedded -r automatically like pyp
-"""
-
-
-##
-
-
-def build_requirements_hash(
-        requirements_txts: ta.Sequence[str],
-) -> str:
-    txt_file_contents: dict = {}
-
-    for txt_file in requirements_txts:
-        txt_file_name = os.path.basename(txt_file)
-        check.not_in(txt_file_name, txt_file_contents)
-        with open(txt_file) as f:
-            txt_contents = f.read()
-        txt_file_contents[txt_file_name] = txt_contents
-
-    #
-
-    lines = []
-    for txt_file, txt_contents in sorted(txt_file_contents.items()):
-        txt_hash = sha256_str(txt_contents)
-        lines.append(f'{txt_file}={txt_hash}')
-
-    return sha256_str('\n'.join(lines))
-
-
-##
-
-
-def download_requirements(
-        image: str,
-        requirements_dir: str,
-        requirements_txts: ta.Sequence[str],
-) -> None:
-    requirements_txt_dir = tempfile.mkdtemp()
-    with defer(lambda: shutil.rmtree(requirements_txt_dir)):
-        for rt in requirements_txts:
-            shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
-
-        subprocesses.check_call(
-            'docker',
-            'run',
-            '--rm',
-            '-i',
-            '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
-            '-v', f'{requirements_txt_dir}:/requirements_txt',
-            image,
-            'pip',
-            'download',
-            '-d', '/requirements',
-            *itertools.chain.from_iterable([
-                ['-r', f'/requirements_txt/{os.path.basename(rt)}']
-                for rt in requirements_txts
-            ]),
-        )
-
-
 ########################################
 # ../ci.py
 
 
-class Ci(ExitStacked):
+class Ci(AsyncExitStacked):
     FILE_NAME_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
@@ -3046,6 +3510,9 @@ class Ci(ExitStacked):
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
+        always_build: bool = False
+
+        no_dependencies: bool = False
 
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
@@ -3065,7 +3532,7 @@ class Ci(ExitStacked):
 
     #
 
-    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
         if self._shell_cache is None:
             return None
 
@@ -3075,9 +3542,9 @@ class Ci(ExitStacked):
 
         get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
 
-        return load_docker_tar_cmd(get_cache_cmd)
+        return await load_docker_tar_cmd(get_cache_cmd)
 
-    def _save_cache_docker_image(self, key: str, image: str) -> None:
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
         if self._shell_cache is None:
             return
 
@@ -3086,12 +3553,12 @@ class Ci(ExitStacked):
 
             put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
 
-            save_docker_tar_cmd(image, put_cache_cmd)
+            await save_docker_tar_cmd(image, put_cache_cmd)
 
     #
 
-    def _load_docker_image(self, image: str) -> None:
-        if not self._cfg.always_pull and is_docker_image_present(image):
+    async def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
 
         dep_suffix = image
@@ -3099,63 +3566,79 @@ class Ci(ExitStacked):
             dep_suffix = dep_suffix.replace(c, '-')
 
         cache_key = f'docker-{dep_suffix}'
-        if self._load_cache_docker_image(cache_key) is not None:
+        if (await self._load_cache_docker_image(cache_key)) is not None:
             return
 
-        pull_docker_image(image)
+        await pull_docker_image(image)
 
-        self._save_cache_docker_image(cache_key, image)
+        await self._save_cache_docker_image(cache_key, image)
 
-    def load_docker_image(self, image: str) -> None:
+    async def load_docker_image(self, image: str) -> None:
         with log_timing_context(f'Load docker image: {image}'):
-            self._load_docker_image(image)
+            await self._load_docker_image(image)
 
-    @cached_nullary
-    def load_compose_service_dependencies(self) -> None:
+    @async_cached_nullary
+    async def load_compose_service_dependencies(self) -> None:
         deps = get_compose_service_dependencies(
             self._cfg.compose_file,
             self._cfg.service,
         )
 
         for dep_image in deps.values():
-            self.load_docker_image(dep_image)
+            await self.load_docker_image(dep_image)
 
     #
 
-    def _resolve_ci_image(self) -> str:
-        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+
+    async def _resolve_ci_image(self) -> str:
+        cache_key = f'ci-{self.docker_file_hash()}'
+        image_tag = f'{self._cfg.service}:{cache_key}'
+
+        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
 
-        cache_key = f'ci-{docker_file_hash}'
-        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
-            return cache_image_id
+        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
 
-        image_id = build_docker_image(
+        image_id = await build_docker_image(
             self._cfg.docker_file,
+            tag=image_tag,
             cwd=self._cfg.project_dir,
         )
 
-        self._save_cache_docker_image(cache_key, image_id)
+        await self._save_cache_docker_image(cache_key, image_id)
 
-        return image_id
+        return image_tag
 
-    @cached_nullary
-    def resolve_ci_image(self) -> str:
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
         with log_timing_context('Resolve ci image') as ltc:
-            image_id = self._resolve_ci_image()
+            image_id = await self._resolve_ci_image()
             ltc.set_description(f'Resolve ci image: {image_id}')
             return image_id
 
     #
 
-    def _resolve_requirements_dir(self) -> str:
-        requirements_txts = [
+    @cached_nullary
+    def requirements_txts(self) -> ta.Sequence[str]:
+        return [
             os.path.join(self._cfg.project_dir, rf)
             for rf in check.not_none(self._cfg.requirements_txts)
         ]
 
-        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+    @cached_nullary
+    def requirements_hash(self) -> str:
+        return build_requirements_hash(self.requirements_txts())[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_key = f'requirements-{requirements_hash}'
+    async def _resolve_requirements_dir(self) -> str:
+        tar_file_key = f'requirements-{self.docker_file_hash()}-{self.requirements_hash()}'
         tar_file_name = f'{tar_file_key}.tar'
 
         temp_dir = tempfile.mkdtemp()
@@ -3171,9 +3654,9 @@ class Ci(ExitStacked):
         os.makedirs(temp_requirements_dir)
 
         download_requirements(
-            self.resolve_ci_image(),
+            await self.resolve_ci_image(),
             temp_requirements_dir,
-            requirements_txts,
+            self.requirements_txts(),
         )
 
         if self._file_cache is not None:
@@ -3190,16 +3673,16 @@ class Ci(ExitStacked):
 
         return temp_requirements_dir
 
-    @cached_nullary
-    def resolve_requirements_dir(self) -> str:
+    @async_cached_nullary
+    async def resolve_requirements_dir(self) -> str:
         with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = self._resolve_requirements_dir()
+            requirements_dir = await self._resolve_requirements_dir()
             ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
             return requirements_dir
 
     #
 
-    def _run_compose_(self) -> None:
+    async def _run_compose_(self) -> None:
         setup_cmds = [
             'pip install --root-user-action ignore --find-links /requirements --no-index uv',
             (
@@ -3217,37 +3700,39 @@ class Ci(ExitStacked):
 
         #
 
-        with DockerComposeRun(DockerComposeRun.Config(
+        async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
-            image=self.resolve_ci_image(),
+            image=await self.resolve_ci_image(),
 
             cmd=ci_cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+                '-v', f'{os.path.abspath(await self.resolve_requirements_dir())}:/requirements',
             ],
 
             cwd=self._cfg.project_dir,
+
+            no_dependencies=self._cfg.no_dependencies,
         )) as ci_compose_run:
-            ci_compose_run.run()
+            await ci_compose_run.run()
 
-    def _run_compose(self) -> None:
+    async def _run_compose(self) -> None:
         with log_timing_context('Run compose'):
-            self._run_compose_()
+            await self._run_compose_()
 
     #
 
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
+    async def run(self) -> None:
+        await self.load_compose_service_dependencies()
 
-        self.resolve_ci_image()
+        await self.resolve_ci_image()
 
-        self.resolve_requirements_dir()
+        await self.resolve_requirements_dir()
 
-        self._run_compose()
+        await self._run_compose()
 
 
 ########################################
@@ -3263,7 +3748,7 @@ class GithubCli(ArgparseCli):
         argparse_arg('key'),
     )
     def get_cache_entry(self) -> None:
-        shell_client = GithubV1CacheShellClient()
+        shell_client = GithubCacheServiceV1ShellClient()
         entry = shell_client.run_get_entry(self.args.key)
         if entry is None:
             return
@@ -3322,18 +3807,21 @@ class CiCli(ArgparseCli):
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
+
         argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
+
         argparse_arg('--always-pull', action='store_true'),
+        argparse_arg('--always-build', action='store_true'),
+
+        argparse_arg('--no-dependencies', action='store_true'),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
-        service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
-        always_pull = self.args.always_pull
 
         #
 
@@ -3358,10 +3846,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:
@@ -3397,14 +3891,14 @@ class CiCli(ArgparseCli):
 
         #
 
-        with Ci(
+        async with Ci(
                 Ci.Config(
                     project_dir=project_dir,
 
                     docker_file=docker_file,
 
                     compose_file=compose_file,
-                    service=service,
+                    service=self.args.service,
 
                     requirements_txts=requirements_txts,
 
@@ -3413,12 +3907,15 @@ class CiCli(ArgparseCli):
                         'python3 -m pytest -svv test.py',
                     ])),
 
-                    always_pull=always_pull,
+                    always_pull=self.args.always_pull,
+                    always_build=self.args.always_build,
+
+                    no_dependencies=self.args.no_dependencies,
                 ),
                 file_cache=file_cache,
                 shell_cache=shell_cache,
         ) as ci:
-            ci.run()
+            await ci.run()
 
 
 async def _async_main() -> ta.Optional[int]: