omdev 0.0.0.dev213__py3-none-any.whl → 0.0.0.dev215__py3-none-any.whl

omdev/scripts/ci.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omlish-amalg-output ../ci/cli.py
-# ruff: noqa: N802 UP006 UP007 UP036
+# ruff: noqa: N802 TC003 UP006 UP007 UP036
 """
 Inputs:
  - requirements.txt
@@ -12,7 +12,7 @@ Inputs:
 
 ==
 
-./python -m ci run --cache-dir ci/cache ci/project omlish-ci
+./python -m omdev.ci run --cache-dir omdev/ci/tests/cache omdev/ci/tests/project omlish-ci
 """
 import abc
 import argparse
@@ -25,6 +25,7 @@ import dataclasses as dc
 import datetime
 import functools
 import hashlib
+import http.client
 import inspect
 import itertools
 import json
@@ -42,6 +43,7 @@ import time
 import types
 import typing as ta
 import urllib.parse
+import urllib.request
 
 
 ########################################
@@ -57,12 +59,12 @@ if sys.version_info < (3, 8):
 # shell.py
 T = ta.TypeVar('T')
 
+# ../../omlish/asyncs/asyncio/asyncio.py
+CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
+
 # ../../omlish/asyncs/asyncio/timeouts.py
 AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
 
-# ../../omlish/lite/cached.py
-CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
-
 # ../../omlish/lite/check.py
 SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
 CheckMessage = ta.Union[str, ta.Callable[..., ta.Optional[str]], None]  # ta.TypeAlias
@@ -82,6 +84,34 @@ AsyncExitStackedT = ta.TypeVar('AsyncExitStackedT', bound='AsyncExitStacked')
 SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
 
 
+########################################
+# ../consts.py
+
+
+CI_CACHE_VERSION = 1
+
+
+########################################
+# ../github/env.py
+
+
+@dc.dataclass(frozen=True)
+class GithubEnvVar:
+    k: str
+
+    def __call__(self) -> ta.Optional[str]:
+        return os.environ.get(self.k)
+
+
+GITHUB_ENV_VARS: ta.Set[GithubEnvVar] = set()
+
+
+def register_github_env_var(k: str) -> GithubEnvVar:
+    ev = GithubEnvVar(k)
+    GITHUB_ENV_VARS.add(ev)
+    return ev
+
+
 ########################################
 # ../shell.py
 
@@ -120,6 +150,71 @@ class ShellCmd:
         )
 
 
+########################################
+# ../../../omlish/asyncs/asyncio/asyncio.py
+
+
+def asyncio_once(fn: CallableT) -> CallableT:
+    future = None
+
+    @functools.wraps(fn)
+    async def inner(*args, **kwargs):
+        nonlocal future
+        if not future:
+            future = asyncio.create_task(fn(*args, **kwargs))
+        return await future
+
+    return ta.cast(CallableT, inner)
+
+
+def drain_tasks(loop=None):
+    if loop is None:
+        loop = asyncio.get_running_loop()
+
+    while loop._ready or loop._scheduled:  # noqa
+        loop._run_once()  # noqa
+
+
+@contextlib.contextmanager
+def draining_asyncio_tasks() -> ta.Iterator[None]:
+    loop = asyncio.get_running_loop()
+    try:
+        yield
+    finally:
+        if loop is not None:
+            drain_tasks(loop)  # noqa
+
+
+async def asyncio_wait_concurrent(
+        coros: ta.Iterable[ta.Awaitable[T]],
+        concurrency: ta.Union[int, asyncio.Semaphore],
+        *,
+        return_when: ta.Any = asyncio.FIRST_EXCEPTION,
+) -> ta.List[T]:
+    if isinstance(concurrency, asyncio.Semaphore):
+        semaphore = concurrency
+    elif isinstance(concurrency, int):
+        semaphore = asyncio.Semaphore(concurrency)
+    else:
+        raise TypeError(concurrency)
+
+    async def limited_task(coro):
+        async with semaphore:
+            return await coro
+
+    tasks = [asyncio.create_task(limited_task(coro)) for coro in coros]
+    done, pending = await asyncio.wait(tasks, return_when=return_when)
+
+    for task in pending:
+        task.cancel()
+
+    for task in done:
+        if task.exception():
+            raise task.exception()  # type: ignore
+
+    return [task.result() for task in done]
+
+
 ########################################
 # ../../../omlish/asyncs/asyncio/timeouts.py
 
@@ -999,170 +1094,183 @@ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
 
 
 ########################################
-# ../cache.py
+# ../../../omlish/os/files.py
 
 
-##
+def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
+    if exist_ok:
+        # First try to bump modification time
+        # Implementation note: GNU touch uses the UTIME_NOW option of the utimensat() / futimens() functions.
+        try:
+            os.utime(self, None)
+        except OSError:
+            pass
+        else:
+            return
+    flags = os.O_CREAT | os.O_WRONLY
+    if not exist_ok:
+        flags |= os.O_EXCL
+    fd = os.open(self, flags, mode)
+    os.close(fd)
 
 
-@abc.abstractmethod
-class FileCache(abc.ABC):
-    @abc.abstractmethod
-    def get_file(self, key: str) -> ta.Optional[str]:
-        raise NotImplementedError
+def unlink_if_exists(path: str) -> None:
+    try:
+        os.unlink(path)
+    except FileNotFoundError:
+        pass
 
-    @abc.abstractmethod
-    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
-        raise NotImplementedError
 
+@contextlib.contextmanager
+def unlinking_if_exists(path: str) -> ta.Iterator[None]:
+    try:
+        yield
+    finally:
+        unlink_if_exists(path)
 
-#
 
+########################################
+# ../cache.py
 
-class DirectoryFileCache(FileCache):
-    def __init__(self, dir: str) -> None:  # noqa
-        super().__init__()
 
-        self._dir = dir
+##
 
-    #
 
-    def get_cache_file_path(
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    def __init__(
             self,
-            key: str,
             *,
-            make_dirs: bool = False,
-    ) -> str:
-        if make_dirs:
-            os.makedirs(self._dir, exist_ok=True)
-        return os.path.join(self._dir, key)
-
-    def format_incomplete_file(self, f: str) -> str:
-        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
-
-    #
-
-    def get_file(self, key: str) -> ta.Optional[str]:
-        cache_file_path = self.get_cache_file_path(key)
-        if not os.path.exists(cache_file_path):
-            return None
-        return cache_file_path
-
-    def put_file(self, key: str, file_path: str) -> None:
-        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
-        shutil.copyfile(file_path, cache_file_path)
+            version: int = CI_CACHE_VERSION,
+    ) -> None:
+        super().__init__()
 
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version = version
 
-##
+    @property
+    def version(self) -> int:
+        return self._version
 
+    #
 
-class ShellCache(abc.ABC):
     @abc.abstractmethod
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
         raise NotImplementedError
 
-    class PutFileCmdContext(abc.ABC):
-        def __init__(self) -> None:
-            super().__init__()
-
-            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
-
-        @property
-        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
-            return self._state
-
-        #
+    @abc.abstractmethod
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
 
-        @property
-        @abc.abstractmethod
-        def cmd(self) -> ShellCmd:
-            raise NotImplementedError
 
-        #
+#
 
-        def __enter__(self):
-            return self
 
-        def __exit__(self, exc_type, exc_val, exc_tb):
-            if exc_val is None:
-                self.commit()
-            else:
-                self.abort()
+class DirectoryFileCache(FileCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            no_create: bool = False,
+            no_purge: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:  # noqa
+        super().__init__(**kwargs)
 
-        #
+        self._dir = dir
+        self._no_create = no_create
+        self._no_purge = no_purge
 
-        @abc.abstractmethod
-        def _commit(self) -> None:
-            raise NotImplementedError
+    #
 
-        def commit(self) -> None:
-            if self._state == 'committed':
-                return
-            elif self._state == 'open':
-                self._commit()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
+    VERSION_FILE_NAME = '.ci-cache-version'
 
-        #
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
 
-        @abc.abstractmethod
-        def _abort(self) -> None:
-            raise NotImplementedError
+        if self._no_create:
+            check.state(os.path.isdir(self._dir))
 
-        def abort(self) -> None:
-            if self._state == 'aborted':
-                return
-            elif self._state == 'open':
-                self._abort()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
+        elif not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
 
-    @abc.abstractmethod
-    def put_file_cmd(self, key: str) -> PutFileCmdContext:
-        raise NotImplementedError
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
 
+        if dir_version == self._version:
+            return
 
-#
+        if self._no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
 
+        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
 
-class DirectoryShellCache(ShellCache):
-    def __init__(self, dfc: DirectoryFileCache) -> None:
-        super().__init__()
+        for n in sorted(os.listdir(self._dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self._dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
 
-        self._dfc = dfc
+        os.unlink(version_file)
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        f = self._dfc.get_file(key)
-        if f is None:
-            return None
-        return ShellCmd(f'cat {shlex.quote(f)}')
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
 
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(self, tf: str, f: str) -> None:
-            super().__init__()
+    #
 
-            self._tf = tf
-            self._f = f
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self._dir, key)
 
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
 
-        def _commit(self) -> None:
-            os.replace(self._tf, self._f)
+    #
 
-        def _abort(self) -> None:
-            os.unlink(self._tf)
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
 
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        f = self._dfc.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path
 
 
 ########################################
-# ../github/cacheapi.py
+# ../github/api.py
 """
 export FILE_SIZE=$(stat --format="%s" $FILE)
 
@@ -1363,16 +1471,27 @@ class GithubCacheServiceV2:
 
 
 ########################################
-# ../utils.py
+# ../github/bootstrap.py
+"""
+sudo rm -rf \
+    /usr/local/.ghcup \
+    /opt/hostedtoolcache \
 
+/usr/local/.ghcup       6.4G, 3391250 files
+/opt/hostedtoolcache    8.0G, 14843980 files
+/usr/local/lib/android  6.4G, 17251667 files
+"""
 
-##
 
+GITHUB_ACTIONS_ENV_VAR = register_github_env_var('GITHUB_ACTIONS')
+
+
+def is_in_github_actions() -> bool:
+    return GITHUB_ACTIONS_ENV_VAR() is not None
 
-def make_temp_file() -> str:
-    file_fd, file = tempfile.mkstemp()
-    os.close(file_fd)
-    return file
+
+########################################
+# ../utils.py
 
 
 ##
@@ -1421,7 +1540,7 @@ class LogTimingContext:
     def __enter__(self) -> 'LogTimingContext':
         self._begin_time = time.time()
 
-        self._log.log(self._level, f'Begin {self._description}')  # noqa
+        self._log.log(self._level, f'Begin : {self._description}')  # noqa
 
         return self
 
@@ -1430,7 +1549,7 @@ class LogTimingContext:
 
         self._log.log(
             self._level,
-            f'End {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
+            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
         )
 
 
@@ -1908,92 +2027,613 @@ class JsonLogFormatter(logging.Formatter):
 
 
 ########################################
-# ../../../omlish/logs/standard.py
-"""
-TODO:
- - structured
- - prefixed
- - debug
- - optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
-"""
+# ../../../omlish/os/temp.py
 
 
-##
+def make_temp_file(**kwargs: ta.Any) -> str:
+    file_fd, file = tempfile.mkstemp(**kwargs)
+    os.close(file_fd)
+    return file
 
 
-STANDARD_LOG_FORMAT_PARTS = [
-    ('asctime', '%(asctime)-15s'),
-    ('process', 'pid=%(process)-6s'),
-    ('thread', 'tid=%(thread)x'),
-    ('levelname', '%(levelname)s'),
-    ('name', '%(name)s'),
-    ('separator', '::'),
-    ('message', '%(message)s'),
-]
+@contextlib.contextmanager
+def temp_file_context(**kwargs: ta.Any) -> ta.Iterator[str]:
+    path = make_temp_file(**kwargs)
+    try:
+        yield path
+    finally:
+        unlink_if_exists(path)
 
 
-class StandardLogFormatter(logging.Formatter):
-    @staticmethod
-    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
-        return ' '.join(v for k, v in parts)
+@contextlib.contextmanager
+def temp_dir_context(
+        root_dir: ta.Optional[str] = None,
+        **kwargs: ta.Any,
+) -> ta.Iterator[str]:
+    path = tempfile.mkdtemp(dir=root_dir, **kwargs)
+    try:
+        yield path
+    finally:
+        shutil.rmtree(path, ignore_errors=True)
 
-    converter = datetime.datetime.fromtimestamp  # type: ignore
 
-    def formatTime(self, record, datefmt=None):
-        ct = self.converter(record.created)  # type: ignore
-        if datefmt:
-            return ct.strftime(datefmt)  # noqa
-        else:
-            t = ct.strftime('%Y-%m-%d %H:%M:%S')
-            return '%s.%03d' % (t, record.msecs)  # noqa
+@contextlib.contextmanager
+def temp_named_file_context(
+        root_dir: ta.Optional[str] = None,
+        cleanup: bool = True,
+        **kwargs: ta.Any,
+) -> ta.Iterator[tempfile._TemporaryFileWrapper]:  # noqa
+    with tempfile.NamedTemporaryFile(dir=root_dir, delete=False, **kwargs) as f:
+        try:
+            yield f
+        finally:
+            if cleanup:
+                shutil.rmtree(f.name, ignore_errors=True)
+
+
+########################################
+# ../github/client.py
 
 
 ##
 
 
-class StandardConfiguredLogHandler(ProxyLogHandler):
-    def __init_subclass__(cls, **kwargs):
-        raise TypeError('This class serves only as a marker and should not be subclassed.')
+class GithubCacheClient(abc.ABC):
+    class Entry(abc.ABC):  # noqa
+        pass
+
+    @abc.abstractmethod
+    def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def upload_file(self, key: str, in_file: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
 
 
 ##
 
 
-@contextlib.contextmanager
-def _locking_logging_module_lock() -> ta.Iterator[None]:
-    if hasattr(logging, '_acquireLock'):
-        logging._acquireLock()  # noqa
-        try:
-            yield
-        finally:
-            logging._releaseLock()  # type: ignore  # noqa
+class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
+    BASE_URL_ENV_VAR = register_github_env_var('ACTIONS_CACHE_URL')
+    AUTH_TOKEN_ENV_VAR = register_github_env_var('ACTIONS_RUNTIME_TOKEN')  # noqa
 
-    elif hasattr(logging, '_lock'):
-        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
-        with logging._lock:  # noqa
-            yield
+    KEY_SUFFIX_ENV_VAR = register_github_env_var('GITHUB_RUN_ID')
 
-    else:
-        raise Exception("Can't find lock in logging module")
+    #
 
+    def __init__(
+            self,
+            *,
+            base_url: ta.Optional[str] = None,
+            auth_token: ta.Optional[str] = None,
 
-def configure_standard_logging(
-        level: ta.Union[int, str] = logging.INFO,
-        *,
-        json: bool = False,
-        target: ta.Optional[logging.Logger] = None,
-        force: bool = False,
-        handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
-) -> ta.Optional[StandardConfiguredLogHandler]:
-    with _locking_logging_module_lock():
-        if target is None:
-            target = logging.root
+            key_prefix: ta.Optional[str] = None,
+            key_suffix: ta.Optional[str] = None,
 
-        #
+            cache_version: int = CI_CACHE_VERSION,
 
-        if not force:
-            if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
-                return None
+            loop: ta.Optional[asyncio.AbstractEventLoop] = None,
+    ) -> None:
+        super().__init__()
+
+        #
+
+        if base_url is None:
+            base_url = check.non_empty_str(self.BASE_URL_ENV_VAR())
+        self._service_url = GithubCacheServiceV1.get_service_url(base_url)
+
+        if auth_token is None:
+            auth_token = self.AUTH_TOKEN_ENV_VAR()
+        self._auth_token = auth_token
+
+        #
+
+        self._key_prefix = key_prefix
+
+        if key_suffix is None:
+            key_suffix = self.KEY_SUFFIX_ENV_VAR()
+        self._key_suffix = check.non_empty_str(key_suffix)
+
+        #
+
+        self._cache_version = check.isinstance(cache_version, int)
+
+        #
+
+        self._given_loop = loop
+
+    #
+
+    def _get_loop(self) -> asyncio.AbstractEventLoop:
+        if (loop := self._given_loop) is not None:
+            return loop
+        return asyncio.get_event_loop()
+
+    #
+
+    def build_request_headers(
+            self,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            *,
+            content_type: ta.Optional[str] = None,
+            json_content: bool = False,
+    ) -> ta.Dict[str, str]:
+        dct = {
+            'Accept': ';'.join([
+                'application/json',
+                f'api-version={GithubCacheServiceV1.API_VERSION}',
+            ]),
+        }
+
+        if (auth_token := self._auth_token):
+            dct['Authorization'] = f'Bearer {auth_token}'
+
+        if content_type is None and json_content:
+            content_type = 'application/json'
+        if content_type is not None:
+            dct['Content-Type'] = content_type
+
+        if headers:
+            dct.update(headers)
+
+        return dct
+
+    #
+
+    def load_json_bytes(self, b: ta.Optional[bytes]) -> ta.Optional[ta.Any]:
+        if not b:
+            return None
+        return json.loads(b.decode('utf-8-sig'))
+
+    #
+
+    async def send_url_request(
+            self,
+            req: urllib.request.Request,
+    ) -> ta.Tuple[http.client.HTTPResponse, ta.Optional[bytes]]:
+        def run_sync():
+            with urllib.request.urlopen(req) as resp:  # noqa
+                body = resp.read()
+            return (resp, body)
+
+        return await self._get_loop().run_in_executor(None, run_sync)  # noqa
+
+    #
+
+    @dc.dataclass()
+    class ServiceRequestError(RuntimeError):
+        status_code: int
+        body: ta.Optional[bytes]
+
+        def __str__(self) -> str:
+            return repr(self)
+
+    async def send_service_request(
+            self,
+            path: str,
+            *,
+            method: ta.Optional[str] = None,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            content_type: ta.Optional[str] = None,
+            content: ta.Optional[bytes] = None,
+            json_content: ta.Optional[ta.Any] = None,
+            success_status_codes: ta.Optional[ta.Container[int]] = None,
+    ) -> ta.Optional[ta.Any]:
+        url = f'{self._service_url}/{path}'
+
+        if content is not None and json_content is not None:
+            raise RuntimeError('Must not pass both content and json_content')
+        elif json_content is not None:
+            content = json_dumps_compact(json_content).encode('utf-8')
+            header_json_content = True
+        else:
+            header_json_content = False
+
+        if method is None:
+            method = 'POST' if content is not None else 'GET'
+
+        #
+
+        req = urllib.request.Request(  # noqa
+            url,
+            method=method,
+            headers=self.build_request_headers(
+                headers,
+                content_type=content_type,
+                json_content=header_json_content,
+            ),
+            data=content,
+        )
+
+        resp, body = await self.send_url_request(req)
+
+        #
+
+        if success_status_codes is not None:
+            is_success = resp.status in success_status_codes
+        else:
+            is_success = (200 <= resp.status <= 300)
+        if not is_success:
+            raise self.ServiceRequestError(resp.status, body)
+
+        return self.load_json_bytes(body)
+
+    #
+
+    KEY_PART_SEPARATOR = '--'
+
+    def fix_key(self, s: str, partial_suffix: bool = False) -> str:
+        return self.KEY_PART_SEPARATOR.join([
+            *([self._key_prefix] if self._key_prefix else []),
+            s,
+            ('' if partial_suffix else self._key_suffix),
+        ])
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Entry(GithubCacheClient.Entry):
+        artifact: GithubCacheServiceV1.ArtifactCacheEntry
+
+    #
+
+    def build_get_entry_url_path(self, *keys: str) -> str:
+        qp = dict(
+            keys=','.join(urllib.parse.quote_plus(k) for k in keys),
+            version=str(self._cache_version),
+        )
+
+        return '?'.join([
+            'cache',
+            '&'.join([
+                f'{k}={v}'
+                for k, v in qp.items()
+            ]),
+        ])
+
+    GET_ENTRY_SUCCESS_STATUS_CODES = (200, 204)
+
+
+##
+
+
+class GithubCacheServiceV1Client(GithubCacheServiceV1BaseClient):
+    DEFAULT_CONCURRENCY = 4
+
+    DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024
+
+    def __init__(
+            self,
+            *,
+            concurrency: int = DEFAULT_CONCURRENCY,
+            chunk_size: int = DEFAULT_CHUNK_SIZE,
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        check.arg(concurrency > 0)
+        self._concurrency = concurrency
+
+        check.arg(chunk_size > 0)
+        self._chunk_size = chunk_size
+
+    #
+
+    async def get_entry(self, key: str) -> ta.Optional[GithubCacheServiceV1BaseClient.Entry]:
+        obj = await self.send_service_request(
+            self.build_get_entry_url_path(self.fix_key(key, partial_suffix=True)),
+        )
+        if obj is None:
+            return None
+
+        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
+            GithubCacheServiceV1.ArtifactCacheEntry,
+            obj,
+        ))
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class _DownloadChunk:
+        key: str
+        url: str
+        out_file: str
+        offset: int
+        size: int
+
+    async def _download_file_chunk_urllib(self, chunk: _DownloadChunk) -> None:
+        req = urllib.request.Request(  # noqa
+            chunk.url,
+            headers={
+                'Range': f'bytes={chunk.offset}-{chunk.offset + chunk.size - 1}',
+            },
+        )
+
+        _, buf_ = await self.send_url_request(req)
+
+        buf = check.not_none(buf_)
+        check.equal(len(buf), chunk.size)
+
+        #
+
+        def write_sync():
+            with open(chunk.out_file, 'r+b') as f:  # noqa
+                f.seek(chunk.offset, os.SEEK_SET)
+                f.write(buf)
+
+        await self._get_loop().run_in_executor(None, write_sync)  # noqa
+
+    # async def _download_file_chunk_curl(self, chunk: _DownloadChunk) -> None:
+    #     async with contextlib.AsyncExitStack() as es:
+    #         f = open(chunk.out_file, 'r+b')
+    #         f.seek(chunk.offset, os.SEEK_SET)
+    #
+    #         tmp_file = es.enter_context(temp_file_context())  # noqa
+    #
+    #         proc = await es.enter_async_context(asyncio_subprocesses.popen(
+    #             'curl',
+    #             '-s',
+    #             '-w', '%{json}',
+    #             '-H', f'Range: bytes={chunk.offset}-{chunk.offset + chunk.size - 1}',
+    #             chunk.url,
+    #             output=subprocess.PIPE,
+    #         ))
+    #
+    #         futs = asyncio.gather(
+    #
+    #         )
+    #
+    #         await proc.wait()
+    #
+    #         with open(tmp_file, 'r') as f:  # noqa
+    #             curl_json = tmp_file.read()
+    #
+    #     curl_res = json.loads(curl_json.decode().strip())
+    #
+    #     status_code = check.isinstance(curl_res['response_code'], int)
+    #
+    #     if not (200 <= status_code <= 300):
+    #         raise RuntimeError(f'Curl chunk download {chunk} failed: {curl_res}')
+
+    async def _download_file_chunk(self, chunk: _DownloadChunk) -> None:
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {chunk.key} '
+                f'file {chunk.out_file} '
+                f'chunk {chunk.offset} - {chunk.offset + chunk.size}',
+        ):
+            await self._download_file_chunk_urllib(chunk)
+
+    async def _download_file(self, entry: GithubCacheServiceV1BaseClient.Entry, out_file: str) -> None:
+        key = check.non_empty_str(entry.artifact.cache_key)
+        url = check.non_empty_str(entry.artifact.archive_location)
+
+        head_resp, _ = await self.send_url_request(urllib.request.Request(  # noqa
+            url,
+            method='HEAD',
+        ))
+        file_size = int(head_resp.headers['Content-Length'])
+
+        #
+
+        with open(out_file, 'xb') as f:  # noqa
+            f.truncate(file_size)
+
+        #
+
+        download_tasks = []
+        chunk_size = self._chunk_size
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            offset = i * chunk_size
+            size = min(chunk_size, file_size - offset)
+            chunk = self._DownloadChunk(
+                key,
+                url,
+                out_file,
+                offset,
+                size,
+            )
+            download_tasks.append(self._download_file_chunk(chunk))
+
+        await asyncio_wait_concurrent(download_tasks, self._concurrency)
+
+    async def download_file(self, entry: GithubCacheClient.Entry, out_file: str) -> None:
+        entry1 = check.isinstance(entry, self.Entry)
+        with log_timing_context(
+                'Downloading github cache '
+                f'key {entry1.artifact.cache_key} '
+                f'version {entry1.artifact.cache_version} '
+                f'to {out_file}',
+        ):
+            await self._download_file(entry1, out_file)
+
+    #
+
+    async def _upload_file_chunk(
+            self,
+            key: str,
+            cache_id: int,
+            in_file: str,
+            offset: int,
+            size: int,
+    ) -> None:
+        with log_timing_context(
+                f'Uploading github cache {key} '
+                f'file {in_file} '
+                f'chunk {offset} - {offset + size}',
+        ):
+            with open(in_file, 'rb') as f:  # noqa
+                f.seek(offset)
+                buf = f.read(size)
+
+            check.equal(len(buf), size)
+
+            await self.send_service_request(
+                f'caches/{cache_id}',
+                method='PATCH',
+                content_type='application/octet-stream',
+                headers={
+                    'Content-Range': f'bytes {offset}-{offset + size - 1}/*',
+                },
+                content=buf,
+                success_status_codes=[204],
+            )
+
+    async def _upload_file(self, key: str, in_file: str) -> None:
+        fixed_key = self.fix_key(key)
+
+        check.state(os.path.isfile(in_file))
+
+        file_size = os.stat(in_file).st_size
+
+        #
+
+        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
+            key=fixed_key,
+            cache_size=file_size,
+            version=str(self._cache_version),
+        )
+        reserve_resp_obj = await self.send_service_request(
+            'caches',
+            json_content=GithubCacheServiceV1.dataclass_to_json(reserve_req),
+            success_status_codes=[201],
+        )
+        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
+            GithubCacheServiceV1.ReserveCacheResponse,
+            reserve_resp_obj,
+        )
+        cache_id = check.isinstance(reserve_resp.cache_id, int)
+
+        log.debug(f'Github cache file {os.path.basename(in_file)} got id {cache_id}')  # noqa
+
+        #
+
+        upload_tasks = []
+        chunk_size = self._chunk_size
+        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
+            offset = i * chunk_size
+            size = min(chunk_size, file_size - offset)
+            upload_tasks.append(self._upload_file_chunk(
+                fixed_key,
+                cache_id,
+                in_file,
+                offset,
+                size,
+            ))
+
+        await asyncio_wait_concurrent(upload_tasks, self._concurrency)
+
+        #
+
+        commit_req = GithubCacheServiceV1.CommitCacheRequest(
+            size=file_size,
+        )
+        await self.send_service_request(
+            f'caches/{cache_id}',
+            json_content=GithubCacheServiceV1.dataclass_to_json(commit_req),
+            success_status_codes=[204],
+        )
+
+    async def upload_file(self, key: str, in_file: str) -> None:
+        with log_timing_context(
+                f'Uploading github cache file {os.path.basename(in_file)} '
+                f'key {key}',
+        ):
+            await self._upload_file(key, in_file)
+
+
+########################################
+# ../../../omlish/logs/standard.py
+"""
+TODO:
+ - structured
+ - prefixed
+ - debug
+ - optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
+"""
+
+
+##
+
+
+STANDARD_LOG_FORMAT_PARTS = [
+    ('asctime', '%(asctime)-15s'),
+    ('process', 'pid=%(process)-6s'),
+    ('thread', 'tid=%(thread)x'),
+    ('levelname', '%(levelname)s'),
+    ('name', '%(name)s'),
+    ('separator', '::'),
+    ('message', '%(message)s'),
+]
+
+
+class StandardLogFormatter(logging.Formatter):
+    @staticmethod
+    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
+        return ' '.join(v for k, v in parts)
+
+    converter = datetime.datetime.fromtimestamp  # type: ignore
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            return ct.strftime(datefmt)  # noqa
+        else:
+            t = ct.strftime('%Y-%m-%d %H:%M:%S')
+            return '%s.%03d' % (t, record.msecs)  # noqa
+
+
+##
+
+
+class StandardConfiguredLogHandler(ProxyLogHandler):
+    def __init_subclass__(cls, **kwargs):
+        raise TypeError('This class serves only as a marker and should not be subclassed.')
+
+
+##
+
+
+@contextlib.contextmanager
+def _locking_logging_module_lock() -> ta.Iterator[None]:
+    if hasattr(logging, '_acquireLock'):
+        logging._acquireLock()  # noqa
+        try:
+            yield
+        finally:
+            logging._releaseLock()  # type: ignore  # noqa
+
+    elif hasattr(logging, '_lock'):
+        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
+        with logging._lock:  # noqa
+            yield
+
+    else:
+        raise Exception("Can't find lock in logging module")
+
+
+def configure_standard_logging(
+        level: ta.Union[int, str] = logging.INFO,
+        *,
+        json: bool = False,
+        target: ta.Optional[logging.Logger] = None,
+        force: bool = False,
+        handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
+) -> ta.Optional[StandardConfiguredLogHandler]:
+    with _locking_logging_module_lock():
+        if target is None:
+            target = logging.root
+
+        #
+
+        if not force:
+            if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
+                return None
 
         #
 
@@ -2358,201 +2998,97 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
 
 
 ########################################
-# ../github/curl.py
+# ../github/cache.py
 
 
 ##
 
 
-class GithubServiceCurlClient:
+class GithubFileCache(FileCache):
     def __init__(
             self,
-            service_url: str,
-            auth_token: ta.Optional[str] = None,
+            dir: str,  # noqa
             *,
-            api_version: ta.Optional[str] = None,
+            client: ta.Optional[GithubCacheClient] = None,
+            **kwargs: ta.Any,
     ) -> None:
-        super().__init__()
-
-        self._service_url = check.non_empty_str(service_url)
-        self._auth_token = auth_token
-        self._api_version = api_version
-
-    #
-
-    _MISSING = object()
-
-    def build_headers(
-            self,
-            headers: ta.Optional[ta.Mapping[str, str]] = None,
-            *,
-            auth_token: ta.Any = _MISSING,
-            content_type: ta.Optional[str] = None,
-    ) -> ta.Dict[str, str]:
-        dct = {
-            'Accept': ';'.join([
-                'application/json',
-                *([f'api-version={self._api_version}'] if self._api_version else []),
-            ]),
-        }
-
-        if auth_token is self._MISSING:
-            auth_token = self._auth_token
-        if auth_token:
-            dct['Authorization'] = f'Bearer {auth_token}'
-
-        if content_type is not None:
-            dct['Content-Type'] = content_type
-
-        if headers:
-            dct.update(headers)
-
-        return dct
-
-    #
-
-    HEADER_AUTH_TOKEN_ENV_KEY_PREFIX = '_GITHUB_SERVICE_AUTH_TOKEN'  # noqa
-
-    @property
-    def header_auth_token_env_key(self) -> str:
-        return f'{self.HEADER_AUTH_TOKEN_ENV_KEY_PREFIX}_{id(self)}'
-
-    def build_cmd(
-            self,
-            method: str,
-            url: str,
-            *,
-            json_content: bool = False,
-            content_type: ta.Optional[str] = None,
-            headers: ta.Optional[ta.Dict[str, str]] = None,
-    ) -> ShellCmd:
-        if content_type is None and json_content:
-            content_type = 'application/json'
-
-        env = {}
+        super().__init__(**kwargs)
 
-        header_auth_token: ta.Optional[str]
-        if self._auth_token:
-            header_env_key = self.header_auth_token_env_key
-            env[header_env_key] = self._auth_token
-            header_auth_token = f'${header_env_key}'
-        else:
-            header_auth_token = None
-
-        built_hdrs = self.build_headers(
-            headers,
-            auth_token=header_auth_token,
-            content_type=content_type,
-        )
-
-        url = f'{self._service_url}/{url}'
-
-        cmd = ' '.join([
-            'curl',
-            '-s',
-            '-X', method,
-            url,
-            *[f'-H "{k}: {v}"' for k, v in built_hdrs.items()],
-        ])
+        self._dir = check.not_none(dir)
 
-        return ShellCmd(
-            cmd,
-            env=env,
-        )
+        if client is None:
+            client = GithubCacheServiceV1Client(
+                cache_version=self._version,
+            )
+        self._client: GithubCacheClient = client
 
-    def build_post_json_cmd(
-            self,
-            url: str,
-            obj: ta.Any,
-            **kwargs: ta.Any,
-    ) -> ShellCmd:
-        curl_cmd = self.build_cmd(
-            'POST',
-            url,
-            json_content=True,
-            **kwargs,
+        self._local = DirectoryFileCache(
+            self._dir,
+            version=self._version,
         )
 
-        obj_json = json_dumps_compact(obj)
-
-        return dc.replace(curl_cmd, s=f'{curl_cmd.s} -d {shlex.quote(obj_json)}')
-
-    #
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return local_file
 
-    @dc.dataclass()
-    class Error(RuntimeError):
-        status_code: int
-        body: ta.Optional[bytes]
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
 
-        def __str__(self) -> str:
-            return repr(self)
+        tmp_file = self._local.format_incomplete_file(local_file)
+        with unlinking_if_exists(tmp_file):
+            await self._client.download_file(entry, tmp_file)
 
-    @dc.dataclass(frozen=True)
-    class Result:
-        status_code: int
-        body: ta.Optional[bytes]
+            os.replace(tmp_file, local_file)
 
-        def as_error(self) -> 'GithubServiceCurlClient.Error':
-            return GithubServiceCurlClient.Error(
-                status_code=self.status_code,
-                body=self.body,
-            )
+        return local_file
 
-    def run_cmd(
+    async def put_file(
             self,
-            cmd: ShellCmd,
+            key: str,
+            file_path: str,
             *,
-            raise_: bool = False,
-            **subprocess_kwargs: ta.Any,
-    ) -> Result:
-        out_file = make_temp_file()
-        with defer(lambda: os.unlink(out_file)):
-            run_cmd = dc.replace(cmd, s=f"{cmd.s} -o {out_file} -w '%{{json}}'")
-
-            out_json_bytes = run_cmd.run(
-                subprocesses.check_output,
-                **subprocess_kwargs,
-            )
-
-            out_json = json.loads(out_json_bytes.decode())
-            status_code = check.isinstance(out_json['response_code'], int)
-
-            with open(out_file, 'rb') as f:
-                body = f.read()
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = await self._local.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )
 
-            result = self.Result(
-                status_code=status_code,
-                body=body,
-            )
+        await self._client.upload_file(key, cache_file_path)
 
-        if raise_ and (500 <= status_code <= 600):
-            raise result.as_error()
+        return cache_file_path
 
-        return result
 
-    def run_json_cmd(
-            self,
-            cmd: ShellCmd,
-            *,
-            success_status_codes: ta.Optional[ta.Container[int]] = None,
-    ) -> ta.Optional[ta.Any]:
-        result = self.run_cmd(cmd, raise_=True)
+########################################
+# ../github/cli.py
+"""
+See:
+ - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
+"""
 
-        if success_status_codes is not None:
-            is_success = result.status_code in success_status_codes
-        else:
-            is_success = 200 <= result.status_code < 300
 
-        if is_success:
-            if not (body := result.body):
-                return None
-            return json.loads(body.decode('utf-8-sig'))
+class GithubCli(ArgparseCli):
+    @argparse_cmd()
+    def list_referenced_env_vars(self) -> None:
+        print('\n'.join(sorted(ev.k for ev in GITHUB_ENV_VARS)))
 
-        elif result.status_code == 404:
-            return None
+    @argparse_cmd(
+        argparse_arg('key'),
+    )
+    async def get_cache_entry(self) -> None:
+        client = GithubCacheServiceV1Client()
+        entry = await client.get_entry(self.args.key)
+        if entry is None:
+            return
+        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
 
-        else:
-            raise result.as_error()
+    @argparse_cmd(
+        argparse_arg('repository-id'),
+    )
+    def list_cache_entries(self) -> None:
+        raise NotImplementedError
 
 
 ########################################
@@ -2937,11 +3473,6 @@ class DockerComposeRun(AsyncExitStacked):
             if k in out_service:
                 del out_service[k]
 
-        out_service['links'] = [
-            f'{l}:{l}' if ':' not in l else l
-            for l in out_service.get('links', [])
-        ]
-
         #
 
         if not self._cfg.no_dependencies:
@@ -2958,7 +3489,6 @@ class DockerComposeRun(AsyncExitStacked):
 
         else:
             out_service['depends_on'] = []
-            out_service['links'] = []
 
         #
 
@@ -3049,444 +3579,134 @@ def read_docker_tar_image_tag(tar_file: str) -> str:
         with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
             m = mf.read()
 
-    manifests = json.loads(m.decode('utf-8'))
-    manifest = check.single(manifests)
-    tag = check.non_empty_str(check.single(manifest['RepoTags']))
-    return tag
-
-
-def read_docker_tar_image_id(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
-            i = mf.read()
-
-    index = json.loads(i.decode('utf-8'))
-    manifest = check.single(index['manifests'])
-    image_id = check.non_empty_str(manifest['digest'])
-    return image_id
-
-
-##
-
-
-async def is_docker_image_present(image: str) -> bool:
-    out = await asyncio_subprocesses.check_output(
-        'docker',
-        'images',
-        '--format', 'json',
-        image,
-    )
-
-    out_s = out.decode('utf-8').strip()
-    if not out_s:
-        return False
-
-    json.loads(out_s)  # noqa
-    return True
-
-
-async def pull_docker_image(
-        image: str,
-) -> None:
-    await asyncio_subprocesses.check_call(
-        'docker',
-        'pull',
-        image,
-    )
-
-
-async def build_docker_image(
-        docker_file: str,
-        *,
-        tag: ta.Optional[str] = None,
-        cwd: ta.Optional[str] = None,
-) -> str:
-    id_file = make_temp_file()
-    with defer(lambda: os.unlink(id_file)):
-        await asyncio_subprocesses.check_call(
-            'docker',
-            'build',
-            '-f', os.path.abspath(docker_file),
-            '--iidfile', id_file,
-            '--squash',
-            *(['--tag', tag] if tag is not None else []),
-            '.',
-            **(dict(cwd=cwd) if cwd is not None else {}),
-        )
-
-        with open(id_file) as f:  # noqa
-            image_id = check.single(f.read().strip().splitlines()).strip()
-
-    return image_id
-
-
-async def tag_docker_image(image: str, tag: str) -> None:
-    await asyncio_subprocesses.check_call(
-        'docker',
-        'tag',
-        image,
-        tag,
-    )
-
-
-async def delete_docker_tag(tag: str) -> None:
-    await asyncio_subprocesses.check_call(
-        'docker',
-        'rmi',
-        tag,
-    )
-
-
-##
-
-
-async def save_docker_tar_cmd(
-        image: str,
-        output_cmd: ShellCmd,
-) -> None:
-    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
-    await cmd.run(asyncio_subprocesses.check_call)
-
-
-async def save_docker_tar(
-        image: str,
-        tar_file: str,
-) -> None:
-    return await save_docker_tar_cmd(
-        image,
-        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
-    )
-
-
-#
-
-
-async def load_docker_tar_cmd(
-        input_cmd: ShellCmd,
-) -> str:
-    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
-
-    out = (await cmd.run(asyncio_subprocesses.check_output)).decode()
-
-    line = check.single(out.strip().splitlines())
-    loaded = line.partition(':')[2].strip()
-    return loaded
-
-
-async def load_docker_tar(
-        tar_file: str,
-) -> str:
-    return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
-
-
-########################################
-# ../github/cache.py
-
-
-##
-
-
-class GithubCacheShellClient(abc.ABC):
-    class Entry(abc.ABC):  # noqa
-        pass
-
-    @abc.abstractmethod
-    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def download_get_entry(self, entry: Entry, out_file: str) -> None:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def upload_cache_entry(self, key: str, in_file: str) -> None:
-        raise NotImplementedError
-
-
-#
-
-
-class GithubCacheServiceV1ShellClient(GithubCacheShellClient):
-    BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
-    AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN'  # noqa
-
-    KEY_SUFFIX_ENV_KEY = 'GITHUB_RUN_ID'
-
-    CACHE_VERSION: ta.ClassVar[int] = 1
-
-    #
-
-    def __init__(
-            self,
-            *,
-            base_url: ta.Optional[str] = None,
-            auth_token: ta.Optional[str] = None,
-
-            key_prefix: ta.Optional[str] = None,
-            key_suffix: ta.Optional[str] = None,
-    ) -> None:
-        super().__init__()
-
-        #
-
-        if base_url is None:
-            base_url = os.environ[self.BASE_URL_ENV_KEY]
-        service_url = GithubCacheServiceV1.get_service_url(base_url)
-
-        if auth_token is None:
-            auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
-
-        self._curl = GithubServiceCurlClient(
-            service_url,
-            auth_token,
-            api_version=GithubCacheServiceV1.API_VERSION,
-        )
-
-        #
-
-        self._key_prefix = key_prefix
-
-        if key_suffix is None:
-            key_suffix = os.environ[self.KEY_SUFFIX_ENV_KEY]
-        self._key_suffix = check.non_empty_str(key_suffix)
-
-    #
-
-    KEY_PART_SEPARATOR = '--'
-
-    def fix_key(self, s: str) -> str:
-        return self.KEY_PART_SEPARATOR.join([
-            *([self._key_prefix] if self._key_prefix else []),
-            s,
-            self._key_suffix,
-        ])
-
-    #
-
-    @dc.dataclass(frozen=True)
-    class Entry(GithubCacheShellClient.Entry):
-        artifact: GithubCacheServiceV1.ArtifactCacheEntry
-
-    #
-
-    def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
-        fixed_key = self.fix_key(key)
-
-        qp = dict(
-            keys=fixed_key,
-            version=str(self.CACHE_VERSION),
-        )
-
-        return self._curl.build_cmd(
-            'GET',
-            shlex.quote('?'.join([
-                'cache',
-                '&'.join([
-                    f'{k}={urllib.parse.quote_plus(v)}'
-                    for k, v in qp.items()
-                ]),
-            ])),
-        )
-
-    def run_get_entry(self, key: str) -> ta.Optional[Entry]:
-        fixed_key = self.fix_key(key)
-        curl_cmd = self.build_get_entry_curl_cmd(fixed_key)
-
-        obj = self._curl.run_json_cmd(
-            curl_cmd,
-            success_status_codes=[200, 204],
-        )
-        if obj is None:
-            return None
-
-        return self.Entry(GithubCacheServiceV1.dataclass_from_json(
-            GithubCacheServiceV1.ArtifactCacheEntry,
-            obj,
-        ))
-
-    #
-
-    def build_download_get_entry_cmd(self, entry: Entry, out_file: str) -> ShellCmd:
-        return ShellCmd(' '.join([
-            'aria2c',
-            '-x', '4',
-            '-o', out_file,
-            check.non_empty_str(entry.artifact.archive_location),
-        ]))
-
-    def download_get_entry(self, entry: GithubCacheShellClient.Entry, out_file: str) -> None:
-        dl_cmd = self.build_download_get_entry_cmd(
-            check.isinstance(entry, GithubCacheServiceV1ShellClient.Entry),
-            out_file,
-        )
-        dl_cmd.run(subprocesses.check_call)
-
-    #
-
-    def upload_cache_entry(self, key: str, in_file: str) -> None:
-        fixed_key = self.fix_key(key)
-
-        check.state(os.path.isfile(in_file))
-
-        file_size = os.stat(in_file).st_size
-
-        #
-
-        reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
-            key=fixed_key,
-            cache_size=file_size,
-            version=str(self.CACHE_VERSION),
-        )
-        reserve_cmd = self._curl.build_post_json_cmd(
-            'caches',
-            GithubCacheServiceV1.dataclass_to_json(reserve_req),
-        )
-        reserve_resp_obj: ta.Any = check.not_none(self._curl.run_json_cmd(
-            reserve_cmd,
-            success_status_codes=[201],
-        ))
-        reserve_resp = GithubCacheServiceV1.dataclass_from_json(  # noqa
-            GithubCacheServiceV1.ReserveCacheResponse,
-            reserve_resp_obj,
-        )
-        cache_id = check.isinstance(reserve_resp.cache_id, int)
-
-        #
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
 
-        tmp_file = make_temp_file()
 
-        print(f'{file_size=}')
-        num_written = 0
-        chunk_size = 32 * 1024 * 1024
-        for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
-            ofs = i * chunk_size
-            sz = min(chunk_size, file_size - ofs)
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
 
-            patch_cmd = self._curl.build_cmd(
-                'PATCH',
-                f'caches/{cache_id}',
-                content_type='application/octet-stream',
-                headers={
-                    'Content-Range': f'bytes {ofs}-{ofs + sz - 1}/*',
-                },
-            )
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id
 
-            #
 
-            # patch_data_cmd = dc.replace(patch_cmd, s=' | '.join([
-            #     f'dd if={in_file} bs={chunk_size} skip={i} count=1 status=none',
-            #     f'{patch_cmd.s} --data-binary -',
-            # ]))
-            # print(f'{patch_data_cmd.s=}')
-            # patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+##
 
-            #
 
-            with open(in_file, 'rb') as f:
-                f.seek(ofs)
-                buf = f.read(sz)
-            with open(tmp_file, 'wb') as f:
-                f.write(buf)
-            num_written += len(buf)
-            print(f'{num_written=}')
-            patch_data_cmd = dc.replace(patch_cmd, s=f'{patch_cmd.s} --data-binary @{tmp_file}')
-            print(f'{patch_data_cmd.s=}')
-            patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
+async def is_docker_image_present(image: str) -> bool:
+    out = await asyncio_subprocesses.check_output(
+        'docker',
+        'images',
+        '--format', 'json',
+        image,
+    )
+
+    out_s = out.decode('utf-8').strip()
+    if not out_s:
+        return False
 
-            #
+    json.loads(out_s)  # noqa
+    return True
 
-            check.equal(patch_result.status_code, 204)
-            ofs += sz
 
-        #
+async def pull_docker_image(
+        image: str,
+) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'pull',
+        image,
+    )
 
-        commit_req = GithubCacheServiceV1.CommitCacheRequest(
-            size=file_size,
-        )
-        commit_cmd = self._curl.build_post_json_cmd(
-            f'caches/{cache_id}',
-            GithubCacheServiceV1.dataclass_to_json(commit_req),
-        )
-        commit_result = self._curl.run_cmd(commit_cmd, raise_=True)
-        check.equal(commit_result.status_code, 204)
 
+async def build_docker_image(
+        docker_file: str,
+        *,
+        tag: ta.Optional[str] = None,
+        cwd: ta.Optional[str] = None,
+        run_options: ta.Optional[ta.Sequence[str]] = None,
+) -> str:
+    with temp_file_context() as id_file:
+        await asyncio_subprocesses.check_call(
+            'docker',
+            'build',
+            '-f', os.path.abspath(docker_file),
+            '--iidfile', id_file,
+            *(['--tag', tag] if tag is not None else []),
+            *(run_options or []),
+            '.',
+            **(dict(cwd=cwd) if cwd is not None else {}),
+        )
 
-##
+        with open(id_file) as f:  # noqa
+            image_id = check.single(f.read().strip().splitlines()).strip()
 
+    return image_id
 
-class GithubShellCache(ShellCache):
-    def __init__(
-            self,
-            dir: str,  # noqa
-            *,
-            client: ta.Optional[GithubCacheShellClient] = None,
-    ) -> None:
-        super().__init__()
 
-        self._dir = check.not_none(dir)
+async def tag_docker_image(image: str, tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'tag',
+        image,
+        tag,
+    )
 
-        if client is None:
-            client = GithubCacheServiceV1ShellClient()
-        self._client: GithubCacheShellClient = client
 
-        self._local = DirectoryFileCache(self._dir)
+async def delete_docker_tag(tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'rmi',
+        tag,
+    )
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        local_file = self._local.get_cache_file_path(key)
-        if os.path.exists(local_file):
-            return ShellCmd(f'cat {shlex.quote(local_file)}')
 
-        if (entry := self._client.run_get_entry(key)) is None:
-            return None
+##
 
-        tmp_file = self._local.format_incomplete_file(local_file)
-        try:
-            self._client.download_get_entry(entry, tmp_file)
 
-            os.replace(tmp_file, local_file)
+async def save_docker_tar_cmd(
+        image: str,
+        output_cmd: ShellCmd,
+) -> None:
+    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
+    await cmd.run(asyncio_subprocesses.check_call)
 
-        except BaseException:  # noqa
-            os.unlink(tmp_file)
 
-            raise
+async def save_docker_tar(
+        image: str,
+        tar_file: str,
+) -> None:
+    return await save_docker_tar_cmd(
+        image,
+        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
+    )
 
-        return ShellCmd(f'cat {shlex.quote(local_file)}')
 
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(
-                self,
-                owner: 'GithubShellCache',
-                key: str,
-                tmp_file: str,
-                local_file: str,
-        ) -> None:
-            super().__init__()
+#
 
-            self._owner = owner
-            self._key = key
-            self._tmp_file = tmp_file
-            self._local_file = local_file
 
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tmp_file)}')
+async def load_docker_tar_cmd(
+        input_cmd: ShellCmd,
+) -> str:
+    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
 
-        def _commit(self) -> None:
-            os.replace(self._tmp_file, self._local_file)
+    out = (await cmd.run(asyncio_subprocesses.check_output)).decode()
 
-            self._owner._client.upload_cache_entry(self._key, self._local_file)  # noqa
+    line = check.single(out.strip().splitlines())
+    loaded = line.partition(':')[2].strip()
+    return loaded
 
-        def _abort(self) -> None:
-            os.unlink(self._tmp_file)
 
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        local_file = self._local.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(
-            self,
-            key,
-            self._local.format_incomplete_file(local_file),
-            local_file,
-        )
+async def load_docker_tar(
+        tar_file: str,
+) -> str:
+    return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
 
 
 ########################################
@@ -3494,7 +3714,7 @@ class GithubShellCache(ShellCache):
 
 
 class Ci(AsyncExitStacked):
-    FILE_NAME_HASH_LEN = 16
+    KEY_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
     class Config:
@@ -3507,6 +3727,8 @@ class Ci(AsyncExitStacked):
 
         cmd: ShellCmd
 
+        #
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
@@ -3514,6 +3736,10 @@ class Ci(AsyncExitStacked):
 
         no_dependencies: bool = False
 
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        #
+
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
 
@@ -3521,42 +3747,15 @@ class Ci(AsyncExitStacked):
             self,
             cfg: Config,
             *,
-            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
-        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
-        if self._shell_cache is None:
-            return None
-
-        get_cache_cmd = self._shell_cache.get_file_cmd(key)
-        if get_cache_cmd is None:
-            return None
-
-        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
-
-        return await load_docker_tar_cmd(get_cache_cmd)
-
-    async def _save_cache_docker_image(self, key: str, image: str) -> None:
-        if self._shell_cache is None:
-            return
-
-        with self._shell_cache.put_file_cmd(key) as put_cache:
-            put_cache_cmd = put_cache.cmd
-
-            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
-
-            await save_docker_tar_cmd(image, put_cache_cmd)
-
-    #
-
     async def _load_docker_image(self, image: str) -> None:
         if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
@@ -3577,24 +3776,38 @@ class Ci(AsyncExitStacked):
         with log_timing_context(f'Load docker image: {image}'):
             await self._load_docker_image(image)
 
-    @async_cached_nullary
-    async def load_compose_service_dependencies(self) -> None:
-        deps = get_compose_service_dependencies(
-            self._cfg.compose_file,
-            self._cfg.service,
-        )
+    #
 
-        for dep_image in deps.values():
-            await self.load_docker_image(dep_image)
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
 
-    #
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
 
-    @cached_nullary
-    def docker_file_hash(self) -> str:
-        return build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
 
-    async def _resolve_ci_image(self) -> str:
-        cache_key = f'ci-{self.docker_file_hash()}'
+        return await load_docker_tar_cmd(get_cache_cmd)
+
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
+
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
+
+            await save_docker_tar_cmd(image, write_tmp_cmd)
+
+            await self._file_cache.put_file(key, tmp_file, steal=True)
+
+    #
+
+    async def _resolve_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
         image_tag = f'{self._cfg.service}:{cache_key}'
 
         if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
@@ -3607,21 +3820,35 @@ class Ci(AsyncExitStacked):
             )
             return image_tag
 
-        image_id = await build_docker_image(
-            self._cfg.docker_file,
-            tag=image_tag,
-            cwd=self._cfg.project_dir,
-        )
+        image_id = await build_and_tag(image_tag)
 
         await self._save_cache_docker_image(cache_key, image_id)
 
         return image_tag
 
+    #
+
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_base_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            return await build_docker_image(
+                self._cfg.docker_file,
+                tag=image_tag,
+                cwd=self._cfg.project_dir,
+            )
+
+        cache_key = f'ci-base-{self.docker_file_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
     @async_cached_nullary
-    async def resolve_ci_image(self) -> str:
-        with log_timing_context('Resolve ci image') as ltc:
-            image_id = await self._resolve_ci_image()
-            ltc.set_description(f'Resolve ci image: {image_id}')
+    async def resolve_ci_base_image(self) -> str:
+        with log_timing_context('Resolve ci base image') as ltc:
+            image_id = await self._resolve_ci_base_image()
+            ltc.set_description(f'Resolve ci base image: {image_id}')
             return image_id
 
     #
@@ -3635,82 +3862,85 @@ class Ci(AsyncExitStacked):
 
     @cached_nullary
     def requirements_hash(self) -> str:
-        return build_requirements_hash(self.requirements_txts())[:self.FILE_NAME_HASH_LEN]
-
-    async def _resolve_requirements_dir(self) -> str:
-        tar_file_key = f'requirements-{self.docker_file_hash()}-{self.requirements_hash()}'
-        tar_file_name = f'{tar_file_key}.tar'
-
-        temp_dir = tempfile.mkdtemp()
-        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
-
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
-            with tarfile.open(cache_tar_file) as tar:
-                tar.extractall(path=temp_dir)  # noqa
+        return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
 
-            return temp_dir
-
-        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
-        os.makedirs(temp_requirements_dir)
+    async def _resolve_ci_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            base_image = await self.resolve_ci_base_image()
+
+            setup_cmds = [
+                ' '.join([
+                    'pip install',
+                    '--no-cache-dir',
+                    '--root-user-action ignore',
+                    'uv',
+                ]),
+                ' '.join([
+                    'uv pip install',
+                    '--no-cache',
+                    '--index-strategy unsafe-best-match',
+                    '--system',
+                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                ]),
+            ]
+            setup_cmd = ' && '.join(setup_cmds)
 
-        download_requirements(
-            await self.resolve_ci_image(),
-            temp_requirements_dir,
-            self.requirements_txts(),
-        )
+            docker_file_lines = [
+                f'FROM {base_image}',
+                'RUN mkdir /project',
+                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                f'RUN {setup_cmd}',
+                'RUN rm /project/*',
+                'WORKDIR /project',
+            ]
 
-        if self._file_cache is not None:
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+            with temp_file_context() as docker_file:
+                with open(docker_file, 'w') as f:  # noqa
+                    f.write('\n'.join(docker_file_lines))
 
-            with tarfile.open(temp_tar_file, 'w') as tar:
-                for requirement_file in os.listdir(temp_requirements_dir):
-                    tar.add(
-                        os.path.join(temp_requirements_dir, requirement_file),
-                        arcname=requirement_file,
-                    )
+                return await build_docker_image(
+                    docker_file,
+                    tag=image_tag,
+                    cwd=self._cfg.project_dir,
+                )
 
-            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
+        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
 
-        return temp_requirements_dir
+        return await self._resolve_docker_image(cache_key, build_and_tag)
 
     @async_cached_nullary
-    async def resolve_requirements_dir(self) -> str:
-        with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = await self._resolve_requirements_dir()
-            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
-            return requirements_dir
+    async def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = await self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
 
     #
 
-    async def _run_compose_(self) -> None:
-        setup_cmds = [
-            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
-            (
-                'uv pip install --system --find-links /requirements ' +
-                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
-            ),
-        ]
-
-        #
+    @async_cached_nullary
+    async def load_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
 
-        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
-            *setup_cmds,
-            f'({self._cfg.cmd.s})',
-        ]))
+        for dep_image in deps.values():
+            await self.load_docker_image(dep_image)
 
-        #
+    #
 
+    async def _run_compose_(self) -> None:
         async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
             image=await self.resolve_ci_image(),
 
-            cmd=ci_cmd,
+            cmd=self._cfg.cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(await self.resolve_requirements_dir())}:/requirements',
+                *(self._cfg.run_options or []),
             ],
 
             cwd=self._cfg.project_dir,
@@ -3726,41 +3956,13 @@ class Ci(AsyncExitStacked):
     #
 
     async def run(self) -> None:
-        await self.load_compose_service_dependencies()
-
         await self.resolve_ci_image()
 
-        await self.resolve_requirements_dir()
+        await self.load_dependencies()
 
         await self._run_compose()
 
 
-########################################
-# ../github/cli.py
-"""
-See:
- - https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
-"""
-
-
-class GithubCli(ArgparseCli):
-    @argparse_cmd(
-        argparse_arg('key'),
-    )
-    def get_cache_entry(self) -> None:
-        shell_client = GithubCacheServiceV1ShellClient()
-        entry = shell_client.run_get_entry(self.args.key)
-        if entry is None:
-            return
-        print(json_dumps_pretty(dc.asdict(entry)))  # noqa
-
-    @argparse_cmd(
-        argparse_arg('repository-id'),
-    )
-    def list_cache_entries(self) -> None:
-        raise NotImplementedError
-
-
 ########################################
 # cli.py
 
@@ -3796,8 +3998,8 @@ class CiCli(ArgparseCli):
     @argparse_cmd(
         accepts_unknown=True,
     )
-    def github(self) -> ta.Optional[int]:
-        return GithubCli(self.unknown_args).cli_run()
+    async def github(self) -> ta.Optional[int]:
+        return await GithubCli(self.unknown_args).async_cli_run()
 
     #
 
@@ -3808,13 +4010,20 @@ class CiCli(ArgparseCli):
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
 
-        argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
 
+        argparse_arg('--github', action='store_true'),
+        argparse_arg('--github-detect', action='store_true'),
+
         argparse_arg('--always-pull', action='store_true'),
         argparse_arg('--always-build', action='store_true'),
 
         argparse_arg('--no-dependencies', action='store_true'),
+
+        argparse_arg('-e', '--env', action='append'),
+        argparse_arg('-v', '--volume', action='append'),
+
+        argparse_arg('cmd', nargs=argparse.REMAINDER),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
@@ -3825,6 +4034,11 @@ class CiCli(ArgparseCli):
 
         #
 
+        cmd = ' '.join(self.args.cmd)
+        check.non_empty_str(cmd)
+
+        #
+
         check.state(os.path.isdir(project_dir))
 
         #
@@ -3833,6 +4047,7 @@ class CiCli(ArgparseCli):
             for alt in alts:
                 alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
+                    log.debug('Using %s', alt_file)
                     return alt_file
             return None
 
@@ -3866,6 +4081,7 @@ class CiCli(ArgparseCli):
                 'requirements-ci.txt',
             ]:
                 if os.path.exists(os.path.join(project_dir, rf)):
+                    log.debug('Using %s', rf)
                     requirements_txts.append(rf)
         else:
             for rf in requirements_txts:
@@ -3873,21 +4089,34 @@ class CiCli(ArgparseCli):
 
         #
 
-        shell_cache: ta.Optional[ShellCache] = None
+        github = self.args.github
+        if not github and self.args.github_detect:
+            github = is_in_github_actions()
+            if github:
+                log.debug('Github detected')
+
+        #
+
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
-            if not os.path.exists(cache_dir):
-                os.makedirs(cache_dir)
-            check.state(os.path.isdir(cache_dir))
-
-            directory_file_cache = DirectoryFileCache(cache_dir)
+            cache_dir = os.path.abspath(cache_dir)
+            log.debug('Using cache dir %s', cache_dir)
+            if github:
+                file_cache = GithubFileCache(cache_dir)
+            else:
+                file_cache = DirectoryFileCache(cache_dir)
 
-            file_cache = directory_file_cache
+        #
 
-            if self.args.github_cache:
-                shell_cache = GithubShellCache(cache_dir)
-            else:
-                shell_cache = DirectoryShellCache(directory_file_cache)
+        run_options: ta.List[str] = []
+        for run_arg, run_arg_vals in [
+            ('-e', self.args.env or []),
+            ('-v', self.args.volume or []),
+        ]:
+            run_options.extend(itertools.chain.from_iterable(
+                [run_arg, run_arg_val]
+                for run_arg_val in run_arg_vals
+            ))
 
         #
 
@@ -3902,18 +4131,16 @@ class CiCli(ArgparseCli):
 
                     requirements_txts=requirements_txts,
 
-                    cmd=ShellCmd(' && '.join([
-                        'cd /project',
-                        'python3 -m pytest -svv test.py',
-                    ])),
+                    cmd=ShellCmd(cmd),
 
                     always_pull=self.args.always_pull,
                     always_build=self.args.always_build,
 
                     no_dependencies=self.args.no_dependencies,
+
+                    run_options=run_options,
                 ),
                 file_cache=file_cache,
-                shell_cache=shell_cache,
         ) as ci:
             await ci.run()