omdev 0.0.0.dev213__py3-none-any.whl → 0.0.0.dev215__py3-none-any.whl

omdev/.manifests.json

@@ -267,7 +267,7 @@
     "module": ".tools.docker",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/docker.py",
-    "line": 258,
+    "line": 264,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "docker",

omdev/ci/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite

omdev/ci/cache.py

@@ -1,12 +1,14 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import abc
 import os.path
-import shlex
 import shutil
 import typing as ta
 
-from .shell import ShellCmd
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.logs import log
+
+from .consts import CI_CACHE_VERSION
 
 
 ##
@@ -14,12 +16,35 @@ from .shell import ShellCmd
 
 @abc.abstractmethod
 class FileCache(abc.ABC):
+    def __init__(
+            self,
+            *,
+            version: int = CI_CACHE_VERSION,
+    ) -> None:
+        super().__init__()
+
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version = version
+
+    @property
+    def version(self) -> int:
+        return self._version
+
+    #
+
     @abc.abstractmethod
-    def get_file(self, key: str) -> ta.Optional[str]:
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
         raise NotImplementedError
 
 
@@ -27,142 +52,96 @@ class FileCache(abc.ABC):
 
 
 class DirectoryFileCache(FileCache):
-    def __init__(self, dir: str) -> None:  # noqa
-        super().__init__()
-
-        self._dir = dir
-
-    #
-
-    def get_cache_file_path(
+    def __init__(
             self,
-            key: str,
+            dir: str,  # noqa
             *,
-            make_dirs: bool = False,
-    ) -> str:
-        if make_dirs:
-            os.makedirs(self._dir, exist_ok=True)
-        return os.path.join(self._dir, key)
+            no_create: bool = False,
+            no_purge: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:  # noqa
+        super().__init__(**kwargs)
 
-    def format_incomplete_file(self, f: str) -> str:
-        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+        self._dir = dir
+        self._no_create = no_create
+        self._no_purge = no_purge
 
     #
 
-    def get_file(self, key: str) -> ta.Optional[str]:
-        cache_file_path = self.get_cache_file_path(key)
-        if not os.path.exists(cache_file_path):
-            return None
-        return cache_file_path
-
-    def put_file(self, key: str, file_path: str) -> None:
-        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
-        shutil.copyfile(file_path, cache_file_path)
-
-
-##
-
-
-class ShellCache(abc.ABC):
-    @abc.abstractmethod
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        raise NotImplementedError
-
-    class PutFileCmdContext(abc.ABC):
-        def __init__(self) -> None:
-            super().__init__()
-
-            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
-
-        @property
-        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
-            return self._state
+    VERSION_FILE_NAME = '.ci-cache-version'
 
-        #
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
 
-        @property
-        @abc.abstractmethod
-        def cmd(self) -> ShellCmd:
-            raise NotImplementedError
+        if self._no_create:
+            check.state(os.path.isdir(self._dir))
 
-        #
+        elif not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
 
-        def __enter__(self):
-            return self
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
 
-        def __exit__(self, exc_type, exc_val, exc_tb):
-            if exc_val is None:
-                self.commit()
-            else:
-                self.abort()
+        if dir_version == self._version:
+            return
 
-        #
+        if self._no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
 
-        @abc.abstractmethod
-        def _commit(self) -> None:
-            raise NotImplementedError
+        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
 
-        def commit(self) -> None:
-            if self._state == 'committed':
-                return
-            elif self._state == 'open':
-                self._commit()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
+        for n in sorted(os.listdir(self._dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self._dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
 
-        #
-
-        @abc.abstractmethod
-        def _abort(self) -> None:
-            raise NotImplementedError
-
-        def abort(self) -> None:
-            if self._state == 'aborted':
-                return
-            elif self._state == 'open':
-                self._abort()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
-
-    @abc.abstractmethod
-    def put_file_cmd(self, key: str) -> PutFileCmdContext:
-        raise NotImplementedError
+        os.unlink(version_file)
 
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
 
-#
+    #
 
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self._dir, key)
 
-class DirectoryShellCache(ShellCache):
-    def __init__(self, dfc: DirectoryFileCache) -> None:
-        super().__init__()
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
 
-        self._dfc = dfc
+    #
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        f = self._dfc.get_file(key)
-        if f is None:
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
             return None
-        return ShellCmd(f'cat {shlex.quote(f)}')
-
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(self, tf: str, f: str) -> None:
-            super().__init__()
-
-            self._tf = tf
-            self._f = f
-
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
-
-        def _commit(self) -> None:
-            os.replace(self._tf, self._f)
-
-        def _abort(self) -> None:
-            os.unlink(self._tf)
+        return cache_file_path
 
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        f = self._dfc.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path

omdev/ci/ci.py

@@ -1,20 +1,15 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import dataclasses as dc
 import os.path
-import shutil
-import tarfile
-import tempfile
 import typing as ta
 
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.contextmanagers import AsyncExitStacked
-from omlish.lite.contextmanagers import defer
+from omlish.os.temp import temp_file_context
 
 from .cache import FileCache
-from .cache import ShellCache
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
 from .docker import build_docker_file_hash
@@ -25,13 +20,12 @@ from .docker import pull_docker_image
 from .docker import save_docker_tar_cmd
 from .docker import tag_docker_image
 from .requirements import build_requirements_hash
-from .requirements import download_requirements
 from .shell import ShellCmd
 from .utils import log_timing_context
 
 
 class Ci(AsyncExitStacked):
-    FILE_NAME_HASH_LEN = 16
+    KEY_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
     class Config:
@@ -44,6 +38,8 @@ class Ci(AsyncExitStacked):
 
         cmd: ShellCmd
 
+        #
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
@@ -51,6 +47,10 @@ class Ci(AsyncExitStacked):
 
         no_dependencies: bool = False
 
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        #
+
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
 
@@ -58,42 +58,15 @@ class Ci(AsyncExitStacked):
             self,
             cfg: Config,
             *,
-            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
-        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
-        if self._shell_cache is None:
-            return None
-
-        get_cache_cmd = self._shell_cache.get_file_cmd(key)
-        if get_cache_cmd is None:
-            return None
-
-        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
-
-        return await load_docker_tar_cmd(get_cache_cmd)
-
-    async def _save_cache_docker_image(self, key: str, image: str) -> None:
-        if self._shell_cache is None:
-            return
-
-        with self._shell_cache.put_file_cmd(key) as put_cache:
-            put_cache_cmd = put_cache.cmd
-
-            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
-
-            await save_docker_tar_cmd(image, put_cache_cmd)
-
-    #
-
     async def _load_docker_image(self, image: str) -> None:
         if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
@@ -114,24 +87,38 @@ class Ci(AsyncExitStacked):
         with log_timing_context(f'Load docker image: {image}'):
             await self._load_docker_image(image)
 
-    @async_cached_nullary
-    async def load_compose_service_dependencies(self) -> None:
-        deps = get_compose_service_dependencies(
-            self._cfg.compose_file,
-            self._cfg.service,
-        )
+    #
 
-        for dep_image in deps.values():
-            await self.load_docker_image(dep_image)
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
 
-    #
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
 
-    @cached_nullary
-    def docker_file_hash(self) -> str:
-        return build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
 
-    async def _resolve_ci_image(self) -> str:
-        cache_key = f'ci-{self.docker_file_hash()}'
+        return await load_docker_tar_cmd(get_cache_cmd)
+
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
+
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
+
+            await save_docker_tar_cmd(image, write_tmp_cmd)
+
+            await self._file_cache.put_file(key, tmp_file, steal=True)
+
+    #
+
+    async def _resolve_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
         image_tag = f'{self._cfg.service}:{cache_key}'
 
         if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
@@ -144,21 +131,35 @@ class Ci(AsyncExitStacked):
             )
             return image_tag
 
-        image_id = await build_docker_image(
-            self._cfg.docker_file,
-            tag=image_tag,
-            cwd=self._cfg.project_dir,
-        )
+        image_id = await build_and_tag(image_tag)
 
         await self._save_cache_docker_image(cache_key, image_id)
 
         return image_tag
 
+    #
+
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_base_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            return await build_docker_image(
+                self._cfg.docker_file,
+                tag=image_tag,
+                cwd=self._cfg.project_dir,
+            )
+
+        cache_key = f'ci-base-{self.docker_file_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
     @async_cached_nullary
-    async def resolve_ci_image(self) -> str:
-        with log_timing_context('Resolve ci image') as ltc:
-            image_id = await self._resolve_ci_image()
-            ltc.set_description(f'Resolve ci image: {image_id}')
+    async def resolve_ci_base_image(self) -> str:
+        with log_timing_context('Resolve ci base image') as ltc:
+            image_id = await self._resolve_ci_base_image()
+            ltc.set_description(f'Resolve ci base image: {image_id}')
             return image_id
 
     #
@@ -172,82 +173,85 @@ class Ci(AsyncExitStacked):
 
     @cached_nullary
     def requirements_hash(self) -> str:
-        return build_requirements_hash(self.requirements_txts())[:self.FILE_NAME_HASH_LEN]
-
-    async def _resolve_requirements_dir(self) -> str:
-        tar_file_key = f'requirements-{self.docker_file_hash()}-{self.requirements_hash()}'
-        tar_file_name = f'{tar_file_key}.tar'
-
-        temp_dir = tempfile.mkdtemp()
-        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+        return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
 
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
-            with tarfile.open(cache_tar_file) as tar:
-                tar.extractall(path=temp_dir)  # noqa
+    async def _resolve_ci_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            base_image = await self.resolve_ci_base_image()
+
+            setup_cmds = [
+                ' '.join([
+                    'pip install',
+                    '--no-cache-dir',
+                    '--root-user-action ignore',
+                    'uv',
+                ]),
+                ' '.join([
+                    'uv pip install',
+                    '--no-cache',
+                    '--index-strategy unsafe-best-match',
+                    '--system',
+                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                ]),
+            ]
+            setup_cmd = ' && '.join(setup_cmds)
+
+            docker_file_lines = [
+                f'FROM {base_image}',
+                'RUN mkdir /project',
+                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                f'RUN {setup_cmd}',
+                'RUN rm /project/*',
+                'WORKDIR /project',
+            ]
+
+            with temp_file_context() as docker_file:
+                with open(docker_file, 'w') as f:  # noqa
+                    f.write('\n'.join(docker_file_lines))
+
+                return await build_docker_image(
+                    docker_file,
+                    tag=image_tag,
+                    cwd=self._cfg.project_dir,
+                )
+
+        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
 
-            return temp_dir
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = await self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
 
-        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
-        os.makedirs(temp_requirements_dir)
+    #
 
-        download_requirements(
-            await self.resolve_ci_image(),
-            temp_requirements_dir,
-            self.requirements_txts(),
+    @async_cached_nullary
+    async def load_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
         )
 
-        if self._file_cache is not None:
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
-
-            with tarfile.open(temp_tar_file, 'w') as tar:
-                for requirement_file in os.listdir(temp_requirements_dir):
-                    tar.add(
-                        os.path.join(temp_requirements_dir, requirement_file),
-                        arcname=requirement_file,
-                    )
-
-            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
-
-        return temp_requirements_dir
-
-    @async_cached_nullary
-    async def resolve_requirements_dir(self) -> str:
-        with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = await self._resolve_requirements_dir()
-            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
-            return requirements_dir
+        for dep_image in deps.values():
+            await self.load_docker_image(dep_image)
 
     #
 
     async def _run_compose_(self) -> None:
-        setup_cmds = [
-            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
-            (
-                'uv pip install --system --find-links /requirements ' +
-                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
-            ),
-        ]
-
-        #
-
-        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
-            *setup_cmds,
-            f'({self._cfg.cmd.s})',
-        ]))
-
-        #
-
         async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
             image=await self.resolve_ci_image(),
 
-            cmd=ci_cmd,
+            cmd=self._cfg.cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(await self.resolve_requirements_dir())}:/requirements',
+                *(self._cfg.run_options or []),
             ],
 
             cwd=self._cfg.project_dir,
@@ -263,10 +267,8 @@ class Ci(AsyncExitStacked):
     #
 
     async def run(self) -> None:
-        await self.load_compose_service_dependencies()
-
         await self.resolve_ci_image()
 
-        await self.resolve_requirements_dir()
+        await self.load_dependencies()
 
         await self._run_compose()