omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/ci/cache.py

@@ -1,34 +1,45 @@
 # ruff: noqa: UP006 UP007
 import abc
+import asyncio
+import dataclasses as dc
+import functools
 import os.path
 import shutil
 import typing as ta
+import urllib.request
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.logs import log
+from omlish.os.temp import make_temp_file
 
 from .consts import CI_CACHE_VERSION
 
 
+CacheVersion = ta.NewType('CacheVersion', int)
+
+
 ##
 
 
-@abc.abstractmethod
 class FileCache(abc.ABC):
+    DEFAULT_CACHE_VERSION: ta.ClassVar[CacheVersion] = CacheVersion(CI_CACHE_VERSION)
+
     def __init__(
             self,
             *,
-            version: int = CI_CACHE_VERSION,
+            version: ta.Optional[CacheVersion] = None,
     ) -> None:
         super().__init__()
 
+        if version is None:
+            version = self.DEFAULT_CACHE_VERSION
         check.isinstance(version, int)
         check.arg(version >= 0)
-        self._version = version
+        self._version: CacheVersion = version
 
     @property
-    def version(self) -> int:
+    def version(self) -> CacheVersion:
         return self._version
 
     #
@@ -52,19 +63,28 @@ class FileCache(abc.ABC):
 
 
 class DirectoryFileCache(FileCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        dir: str
+
+        no_create: bool = False
+        no_purge: bool = False
+
     def __init__(
             self,
-            dir: str,  # noqa
+            config: Config,
             *,
-            no_create: bool = False,
-            no_purge: bool = False,
-            **kwargs: ta.Any,
+            version: ta.Optional[CacheVersion] = None,
     ) -> None:  # noqa
-        super().__init__(**kwargs)
+        super().__init__(
+            version=version,
+        )
+
+        self._config = config
 
-        self._dir = dir
-        self._no_create = no_create
-        self._no_purge = no_purge
+    @property
+    def dir(self) -> str:
+        return self._config.dir
 
     #
 
@@ -72,37 +92,38 @@ class DirectoryFileCache(FileCache):
 
     @cached_nullary
     def setup_dir(self) -> None:
-        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
+        version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
 
-        if self._no_create:
-            check.state(os.path.isdir(self._dir))
+        if self._config.no_create:
+            check.state(os.path.isdir(self.dir))
 
-        elif not os.path.isdir(self._dir):
-            os.makedirs(self._dir)
+        elif not os.path.isdir(self.dir):
+            os.makedirs(self.dir)
             with open(version_file, 'w') as f:
                 f.write(str(self._version))
             return
 
+        # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.
         with open(version_file) as f:
             dir_version = int(f.read().strip())
 
         if dir_version == self._version:
             return
 
-        if self._no_purge:
+        if self._config.no_purge:
             raise RuntimeError(f'{dir_version=} != {self._version=}')
 
-        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        dirs = [n for n in sorted(os.listdir(self.dir)) if os.path.isdir(os.path.join(self.dir, n))]
         if dirs:
             raise RuntimeError(
-                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'Refusing to remove stale cache dir {self.dir!r} '
                 f'due to present directories: {", ".join(dirs)}',
             )
 
-        for n in sorted(os.listdir(self._dir)):
+        for n in sorted(os.listdir(self.dir)):
             if n.startswith('.'):
                 continue
-            fp = os.path.join(self._dir, n)
+            fp = os.path.join(self.dir, n)
             check.state(os.path.isfile(fp))
             log.debug('Purging stale cache file: %s', fp)
             os.unlink(fp)
@@ -119,7 +140,7 @@ class DirectoryFileCache(FileCache):
             key: str,
     ) -> str:
         self.setup_dir()
-        return os.path.join(self._dir, key)
+        return os.path.join(self.dir, key)
 
     def format_incomplete_file(self, f: str) -> str:
         return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
@@ -145,3 +166,107 @@ class DirectoryFileCache(FileCache):
         else:
             shutil.copyfile(file_path, cache_file_path)
         return cache_file_path
+
+
+##
+
+
+class DataCache:
+    @dc.dataclass(frozen=True)
+    class Data(abc.ABC):  # noqa
+        pass
+
+    @dc.dataclass(frozen=True)
+    class BytesData(Data):
+        data: bytes
+
+    @dc.dataclass(frozen=True)
+    class FileData(Data):
+        file_path: str
+
+    @dc.dataclass(frozen=True)
+    class UrlData(Data):
+        url: str
+
+    #
+
+    @abc.abstractmethod
+    def get_data(self, key: str) -> ta.Awaitable[ta.Optional[Data]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_data(self, key: str, data: Data) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+#
+
+
+@functools.singledispatch
+async def read_data_cache_data(data: DataCache.Data) -> bytes:
+    raise TypeError(data)
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.BytesData) -> bytes:
+    return data.data
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.FileData) -> bytes:
+    with open(data.file_path, 'rb') as f:  # noqa
+        return f.read()
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.UrlData) -> bytes:
+    def inner() -> bytes:
+        with urllib.request.urlopen(urllib.request.Request(  # noqa
+            data.url,
+        )) as resp:
+            return resp.read()
+
+    return await asyncio.get_running_loop().run_in_executor(None, inner)
+
+
+#
+
+
+class FileCacheDataCache(DataCache):
+    def __init__(
+            self,
+            file_cache: FileCache,
+    ) -> None:
+        super().__init__()
+
+        self._file_cache = file_cache
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        if (file_path := await self._file_cache.get_file(key)) is None:
+            return None
+
+        return DataCache.FileData(file_path)
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        steal = False
+
+        if isinstance(data, DataCache.BytesData):
+            file_path = make_temp_file()
+            with open(file_path, 'wb') as f:  # noqa
+                f.write(data.data)
+            steal = True
+
+        elif isinstance(data, DataCache.FileData):
+            file_path = data.file_path
+
+        elif isinstance(data, DataCache.UrlData):
+            raise NotImplementedError
+
+        else:
+            raise TypeError(data)
+
+        await self._file_cache.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )

omdev/ci/ci.py

@@ -7,21 +7,20 @@ from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.lite.timing import log_timing_context
 from omlish.os.temp import temp_file_context
 
-from .cache import FileCache
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
-from .docker import build_docker_file_hash
-from .docker import build_docker_image
-from .docker import is_docker_image_present
-from .docker import load_docker_tar_cmd
-from .docker import pull_docker_image
-from .docker import save_docker_tar_cmd
-from .docker import tag_docker_image
+from .docker.buildcaching import DockerBuildCaching
+from .docker.cmds import build_docker_image
+from .docker.imagepulling import DockerImagePulling
+from .docker.utils import build_docker_file_hash
 from .requirements import build_requirements_hash
 from .shell import ShellCmd
-from .utils import log_timing_context
+
+
+##
 
 
 class Ci(AsyncExitStacked):
@@ -56,104 +55,40 @@ class Ci(AsyncExitStacked):
 
     def __init__(
             self,
-            cfg: Config,
+            config: Config,
             *,
-            file_cache: ta.Optional[FileCache] = None,
+            docker_build_caching: DockerBuildCaching,
+            docker_image_pulling: DockerImagePulling,
     ) -> None:
         super().__init__()
 
-        self._cfg = cfg
-        self._file_cache = file_cache
-
-    #
-
-    async def _load_docker_image(self, image: str) -> None:
-        if not self._cfg.always_pull and (await is_docker_image_present(image)):
-            return
-
-        dep_suffix = image
-        for c in '/:.-_':
-            dep_suffix = dep_suffix.replace(c, '-')
-
-        cache_key = f'docker-{dep_suffix}'
-        if (await self._load_cache_docker_image(cache_key)) is not None:
-            return
-
-        await pull_docker_image(image)
-
-        await self._save_cache_docker_image(cache_key, image)
+        self._config = config
 
-    async def load_docker_image(self, image: str) -> None:
-        with log_timing_context(f'Load docker image: {image}'):
-            await self._load_docker_image(image)
-
-    #
-
-    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
-        if self._file_cache is None:
-            return None
-
-        cache_file = await self._file_cache.get_file(key)
-        if cache_file is None:
-            return None
-
-        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
-
-        return await load_docker_tar_cmd(get_cache_cmd)
-
-    async def _save_cache_docker_image(self, key: str, image: str) -> None:
-        if self._file_cache is None:
-            return
-
-        with temp_file_context() as tmp_file:
-            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
-
-            await save_docker_tar_cmd(image, write_tmp_cmd)
-
-            await self._file_cache.put_file(key, tmp_file, steal=True)
-
-    #
-
-    async def _resolve_docker_image(
-            self,
-            cache_key: str,
-            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
-    ) -> str:
-        image_tag = f'{self._cfg.service}:{cache_key}'
-
-        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
-            return image_tag
-
-        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
-            await tag_docker_image(
-                cache_image_id,
-                image_tag,
-            )
-            return image_tag
-
-        image_id = await build_and_tag(image_tag)
-
-        await self._save_cache_docker_image(cache_key, image_id)
-
-        return image_tag
+        self._docker_build_caching = docker_build_caching
+        self._docker_image_pulling = docker_image_pulling
 
     #
 
     @cached_nullary
     def docker_file_hash(self) -> str:
-        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+        return build_docker_file_hash(self._config.docker_file)[:self.KEY_HASH_LEN]
+
+    @cached_nullary
+    def ci_base_image_cache_key(self) -> str:
+        return f'ci-base-{self.docker_file_hash()}'
 
     async def _resolve_ci_base_image(self) -> str:
         async def build_and_tag(image_tag: str) -> str:
             return await build_docker_image(
-                self._cfg.docker_file,
+                self._config.docker_file,
                 tag=image_tag,
-                cwd=self._cfg.project_dir,
+                cwd=self._config.project_dir,
             )
 
-        cache_key = f'ci-base-{self.docker_file_hash()}'
-
-        return await self._resolve_docker_image(cache_key, build_and_tag)
+        return await self._docker_build_caching.cached_build_docker_image(
+            self.ci_base_image_cache_key(),
+            build_and_tag,
+        )
 
     @async_cached_nullary
     async def resolve_ci_base_image(self) -> str:
@@ -167,14 +102,18 @@ class Ci(AsyncExitStacked):
     @cached_nullary
     def requirements_txts(self) -> ta.Sequence[str]:
         return [
-            os.path.join(self._cfg.project_dir, rf)
-            for rf in check.not_none(self._cfg.requirements_txts)
+            os.path.join(self._config.project_dir, rf)
+            for rf in check.not_none(self._config.requirements_txts)
         ]
 
     @cached_nullary
     def requirements_hash(self) -> str:
         return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
 
+    @cached_nullary
+    def ci_image_cache_key(self) -> str:
+        return f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+
     async def _resolve_ci_image(self) -> str:
         async def build_and_tag(image_tag: str) -> str:
             base_image = await self.resolve_ci_base_image()
@@ -191,7 +130,7 @@ class Ci(AsyncExitStacked):
                     '--no-cache',
                     '--index-strategy unsafe-best-match',
                     '--system',
-                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                    *[f'-r /project/{rf}' for rf in self._config.requirements_txts or []],
                 ]),
             ]
             setup_cmd = ' && '.join(setup_cmds)
@@ -199,7 +138,7 @@ class Ci(AsyncExitStacked):
             docker_file_lines = [
                 f'FROM {base_image}',
                 'RUN mkdir /project',
-                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                *[f'COPY {rf} /project/{rf}' for rf in self._config.requirements_txts or []],
                 f'RUN {setup_cmd}',
                 'RUN rm /project/*',
                 'WORKDIR /project',
@@ -212,12 +151,13 @@ class Ci(AsyncExitStacked):
                 return await build_docker_image(
                     docker_file,
                     tag=image_tag,
-                    cwd=self._cfg.project_dir,
+                    cwd=self._config.project_dir,
                 )
 
-        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
-
-        return await self._resolve_docker_image(cache_key, build_and_tag)
+        return await self._docker_build_caching.cached_build_docker_image(
+            self.ci_image_cache_key(),
+            build_and_tag,
+        )
 
     @async_cached_nullary
     async def resolve_ci_image(self) -> str:
@@ -229,34 +169,34 @@ class Ci(AsyncExitStacked):
     #
 
     @async_cached_nullary
-    async def load_dependencies(self) -> None:
+    async def pull_dependencies(self) -> None:
         deps = get_compose_service_dependencies(
-            self._cfg.compose_file,
-            self._cfg.service,
+            self._config.compose_file,
+            self._config.service,
         )
 
         for dep_image in deps.values():
-            await self.load_docker_image(dep_image)
+            await self._docker_image_pulling.pull_docker_image(dep_image)
 
     #
 
     async def _run_compose_(self) -> None:
         async with DockerComposeRun(DockerComposeRun.Config(
-            compose_file=self._cfg.compose_file,
-            service=self._cfg.service,
+            compose_file=self._config.compose_file,
+            service=self._config.service,
 
             image=await self.resolve_ci_image(),
 
-            cmd=self._cfg.cmd,
+            cmd=self._config.cmd,
 
             run_options=[
-                '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                *(self._cfg.run_options or []),
+                '-v', f'{os.path.abspath(self._config.project_dir)}:/project',
+                *(self._config.run_options or []),
             ],
 
-            cwd=self._cfg.project_dir,
+            cwd=self._config.project_dir,
 
-            no_dependencies=self._cfg.no_dependencies,
+            no_dependencies=self._config.no_dependencies,
         )) as ci_compose_run:
             await ci_compose_run.run()
 
@@ -269,6 +209,6 @@ class Ci(AsyncExitStacked):
     async def run(self) -> None:
         await self.resolve_ci_image()
 
-        await self.load_dependencies()
+        await self.pull_dependencies()
 
         await self._run_compose()

omdev/ci/cli.py

@@ -21,16 +21,15 @@ from omlish.argparse.cli import ArgparseCli
 from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.check import check
+from omlish.lite.inject import inj
 from omlish.lite.logs import log
 from omlish.logs.standard import configure_standard_logging
 
-from .cache import DirectoryFileCache
-from .cache import FileCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
 from .github.bootstrap import is_in_github_actions
-from .github.cache import GithubFileCache
 from .github.cli import GithubCli
+from .inject import bind_ci
 from .requirements import build_requirements_hash
 from .shell import ShellCmd
 
@@ -165,14 +164,9 @@ class CiCli(ArgparseCli):
 
         #
 
-        file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
             cache_dir = os.path.abspath(cache_dir)
             log.debug('Using cache dir %s', cache_dir)
-            if github:
-                file_cache = GithubFileCache(cache_dir)
-            else:
-                file_cache = DirectoryFileCache(cache_dir)
 
         #
 
@@ -188,28 +182,35 @@ class CiCli(ArgparseCli):
 
         #
 
-        async with Ci(
-                Ci.Config(
-                    project_dir=project_dir,
+        config = Ci.Config(
+            project_dir=project_dir,
+
+            docker_file=docker_file,
+
+            compose_file=compose_file,
+            service=self.args.service,
 
-                    docker_file=docker_file,
+            requirements_txts=requirements_txts,
 
-                    compose_file=compose_file,
-                    service=self.args.service,
+            cmd=ShellCmd(cmd),
 
-                    requirements_txts=requirements_txts,
+            always_pull=self.args.always_pull,
+            always_build=self.args.always_build,
 
-                    cmd=ShellCmd(cmd),
+            no_dependencies=self.args.no_dependencies,
 
-                    always_pull=self.args.always_pull,
-                    always_build=self.args.always_build,
+            run_options=run_options,
+        )
 
-                    no_dependencies=self.args.no_dependencies,
+        injector = inj.create_injector(bind_ci(
+            config=config,
+
+            github=github,
+
+            cache_dir=cache_dir,
+        ))
 
-                    run_options=run_options,
-                ),
-                file_cache=file_cache,
-        ) as ci:
+        async with injector[Ci] as ci:
             await ci.run()
 
 

omdev/ci/docker/buildcaching.py

@@ -0,0 +1,69 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import typing as ta
+
+from .cache import DockerCache
+from .cmds import is_docker_image_present
+from .cmds import tag_docker_image
+
+
+##
+
+
+class DockerBuildCaching(abc.ABC):
+    @abc.abstractmethod
+    def cached_build_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],  # image_tag -> image_id
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
+
+
+class DockerBuildCachingImpl(DockerBuildCaching):
+    @dc.dataclass(frozen=True)
+    class Config:
+        service: str
+
+        always_build: bool = False
+
+    def __init__(
+            self,
+            *,
+            config: Config,
+
+            docker_cache: ta.Optional[DockerCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self._docker_cache = docker_cache
+
+    async def cached_build_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
+        image_tag = f'{self._config.service}:{cache_key}'
+
+        if not self._config.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
+
+        if (
+                self._docker_cache is not None and
+                (cache_image_id := await self._docker_cache.load_cache_docker_image(cache_key)) is not None
+        ):
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
+
+        image_id = await build_and_tag(image_tag)
+
+        if self._docker_cache is not None:
+            await self._docker_cache.save_cache_docker_image(cache_key, image_id)
+
+        return image_tag