omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev214__py3-none-any.whl

omdev/ci/ci.py

@@ -1,19 +1,15 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import dataclasses as dc
 import os.path
-import shutil
-import tarfile
-import tempfile
 import typing as ta
 
+from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import ExitStacked
-from omlish.lite.contextmanagers import defer
+from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.os.temp import temp_file_context
 
 from .cache import FileCache
-from .cache import ShellCache
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
 from .docker import build_docker_file_hash
@@ -22,14 +18,14 @@ from .docker import is_docker_image_present
 from .docker import load_docker_tar_cmd
 from .docker import pull_docker_image
 from .docker import save_docker_tar_cmd
+from .docker import tag_docker_image
 from .requirements import build_requirements_hash
-from .requirements import download_requirements
 from .shell import ShellCmd
 from .utils import log_timing_context
 
 
-class Ci(ExitStacked):
-    FILE_NAME_HASH_LEN = 16
+class Ci(AsyncExitStacked):
+    KEY_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
     class Config:
@@ -42,9 +38,18 @@ class Ci(ExitStacked):
 
         cmd: ShellCmd
 
+        #
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
+        always_build: bool = False
+
+        no_dependencies: bool = False
+
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        #
 
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
@@ -53,44 +58,17 @@ class Ci(ExitStacked):
             self,
             cfg: Config,
             *,
-            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
-        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
-        if self._shell_cache is None:
-            return None
-
-        get_cache_cmd = self._shell_cache.get_file_cmd(key)
-        if get_cache_cmd is None:
-            return None
-
-        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
-
-        return load_docker_tar_cmd(get_cache_cmd)
-
-    def _save_cache_docker_image(self, key: str, image: str) -> None:
-        if self._shell_cache is None:
-            return
-
-        with self._shell_cache.put_file_cmd(key) as put_cache:
-            put_cache_cmd = put_cache.cmd
-
-            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
-
-            save_docker_tar_cmd(image, put_cache_cmd)
-
-    #
-
-    def _load_docker_image(self, image: str) -> None:
-        if not self._cfg.always_pull and is_docker_image_present(image):
+    async def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
 
         dep_suffix = image
@@ -98,152 +76,199 @@ class Ci(ExitStacked):
             dep_suffix = dep_suffix.replace(c, '-')
 
         cache_key = f'docker-{dep_suffix}'
-        if self._load_cache_docker_image(cache_key) is not None:
+        if (await self._load_cache_docker_image(cache_key)) is not None:
             return
 
-        pull_docker_image(image)
+        await pull_docker_image(image)
 
-        self._save_cache_docker_image(cache_key, image)
+        await self._save_cache_docker_image(cache_key, image)
 
-    def load_docker_image(self, image: str) -> None:
+    async def load_docker_image(self, image: str) -> None:
         with log_timing_context(f'Load docker image: {image}'):
-            self._load_docker_image(image)
-
-    @cached_nullary
-    def load_compose_service_dependencies(self) -> None:
-        deps = get_compose_service_dependencies(
-            self._cfg.compose_file,
-            self._cfg.service,
-        )
-
-        for dep_image in deps.values():
-            self.load_docker_image(dep_image)
+            await self._load_docker_image(image)
 
     #
 
-    def _resolve_ci_image(self) -> str:
-        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
 
-        cache_key = f'ci-{docker_file_hash}'
-        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
-            return cache_image_id
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
 
-        image_id = build_docker_image(
-            self._cfg.docker_file,
-            cwd=self._cfg.project_dir,
-        )
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
 
-        self._save_cache_docker_image(cache_key, image_id)
+        return await load_docker_tar_cmd(get_cache_cmd)
 
-        return image_id
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
 
-    @cached_nullary
-    def resolve_ci_image(self) -> str:
-        with log_timing_context('Resolve ci image') as ltc:
-            image_id = self._resolve_ci_image()
-            ltc.set_description(f'Resolve ci image: {image_id}')
-            return image_id
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
 
-    #
+            await save_docker_tar_cmd(image, write_tmp_cmd)
 
-    def _resolve_requirements_dir(self) -> str:
-        requirements_txts = [
-            os.path.join(self._cfg.project_dir, rf)
-            for rf in check.not_none(self._cfg.requirements_txts)
-        ]
+            await self._file_cache.put_file(key, tmp_file, steal=True)
 
-        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+    #
 
-        tar_file_key = f'requirements-{requirements_hash}'
-        tar_file_name = f'{tar_file_key}.tar'
+    async def _resolve_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
+        image_tag = f'{self._cfg.service}:{cache_key}'
 
-        temp_dir = tempfile.mkdtemp()
-        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
 
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
-            with tarfile.open(cache_tar_file) as tar:
-                tar.extractall(path=temp_dir)  # noqa
+        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
 
-            return temp_dir
+        image_id = await build_and_tag(image_tag)
 
-        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
-        os.makedirs(temp_requirements_dir)
+        await self._save_cache_docker_image(cache_key, image_id)
 
-        download_requirements(
-            self.resolve_ci_image(),
-            temp_requirements_dir,
-            requirements_txts,
-        )
+        return image_tag
 
-        if self._file_cache is not None:
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+    #
 
-            with tarfile.open(temp_tar_file, 'w') as tar:
-                for requirement_file in os.listdir(temp_requirements_dir):
-                    tar.add(
-                        os.path.join(temp_requirements_dir, requirement_file),
-                        arcname=requirement_file,
-                    )
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_base_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            return await build_docker_image(
+                self._cfg.docker_file,
+                tag=image_tag,
+                cwd=self._cfg.project_dir,
+            )
+
+        cache_key = f'ci-base-{self.docker_file_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
+    @async_cached_nullary
+    async def resolve_ci_base_image(self) -> str:
+        with log_timing_context('Resolve ci base image') as ltc:
+            image_id = await self._resolve_ci_base_image()
+            ltc.set_description(f'Resolve ci base image: {image_id}')
+            return image_id
 
-            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
+    #
 
-        return temp_requirements_dir
+    @cached_nullary
+    def requirements_txts(self) -> ta.Sequence[str]:
+        return [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
 
     @cached_nullary
-    def resolve_requirements_dir(self) -> str:
-        with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = self._resolve_requirements_dir()
-            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
-            return requirements_dir
+    def requirements_hash(self) -> str:
+        return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            base_image = await self.resolve_ci_base_image()
+
+            setup_cmds = [
+                ' '.join([
+                    'pip install',
+                    '--no-cache-dir',
+                    '--root-user-action ignore',
+                    'uv',
+                ]),
+                ' '.join([
+                    'uv pip install',
+                    '--no-cache',
+                    '--index-strategy unsafe-best-match',
+                    '--system',
+                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                ]),
+            ]
+            setup_cmd = ' && '.join(setup_cmds)
+
+            docker_file_lines = [
+                f'FROM {base_image}',
+                'RUN mkdir /project',
+                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                f'RUN {setup_cmd}',
+                'RUN rm /project/*',
+                'WORKDIR /project',
+            ]
+
+            with temp_file_context() as docker_file:
+                with open(docker_file, 'w') as f:  # noqa
+                    f.write('\n'.join(docker_file_lines))
+
+                return await build_docker_image(
+                    docker_file,
+                    tag=image_tag,
+                    cwd=self._cfg.project_dir,
+                )
+
+        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = await self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
 
     #
 
-    def _run_compose_(self) -> None:
-        setup_cmds = [
-            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
-            (
-                'uv pip install --system --find-links /requirements ' +
-                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
-            ),
-        ]
-
-        #
+    @async_cached_nullary
+    async def load_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
 
-        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
-            *setup_cmds,
-            f'({self._cfg.cmd.s})',
-        ]))
+        for dep_image in deps.values():
+            await self.load_docker_image(dep_image)
 
-        #
+    #
 
-        with DockerComposeRun(DockerComposeRun.Config(
+    async def _run_compose_(self) -> None:
+        async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
-            image=self.resolve_ci_image(),
+            image=await self.resolve_ci_image(),
 
-            cmd=ci_cmd,
+            cmd=self._cfg.cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+                *(self._cfg.run_options or []),
             ],
 
             cwd=self._cfg.project_dir,
+
+            no_dependencies=self._cfg.no_dependencies,
         )) as ci_compose_run:
-            ci_compose_run.run()
+            await ci_compose_run.run()
 
-    def _run_compose(self) -> None:
+    async def _run_compose(self) -> None:
         with log_timing_context('Run compose'):
-            self._run_compose_()
+            await self._run_compose_()
 
     #
 
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
-
-        self.resolve_ci_image()
+    async def run(self) -> None:
+        await self.resolve_ci_image()
 
-        self.resolve_requirements_dir()
+        await self.load_dependencies()
 
-        self._run_compose()
+        await self._run_compose()

omdev/ci/cli.py

@@ -1,6 +1,5 @@
 # @omlish-amalg ../scripts/ci.py
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 """
 Inputs:
  - requirements.txt
@@ -9,9 +8,11 @@ Inputs:
 
 ==
 
-./python -m ci run --cache-dir ci/cache ci/project omlish-ci
+./python -m omdev.ci run --cache-dir omdev/ci/tests/cache omdev/ci/tests/project omlish-ci
 """
+import argparse
 import asyncio
+import itertools
 import os.path
 import sys
 import typing as ta
@@ -20,15 +21,15 @@ from omlish.argparse.cli import ArgparseCli
 from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.check import check
+from omlish.lite.logs import log
 from omlish.logs.standard import configure_standard_logging
 
 from .cache import DirectoryFileCache
-from .cache import DirectoryShellCache
 from .cache import FileCache
-from .cache import ShellCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
-from .github.cache import GithubShellCache
+from .github.bootstrap import is_in_github_actions
+from .github.cache import GithubFileCache
 from .github.cli import GithubCli
 from .requirements import build_requirements_hash
 from .shell import ShellCmd
@@ -65,8 +66,8 @@ class CiCli(ArgparseCli):
     @argparse_cmd(
         accepts_unknown=True,
     )
-    def github(self) -> ta.Optional[int]:
-        return GithubCli(self.unknown_args).cli_run()
+    async def github(self) -> ta.Optional[int]:
+        return await GithubCli(self.unknown_args).async_cli_run()
 
     #
 
@@ -76,18 +77,33 @@ class CiCli(ArgparseCli):
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
-        argparse_arg('--github-cache', action='store_true'),
+
         argparse_arg('--cache-dir'),
+
+        argparse_arg('--github', action='store_true'),
+        argparse_arg('--github-detect', action='store_true'),
+
         argparse_arg('--always-pull', action='store_true'),
+        argparse_arg('--always-build', action='store_true'),
+
+        argparse_arg('--no-dependencies', action='store_true'),
+
+        argparse_arg('-e', '--env', action='append'),
+        argparse_arg('-v', '--volume', action='append'),
+
+        argparse_arg('cmd', nargs=argparse.REMAINDER),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
-        service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
-        always_pull = self.args.always_pull
+
+        #
+
+        cmd = ' '.join(self.args.cmd)
+        check.non_empty_str(cmd)
 
         #
 
@@ -99,6 +115,7 @@ class CiCli(ArgparseCli):
             for alt in alts:
                 alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
+                    log.debug('Using %s', alt_file)
                     return alt_file
             return None
 
@@ -132,6 +149,7 @@ class CiCli(ArgparseCli):
                 'requirements-ci.txt',
             ]:
                 if os.path.exists(os.path.join(project_dir, rf)):
+                    log.debug('Using %s', rf)
                     requirements_txts.append(rf)
         else:
             for rf in requirements_txts:
@@ -139,46 +157,60 @@ class CiCli(ArgparseCli):
 
         #
 
-        shell_cache: ta.Optional[ShellCache] = None
+        github = self.args.github
+        if not github and self.args.github_detect:
+            github = is_in_github_actions()
+            if github:
+                log.debug('Github detected')
+
+        #
+
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
-            if not os.path.exists(cache_dir):
-                os.makedirs(cache_dir)
-            check.state(os.path.isdir(cache_dir))
-
-            directory_file_cache = DirectoryFileCache(cache_dir)
+            cache_dir = os.path.abspath(cache_dir)
+            log.debug('Using cache dir %s', cache_dir)
+            if github:
+                file_cache = GithubFileCache(cache_dir)
+            else:
+                file_cache = DirectoryFileCache(cache_dir)
 
-            file_cache = directory_file_cache
+        #
 
-            if self.args.github_cache:
-                shell_cache = GithubShellCache(cache_dir)
-            else:
-                shell_cache = DirectoryShellCache(directory_file_cache)
+        run_options: ta.List[str] = []
+        for run_arg, run_arg_vals in [
+            ('-e', self.args.env or []),
+            ('-v', self.args.volume or []),
+        ]:
+            run_options.extend(itertools.chain.from_iterable(
+                [run_arg, run_arg_val]
+                for run_arg_val in run_arg_vals
+            ))
 
         #
 
-        with Ci(
+        async with Ci(
                 Ci.Config(
                     project_dir=project_dir,
 
                     docker_file=docker_file,
 
                     compose_file=compose_file,
-                    service=service,
+                    service=self.args.service,
 
                     requirements_txts=requirements_txts,
 
-                    cmd=ShellCmd(' && '.join([
-                        'cd /project',
-                        'python3 -m pytest -svv test.py',
-                    ])),
+                    cmd=ShellCmd(cmd),
+
+                    always_pull=self.args.always_pull,
+                    always_build=self.args.always_build,
+
+                    no_dependencies=self.args.no_dependencies,
 
-                    always_pull=always_pull,
+                    run_options=run_options,
                 ),
                 file_cache=file_cache,
-                shell_cache=shell_cache,
         ) as ci:
-            ci.run()
+            await ci.run()
 
 
 async def _async_main() -> ta.Optional[int]: