omdev 0.0.0.dev211__py3-none-any.whl → 0.0.0.dev213__py3-none-any.whl

omdev/ci/ci.py

@@ -7,9 +7,10 @@ import tarfile
 import tempfile
 import typing as ta
 
+from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import AsyncExitStacked
 from omlish.lite.contextmanagers import defer
 
 from .cache import FileCache
@@ -22,13 +23,14 @@ from .docker import is_docker_image_present
 from .docker import load_docker_tar_cmd
 from .docker import pull_docker_image
 from .docker import save_docker_tar_cmd
+from .docker import tag_docker_image
 from .requirements import build_requirements_hash
 from .requirements import download_requirements
 from .shell import ShellCmd
 from .utils import log_timing_context
 
 
-class Ci(ExitStacked):
+class Ci(AsyncExitStacked):
     FILE_NAME_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
@@ -45,6 +47,9 @@ class Ci(ExitStacked):
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
+        always_build: bool = False
+
+        no_dependencies: bool = False
 
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
@@ -64,7 +69,7 @@ class Ci(ExitStacked):
 
     #
 
-    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
         if self._shell_cache is None:
             return None
 
@@ -74,9 +79,9 @@ class Ci(ExitStacked):
 
         get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
 
-        return load_docker_tar_cmd(get_cache_cmd)
+        return await load_docker_tar_cmd(get_cache_cmd)
 
-    def _save_cache_docker_image(self, key: str, image: str) -> None:
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
         if self._shell_cache is None:
             return
 
@@ -85,12 +90,12 @@ class Ci(ExitStacked):
 
             put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
 
-            save_docker_tar_cmd(image, put_cache_cmd)
+            await save_docker_tar_cmd(image, put_cache_cmd)
 
     #
 
-    def _load_docker_image(self, image: str) -> None:
-        if not self._cfg.always_pull and is_docker_image_present(image):
+    async def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
 
         dep_suffix = image
@@ -98,63 +103,79 @@ class Ci(ExitStacked):
             dep_suffix = dep_suffix.replace(c, '-')
 
         cache_key = f'docker-{dep_suffix}'
-        if self._load_cache_docker_image(cache_key) is not None:
+        if (await self._load_cache_docker_image(cache_key)) is not None:
             return
 
-        pull_docker_image(image)
+        await pull_docker_image(image)
 
-        self._save_cache_docker_image(cache_key, image)
+        await self._save_cache_docker_image(cache_key, image)
 
-    def load_docker_image(self, image: str) -> None:
+    async def load_docker_image(self, image: str) -> None:
         with log_timing_context(f'Load docker image: {image}'):
-            self._load_docker_image(image)
+            await self._load_docker_image(image)
 
-    @cached_nullary
-    def load_compose_service_dependencies(self) -> None:
+    @async_cached_nullary
+    async def load_compose_service_dependencies(self) -> None:
         deps = get_compose_service_dependencies(
             self._cfg.compose_file,
             self._cfg.service,
         )
 
         for dep_image in deps.values():
-            self.load_docker_image(dep_image)
+            await self.load_docker_image(dep_image)
 
     #
 
-    def _resolve_ci_image(self) -> str:
-        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+
+    async def _resolve_ci_image(self) -> str:
+        cache_key = f'ci-{self.docker_file_hash()}'
+        image_tag = f'{self._cfg.service}:{cache_key}'
+
+        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
 
-        cache_key = f'ci-{docker_file_hash}'
-        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
-            return cache_image_id
+        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
 
-        image_id = build_docker_image(
+        image_id = await build_docker_image(
             self._cfg.docker_file,
+            tag=image_tag,
             cwd=self._cfg.project_dir,
         )
 
-        self._save_cache_docker_image(cache_key, image_id)
+        await self._save_cache_docker_image(cache_key, image_id)
 
-        return image_id
+        return image_tag
 
-    @cached_nullary
-    def resolve_ci_image(self) -> str:
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
         with log_timing_context('Resolve ci image') as ltc:
-            image_id = self._resolve_ci_image()
+            image_id = await self._resolve_ci_image()
             ltc.set_description(f'Resolve ci image: {image_id}')
             return image_id
 
     #
 
-    def _resolve_requirements_dir(self) -> str:
-        requirements_txts = [
+    @cached_nullary
+    def requirements_txts(self) -> ta.Sequence[str]:
+        return [
             os.path.join(self._cfg.project_dir, rf)
             for rf in check.not_none(self._cfg.requirements_txts)
         ]
 
-        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+    @cached_nullary
+    def requirements_hash(self) -> str:
+        return build_requirements_hash(self.requirements_txts())[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_key = f'requirements-{requirements_hash}'
+    async def _resolve_requirements_dir(self) -> str:
+        tar_file_key = f'requirements-{self.docker_file_hash()}-{self.requirements_hash()}'
         tar_file_name = f'{tar_file_key}.tar'
 
         temp_dir = tempfile.mkdtemp()
@@ -170,9 +191,9 @@ class Ci(ExitStacked):
         os.makedirs(temp_requirements_dir)
 
         download_requirements(
-            self.resolve_ci_image(),
+            await self.resolve_ci_image(),
             temp_requirements_dir,
-            requirements_txts,
+            self.requirements_txts(),
         )
 
         if self._file_cache is not None:
@@ -189,16 +210,16 @@ class Ci(ExitStacked):
 
         return temp_requirements_dir
 
-    @cached_nullary
-    def resolve_requirements_dir(self) -> str:
+    @async_cached_nullary
+    async def resolve_requirements_dir(self) -> str:
         with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = self._resolve_requirements_dir()
+            requirements_dir = await self._resolve_requirements_dir()
             ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
             return requirements_dir
 
     #
 
-    def _run_compose_(self) -> None:
+    async def _run_compose_(self) -> None:
         setup_cmds = [
             'pip install --root-user-action ignore --find-links /requirements --no-index uv',
             (
@@ -216,34 +237,36 @@ class Ci(ExitStacked):
 
         #
 
-        with DockerComposeRun(DockerComposeRun.Config(
+        async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
-            image=self.resolve_ci_image(),
+            image=await self.resolve_ci_image(),
 
             cmd=ci_cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+                '-v', f'{os.path.abspath(await self.resolve_requirements_dir())}:/requirements',
             ],
 
             cwd=self._cfg.project_dir,
+
+            no_dependencies=self._cfg.no_dependencies,
         )) as ci_compose_run:
-            ci_compose_run.run()
+            await ci_compose_run.run()
 
-    def _run_compose(self) -> None:
+    async def _run_compose(self) -> None:
         with log_timing_context('Run compose'):
-            self._run_compose_()
+            await self._run_compose_()
 
     #
 
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
+    async def run(self) -> None:
+        await self.load_compose_service_dependencies()
 
-        self.resolve_ci_image()
+        await self.resolve_ci_image()
 
-        self.resolve_requirements_dir()
+        await self.resolve_requirements_dir()
 
-        self._run_compose()
+        await self._run_compose()

omdev/ci/cli.py

@@ -76,18 +76,21 @@ class CiCli(ArgparseCli):
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
+
         argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
+
         argparse_arg('--always-pull', action='store_true'),
+        argparse_arg('--always-build', action='store_true'),
+
+        argparse_arg('--no-dependencies', action='store_true'),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
-        service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
-        always_pull = self.args.always_pull
 
         #
 
@@ -112,10 +115,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:
@@ -151,14 +160,14 @@ class CiCli(ArgparseCli):
 
         #
 
-        with Ci(
+        async with Ci(
                 Ci.Config(
                     project_dir=project_dir,
 
                     docker_file=docker_file,
 
                     compose_file=compose_file,
-                    service=service,
+                    service=self.args.service,
 
                     requirements_txts=requirements_txts,
 
@@ -167,12 +176,15 @@ class CiCli(ArgparseCli):
                         'python3 -m pytest -svv test.py',
                     ])),
 
-                    always_pull=always_pull,
+                    always_pull=self.args.always_pull,
+                    always_build=self.args.always_build,
+
+                    no_dependencies=self.args.no_dependencies,
                 ),
                 file_cache=file_cache,
                 shell_cache=shell_cache,
         ) as ci:
-            ci.run()
+            await ci.run()
 
 
 async def _async_main() -> ta.Optional[int]:

omdev/ci/compose.py

@@ -11,12 +11,13 @@ import os.path
 import shlex
 import typing as ta
 
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.lite.contextmanagers import adefer
 from omlish.lite.contextmanagers import defer
 from omlish.lite.json import json_dumps_pretty
-from omlish.subprocesses import subprocesses
 
 from .shell import ShellCmd
 from .utils import make_temp_file
@@ -46,7 +47,7 @@ def get_compose_service_dependencies(
 ##
 
 
-class DockerComposeRun(ExitStacked):
+class DockerComposeRun(AsyncExitStacked):
     @dc.dataclass(frozen=True)
     class Config:
         compose_file: str
@@ -64,6 +65,7 @@ class DockerComposeRun(ExitStacked):
 
         #
 
+        no_dependencies: bool = False
         no_dependency_cleanup: bool = False
 
         #
@@ -82,40 +84,6 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    @property
-    def image_tag(self) -> str:
-        pfx = 'sha256:'
-        if (image := self._cfg.image).startswith(pfx):
-            image = image[len(pfx):]
-
-        return f'{self._cfg.service}:{image}'
-
-    @cached_nullary
-    def tag_image(self) -> str:
-        image_tag = self.image_tag
-
-        subprocesses.check_call(
-            'docker',
-            'tag',
-            self._cfg.image,
-            image_tag,
-            **self._subprocess_kwargs,
-        )
-
-        def delete_tag() -> None:
-            subprocesses.check_call(
-                'docker',
-                'rmi',
-                image_tag,
-                **self._subprocess_kwargs,
-            )
-
-        self._enter_context(defer(delete_tag))  # noqa
-
-        return image_tag
-
-    #
-
     def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
         out = dict(in_dct)
 
@@ -129,7 +97,7 @@ class DockerComposeRun(ExitStacked):
         in_service: dict = in_services[self._cfg.service]
         out_services[self._cfg.service] = out_service = dict(in_service)
 
-        out_service['image'] = self.image_tag
+        out_service['image'] = self._cfg.image
 
         for k in ['build', 'platform']:
             if k in out_service:
@@ -142,16 +110,21 @@ class DockerComposeRun(ExitStacked):
 
         #
 
-        depends_on = in_service.get('depends_on', [])
+        if not self._cfg.no_dependencies:
+            depends_on = in_service.get('depends_on', [])
 
-        for dep_service, in_dep_service_dct in list(in_services.items()):
-            if dep_service not in depends_on:
-                continue
+            for dep_service, in_dep_service_dct in list(in_services.items()):
+                if dep_service not in depends_on:
+                    continue
 
-            out_dep_service: dict = dict(in_dep_service_dct)
-            out_services[dep_service] = out_dep_service
+                out_dep_service: dict = dict(in_dep_service_dct)
+                out_services[dep_service] = out_dep_service
 
-            out_dep_service['ports'] = []
+                out_dep_service['ports'] = []
+
+        else:
+            out_service['depends_on'] = []
+            out_service['links'] = []
 
         #
 
@@ -177,22 +150,20 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    def _cleanup_dependencies(self) -> None:
-        subprocesses.check_call(
+    async def _cleanup_dependencies(self) -> None:
+        await asyncio_subprocesses.check_call(
             'docker',
             'compose',
             '-f', self.rewrite_compose_file(),
             'down',
         )
 
-    def run(self) -> None:
-        self.tag_image()
-
+    async def run(self) -> None:
         compose_file = self.rewrite_compose_file()
 
-        with contextlib.ExitStack() as es:
-            if not self._cfg.no_dependency_cleanup:
-                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+        async with contextlib.AsyncExitStack() as es:
+            if not (self._cfg.no_dependencies or self._cfg.no_dependency_cleanup):
+                await es.enter_async_context(adefer(self._cleanup_dependencies))  # noqa
 
             sh_cmd = ' '.join([
                 'docker',
@@ -200,7 +171,10 @@ class DockerComposeRun(ExitStacked):
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
                 *(self._cfg.run_options or []),
                 self._cfg.service,
                 'sh', '-c', shlex.quote(self._cfg.cmd.s),
@@ -208,7 +182,7 @@ class DockerComposeRun(ExitStacked):
 
             run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
 
-            run_cmd.run(
-                subprocesses.check_call,
+            await run_cmd.run(
+                asyncio_subprocesses.check_call,
                 **self._subprocess_kwargs,
             )

omdev/ci/docker.py

@@ -13,9 +13,9 @@ import shlex
 import tarfile
 import typing as ta
 
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.lite.contextmanagers import defer
-from omlish.subprocesses import subprocesses
 
 from .shell import ShellCmd
 from .utils import make_temp_file
@@ -60,8 +60,8 @@ def read_docker_tar_image_id(tar_file: str) -> str:
 ##
 
 
-def is_docker_image_present(image: str) -> bool:
-    out = subprocesses.check_output(
+async def is_docker_image_present(image: str) -> bool:
+    out = await asyncio_subprocesses.check_output(
         'docker',
         'images',
         '--format', 'json',
@@ -76,55 +76,74 @@ def is_docker_image_present(image: str) -> bool:
     return True
 
 
-def pull_docker_image(
+async def pull_docker_image(
         image: str,
 ) -> None:
-    subprocesses.check_call(
+    await asyncio_subprocesses.check_call(
         'docker',
         'pull',
         image,
     )
 
 
-def build_docker_image(
+async def build_docker_image(
         docker_file: str,
         *,
+        tag: ta.Optional[str] = None,
         cwd: ta.Optional[str] = None,
 ) -> str:
     id_file = make_temp_file()
     with defer(lambda: os.unlink(id_file)):
-        subprocesses.check_call(
+        await asyncio_subprocesses.check_call(
             'docker',
             'build',
             '-f', os.path.abspath(docker_file),
             '--iidfile', id_file,
             '--squash',
+            *(['--tag', tag] if tag is not None else []),
             '.',
             **(dict(cwd=cwd) if cwd is not None else {}),
         )
 
-        with open(id_file) as f:
+        with open(id_file) as f:  # noqa
             image_id = check.single(f.read().strip().splitlines()).strip()
 
     return image_id
 
 
+async def tag_docker_image(image: str, tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'tag',
+        image,
+        tag,
+    )
+
+
+async def delete_docker_tag(tag: str) -> None:
+    await asyncio_subprocesses.check_call(
+        'docker',
+        'rmi',
+        tag,
+    )
+
+
 ##
 
 
-def save_docker_tar_cmd(
+async def save_docker_tar_cmd(
         image: str,
         output_cmd: ShellCmd,
 ) -> None:
     cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
-    cmd.run(subprocesses.check_call)
+    await cmd.run(asyncio_subprocesses.check_call)
 
 
-def save_docker_tar(
+async def save_docker_tar(
         image: str,
         tar_file: str,
 ) -> None:
-    return save_docker_tar_cmd(
+    return await save_docker_tar_cmd(
         image,
         ShellCmd(f'cat > {shlex.quote(tar_file)}'),
     )
@@ -133,19 +152,19 @@ def save_docker_tar(
 #
 
 
-def load_docker_tar_cmd(
+async def load_docker_tar_cmd(
         input_cmd: ShellCmd,
 ) -> str:
     cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
 
-    out = cmd.run(subprocesses.check_output).decode()
+    out = (await cmd.run(asyncio_subprocesses.check_output)).decode()
 
     line = check.single(out.strip().splitlines())
     loaded = line.partition(':')[2].strip()
     return loaded
 
 
-def load_docker_tar(
+async def load_docker_tar(
         tar_file: str,
 ) -> str:
-    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))
+    return await load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))