omdev 0.0.0.dev210__py3-none-any.whl → 0.0.0.dev211__py3-none-any.whl

omdev/ci/cache.py

@@ -2,21 +2,24 @@
 # @omlish-lite
 import abc
 import os.path
+import shlex
 import shutil
 import typing as ta
 
+from .shell import ShellCmd
 
-#
+
+##
 
 
 @abc.abstractmethod
 class FileCache(abc.ABC):
     @abc.abstractmethod
-    def get_file(self, name: str) -> ta.Optional[str]:
+    def get_file(self, key: str) -> ta.Optional[str]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def put_file(self, name: str) -> ta.Optional[str]:
+    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
         raise NotImplementedError
 
 
@@ -29,13 +32,137 @@ class DirectoryFileCache(FileCache):
 
         self._dir = dir
 
-    def get_file(self, name: str) -> ta.Optional[str]:
-        file_path = os.path.join(self._dir, name)
-        if not os.path.exists(file_path):
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+            *,
+            make_dirs: bool = False,
+    ) -> str:
+        if make_dirs:
+            os.makedirs(self._dir, exist_ok=True)
+        return os.path.join(self._dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
             return None
-        return file_path
+        return cache_file_path
 
-    def put_file(self, file_path: str) -> None:
-        os.makedirs(self._dir, exist_ok=True)
-        cache_file_path = os.path.join(self._dir, os.path.basename(file_path))
+    def put_file(self, key: str, file_path: str) -> None:
+        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
         shutil.copyfile(file_path, cache_file_path)
+
+
+##
+
+
+class ShellCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        raise NotImplementedError
+
+    class PutFileCmdContext(abc.ABC):
+        def __init__(self) -> None:
+            super().__init__()
+
+            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
+
+        @property
+        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
+            return self._state
+
+        #
+
+        @property
+        @abc.abstractmethod
+        def cmd(self) -> ShellCmd:
+            raise NotImplementedError
+
+        #
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            if exc_val is None:
+                self.commit()
+            else:
+                self.abort()
+
+        #
+
+        @abc.abstractmethod
+        def _commit(self) -> None:
+            raise NotImplementedError
+
+        def commit(self) -> None:
+            if self._state == 'committed':
+                return
+            elif self._state == 'open':
+                self._commit()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+        #
+
+        @abc.abstractmethod
+        def _abort(self) -> None:
+            raise NotImplementedError
+
+        def abort(self) -> None:
+            if self._state == 'aborted':
+                return
+            elif self._state == 'open':
+                self._abort()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+    @abc.abstractmethod
+    def put_file_cmd(self, key: str) -> PutFileCmdContext:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryShellCache(ShellCache):
+    def __init__(self, dfc: DirectoryFileCache) -> None:
+        super().__init__()
+
+        self._dfc = dfc
+
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        f = self._dfc.get_file(key)
+        if f is None:
+            return None
+        return ShellCmd(f'cat {shlex.quote(f)}')
+
+    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
+        def __init__(self, tf: str, f: str) -> None:
+            super().__init__()
+
+            self._tf = tf
+            self._f = f
+
+        @property
+        def cmd(self) -> ShellCmd:
+            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
+
+        def _commit(self) -> None:
+            os.replace(self._tf, self._f)
+
+        def _abort(self) -> None:
+            os.unlink(self._tf)
+
+    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
+        f = self._dfc.get_cache_file_path(key, make_dirs=True)
+        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)

omdev/ci/ci.py

@@ -13,19 +13,19 @@ from omlish.lite.contextmanagers import ExitStacked
 from omlish.lite.contextmanagers import defer
 
 from .cache import FileCache
+from .cache import ShellCache
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
-from .dockertars import build_docker_file_hash
-from .dockertars import build_docker_tar
-from .dockertars import is_docker_image_present
-from .dockertars import load_docker_tar
-from .dockertars import pull_docker_tar
-from .dockertars import read_docker_tar_image_id
+from .docker import build_docker_file_hash
+from .docker import build_docker_image
+from .docker import is_docker_image_present
+from .docker import load_docker_tar_cmd
+from .docker import pull_docker_image
+from .docker import save_docker_tar_cmd
 from .requirements import build_requirements_hash
 from .requirements import download_requirements
-
-
-##
+from .shell import ShellCmd
+from .utils import log_timing_context
 
 
 class Ci(ExitStacked):
@@ -40,8 +40,12 @@ class Ci(ExitStacked):
         compose_file: str
         service: str
 
+        cmd: ShellCmd
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
+        always_pull: bool = False
+
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
 
@@ -49,40 +53,61 @@ class Ci(ExitStacked):
             self,
             cfg: Config,
             *,
+            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
+        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    def load_docker_image(self, image: str) -> None:
-        if is_docker_image_present(image):
+    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._shell_cache is None:
+            return None
+
+        get_cache_cmd = self._shell_cache.get_file_cmd(key)
+        if get_cache_cmd is None:
+            return None
+
+        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
+
+        return load_docker_tar_cmd(get_cache_cmd)
+
+    def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._shell_cache is None:
+            return
+
+        with self._shell_cache.put_file_cmd(key) as put_cache:
+            put_cache_cmd = put_cache.cmd
+
+            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
+
+            save_docker_tar_cmd(image, put_cache_cmd)
+
+    #
+
+    def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and is_docker_image_present(image):
             return
 
         dep_suffix = image
         for c in '/:.-_':
             dep_suffix = dep_suffix.replace(c, '-')
 
-        tar_file_name = f'docker-{dep_suffix}.tar'
-
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
-            load_docker_tar(cache_tar_file)
+        cache_key = f'docker-{dep_suffix}'
+        if self._load_cache_docker_image(cache_key) is not None:
             return
 
-        temp_dir = tempfile.mkdtemp()
-        with defer(lambda: shutil.rmtree(temp_dir)):
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+        pull_docker_image(image)
 
-            pull_docker_tar(
-                image,
-                temp_tar_file,
-            )
+        self._save_cache_docker_image(cache_key, image)
 
-            if self._file_cache is not None:
-                self._file_cache.put_file(temp_tar_file)
+    def load_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            self._load_docker_image(image)
 
     @cached_nullary
     def load_compose_service_dependencies(self) -> None:
@@ -96,46 +121,46 @@ class Ci(ExitStacked):
 
     #
 
-    @cached_nullary
-    def build_ci_image(self) -> str:
+    def _resolve_ci_image(self) -> str:
         docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_name = f'ci-{docker_file_hash}.tar'
-
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
-            image_id = read_docker_tar_image_id(cache_tar_file)
-            load_docker_tar(cache_tar_file)
-            return image_id
+        cache_key = f'ci-{docker_file_hash}'
+        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
+            return cache_image_id
 
-        temp_dir = tempfile.mkdtemp()
-        with defer(lambda: shutil.rmtree(temp_dir)):
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+        image_id = build_docker_image(
+            self._cfg.docker_file,
+            cwd=self._cfg.project_dir,
+        )
 
-            image_id = build_docker_tar(
-                self._cfg.docker_file,
-                temp_tar_file,
-                cwd=self._cfg.project_dir,
-            )
+        self._save_cache_docker_image(cache_key, image_id)
 
-            if self._file_cache is not None:
-                self._file_cache.put_file(temp_tar_file)
+        return image_id
 
+    @cached_nullary
+    def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
             return image_id
 
     #
 
-    @cached_nullary
-    def build_requirements_dir(self) -> str:
-        requirements_txts = check.not_none(self._cfg.requirements_txts)
+    def _resolve_requirements_dir(self) -> str:
+        requirements_txts = [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
 
         requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_name = f'requirements-{requirements_hash}.tar'
+        tar_file_key = f'requirements-{requirements_hash}'
+        tar_file_name = f'{tar_file_key}.tar'
 
         temp_dir = tempfile.mkdtemp()
         self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
 
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
             with tarfile.open(cache_tar_file) as tar:
                 tar.extractall(path=temp_dir)  # noqa
 
@@ -145,7 +170,7 @@ class Ci(ExitStacked):
         os.makedirs(temp_requirements_dir)
 
         download_requirements(
-            self.build_ci_image(),
+            self.resolve_ci_image(),
             temp_requirements_dir,
             requirements_txts,
         )
@@ -160,21 +185,20 @@ class Ci(ExitStacked):
                         arcname=requirement_file,
                     )
 
-            self._file_cache.put_file(temp_tar_file)
+            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
 
         return temp_requirements_dir
 
-    #
-
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
-
-        ci_image = self.build_ci_image()
-
-        requirements_dir = self.build_requirements_dir()
+    @cached_nullary
+    def resolve_requirements_dir(self) -> str:
+        with log_timing_context('Resolve requirements dir') as ltc:
+            requirements_dir = self._resolve_requirements_dir()
+            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
+            return requirements_dir
 
-        #
+    #
 
+    def _run_compose_(self) -> None:
         setup_cmds = [
             'pip install --root-user-action ignore --find-links /requirements --no-index uv',
             (
@@ -185,30 +209,41 @@ class Ci(ExitStacked):
 
         #
 
-        test_cmds = [
-            '(cd /project && python3 -m pytest -svv test.py)',
-        ]
+        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
+            *setup_cmds,
+            f'({self._cfg.cmd.s})',
+        ]))
 
         #
 
-        bash_src = ' && '.join([
-            *setup_cmds,
-            *test_cmds,
-        ])
-
         with DockerComposeRun(DockerComposeRun.Config(
-                compose_file=self._cfg.compose_file,
-                service=self._cfg.service,
+            compose_file=self._cfg.compose_file,
+            service=self._cfg.service,
 
-                image=ci_image,
+            image=self.resolve_ci_image(),
 
-                run_cmd=['bash', '-c', bash_src],
+            cmd=ci_cmd,
 
-                run_options=[
-                    '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                    '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
-                ],
+            run_options=[
+                '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
+                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+            ],
 
-                cwd=self._cfg.project_dir,
+            cwd=self._cfg.project_dir,
         )) as ci_compose_run:
             ci_compose_run.run()
+
+    def _run_compose(self) -> None:
+        with log_timing_context('Run compose'):
+            self._run_compose_()
+
+    #
+
+    def run(self) -> None:
+        self.load_compose_service_dependencies()
+
+        self.resolve_ci_image()
+
+        self.resolve_requirements_dir()
+
+        self._run_compose()

omdev/ci/cli.py

@@ -20,15 +20,18 @@ from omlish.argparse.cli import ArgparseCli
 from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.check import check
+from omlish.logs.standard import configure_standard_logging
 
 from .cache import DirectoryFileCache
+from .cache import DirectoryShellCache
 from .cache import FileCache
+from .cache import ShellCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
+from .github.cache import GithubShellCache
+from .github.cli import GithubCli
 from .requirements import build_requirements_hash
-
-
-##
+from .shell import ShellCmd
 
 
 class CiCli(ArgparseCli):
@@ -59,23 +62,32 @@ class CiCli(ArgparseCli):
 
     #
 
+    @argparse_cmd(
+        accepts_unknown=True,
+    )
+    def github(self) -> ta.Optional[int]:
+        return GithubCli(self.unknown_args).cli_run()
+
+    #
+
     @argparse_cmd(
         argparse_arg('project-dir'),
         argparse_arg('service'),
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
+        argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
+        argparse_arg('--always-pull', action='store_true'),
     )
     async def run(self) -> None:
-        await asyncio.sleep(1)
-
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
         service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
+        always_pull = self.args.always_pull
 
         #
 
@@ -85,7 +97,7 @@ class CiCli(ArgparseCli):
 
         def find_alt_file(*alts: str) -> ta.Optional[str]:
             for alt in alts:
-                alt_file = os.path.join(project_dir, alt)
+                alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
                     return alt_file
             return None
@@ -121,24 +133,44 @@ class CiCli(ArgparseCli):
 
         #
 
+        shell_cache: ta.Optional[ShellCache] = None
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
             if not os.path.exists(cache_dir):
                 os.makedirs(cache_dir)
             check.state(os.path.isdir(cache_dir))
-            file_cache = DirectoryFileCache(cache_dir)
+
+            directory_file_cache = DirectoryFileCache(cache_dir)
+
+            file_cache = directory_file_cache
+
+            if self.args.github_cache:
+                shell_cache = GithubShellCache(cache_dir)
+            else:
+                shell_cache = DirectoryShellCache(directory_file_cache)
 
         #
 
         with Ci(
                 Ci.Config(
                     project_dir=project_dir,
+
                     docker_file=docker_file,
+
                     compose_file=compose_file,
                     service=service,
+
                     requirements_txts=requirements_txts,
+
+                    cmd=ShellCmd(' && '.join([
+                        'cd /project',
+                        'python3 -m pytest -svv test.py',
+                    ])),
+
+                    always_pull=always_pull,
                 ),
                 file_cache=file_cache,
+                shell_cache=shell_cache,
         ) as ci:
             ci.run()
 
@@ -148,6 +180,8 @@ async def _async_main() -> ta.Optional[int]:
 
 
 def _main() -> None:
+    configure_standard_logging('DEBUG')
+
     sys.exit(rc if isinstance(rc := asyncio.run(_async_main()), int) else 0)
 
 

omdev/ci/compose.py

@@ -4,8 +4,11 @@
 TODO:
  - fix rmi - only when not referenced anymore
 """
+import contextlib
 import dataclasses as dc
+import itertools
 import os.path
+import shlex
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
@@ -15,6 +18,7 @@ from omlish.lite.contextmanagers import defer
 from omlish.lite.json import json_dumps_pretty
 from omlish.subprocesses import subprocesses
 
+from .shell import ShellCmd
 from .utils import make_temp_file
 from .utils import read_yaml_file
 
@@ -50,7 +54,7 @@ class DockerComposeRun(ExitStacked):
 
         image: str
 
-        run_cmd: ta.Sequence[str]
+        cmd: ShellCmd
 
         #
 
@@ -60,9 +64,11 @@ class DockerComposeRun(ExitStacked):
 
         #
 
-        def __post_init__(self) -> None:
-            check.not_isinstance(self.run_cmd, str)
+        no_dependency_cleanup: bool = False
+
+        #
 
+        def __post_init__(self) -> None:
             check.not_isinstance(self.run_options, str)
 
     def __init__(self, cfg: Config) -> None:
@@ -171,28 +177,38 @@ class DockerComposeRun(ExitStacked):
 
     #
 
+    def _cleanup_dependencies(self) -> None:
+        subprocesses.check_call(
+            'docker',
+            'compose',
+            '-f', self.rewrite_compose_file(),
+            'down',
+        )
+
     def run(self) -> None:
         self.tag_image()
 
         compose_file = self.rewrite_compose_file()
 
-        try:
-            subprocesses.check_call(
+        with contextlib.ExitStack() as es:
+            if not self._cfg.no_dependency_cleanup:
+                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+
+            sh_cmd = ' '.join([
                 'docker',
                 'compose',
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *self._cfg.run_options or [],
+                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *(self._cfg.run_options or []),
                 self._cfg.service,
-                *self._cfg.run_cmd,
-                **self._subprocess_kwargs,
-            )
+                'sh', '-c', shlex.quote(self._cfg.cmd.s),
+            ])
 
-        finally:
-            subprocesses.check_call(
-                'docker',
-                'compose',
-                '-f', compose_file,
-                'down',
+            run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
+
+            run_cmd.run(
+                subprocesses.check_call,
+                **self._subprocess_kwargs,
             )