omdev 0.0.0.dev210__py3-none-any.whl → 0.0.0.dev212__py3-none-any.whl

omdev/ci/ci.py

@@ -13,19 +13,19 @@ from omlish.lite.contextmanagers import ExitStacked
 from omlish.lite.contextmanagers import defer
 
 from .cache import FileCache
+from .cache import ShellCache
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
-from .dockertars import build_docker_file_hash
-from .dockertars import build_docker_tar
-from .dockertars import is_docker_image_present
-from .dockertars import load_docker_tar
-from .dockertars import pull_docker_tar
-from .dockertars import read_docker_tar_image_id
+from .docker import build_docker_file_hash
+from .docker import build_docker_image
+from .docker import is_docker_image_present
+from .docker import load_docker_tar_cmd
+from .docker import pull_docker_image
+from .docker import save_docker_tar_cmd
 from .requirements import build_requirements_hash
 from .requirements import download_requirements
-
-
-##
+from .shell import ShellCmd
+from .utils import log_timing_context
 
 
 class Ci(ExitStacked):
@@ -40,8 +40,12 @@ class Ci(ExitStacked):
         compose_file: str
         service: str
 
+        cmd: ShellCmd
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
+        always_pull: bool = False
+
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
 
@@ -49,40 +53,61 @@ class Ci(ExitStacked):
             self,
             cfg: Config,
             *,
+            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
+        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    def load_docker_image(self, image: str) -> None:
-        if is_docker_image_present(image):
+    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._shell_cache is None:
+            return None
+
+        get_cache_cmd = self._shell_cache.get_file_cmd(key)
+        if get_cache_cmd is None:
+            return None
+
+        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
+
+        return load_docker_tar_cmd(get_cache_cmd)
+
+    def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._shell_cache is None:
+            return
+
+        with self._shell_cache.put_file_cmd(key) as put_cache:
+            put_cache_cmd = put_cache.cmd
+
+            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
+
+            save_docker_tar_cmd(image, put_cache_cmd)
+
+    #
+
+    def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and is_docker_image_present(image):
             return
 
         dep_suffix = image
         for c in '/:.-_':
             dep_suffix = dep_suffix.replace(c, '-')
 
-        tar_file_name = f'docker-{dep_suffix}.tar'
-
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
-            load_docker_tar(cache_tar_file)
+        cache_key = f'docker-{dep_suffix}'
+        if self._load_cache_docker_image(cache_key) is not None:
             return
 
-        temp_dir = tempfile.mkdtemp()
-        with defer(lambda: shutil.rmtree(temp_dir)):
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+        pull_docker_image(image)
 
-            pull_docker_tar(
-                image,
-                temp_tar_file,
-            )
+        self._save_cache_docker_image(cache_key, image)
 
-            if self._file_cache is not None:
-                self._file_cache.put_file(temp_tar_file)
+    def load_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            self._load_docker_image(image)
 
     @cached_nullary
     def load_compose_service_dependencies(self) -> None:
@@ -96,46 +121,46 @@ class Ci(ExitStacked):
 
     #
 
-    @cached_nullary
-    def build_ci_image(self) -> str:
+    def _resolve_ci_image(self) -> str:
         docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_name = f'ci-{docker_file_hash}.tar'
-
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
-            image_id = read_docker_tar_image_id(cache_tar_file)
-            load_docker_tar(cache_tar_file)
-            return image_id
+        cache_key = f'ci-{docker_file_hash}'
+        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
+            return cache_image_id
 
-        temp_dir = tempfile.mkdtemp()
-        with defer(lambda: shutil.rmtree(temp_dir)):
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+        image_id = build_docker_image(
+            self._cfg.docker_file,
+            cwd=self._cfg.project_dir,
+        )
 
-            image_id = build_docker_tar(
-                self._cfg.docker_file,
-                temp_tar_file,
-                cwd=self._cfg.project_dir,
-            )
+        self._save_cache_docker_image(cache_key, image_id)
 
-            if self._file_cache is not None:
-                self._file_cache.put_file(temp_tar_file)
+        return image_id
 
+    @cached_nullary
+    def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
             return image_id
 
     #
 
-    @cached_nullary
-    def build_requirements_dir(self) -> str:
-        requirements_txts = check.not_none(self._cfg.requirements_txts)
+    def _resolve_requirements_dir(self) -> str:
+        requirements_txts = [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
 
         requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_name = f'requirements-{requirements_hash}.tar'
+        tar_file_key = f'requirements-{requirements_hash}'
+        tar_file_name = f'{tar_file_key}.tar'
 
         temp_dir = tempfile.mkdtemp()
         self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
 
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
             with tarfile.open(cache_tar_file) as tar:
                 tar.extractall(path=temp_dir)  # noqa
 
@@ -145,7 +170,7 @@ class Ci(ExitStacked):
         os.makedirs(temp_requirements_dir)
 
         download_requirements(
-            self.build_ci_image(),
+            self.resolve_ci_image(),
             temp_requirements_dir,
             requirements_txts,
         )
@@ -160,21 +185,20 @@ class Ci(ExitStacked):
                         arcname=requirement_file,
                     )
 
-            self._file_cache.put_file(temp_tar_file)
+            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
 
         return temp_requirements_dir
 
-    #
-
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
-
-        ci_image = self.build_ci_image()
-
-        requirements_dir = self.build_requirements_dir()
+    @cached_nullary
+    def resolve_requirements_dir(self) -> str:
+        with log_timing_context('Resolve requirements dir') as ltc:
+            requirements_dir = self._resolve_requirements_dir()
+            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
+            return requirements_dir
 
-        #
+    #
 
+    def _run_compose_(self) -> None:
         setup_cmds = [
             'pip install --root-user-action ignore --find-links /requirements --no-index uv',
             (
@@ -185,30 +209,41 @@ class Ci(ExitStacked):
 
         #
 
-        test_cmds = [
-            '(cd /project && python3 -m pytest -svv test.py)',
-        ]
+        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
+            *setup_cmds,
+            f'({self._cfg.cmd.s})',
+        ]))
 
         #
 
-        bash_src = ' && '.join([
-            *setup_cmds,
-            *test_cmds,
-        ])
-
         with DockerComposeRun(DockerComposeRun.Config(
-                compose_file=self._cfg.compose_file,
-                service=self._cfg.service,
+            compose_file=self._cfg.compose_file,
+            service=self._cfg.service,
 
-                image=ci_image,
+            image=self.resolve_ci_image(),
 
-                run_cmd=['bash', '-c', bash_src],
+            cmd=ci_cmd,
 
-                run_options=[
-                    '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                    '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
-                ],
+            run_options=[
+                '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
+                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+            ],
 
-                cwd=self._cfg.project_dir,
+            cwd=self._cfg.project_dir,
         )) as ci_compose_run:
             ci_compose_run.run()
+
+    def _run_compose(self) -> None:
+        with log_timing_context('Run compose'):
+            self._run_compose_()
+
+    #
+
+    def run(self) -> None:
+        self.load_compose_service_dependencies()
+
+        self.resolve_ci_image()
+
+        self.resolve_requirements_dir()
+
+        self._run_compose()

omdev/ci/cli.py

@@ -20,15 +20,18 @@ from omlish.argparse.cli import ArgparseCli
 from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.check import check
+from omlish.logs.standard import configure_standard_logging
 
 from .cache import DirectoryFileCache
+from .cache import DirectoryShellCache
 from .cache import FileCache
+from .cache import ShellCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
+from .github.cache import GithubShellCache
+from .github.cli import GithubCli
 from .requirements import build_requirements_hash
-
-
-##
+from .shell import ShellCmd
 
 
 class CiCli(ArgparseCli):
@@ -59,23 +62,32 @@ class CiCli(ArgparseCli):
 
     #
 
+    @argparse_cmd(
+        accepts_unknown=True,
+    )
+    def github(self) -> ta.Optional[int]:
+        return GithubCli(self.unknown_args).cli_run()
+
+    #
+
     @argparse_cmd(
         argparse_arg('project-dir'),
         argparse_arg('service'),
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
+        argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
+        argparse_arg('--always-pull', action='store_true'),
     )
     async def run(self) -> None:
-        await asyncio.sleep(1)
-
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
         service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
+        always_pull = self.args.always_pull
 
         #
 
@@ -85,7 +97,7 @@ class CiCli(ArgparseCli):
 
         def find_alt_file(*alts: str) -> ta.Optional[str]:
             for alt in alts:
-                alt_file = os.path.join(project_dir, alt)
+                alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
                     return alt_file
             return None
@@ -100,10 +112,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:
@@ -121,24 +139,44 @@ class CiCli(ArgparseCli):
 
         #
 
+        shell_cache: ta.Optional[ShellCache] = None
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
             if not os.path.exists(cache_dir):
                 os.makedirs(cache_dir)
             check.state(os.path.isdir(cache_dir))
-            file_cache = DirectoryFileCache(cache_dir)
+
+            directory_file_cache = DirectoryFileCache(cache_dir)
+
+            file_cache = directory_file_cache
+
+            if self.args.github_cache:
+                shell_cache = GithubShellCache(cache_dir)
+            else:
+                shell_cache = DirectoryShellCache(directory_file_cache)
 
         #
 
         with Ci(
                 Ci.Config(
                     project_dir=project_dir,
+
                     docker_file=docker_file,
+
                     compose_file=compose_file,
                     service=service,
+
                     requirements_txts=requirements_txts,
+
+                    cmd=ShellCmd(' && '.join([
+                        'cd /project',
+                        'python3 -m pytest -svv test.py',
+                    ])),
+
+                    always_pull=always_pull,
                 ),
                 file_cache=file_cache,
+                shell_cache=shell_cache,
         ) as ci:
             ci.run()
 
@@ -148,6 +186,8 @@ async def _async_main() -> ta.Optional[int]:
 
 
 def _main() -> None:
+    configure_standard_logging('DEBUG')
+
     sys.exit(rc if isinstance(rc := asyncio.run(_async_main()), int) else 0)
 
 

omdev/ci/compose.py

@@ -4,8 +4,11 @@
 TODO:
  - fix rmi - only when not referenced anymore
 """
+import contextlib
 import dataclasses as dc
+import itertools
 import os.path
+import shlex
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
@@ -15,6 +18,7 @@ from omlish.lite.contextmanagers import defer
 from omlish.lite.json import json_dumps_pretty
 from omlish.subprocesses import subprocesses
 
+from .shell import ShellCmd
 from .utils import make_temp_file
 from .utils import read_yaml_file
 
@@ -50,7 +54,7 @@ class DockerComposeRun(ExitStacked):
 
         image: str
 
-        run_cmd: ta.Sequence[str]
+        cmd: ShellCmd
 
         #
 
@@ -60,9 +64,11 @@ class DockerComposeRun(ExitStacked):
 
         #
 
-        def __post_init__(self) -> None:
-            check.not_isinstance(self.run_cmd, str)
+        no_dependency_cleanup: bool = False
+
+        #
 
+        def __post_init__(self) -> None:
             check.not_isinstance(self.run_options, str)
 
     def __init__(self, cfg: Config) -> None:
@@ -171,28 +177,41 @@ class DockerComposeRun(ExitStacked):
 
     #
 
+    def _cleanup_dependencies(self) -> None:
+        subprocesses.check_call(
+            'docker',
+            'compose',
+            '-f', self.rewrite_compose_file(),
+            'down',
+        )
+
     def run(self) -> None:
         self.tag_image()
 
         compose_file = self.rewrite_compose_file()
 
-        try:
-            subprocesses.check_call(
+        with contextlib.ExitStack() as es:
+            if not self._cfg.no_dependency_cleanup:
+                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+
+            sh_cmd = ' '.join([
                 'docker',
                 'compose',
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *self._cfg.run_options or [],
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
+                *(self._cfg.run_options or []),
                 self._cfg.service,
-                *self._cfg.run_cmd,
-                **self._subprocess_kwargs,
-            )
+                'sh', '-c', shlex.quote(self._cfg.cmd.s),
+            ])
 
-        finally:
-            subprocesses.check_call(
-                'docker',
-                'compose',
-                '-f', compose_file,
-                'down',
+            run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
+
+            run_cmd.run(
+                subprocesses.check_call,
+                **self._subprocess_kwargs,
             )

omdev/ci/{dockertars.py → docker.py}

@@ -6,8 +6,10 @@ TODO:
   - doesn't change too much though
 """
 import contextlib
+import dataclasses as dc
 import json
 import os.path
+import shlex
 import tarfile
 import typing as ta
 
@@ -15,6 +17,7 @@ from omlish.lite.check import check
 from omlish.lite.contextmanagers import defer
 from omlish.subprocesses import subprocesses
 
+from .shell import ShellCmd
 from .utils import make_temp_file
 from .utils import sha256_str
 
@@ -73,12 +76,8 @@ def is_docker_image_present(image: str) -> bool:
     return True
 
 
-##
-
-
-def pull_docker_tar(
+def pull_docker_image(
         image: str,
-        tar_file: str,
 ) -> None:
     subprocesses.check_call(
         'docker',
@@ -86,19 +85,11 @@ def pull_docker_tar(
         image,
     )
 
-    subprocesses.check_call(
-        'docker',
-        'save',
-        image,
-        '-o', tar_file,
-    )
-
 
-def build_docker_tar(
-    docker_file: str,
-    tar_file: str,
-    *,
-    cwd: ta.Optional[str] = None,
+def build_docker_image(
+        docker_file: str,
+        *,
+        cwd: ta.Optional[str] = None,
 ) -> str:
     id_file = make_temp_file()
     with defer(lambda: os.unlink(id_file)):
@@ -115,24 +106,46 @@ def build_docker_tar(
         with open(id_file) as f:
             image_id = check.single(f.read().strip().splitlines()).strip()
 
-        subprocesses.check_call(
-            'docker',
-            'save',
-            image_id,
-            '-o', tar_file,
-        )
-
-        return image_id
+    return image_id
 
 
 ##
 
 
-def load_docker_tar(
+def save_docker_tar_cmd(
+        image: str,
+        output_cmd: ShellCmd,
+) -> None:
+    cmd = dc.replace(output_cmd, s=f'docker save {image} | {output_cmd.s}')
+    cmd.run(subprocesses.check_call)
+
+
+def save_docker_tar(
+        image: str,
         tar_file: str,
 ) -> None:
-    subprocesses.check_call(
-        'docker',
-        'load',
-        '-i', tar_file,
+    return save_docker_tar_cmd(
+        image,
+        ShellCmd(f'cat > {shlex.quote(tar_file)}'),
     )
+
+
+#
+
+
+def load_docker_tar_cmd(
+        input_cmd: ShellCmd,
+) -> str:
+    cmd = dc.replace(input_cmd, s=f'{input_cmd.s} | docker load')
+
+    out = cmd.run(subprocesses.check_output).decode()
+
+    line = check.single(out.strip().splitlines())
+    loaded = line.partition(':')[2].strip()
+    return loaded
+
+
+def load_docker_tar(
+        tar_file: str,
+) -> str:
+    return load_docker_tar_cmd(ShellCmd(f'cat {shlex.quote(tar_file)}'))

omdev/ci/github/bootstrap.py

@@ -0,0 +1,11 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+sudo rm -rf \
+    /usr/local/.ghcup \
+    /opt/hostedtoolcache \
+
+/usr/local/.ghcup       6.4G, 3391250 files
+/opt/hostedtoolcache    8.0G, 14843980 files
+/usr/local/lib/android  6.4G, 17251667 files
+"""