omdev 0.0.0.dev210__tar.gz → 0.0.0.dev211__tar.gz

{omdev-0.0.0.dev210/omdev.egg-info → omdev-0.0.0.dev211}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev210
+Version: 0.0.0.dev211
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev210
+Requires-Dist: omlish==0.0.0.dev211
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

omdev-0.0.0.dev211/omdev/ci/cache.py

@@ -0,0 +1,168 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import os.path
+import shlex
+import shutil
+import typing as ta
+
+from .shell import ShellCmd
+
+
+##
+
+
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file(self, key: str) -> ta.Optional[str]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryFileCache(FileCache):
+    def __init__(self, dir: str) -> None:  # noqa
+        super().__init__()
+
+        self._dir = dir
+
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+            *,
+            make_dirs: bool = False,
+    ) -> str:
+        if make_dirs:
+            os.makedirs(self._dir, exist_ok=True)
+        return os.path.join(self._dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
+
+    def put_file(self, key: str, file_path: str) -> None:
+        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
+        shutil.copyfile(file_path, cache_file_path)
+
+
+##
+
+
+class ShellCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        raise NotImplementedError
+
+    class PutFileCmdContext(abc.ABC):
+        def __init__(self) -> None:
+            super().__init__()
+
+            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
+
+        @property
+        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
+            return self._state
+
+        #
+
+        @property
+        @abc.abstractmethod
+        def cmd(self) -> ShellCmd:
+            raise NotImplementedError
+
+        #
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            if exc_val is None:
+                self.commit()
+            else:
+                self.abort()
+
+        #
+
+        @abc.abstractmethod
+        def _commit(self) -> None:
+            raise NotImplementedError
+
+        def commit(self) -> None:
+            if self._state == 'committed':
+                return
+            elif self._state == 'open':
+                self._commit()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+        #
+
+        @abc.abstractmethod
+        def _abort(self) -> None:
+            raise NotImplementedError
+
+        def abort(self) -> None:
+            if self._state == 'aborted':
+                return
+            elif self._state == 'open':
+                self._abort()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+    @abc.abstractmethod
+    def put_file_cmd(self, key: str) -> PutFileCmdContext:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryShellCache(ShellCache):
+    def __init__(self, dfc: DirectoryFileCache) -> None:
+        super().__init__()
+
+        self._dfc = dfc
+
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        f = self._dfc.get_file(key)
+        if f is None:
+            return None
+        return ShellCmd(f'cat {shlex.quote(f)}')
+
+    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
+        def __init__(self, tf: str, f: str) -> None:
+            super().__init__()
+
+            self._tf = tf
+            self._f = f
+
+        @property
+        def cmd(self) -> ShellCmd:
+            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
+
+        def _commit(self) -> None:
+            os.replace(self._tf, self._f)
+
+        def _abort(self) -> None:
+            os.unlink(self._tf)
+
+    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
+        f = self._dfc.get_cache_file_path(key, make_dirs=True)
+        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)

omdev-0.0.0.dev211/omdev/ci/ci.py

@@ -0,0 +1,249 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import os.path
+import shutil
+import tarfile
+import tempfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import defer
+
+from .cache import FileCache
+from .cache import ShellCache
+from .compose import DockerComposeRun
+from .compose import get_compose_service_dependencies
+from .docker import build_docker_file_hash
+from .docker import build_docker_image
+from .docker import is_docker_image_present
+from .docker import load_docker_tar_cmd
+from .docker import pull_docker_image
+from .docker import save_docker_tar_cmd
+from .requirements import build_requirements_hash
+from .requirements import download_requirements
+from .shell import ShellCmd
+from .utils import log_timing_context
+
+
+class Ci(ExitStacked):
+    FILE_NAME_HASH_LEN = 16
+
+    @dc.dataclass(frozen=True)
+    class Config:
+        project_dir: str
+
+        docker_file: str
+
+        compose_file: str
+        service: str
+
+        cmd: ShellCmd
+
+        requirements_txts: ta.Optional[ta.Sequence[str]] = None
+
+        always_pull: bool = False
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.requirements_txts, str)
+
+    def __init__(
+            self,
+            cfg: Config,
+            *,
+            shell_cache: ta.Optional[ShellCache] = None,
+            file_cache: ta.Optional[FileCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+        self._shell_cache = shell_cache
+        self._file_cache = file_cache
+
+    #
+
+    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._shell_cache is None:
+            return None
+
+        get_cache_cmd = self._shell_cache.get_file_cmd(key)
+        if get_cache_cmd is None:
+            return None
+
+        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
+
+        return load_docker_tar_cmd(get_cache_cmd)
+
+    def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._shell_cache is None:
+            return
+
+        with self._shell_cache.put_file_cmd(key) as put_cache:
+            put_cache_cmd = put_cache.cmd
+
+            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
+
+            save_docker_tar_cmd(image, put_cache_cmd)
+
+    #
+
+    def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and is_docker_image_present(image):
+            return
+
+        dep_suffix = image
+        for c in '/:.-_':
+            dep_suffix = dep_suffix.replace(c, '-')
+
+        cache_key = f'docker-{dep_suffix}'
+        if self._load_cache_docker_image(cache_key) is not None:
+            return
+
+        pull_docker_image(image)
+
+        self._save_cache_docker_image(cache_key, image)
+
+    def load_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            self._load_docker_image(image)
+
+    @cached_nullary
+    def load_compose_service_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
+
+        for dep_image in deps.values():
+            self.load_docker_image(dep_image)
+
+    #
+
+    def _resolve_ci_image(self) -> str:
+        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+
+        cache_key = f'ci-{docker_file_hash}'
+        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
+            return cache_image_id
+
+        image_id = build_docker_image(
+            self._cfg.docker_file,
+            cwd=self._cfg.project_dir,
+        )
+
+        self._save_cache_docker_image(cache_key, image_id)
+
+        return image_id
+
+    @cached_nullary
+    def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
+
+    #
+
+    def _resolve_requirements_dir(self) -> str:
+        requirements_txts = [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
+
+        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+
+        tar_file_key = f'requirements-{requirements_hash}'
+        tar_file_name = f'{tar_file_key}.tar'
+
+        temp_dir = tempfile.mkdtemp()
+        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
+            with tarfile.open(cache_tar_file) as tar:
+                tar.extractall(path=temp_dir)  # noqa
+
+            return temp_dir
+
+        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
+        os.makedirs(temp_requirements_dir)
+
+        download_requirements(
+            self.resolve_ci_image(),
+            temp_requirements_dir,
+            requirements_txts,
+        )
+
+        if self._file_cache is not None:
+            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+
+            with tarfile.open(temp_tar_file, 'w') as tar:
+                for requirement_file in os.listdir(temp_requirements_dir):
+                    tar.add(
+                        os.path.join(temp_requirements_dir, requirement_file),
+                        arcname=requirement_file,
+                    )
+
+            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
+
+        return temp_requirements_dir
+
+    @cached_nullary
+    def resolve_requirements_dir(self) -> str:
+        with log_timing_context('Resolve requirements dir') as ltc:
+            requirements_dir = self._resolve_requirements_dir()
+            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
+            return requirements_dir
+
+    #
+
+    def _run_compose_(self) -> None:
+        setup_cmds = [
+            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
+            (
+                'uv pip install --system --find-links /requirements ' +
+                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
+            ),
+        ]
+
+        #
+
+        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
+            *setup_cmds,
+            f'({self._cfg.cmd.s})',
+        ]))
+
+        #
+
+        with DockerComposeRun(DockerComposeRun.Config(
+            compose_file=self._cfg.compose_file,
+            service=self._cfg.service,
+
+            image=self.resolve_ci_image(),
+
+            cmd=ci_cmd,
+
+            run_options=[
+                '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
+                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+            ],
+
+            cwd=self._cfg.project_dir,
+        )) as ci_compose_run:
+            ci_compose_run.run()
+
+    def _run_compose(self) -> None:
+        with log_timing_context('Run compose'):
+            self._run_compose_()
+
+    #
+
+    def run(self) -> None:
+        self.load_compose_service_dependencies()
+
+        self.resolve_ci_image()
+
+        self.resolve_requirements_dir()
+
+        self._run_compose()

{omdev-0.0.0.dev210 → omdev-0.0.0.dev211}/omdev/ci/cli.py

@@ -20,15 +20,18 @@ from omlish.argparse.cli import ArgparseCli
 from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.check import check
+from omlish.logs.standard import configure_standard_logging
 
 from .cache import DirectoryFileCache
+from .cache import DirectoryShellCache
 from .cache import FileCache
+from .cache import ShellCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
+from .github.cache import GithubShellCache
+from .github.cli import GithubCli
 from .requirements import build_requirements_hash
-
-
-##
+from .shell import ShellCmd
 
 
 class CiCli(ArgparseCli):
@@ -59,23 +62,32 @@ class CiCli(ArgparseCli):
 
     #
 
+    @argparse_cmd(
+        accepts_unknown=True,
+    )
+    def github(self) -> ta.Optional[int]:
+        return GithubCli(self.unknown_args).cli_run()
+
+    #
+
     @argparse_cmd(
         argparse_arg('project-dir'),
         argparse_arg('service'),
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
+        argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
+        argparse_arg('--always-pull', action='store_true'),
     )
     async def run(self) -> None:
-        await asyncio.sleep(1)
-
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
         service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
+        always_pull = self.args.always_pull
 
         #
 
@@ -85,7 +97,7 @@ class CiCli(ArgparseCli):
 
         def find_alt_file(*alts: str) -> ta.Optional[str]:
             for alt in alts:
-                alt_file = os.path.join(project_dir, alt)
+                alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
                     return alt_file
             return None
@@ -121,24 +133,44 @@ class CiCli(ArgparseCli):
 
         #
 
+        shell_cache: ta.Optional[ShellCache] = None
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
             if not os.path.exists(cache_dir):
                 os.makedirs(cache_dir)
             check.state(os.path.isdir(cache_dir))
-            file_cache = DirectoryFileCache(cache_dir)
+
+            directory_file_cache = DirectoryFileCache(cache_dir)
+
+            file_cache = directory_file_cache
+
+            if self.args.github_cache:
+                shell_cache = GithubShellCache(cache_dir)
+            else:
+                shell_cache = DirectoryShellCache(directory_file_cache)
 
         #
 
         with Ci(
                 Ci.Config(
                     project_dir=project_dir,
+
                     docker_file=docker_file,
+
                     compose_file=compose_file,
                     service=service,
+
                     requirements_txts=requirements_txts,
+
+                    cmd=ShellCmd(' && '.join([
+                        'cd /project',
+                        'python3 -m pytest -svv test.py',
+                    ])),
+
+                    always_pull=always_pull,
                 ),
                 file_cache=file_cache,
+                shell_cache=shell_cache,
         ) as ci:
             ci.run()
 
@@ -148,6 +180,8 @@ async def _async_main() -> ta.Optional[int]:
 
 
 def _main() -> None:
+    configure_standard_logging('DEBUG')
+
     sys.exit(rc if isinstance(rc := asyncio.run(_async_main()), int) else 0)
 
 

{omdev-0.0.0.dev210 → omdev-0.0.0.dev211}/omdev/ci/compose.py

@@ -4,8 +4,11 @@
 TODO:
  - fix rmi - only when not referenced anymore
 """
+import contextlib
 import dataclasses as dc
+import itertools
 import os.path
+import shlex
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
@@ -15,6 +18,7 @@ from omlish.lite.contextmanagers import defer
 from omlish.lite.json import json_dumps_pretty
 from omlish.subprocesses import subprocesses
 
+from .shell import ShellCmd
 from .utils import make_temp_file
 from .utils import read_yaml_file
 
@@ -50,7 +54,7 @@ class DockerComposeRun(ExitStacked):
 
         image: str
 
-        run_cmd: ta.Sequence[str]
+        cmd: ShellCmd
 
         #
 
@@ -60,9 +64,11 @@ class DockerComposeRun(ExitStacked):
 
         #
 
-        def __post_init__(self) -> None:
-            check.not_isinstance(self.run_cmd, str)
+        no_dependency_cleanup: bool = False
+
+        #
 
+        def __post_init__(self) -> None:
             check.not_isinstance(self.run_options, str)
 
     def __init__(self, cfg: Config) -> None:
@@ -171,28 +177,38 @@ class DockerComposeRun(ExitStacked):
 
     #
 
+    def _cleanup_dependencies(self) -> None:
+        subprocesses.check_call(
+            'docker',
+            'compose',
+            '-f', self.rewrite_compose_file(),
+            'down',
+        )
+
     def run(self) -> None:
         self.tag_image()
 
         compose_file = self.rewrite_compose_file()
 
-        try:
-            subprocesses.check_call(
+        with contextlib.ExitStack() as es:
+            if not self._cfg.no_dependency_cleanup:
+                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+
+            sh_cmd = ' '.join([
                 'docker',
                 'compose',
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *self._cfg.run_options or [],
+                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *(self._cfg.run_options or []),
                 self._cfg.service,
-                *self._cfg.run_cmd,
-                **self._subprocess_kwargs,
-            )
+                'sh', '-c', shlex.quote(self._cfg.cmd.s),
+            ])
 
-        finally:
-            subprocesses.check_call(
-                'docker',
-                'compose',
-                '-f', compose_file,
-                'down',
+            run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
+
+            run_cmd.run(
+                subprocesses.check_call,
+                **self._subprocess_kwargs,
             )