omdev 0.0.0.dev213__tar.gz → 0.0.0.dev214__tar.gz

{omdev-0.0.0.dev213/omdev.egg-info → omdev-0.0.0.dev214}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev213
+Version: 0.0.0.dev214
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev213
+Requires-Dist: omlish==0.0.0.dev214
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev213 → omdev-0.0.0.dev214}/omdev/.manifests.json

@@ -267,7 +267,7 @@
     "module": ".tools.docker",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/docker.py",
-    "line": 258,
+    "line": 264,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "docker",

omdev-0.0.0.dev214/omdev/ci/cache.py

@@ -0,0 +1,147 @@
+# ruff: noqa: UP006 UP007
+import abc
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.logs import log
+
+from .consts import CI_CACHE_VERSION
+
+
+##
+
+
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    def __init__(
+            self,
+            *,
+            version: int = CI_CACHE_VERSION,
+    ) -> None:
+        super().__init__()
+
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version = version
+
+    @property
+    def version(self) -> int:
+        return self._version
+
+    #
+
+    @abc.abstractmethod
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryFileCache(FileCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            no_create: bool = False,
+            no_purge: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:  # noqa
+        super().__init__(**kwargs)
+
+        self._dir = dir
+        self._no_create = no_create
+        self._no_purge = no_purge
+
+    #
+
+    VERSION_FILE_NAME = '.ci-cache-version'
+
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
+
+        if self._no_create:
+            check.state(os.path.isdir(self._dir))
+
+        elif not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
+
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
+
+        if dir_version == self._version:
+            return
+
+        if self._no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
+
+        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
+
+        for n in sorted(os.listdir(self._dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self._dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
+
+        os.unlink(version_file)
+
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
+
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self._dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
+
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path

{omdev-0.0.0.dev213 → omdev-0.0.0.dev214}/omdev/ci/ci.py

@@ -1,20 +1,15 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import dataclasses as dc
 import os.path
-import shutil
-import tarfile
-import tempfile
 import typing as ta
 
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.contextmanagers import AsyncExitStacked
-from omlish.lite.contextmanagers import defer
+from omlish.os.temp import temp_file_context
 
 from .cache import FileCache
-from .cache import ShellCache
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
 from .docker import build_docker_file_hash
@@ -25,13 +20,12 @@ from .docker import pull_docker_image
 from .docker import save_docker_tar_cmd
 from .docker import tag_docker_image
 from .requirements import build_requirements_hash
-from .requirements import download_requirements
 from .shell import ShellCmd
 from .utils import log_timing_context
 
 
 class Ci(AsyncExitStacked):
-    FILE_NAME_HASH_LEN = 16
+    KEY_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
     class Config:
@@ -44,6 +38,8 @@ class Ci(AsyncExitStacked):
 
         cmd: ShellCmd
 
+        #
+
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
@@ -51,6 +47,10 @@ class Ci(AsyncExitStacked):
 
         no_dependencies: bool = False
 
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        #
+
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
 
@@ -58,42 +58,15 @@ class Ci(AsyncExitStacked):
             self,
             cfg: Config,
             *,
-            shell_cache: ta.Optional[ShellCache] = None,
             file_cache: ta.Optional[FileCache] = None,
     ) -> None:
         super().__init__()
 
         self._cfg = cfg
-        self._shell_cache = shell_cache
         self._file_cache = file_cache
 
     #
 
-    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
-        if self._shell_cache is None:
-            return None
-
-        get_cache_cmd = self._shell_cache.get_file_cmd(key)
-        if get_cache_cmd is None:
-            return None
-
-        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
-
-        return await load_docker_tar_cmd(get_cache_cmd)
-
-    async def _save_cache_docker_image(self, key: str, image: str) -> None:
-        if self._shell_cache is None:
-            return
-
-        with self._shell_cache.put_file_cmd(key) as put_cache:
-            put_cache_cmd = put_cache.cmd
-
-            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
-
-            await save_docker_tar_cmd(image, put_cache_cmd)
-
-    #
-
     async def _load_docker_image(self, image: str) -> None:
         if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
@@ -114,24 +87,38 @@ class Ci(AsyncExitStacked):
         with log_timing_context(f'Load docker image: {image}'):
             await self._load_docker_image(image)
 
-    @async_cached_nullary
-    async def load_compose_service_dependencies(self) -> None:
-        deps = get_compose_service_dependencies(
-            self._cfg.compose_file,
-            self._cfg.service,
-        )
+    #
 
-        for dep_image in deps.values():
-            await self.load_docker_image(dep_image)
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
 
-    #
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
 
-    @cached_nullary
-    def docker_file_hash(self) -> str:
-        return build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
 
-    async def _resolve_ci_image(self) -> str:
-        cache_key = f'ci-{self.docker_file_hash()}'
+        return await load_docker_tar_cmd(get_cache_cmd)
+
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
+
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
+
+            await save_docker_tar_cmd(image, write_tmp_cmd)
+
+            await self._file_cache.put_file(key, tmp_file, steal=True)
+
+    #
+
+    async def _resolve_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
         image_tag = f'{self._cfg.service}:{cache_key}'
 
         if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
@@ -144,21 +131,35 @@ class Ci(AsyncExitStacked):
             )
             return image_tag
 
-        image_id = await build_docker_image(
-            self._cfg.docker_file,
-            tag=image_tag,
-            cwd=self._cfg.project_dir,
-        )
+        image_id = await build_and_tag(image_tag)
 
         await self._save_cache_docker_image(cache_key, image_id)
 
         return image_tag
 
+    #
+
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_base_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            return await build_docker_image(
+                self._cfg.docker_file,
+                tag=image_tag,
+                cwd=self._cfg.project_dir,
+            )
+
+        cache_key = f'ci-base-{self.docker_file_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
     @async_cached_nullary
-    async def resolve_ci_image(self) -> str:
-        with log_timing_context('Resolve ci image') as ltc:
-            image_id = await self._resolve_ci_image()
-            ltc.set_description(f'Resolve ci image: {image_id}')
+    async def resolve_ci_base_image(self) -> str:
+        with log_timing_context('Resolve ci base image') as ltc:
+            image_id = await self._resolve_ci_base_image()
+            ltc.set_description(f'Resolve ci base image: {image_id}')
             return image_id
 
     #
@@ -172,82 +173,85 @@ class Ci(AsyncExitStacked):
 
     @cached_nullary
     def requirements_hash(self) -> str:
-        return build_requirements_hash(self.requirements_txts())[:self.FILE_NAME_HASH_LEN]
-
-    async def _resolve_requirements_dir(self) -> str:
-        tar_file_key = f'requirements-{self.docker_file_hash()}-{self.requirements_hash()}'
-        tar_file_name = f'{tar_file_key}.tar'
-
-        temp_dir = tempfile.mkdtemp()
-        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+        return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
 
-        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
-            with tarfile.open(cache_tar_file) as tar:
-                tar.extractall(path=temp_dir)  # noqa
+    async def _resolve_ci_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            base_image = await self.resolve_ci_base_image()
+
+            setup_cmds = [
+                ' '.join([
+                    'pip install',
+                    '--no-cache-dir',
+                    '--root-user-action ignore',
+                    'uv',
+                ]),
+                ' '.join([
+                    'uv pip install',
+                    '--no-cache',
+                    '--index-strategy unsafe-best-match',
+                    '--system',
+                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                ]),
+            ]
+            setup_cmd = ' && '.join(setup_cmds)
+
+            docker_file_lines = [
+                f'FROM {base_image}',
+                'RUN mkdir /project',
+                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                f'RUN {setup_cmd}',
+                'RUN rm /project/*',
+                'WORKDIR /project',
+            ]
+
+            with temp_file_context() as docker_file:
+                with open(docker_file, 'w') as f:  # noqa
+                    f.write('\n'.join(docker_file_lines))
+
+                return await build_docker_image(
+                    docker_file,
+                    tag=image_tag,
+                    cwd=self._cfg.project_dir,
+                )
+
+        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
 
-            return temp_dir
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = await self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
 
-        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
-        os.makedirs(temp_requirements_dir)
+    #
 
-        download_requirements(
-            await self.resolve_ci_image(),
-            temp_requirements_dir,
-            self.requirements_txts(),
+    @async_cached_nullary
+    async def load_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
         )
 
-        if self._file_cache is not None:
-            temp_tar_file = os.path.join(temp_dir, tar_file_name)
-
-            with tarfile.open(temp_tar_file, 'w') as tar:
-                for requirement_file in os.listdir(temp_requirements_dir):
-                    tar.add(
-                        os.path.join(temp_requirements_dir, requirement_file),
-                        arcname=requirement_file,
-                    )
-
-            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
-
-        return temp_requirements_dir
-
-    @async_cached_nullary
-    async def resolve_requirements_dir(self) -> str:
-        with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = await self._resolve_requirements_dir()
-            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
-            return requirements_dir
+        for dep_image in deps.values():
+            await self.load_docker_image(dep_image)
 
     #
 
     async def _run_compose_(self) -> None:
-        setup_cmds = [
-            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
-            (
-                'uv pip install --system --find-links /requirements ' +
-                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
-            ),
-        ]
-
-        #
-
-        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
-            *setup_cmds,
-            f'({self._cfg.cmd.s})',
-        ]))
-
-        #
-
         async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
             image=await self.resolve_ci_image(),
 
-            cmd=ci_cmd,
+            cmd=self._cfg.cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(await self.resolve_requirements_dir())}:/requirements',
+                *(self._cfg.run_options or []),
             ],
 
             cwd=self._cfg.project_dir,
@@ -263,10 +267,8 @@ class Ci(AsyncExitStacked):
     #
 
     async def run(self) -> None:
-        await self.load_compose_service_dependencies()
-
         await self.resolve_ci_image()
 
-        await self.resolve_requirements_dir()
+        await self.load_dependencies()
 
         await self._run_compose()

{omdev-0.0.0.dev213 → omdev-0.0.0.dev214}/omdev/ci/cli.py

@@ -1,6 +1,5 @@
 # @omlish-amalg ../scripts/ci.py
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 """
 Inputs:
  - requirements.txt
@@ -9,9 +8,11 @@ Inputs:
 
 ==
 
-./python -m ci run --cache-dir ci/cache ci/project omlish-ci
+./python -m omdev.ci run --cache-dir omdev/ci/tests/cache omdev/ci/tests/project omlish-ci
 """
+import argparse
 import asyncio
+import itertools
 import os.path
 import sys
 import typing as ta
@@ -20,15 +21,15 @@ from omlish.argparse.cli import ArgparseCli
 from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.check import check
+from omlish.lite.logs import log
 from omlish.logs.standard import configure_standard_logging
 
 from .cache import DirectoryFileCache
-from .cache import DirectoryShellCache
 from .cache import FileCache
-from .cache import ShellCache
 from .ci import Ci
 from .compose import get_compose_service_dependencies
-from .github.cache import GithubShellCache
+from .github.bootstrap import is_in_github_actions
+from .github.cache import GithubFileCache
 from .github.cli import GithubCli
 from .requirements import build_requirements_hash
 from .shell import ShellCmd
@@ -65,8 +66,8 @@ class CiCli(ArgparseCli):
     @argparse_cmd(
         accepts_unknown=True,
     )
-    def github(self) -> ta.Optional[int]:
-        return GithubCli(self.unknown_args).cli_run()
+    async def github(self) -> ta.Optional[int]:
+        return await GithubCli(self.unknown_args).async_cli_run()
 
     #
 
@@ -77,13 +78,20 @@ class CiCli(ArgparseCli):
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
 
-        argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
 
+        argparse_arg('--github', action='store_true'),
+        argparse_arg('--github-detect', action='store_true'),
+
         argparse_arg('--always-pull', action='store_true'),
         argparse_arg('--always-build', action='store_true'),
 
         argparse_arg('--no-dependencies', action='store_true'),
+
+        argparse_arg('-e', '--env', action='append'),
+        argparse_arg('-v', '--volume', action='append'),
+
+        argparse_arg('cmd', nargs=argparse.REMAINDER),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
@@ -94,6 +102,11 @@ class CiCli(ArgparseCli):
 
         #
 
+        cmd = ' '.join(self.args.cmd)
+        check.non_empty_str(cmd)
+
+        #
+
         check.state(os.path.isdir(project_dir))
 
         #
@@ -102,6 +115,7 @@ class CiCli(ArgparseCli):
             for alt in alts:
                 alt_file = os.path.abspath(os.path.join(project_dir, alt))
                 if os.path.isfile(alt_file):
+                    log.debug('Using %s', alt_file)
                     return alt_file
             return None
 
@@ -135,6 +149,7 @@ class CiCli(ArgparseCli):
                 'requirements-ci.txt',
             ]:
                 if os.path.exists(os.path.join(project_dir, rf)):
+                    log.debug('Using %s', rf)
                     requirements_txts.append(rf)
         else:
             for rf in requirements_txts:
@@ -142,21 +157,34 @@ class CiCli(ArgparseCli):
 
         #
 
-        shell_cache: ta.Optional[ShellCache] = None
+        github = self.args.github
+        if not github and self.args.github_detect:
+            github = is_in_github_actions()
+            if github:
+                log.debug('Github detected')
+
+        #
+
         file_cache: ta.Optional[FileCache] = None
         if cache_dir is not None:
-            if not os.path.exists(cache_dir):
-                os.makedirs(cache_dir)
-            check.state(os.path.isdir(cache_dir))
-
-            directory_file_cache = DirectoryFileCache(cache_dir)
+            cache_dir = os.path.abspath(cache_dir)
+            log.debug('Using cache dir %s', cache_dir)
+            if github:
+                file_cache = GithubFileCache(cache_dir)
+            else:
+                file_cache = DirectoryFileCache(cache_dir)
 
-            file_cache = directory_file_cache
+        #
 
-            if self.args.github_cache:
-                shell_cache = GithubShellCache(cache_dir)
-            else:
-                shell_cache = DirectoryShellCache(directory_file_cache)
+        run_options: ta.List[str] = []
+        for run_arg, run_arg_vals in [
+            ('-e', self.args.env or []),
+            ('-v', self.args.volume or []),
+        ]:
+            run_options.extend(itertools.chain.from_iterable(
+                [run_arg, run_arg_val]
+                for run_arg_val in run_arg_vals
+            ))
 
         #
 
@@ -171,18 +199,16 @@ class CiCli(ArgparseCli):
 
                     requirements_txts=requirements_txts,
 
-                    cmd=ShellCmd(' && '.join([
-                        'cd /project',
-                        'python3 -m pytest -svv test.py',
-                    ])),
+                    cmd=ShellCmd(cmd),
 
                     always_pull=self.args.always_pull,
                     always_build=self.args.always_build,
 
                     no_dependencies=self.args.no_dependencies,
+
+                    run_options=run_options,
                 ),
                 file_cache=file_cache,
-                shell_cache=shell_cache,
         ) as ci:
             await ci.run()