omdev 0.0.0.dev209__py3-none-any.whl → 0.0.0.dev211__py3-none-any.whl

omdev/.manifests.json

@@ -357,7 +357,7 @@
     "module": ".tools.pawk.pawk",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/pawk/pawk.py",
-    "line": 390,
+    "line": 389,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "pawk",

omdev/amalg/main.py

@@ -28,6 +28,7 @@ Targets:
 import argparse
 import logging
 import os.path
+import stat
 import typing as ta
 
 from omlish import check
@@ -60,7 +61,15 @@ def _gen_one(
     if output_path is not None:
         with open(output_path, 'w') as f:
             f.write(src)
-        os.chmod(output_path, os.stat(input_path).st_mode)
+
+        src_mode = os.stat(input_path).st_mode
+        out_mode = (
+            src_mode
+            | (stat.S_IXUSR if src_mode & stat.S_IRUSR else 0)
+            | (stat.S_IXGRP if src_mode & stat.S_IRGRP else 0)
+            | (stat.S_IXOTH if src_mode & stat.S_IROTH else 0)
+        )
+        os.chmod(output_path, out_mode)
 
     else:
         print(src)

omdev/cc/cdeps.py

@@ -0,0 +1,38 @@
+import dataclasses as dc
+import tomllib
+import typing as ta
+
+from omlish import cached
+from omlish import lang
+from omlish import marshal as msh
+
+
+@dc.dataclass(frozen=True)
+class Cdep:
+    @dc.dataclass(frozen=True)
+    class Git:
+        url: str
+        rev: str
+
+        subtrees: ta.Sequence[str] | None = None
+
+    git: Git
+
+    #
+
+    include: ta.Sequence[str] | None = None
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Cmake:
+        fetch_content_url: str | None = None
+
+    cmake: Cmake | None = None
+
+
+@cached.function
+def load_cdeps() -> ta.Mapping[str, Cdep]:
+    src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
+    dct = tomllib.loads(src)
+    return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore

omdev/cc/cdeps.toml

@@ -3,9 +3,12 @@ include = ['single_include']
 
 [deps.json.git]
 url = 'https://github.com/nlohmann/json'
-rev = '2d42229f4d68c6f86f37468b84ac65e86b815bbb'
+rev = '9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03'
 subtrees = ['single_include']
 
+[deps.json.cmake]
+fetch_content_url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+
 #
 
 [deps.pybind11]

omdev/cc/cli.py

@@ -19,45 +19,22 @@ TODO:
  - cext interop
  - gen cmake
 """
-import dataclasses as dc
 import os
 import shlex
 import shutil
 import subprocess
 import tempfile
-import tomllib
 import typing as ta
 
-from omlish import cached
 from omlish import check
-from omlish import lang
 from omlish import marshal as msh
 from omlish.argparse import all as ap
 from omlish.formats import json
 
 from .. import magic
 from ..cache import data as dcache
-
-
-@dc.dataclass(frozen=True)
-class Cdep:
-    @dc.dataclass(frozen=True)
-    class Git:
-        url: str
-        rev: str
-
-        subtrees: ta.Sequence[str] | None = None
-
-    git: Git
-
-    include: ta.Sequence[str] | None = None
-
-
-@cached.function
-def load_cdeps() -> ta.Mapping[str, Cdep]:
-    src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
-    dct = tomllib.loads(src)
-    return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore
+from .cdeps import Cdep
+from .cdeps import load_cdeps
 
 
 class Cli(ap.Cli):

omdev/ci/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == '__main__':
+    from .cli import _main  # noqa
+
+    _main()

omdev/ci/cache.py

@@ -0,0 +1,168 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import os.path
+import shlex
+import shutil
+import typing as ta
+
+from .shell import ShellCmd
+
+
+##
+
+
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file(self, key: str) -> ta.Optional[str]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryFileCache(FileCache):
+    def __init__(self, dir: str) -> None:  # noqa
+        super().__init__()
+
+        self._dir = dir
+
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+            *,
+            make_dirs: bool = False,
+    ) -> str:
+        if make_dirs:
+            os.makedirs(self._dir, exist_ok=True)
+        return os.path.join(self._dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
+
+    def put_file(self, key: str, file_path: str) -> None:
+        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
+        shutil.copyfile(file_path, cache_file_path)
+
+
+##
+
+
+class ShellCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        raise NotImplementedError
+
+    class PutFileCmdContext(abc.ABC):
+        def __init__(self) -> None:
+            super().__init__()
+
+            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
+
+        @property
+        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
+            return self._state
+
+        #
+
+        @property
+        @abc.abstractmethod
+        def cmd(self) -> ShellCmd:
+            raise NotImplementedError
+
+        #
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            if exc_val is None:
+                self.commit()
+            else:
+                self.abort()
+
+        #
+
+        @abc.abstractmethod
+        def _commit(self) -> None:
+            raise NotImplementedError
+
+        def commit(self) -> None:
+            if self._state == 'committed':
+                return
+            elif self._state == 'open':
+                self._commit()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+        #
+
+        @abc.abstractmethod
+        def _abort(self) -> None:
+            raise NotImplementedError
+
+        def abort(self) -> None:
+            if self._state == 'aborted':
+                return
+            elif self._state == 'open':
+                self._abort()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+    @abc.abstractmethod
+    def put_file_cmd(self, key: str) -> PutFileCmdContext:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryShellCache(ShellCache):
+    def __init__(self, dfc: DirectoryFileCache) -> None:
+        super().__init__()
+
+        self._dfc = dfc
+
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        f = self._dfc.get_file(key)
+        if f is None:
+            return None
+        return ShellCmd(f'cat {shlex.quote(f)}')
+
+    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
+        def __init__(self, tf: str, f: str) -> None:
+            super().__init__()
+
+            self._tf = tf
+            self._f = f
+
+        @property
+        def cmd(self) -> ShellCmd:
+            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
+
+        def _commit(self) -> None:
+            os.replace(self._tf, self._f)
+
+        def _abort(self) -> None:
+            os.unlink(self._tf)
+
+    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
+        f = self._dfc.get_cache_file_path(key, make_dirs=True)
+        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)

omdev/ci/ci.py

@@ -0,0 +1,249 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import os.path
+import shutil
+import tarfile
+import tempfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import defer
+
+from .cache import FileCache
+from .cache import ShellCache
+from .compose import DockerComposeRun
+from .compose import get_compose_service_dependencies
+from .docker import build_docker_file_hash
+from .docker import build_docker_image
+from .docker import is_docker_image_present
+from .docker import load_docker_tar_cmd
+from .docker import pull_docker_image
+from .docker import save_docker_tar_cmd
+from .requirements import build_requirements_hash
+from .requirements import download_requirements
+from .shell import ShellCmd
+from .utils import log_timing_context
+
+
+class Ci(ExitStacked):
+    FILE_NAME_HASH_LEN = 16
+
+    @dc.dataclass(frozen=True)
+    class Config:
+        project_dir: str
+
+        docker_file: str
+
+        compose_file: str
+        service: str
+
+        cmd: ShellCmd
+
+        requirements_txts: ta.Optional[ta.Sequence[str]] = None
+
+        always_pull: bool = False
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.requirements_txts, str)
+
+    def __init__(
+            self,
+            cfg: Config,
+            *,
+            shell_cache: ta.Optional[ShellCache] = None,
+            file_cache: ta.Optional[FileCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+        self._shell_cache = shell_cache
+        self._file_cache = file_cache
+
+    #
+
+    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._shell_cache is None:
+            return None
+
+        get_cache_cmd = self._shell_cache.get_file_cmd(key)
+        if get_cache_cmd is None:
+            return None
+
+        get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
+
+        return load_docker_tar_cmd(get_cache_cmd)
+
+    def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._shell_cache is None:
+            return
+
+        with self._shell_cache.put_file_cmd(key) as put_cache:
+            put_cache_cmd = put_cache.cmd
+
+            put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
+
+            save_docker_tar_cmd(image, put_cache_cmd)
+
+    #
+
+    def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and is_docker_image_present(image):
+            return
+
+        dep_suffix = image
+        for c in '/:.-_':
+            dep_suffix = dep_suffix.replace(c, '-')
+
+        cache_key = f'docker-{dep_suffix}'
+        if self._load_cache_docker_image(cache_key) is not None:
+            return
+
+        pull_docker_image(image)
+
+        self._save_cache_docker_image(cache_key, image)
+
+    def load_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            self._load_docker_image(image)
+
+    @cached_nullary
+    def load_compose_service_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
+
+        for dep_image in deps.values():
+            self.load_docker_image(dep_image)
+
+    #
+
+    def _resolve_ci_image(self) -> str:
+        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+
+        cache_key = f'ci-{docker_file_hash}'
+        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
+            return cache_image_id
+
+        image_id = build_docker_image(
+            self._cfg.docker_file,
+            cwd=self._cfg.project_dir,
+        )
+
+        self._save_cache_docker_image(cache_key, image_id)
+
+        return image_id
+
+    @cached_nullary
+    def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
+
+    #
+
+    def _resolve_requirements_dir(self) -> str:
+        requirements_txts = [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
+
+        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+
+        tar_file_key = f'requirements-{requirements_hash}'
+        tar_file_name = f'{tar_file_key}.tar'
+
+        temp_dir = tempfile.mkdtemp()
+        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_key)):
+            with tarfile.open(cache_tar_file) as tar:
+                tar.extractall(path=temp_dir)  # noqa
+
+            return temp_dir
+
+        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
+        os.makedirs(temp_requirements_dir)
+
+        download_requirements(
+            self.resolve_ci_image(),
+            temp_requirements_dir,
+            requirements_txts,
+        )
+
+        if self._file_cache is not None:
+            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+
+            with tarfile.open(temp_tar_file, 'w') as tar:
+                for requirement_file in os.listdir(temp_requirements_dir):
+                    tar.add(
+                        os.path.join(temp_requirements_dir, requirement_file),
+                        arcname=requirement_file,
+                    )
+
+            self._file_cache.put_file(os.path.basename(tar_file_key), temp_tar_file)
+
+        return temp_requirements_dir
+
+    @cached_nullary
+    def resolve_requirements_dir(self) -> str:
+        with log_timing_context('Resolve requirements dir') as ltc:
+            requirements_dir = self._resolve_requirements_dir()
+            ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
+            return requirements_dir
+
+    #
+
+    def _run_compose_(self) -> None:
+        setup_cmds = [
+            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
+            (
+                'uv pip install --system --find-links /requirements ' +
+                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
+            ),
+        ]
+
+        #
+
+        ci_cmd = dc.replace(self._cfg.cmd, s=' && '.join([
+            *setup_cmds,
+            f'({self._cfg.cmd.s})',
+        ]))
+
+        #
+
+        with DockerComposeRun(DockerComposeRun.Config(
+            compose_file=self._cfg.compose_file,
+            service=self._cfg.service,
+
+            image=self.resolve_ci_image(),
+
+            cmd=ci_cmd,
+
+            run_options=[
+                '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
+                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+            ],
+
+            cwd=self._cfg.project_dir,
+        )) as ci_compose_run:
+            ci_compose_run.run()
+
+    def _run_compose(self) -> None:
+        with log_timing_context('Run compose'):
+            self._run_compose_()
+
+    #
+
+    def run(self) -> None:
+        self.load_compose_service_dependencies()
+
+        self.resolve_ci_image()
+
+        self.resolve_requirements_dir()
+
+        self._run_compose()