omdev 0.0.0.dev209__py3-none-any.whl → 0.0.0.dev210__py3-none-any.whl

omdev/.manifests.json

@@ -357,7 +357,7 @@
     "module": ".tools.pawk.pawk",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/pawk/pawk.py",
-    "line": 390,
+    "line": 389,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "pawk",

omdev/amalg/main.py

@@ -28,6 +28,7 @@ Targets:
 import argparse
 import logging
 import os.path
+import stat
 import typing as ta
 
 from omlish import check
@@ -60,7 +61,15 @@ def _gen_one(
     if output_path is not None:
         with open(output_path, 'w') as f:
             f.write(src)
-        os.chmod(output_path, os.stat(input_path).st_mode)
+
+        src_mode = os.stat(input_path).st_mode
+        out_mode = (
+            src_mode
+            | (stat.S_IXUSR if src_mode & stat.S_IRUSR else 0)
+            | (stat.S_IXGRP if src_mode & stat.S_IRGRP else 0)
+            | (stat.S_IXOTH if src_mode & stat.S_IROTH else 0)
+        )
+        os.chmod(output_path, out_mode)
 
     else:
         print(src)

omdev/cc/cdeps.py

@@ -0,0 +1,38 @@
+import dataclasses as dc
+import tomllib
+import typing as ta
+
+from omlish import cached
+from omlish import lang
+from omlish import marshal as msh
+
+
+@dc.dataclass(frozen=True)
+class Cdep:
+    @dc.dataclass(frozen=True)
+    class Git:
+        url: str
+        rev: str
+
+        subtrees: ta.Sequence[str] | None = None
+
+    git: Git
+
+    #
+
+    include: ta.Sequence[str] | None = None
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Cmake:
+        fetch_content_url: str | None = None
+
+    cmake: Cmake | None = None
+
+
+@cached.function
+def load_cdeps() -> ta.Mapping[str, Cdep]:
+    src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
+    dct = tomllib.loads(src)
+    return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore

omdev/cc/cdeps.toml

@@ -3,9 +3,12 @@ include = ['single_include']
 
 [deps.json.git]
 url = 'https://github.com/nlohmann/json'
-rev = '2d42229f4d68c6f86f37468b84ac65e86b815bbb'
+rev = '9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03'
 subtrees = ['single_include']
 
+[deps.json.cmake]
+fetch_content_url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+
 #
 
 [deps.pybind11]

omdev/cc/cli.py

@@ -19,45 +19,22 @@ TODO:
  - cext interop
  - gen cmake
 """
-import dataclasses as dc
 import os
 import shlex
 import shutil
 import subprocess
 import tempfile
-import tomllib
 import typing as ta
 
-from omlish import cached
 from omlish import check
-from omlish import lang
 from omlish import marshal as msh
 from omlish.argparse import all as ap
 from omlish.formats import json
 
 from .. import magic
 from ..cache import data as dcache
-
-
-@dc.dataclass(frozen=True)
-class Cdep:
-    @dc.dataclass(frozen=True)
-    class Git:
-        url: str
-        rev: str
-
-        subtrees: ta.Sequence[str] | None = None
-
-    git: Git
-
-    include: ta.Sequence[str] | None = None
-
-
-@cached.function
-def load_cdeps() -> ta.Mapping[str, Cdep]:
-    src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
-    dct = tomllib.loads(src)
-    return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore
+from .cdeps import Cdep
+from .cdeps import load_cdeps
 
 
 class Cli(ap.Cli):

omdev/ci/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == '__main__':
+    from .cli import _main  # noqa
+
+    _main()

omdev/ci/cache.py

@@ -0,0 +1,41 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import os.path
+import shutil
+import typing as ta
+
+
+#
+
+
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file(self, name: str) -> ta.Optional[str]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_file(self, name: str) -> ta.Optional[str]:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryFileCache(FileCache):
+    def __init__(self, dir: str) -> None:  # noqa
+        super().__init__()
+
+        self._dir = dir
+
+    def get_file(self, name: str) -> ta.Optional[str]:
+        file_path = os.path.join(self._dir, name)
+        if not os.path.exists(file_path):
+            return None
+        return file_path
+
+    def put_file(self, file_path: str) -> None:
+        os.makedirs(self._dir, exist_ok=True)
+        cache_file_path = os.path.join(self._dir, os.path.basename(file_path))
+        shutil.copyfile(file_path, cache_file_path)

omdev/ci/ci.py

@@ -0,0 +1,214 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import os.path
+import shutil
+import tarfile
+import tempfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import defer
+
+from .cache import FileCache
+from .compose import DockerComposeRun
+from .compose import get_compose_service_dependencies
+from .dockertars import build_docker_file_hash
+from .dockertars import build_docker_tar
+from .dockertars import is_docker_image_present
+from .dockertars import load_docker_tar
+from .dockertars import pull_docker_tar
+from .dockertars import read_docker_tar_image_id
+from .requirements import build_requirements_hash
+from .requirements import download_requirements
+
+
+##
+
+
+class Ci(ExitStacked):
+    FILE_NAME_HASH_LEN = 16
+
+    @dc.dataclass(frozen=True)
+    class Config:
+        project_dir: str
+
+        docker_file: str
+
+        compose_file: str
+        service: str
+
+        requirements_txts: ta.Optional[ta.Sequence[str]] = None
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.requirements_txts, str)
+
+    def __init__(
+            self,
+            cfg: Config,
+            *,
+            file_cache: ta.Optional[FileCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+        self._file_cache = file_cache
+
+    #
+
+    def load_docker_image(self, image: str) -> None:
+        if is_docker_image_present(image):
+            return
+
+        dep_suffix = image
+        for c in '/:.-_':
+            dep_suffix = dep_suffix.replace(c, '-')
+
+        tar_file_name = f'docker-{dep_suffix}.tar'
+
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
+            load_docker_tar(cache_tar_file)
+            return
+
+        temp_dir = tempfile.mkdtemp()
+        with defer(lambda: shutil.rmtree(temp_dir)):
+            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+
+            pull_docker_tar(
+                image,
+                temp_tar_file,
+            )
+
+            if self._file_cache is not None:
+                self._file_cache.put_file(temp_tar_file)
+
+    @cached_nullary
+    def load_compose_service_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
+
+        for dep_image in deps.values():
+            self.load_docker_image(dep_image)
+
+    #
+
+    @cached_nullary
+    def build_ci_image(self) -> str:
+        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+
+        tar_file_name = f'ci-{docker_file_hash}.tar'
+
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
+            image_id = read_docker_tar_image_id(cache_tar_file)
+            load_docker_tar(cache_tar_file)
+            return image_id
+
+        temp_dir = tempfile.mkdtemp()
+        with defer(lambda: shutil.rmtree(temp_dir)):
+            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+
+            image_id = build_docker_tar(
+                self._cfg.docker_file,
+                temp_tar_file,
+                cwd=self._cfg.project_dir,
+            )
+
+            if self._file_cache is not None:
+                self._file_cache.put_file(temp_tar_file)
+
+            return image_id
+
+    #
+
+    @cached_nullary
+    def build_requirements_dir(self) -> str:
+        requirements_txts = check.not_none(self._cfg.requirements_txts)
+
+        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+
+        tar_file_name = f'requirements-{requirements_hash}.tar'
+
+        temp_dir = tempfile.mkdtemp()
+        self._enter_context(defer(lambda: shutil.rmtree(temp_dir)))  # noqa
+
+        if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
+            with tarfile.open(cache_tar_file) as tar:
+                tar.extractall(path=temp_dir)  # noqa
+
+            return temp_dir
+
+        temp_requirements_dir = os.path.join(temp_dir, 'requirements')
+        os.makedirs(temp_requirements_dir)
+
+        download_requirements(
+            self.build_ci_image(),
+            temp_requirements_dir,
+            requirements_txts,
+        )
+
+        if self._file_cache is not None:
+            temp_tar_file = os.path.join(temp_dir, tar_file_name)
+
+            with tarfile.open(temp_tar_file, 'w') as tar:
+                for requirement_file in os.listdir(temp_requirements_dir):
+                    tar.add(
+                        os.path.join(temp_requirements_dir, requirement_file),
+                        arcname=requirement_file,
+                    )
+
+            self._file_cache.put_file(temp_tar_file)
+
+        return temp_requirements_dir
+
+    #
+
+    def run(self) -> None:
+        self.load_compose_service_dependencies()
+
+        ci_image = self.build_ci_image()
+
+        requirements_dir = self.build_requirements_dir()
+
+        #
+
+        setup_cmds = [
+            'pip install --root-user-action ignore --find-links /requirements --no-index uv',
+            (
+                'uv pip install --system --find-links /requirements ' +
+                ' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
+            ),
+        ]
+
+        #
+
+        test_cmds = [
+            '(cd /project && python3 -m pytest -svv test.py)',
+        ]
+
+        #
+
+        bash_src = ' && '.join([
+            *setup_cmds,
+            *test_cmds,
+        ])
+
+        with DockerComposeRun(DockerComposeRun.Config(
+                compose_file=self._cfg.compose_file,
+                service=self._cfg.service,
+
+                image=ci_image,
+
+                run_cmd=['bash', '-c', bash_src],
+
+                run_options=[
+                    '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
+                    '-v', f'{os.path.abspath(requirements_dir)}:/requirements',
+                ],
+
+                cwd=self._cfg.project_dir,
+        )) as ci_compose_run:
+            ci_compose_run.run()

omdev/ci/cli.py

@@ -0,0 +1,155 @@
+# @omlish-amalg ../scripts/ci.py
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+Inputs:
+ - requirements.txt
+ - ci.Dockerfile
+ - compose.yml
+
+==
+
+./python -m ci run --cache-dir ci/cache ci/project omlish-ci
+"""
+import asyncio
+import os.path
+import sys
+import typing as ta
+
+from omlish.argparse.cli import ArgparseCli
+from omlish.argparse.cli import argparse_arg
+from omlish.argparse.cli import argparse_cmd
+from omlish.lite.check import check
+
+from .cache import DirectoryFileCache
+from .cache import FileCache
+from .ci import Ci
+from .compose import get_compose_service_dependencies
+from .requirements import build_requirements_hash
+
+
+##
+
+
+class CiCli(ArgparseCli):
+    #
+
+    @argparse_cmd(
+        argparse_arg('requirements-txt', nargs='+'),
+    )
+    def print_requirements_hash(self) -> None:
+        requirements_txts = self.args.requirements_txt
+
+        print(build_requirements_hash(requirements_txts))
+
+    #
+
+    @argparse_cmd(
+        argparse_arg('compose-file'),
+        argparse_arg('service'),
+    )
+    def dump_compose_deps(self) -> None:
+        compose_file = self.args.compose_file
+        service = self.args.service
+
+        print(get_compose_service_dependencies(
+            compose_file,
+            service,
+        ))
+
+    #
+
+    @argparse_cmd(
+        argparse_arg('project-dir'),
+        argparse_arg('service'),
+        argparse_arg('--docker-file'),
+        argparse_arg('--compose-file'),
+        argparse_arg('-r', '--requirements-txt', action='append'),
+        argparse_arg('--cache-dir'),
+    )
+    async def run(self) -> None:
+        await asyncio.sleep(1)
+
+        project_dir = self.args.project_dir
+        docker_file = self.args.docker_file
+        compose_file = self.args.compose_file
+        service = self.args.service
+        requirements_txts = self.args.requirements_txt
+        cache_dir = self.args.cache_dir
+
+        #
+
+        check.state(os.path.isdir(project_dir))
+
+        #
+
+        def find_alt_file(*alts: str) -> ta.Optional[str]:
+            for alt in alts:
+                alt_file = os.path.join(project_dir, alt)
+                if os.path.isfile(alt_file):
+                    return alt_file
+            return None
+
+        if docker_file is None:
+            docker_file = find_alt_file(
+                'docker/ci/Dockerfile',
+                'docker/ci.Dockerfile',
+                'ci.Dockerfile',
+                'Dockerfile',
+            )
+        check.state(os.path.isfile(docker_file))
+
+        if compose_file is None:
+            compose_file = find_alt_file(
+                'docker/compose.yml',
+                'compose.yml',
+            )
+        check.state(os.path.isfile(compose_file))
+
+        if not requirements_txts:
+            requirements_txts = []
+            for rf in [
+                'requirements.txt',
+                'requirements-dev.txt',
+                'requirements-ci.txt',
+            ]:
+                if os.path.exists(os.path.join(project_dir, rf)):
+                    requirements_txts.append(rf)
+        else:
+            for rf in requirements_txts:
+                check.state(os.path.isfile(rf))
+
+        #
+
+        file_cache: ta.Optional[FileCache] = None
+        if cache_dir is not None:
+            if not os.path.exists(cache_dir):
+                os.makedirs(cache_dir)
+            check.state(os.path.isdir(cache_dir))
+            file_cache = DirectoryFileCache(cache_dir)
+
+        #
+
+        with Ci(
+                Ci.Config(
+                    project_dir=project_dir,
+                    docker_file=docker_file,
+                    compose_file=compose_file,
+                    service=service,
+                    requirements_txts=requirements_txts,
+                ),
+                file_cache=file_cache,
+        ) as ci:
+            ci.run()
+
+
+async def _async_main() -> ta.Optional[int]:
+    return await CiCli().async_cli_run()
+
+
+def _main() -> None:
+    sys.exit(rc if isinstance(rc := asyncio.run(_async_main()), int) else 0)
+
+
+if __name__ == '__main__':
+    _main()