omdev 0.0.0.dev212__tar.gz → 0.0.0.dev214__tar.gz

{omdev-0.0.0.dev212/omdev.egg-info → omdev-0.0.0.dev214}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev212
+Version: 0.0.0.dev214
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev212
+Requires-Dist: omlish==0.0.0.dev214
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev212 → omdev-0.0.0.dev214}/omdev/.manifests.json

@@ -267,7 +267,7 @@
     "module": ".tools.docker",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/docker.py",
-    "line": 258,
+    "line": 264,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "docker",

omdev-0.0.0.dev214/omdev/cc/cdeps.py

@@ -0,0 +1,71 @@
+import dataclasses as dc
+import tomllib
+import typing as ta
+
+from omlish import cached
+from omlish import lang
+from omlish import marshal as msh
+from omlish.configs import all as configs
+
+
+@dc.dataclass(frozen=True)
+class Cdep:
+    @dc.dataclass(frozen=True)
+    class Git:
+        url: str
+        rev: str
+
+        subtrees: ta.Sequence[str] | None = None
+
+    git: Git
+
+    #
+
+    sources: ta.Sequence[str] | None = None
+    include: ta.Sequence[str] | None = None
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Cmake:
+        @dc.dataclass(frozen=True)
+        class FetchContent:
+            url: str | None = None
+
+            @dc.dataclass(frozen=True)
+            class Git:
+                repository: str | None = None
+                tag: str | None = None
+
+            git: Git | None = None
+
+        fetch_content: FetchContent | None = None
+
+    cmake: Cmake | None = None
+
+
+def process_marshaled_cdep(obj: ta.Any) -> ta.Any:
+    obj = configs.processing.matched_rewrite(
+        lambda s: s if isinstance(s, str) else ''.join(s),
+        obj,
+        ('sources', None),
+        ('include', None),
+    )
+
+    return obj
+
+
+@cached.function
+def load_cdeps() -> ta.Mapping[str, Cdep]:
+    src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
+    dct = tomllib.loads(src)
+
+    dct = {
+        **dct,
+        'deps': {
+            k: process_marshaled_cdep(v)
+            for k, v in dct.get('deps', {}).items()
+        },
+    }
+
+    return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore

{omdev-0.0.0.dev212 → omdev-0.0.0.dev214}/omdev/cc/cdeps.toml

@@ -1,3 +1,18 @@
+[deps.httplib]
+include = ['.']
+
+[deps.httplib.git]
+url = 'https://github.com/yhirose/cpp-httplib'
+rev = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+subtrees = ['httplib.h']
+
+[deps.httplib.cmake.fetch_content.git]
+repository = 'https://github.com/yhirose/cpp-httplib.git'
+tag = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+
+
+#
+
 [deps.json]
 include = ['single_include']
 
@@ -6,8 +21,9 @@ url = 'https://github.com/nlohmann/json'
 rev = '9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03'
 subtrees = ['single_include']
 
-[deps.json.cmake]
-fetch_content_url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+[deps.json.cmake.fetch_content]
+url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+
 
 #
 
@@ -19,6 +35,7 @@ url = 'https://github.com/pybind/pybind11'
 rev = '945e251a6ce7273058b36214d94415e9c9530b8e'
 subtrees = ['include']
 
+
 #
 
 [deps.ut]

{omdev-0.0.0.dev212 → omdev-0.0.0.dev214}/omdev/cc/cli.py

@@ -18,6 +18,8 @@ Freestanding options:
 TODO:
  - cext interop
  - gen cmake
+ - fix CFLAGS/CCFLAGS/CPPFLAGS/CXXFLAGS
+ - jit-gen cmake mode? multi-src builds
 """
 import os
 import shlex
@@ -35,6 +37,7 @@ from .. import magic
 from ..cache import data as dcache
 from .cdeps import Cdep
 from .cdeps import load_cdeps
+from .cdeps import process_marshaled_cdep
 
 
 class Cli(ap.Cli):
@@ -64,16 +67,22 @@ class Cli(ap.Cli):
             if src_magic.key == '@omlish-cdeps':
                 for dep in check.isinstance(src_magic.prepared, ta.Sequence):
                     if isinstance(dep, ta.Mapping):
-                        dep = msh.unmarshal(dep, Cdep)  # type: ignore
+                        dep = process_marshaled_cdep(dep)
+                        dep = msh.unmarshal(dep, Cdep)
                     else:
                         dep = load_cdeps()[check.isinstance(dep, str)]
 
+                    if dep.sources:
+                        # TODO
+                        raise NotImplementedError
+
                     dep_spec = dcache.GitSpec(
                         url=dep.git.url,
                         rev=dep.git.rev,
                         subtrees=dep.git.subtrees,
                     )
                     dep_dir = dcache.default().get(dep_spec)
+
                     for dep_inc in dep.include or []:
                         inc_dir = os.path.join(dep_dir, dep_inc)
                         check.state(os.path.isdir(inc_dir))
@@ -96,6 +105,9 @@ class Cli(ap.Cli):
         if cflags := os.environ.get('CFLAGS'):
             sh_parts.append(cflags)  # Explicitly shell-unquoted
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            sh_parts.append(ldflags)  # Explicitly shell-unquoted
+
         sh_parts.extend([
             '-std=c++20',
             shlex.quote(os.path.abspath(src_file)),

omdev-0.0.0.dev214/omdev/ci/cache.py

@@ -0,0 +1,147 @@
+# ruff: noqa: UP006 UP007
+import abc
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.logs import log
+
+from .consts import CI_CACHE_VERSION
+
+
+##
+
+
+@abc.abstractmethod
+class FileCache(abc.ABC):
+    def __init__(
+            self,
+            *,
+            version: int = CI_CACHE_VERSION,
+    ) -> None:
+        super().__init__()
+
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version = version
+
+    @property
+    def version(self) -> int:
+        return self._version
+
+    #
+
+    @abc.abstractmethod
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryFileCache(FileCache):
+    def __init__(
+            self,
+            dir: str,  # noqa
+            *,
+            no_create: bool = False,
+            no_purge: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:  # noqa
+        super().__init__(**kwargs)
+
+        self._dir = dir
+        self._no_create = no_create
+        self._no_purge = no_purge
+
+    #
+
+    VERSION_FILE_NAME = '.ci-cache-version'
+
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
+
+        if self._no_create:
+            check.state(os.path.isdir(self._dir))
+
+        elif not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
+
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
+
+        if dir_version == self._version:
+            return
+
+        if self._no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
+
+        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
+
+        for n in sorted(os.listdir(self._dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self._dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
+
+        os.unlink(version_file)
+
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
+
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self._dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
+            return None
+        return cache_file_path
+
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path

omdev-0.0.0.dev214/omdev/ci/ci.py

@@ -0,0 +1,274 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.os.temp import temp_file_context
+
+from .cache import FileCache
+from .compose import DockerComposeRun
+from .compose import get_compose_service_dependencies
+from .docker import build_docker_file_hash
+from .docker import build_docker_image
+from .docker import is_docker_image_present
+from .docker import load_docker_tar_cmd
+from .docker import pull_docker_image
+from .docker import save_docker_tar_cmd
+from .docker import tag_docker_image
+from .requirements import build_requirements_hash
+from .shell import ShellCmd
+from .utils import log_timing_context
+
+
+class Ci(AsyncExitStacked):
+    KEY_HASH_LEN = 16
+
+    @dc.dataclass(frozen=True)
+    class Config:
+        project_dir: str
+
+        docker_file: str
+
+        compose_file: str
+        service: str
+
+        cmd: ShellCmd
+
+        #
+
+        requirements_txts: ta.Optional[ta.Sequence[str]] = None
+
+        always_pull: bool = False
+        always_build: bool = False
+
+        no_dependencies: bool = False
+
+        run_options: ta.Optional[ta.Sequence[str]] = None
+
+        #
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.requirements_txts, str)
+
+    def __init__(
+            self,
+            cfg: Config,
+            *,
+            file_cache: ta.Optional[FileCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._cfg = cfg
+        self._file_cache = file_cache
+
+    #
+
+    async def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and (await is_docker_image_present(image)):
+            return
+
+        dep_suffix = image
+        for c in '/:.-_':
+            dep_suffix = dep_suffix.replace(c, '-')
+
+        cache_key = f'docker-{dep_suffix}'
+        if (await self._load_cache_docker_image(cache_key)) is not None:
+            return
+
+        await pull_docker_image(image)
+
+        await self._save_cache_docker_image(cache_key, image)
+
+    async def load_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            await self._load_docker_image(image)
+
+    #
+
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
+
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
+
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
+
+        return await load_docker_tar_cmd(get_cache_cmd)
+
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
+
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
+
+            await save_docker_tar_cmd(image, write_tmp_cmd)
+
+            await self._file_cache.put_file(key, tmp_file, steal=True)
+
+    #
+
+    async def _resolve_docker_image(
+            self,
+            cache_key: str,
+            build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
+    ) -> str:
+        image_tag = f'{self._cfg.service}:{cache_key}'
+
+        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
+
+        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
+
+        image_id = await build_and_tag(image_tag)
+
+        await self._save_cache_docker_image(cache_key, image_id)
+
+        return image_tag
+
+    #
+
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_base_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            return await build_docker_image(
+                self._cfg.docker_file,
+                tag=image_tag,
+                cwd=self._cfg.project_dir,
+            )
+
+        cache_key = f'ci-base-{self.docker_file_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
+    @async_cached_nullary
+    async def resolve_ci_base_image(self) -> str:
+        with log_timing_context('Resolve ci base image') as ltc:
+            image_id = await self._resolve_ci_base_image()
+            ltc.set_description(f'Resolve ci base image: {image_id}')
+            return image_id
+
+    #
+
+    @cached_nullary
+    def requirements_txts(self) -> ta.Sequence[str]:
+        return [
+            os.path.join(self._cfg.project_dir, rf)
+            for rf in check.not_none(self._cfg.requirements_txts)
+        ]
+
+    @cached_nullary
+    def requirements_hash(self) -> str:
+        return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
+
+    async def _resolve_ci_image(self) -> str:
+        async def build_and_tag(image_tag: str) -> str:
+            base_image = await self.resolve_ci_base_image()
+
+            setup_cmds = [
+                ' '.join([
+                    'pip install',
+                    '--no-cache-dir',
+                    '--root-user-action ignore',
+                    'uv',
+                ]),
+                ' '.join([
+                    'uv pip install',
+                    '--no-cache',
+                    '--index-strategy unsafe-best-match',
+                    '--system',
+                    *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                ]),
+            ]
+            setup_cmd = ' && '.join(setup_cmds)
+
+            docker_file_lines = [
+                f'FROM {base_image}',
+                'RUN mkdir /project',
+                *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
+                f'RUN {setup_cmd}',
+                'RUN rm /project/*',
+                'WORKDIR /project',
+            ]
+
+            with temp_file_context() as docker_file:
+                with open(docker_file, 'w') as f:  # noqa
+                    f.write('\n'.join(docker_file_lines))
+
+                return await build_docker_image(
+                    docker_file,
+                    tag=image_tag,
+                    cwd=self._cfg.project_dir,
+                )
+
+        cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+
+        return await self._resolve_docker_image(cache_key, build_and_tag)
+
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
+        with log_timing_context('Resolve ci image') as ltc:
+            image_id = await self._resolve_ci_image()
+            ltc.set_description(f'Resolve ci image: {image_id}')
+            return image_id
+
+    #
+
+    @async_cached_nullary
+    async def load_dependencies(self) -> None:
+        deps = get_compose_service_dependencies(
+            self._cfg.compose_file,
+            self._cfg.service,
+        )
+
+        for dep_image in deps.values():
+            await self.load_docker_image(dep_image)
+
+    #
+
+    async def _run_compose_(self) -> None:
+        async with DockerComposeRun(DockerComposeRun.Config(
+            compose_file=self._cfg.compose_file,
+            service=self._cfg.service,
+
+            image=await self.resolve_ci_image(),
+
+            cmd=self._cfg.cmd,
+
+            run_options=[
+                '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
+                *(self._cfg.run_options or []),
+            ],
+
+            cwd=self._cfg.project_dir,
+
+            no_dependencies=self._cfg.no_dependencies,
+        )) as ci_compose_run:
+            await ci_compose_run.run()
+
+    async def _run_compose(self) -> None:
+        with log_timing_context('Run compose'):
+            await self._run_compose_()
+
+    #
+
+    async def run(self) -> None:
+        await self.resolve_ci_image()
+
+        await self.load_dependencies()
+
+        await self._run_compose()