omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev213__py3-none-any.whl

omdev/cc/cdeps.py

@@ -5,6 +5,7 @@ import typing as ta
 from omlish import cached
 from omlish import lang
 from omlish import marshal as msh
+from omlish.configs import all as configs
 
 
 @dc.dataclass(frozen=True)
@@ -20,19 +21,51 @@ class Cdep:
 
     #
 
+    sources: ta.Sequence[str] | None = None
     include: ta.Sequence[str] | None = None
 
     #
 
     @dc.dataclass(frozen=True)
     class Cmake:
-        fetch_content_url: str | None = None
+        @dc.dataclass(frozen=True)
+        class FetchContent:
+            url: str | None = None
+
+            @dc.dataclass(frozen=True)
+            class Git:
+                repository: str | None = None
+                tag: str | None = None
+
+            git: Git | None = None
+
+        fetch_content: FetchContent | None = None
 
     cmake: Cmake | None = None
 
 
+def process_marshaled_cdep(obj: ta.Any) -> ta.Any:
+    obj = configs.processing.matched_rewrite(
+        lambda s: s if isinstance(s, str) else ''.join(s),
+        obj,
+        ('sources', None),
+        ('include', None),
+    )
+
+    return obj
+
+
 @cached.function
 def load_cdeps() -> ta.Mapping[str, Cdep]:
     src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
     dct = tomllib.loads(src)
+
+    dct = {
+        **dct,
+        'deps': {
+            k: process_marshaled_cdep(v)
+            for k, v in dct.get('deps', {}).items()
+        },
+    }
+
     return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore

omdev/cc/cdeps.toml

@@ -1,3 +1,18 @@
+[deps.httplib]
+include = ['.']
+
+[deps.httplib.git]
+url = 'https://github.com/yhirose/cpp-httplib'
+rev = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+subtrees = ['httplib.h']
+
+[deps.httplib.cmake.fetch_content.git]
+repository = 'https://github.com/yhirose/cpp-httplib.git'
+tag = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+
+
+#
+
 [deps.json]
 include = ['single_include']
 
@@ -6,8 +21,9 @@ url = 'https://github.com/nlohmann/json'
 rev = '9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03'
 subtrees = ['single_include']
 
-[deps.json.cmake]
-fetch_content_url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+[deps.json.cmake.fetch_content]
+url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+
 
 #
 
@@ -19,6 +35,7 @@ url = 'https://github.com/pybind/pybind11'
 rev = '945e251a6ce7273058b36214d94415e9c9530b8e'
 subtrees = ['include']
 
+
 #
 
 [deps.ut]

omdev/cc/cli.py

@@ -18,6 +18,8 @@ Freestanding options:
 TODO:
  - cext interop
  - gen cmake
+ - fix CFLAGS/CCFLAGS/CPPFLAGS/CXXFLAGS
+ - jit-gen cmake mode? multi-src builds
 """
 import os
 import shlex
@@ -35,6 +37,7 @@ from .. import magic
 from ..cache import data as dcache
 from .cdeps import Cdep
 from .cdeps import load_cdeps
+from .cdeps import process_marshaled_cdep
 
 
 class Cli(ap.Cli):
@@ -64,16 +67,22 @@ class Cli(ap.Cli):
             if src_magic.key == '@omlish-cdeps':
                 for dep in check.isinstance(src_magic.prepared, ta.Sequence):
                     if isinstance(dep, ta.Mapping):
-                        dep = msh.unmarshal(dep, Cdep)  # type: ignore
+                        dep = process_marshaled_cdep(dep)
+                        dep = msh.unmarshal(dep, Cdep)
                     else:
                         dep = load_cdeps()[check.isinstance(dep, str)]
 
+                    if dep.sources:
+                        # TODO
+                        raise NotImplementedError
+
                     dep_spec = dcache.GitSpec(
                         url=dep.git.url,
                         rev=dep.git.rev,
                         subtrees=dep.git.subtrees,
                     )
                     dep_dir = dcache.default().get(dep_spec)
+
                     for dep_inc in dep.include or []:
                         inc_dir = os.path.join(dep_dir, dep_inc)
                         check.state(os.path.isdir(inc_dir))
@@ -96,6 +105,9 @@ class Cli(ap.Cli):
         if cflags := os.environ.get('CFLAGS'):
             sh_parts.append(cflags)  # Explicitly shell-unquoted
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            sh_parts.append(ldflags)  # Explicitly shell-unquoted
+
         sh_parts.extend([
             '-std=c++20',
             shlex.quote(os.path.abspath(src_file)),

omdev/ci/ci.py

@@ -7,9 +7,10 @@ import tarfile
 import tempfile
 import typing as ta
 
+from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import AsyncExitStacked
 from omlish.lite.contextmanagers import defer
 
 from .cache import FileCache
@@ -22,13 +23,14 @@ from .docker import is_docker_image_present
 from .docker import load_docker_tar_cmd
 from .docker import pull_docker_image
 from .docker import save_docker_tar_cmd
+from .docker import tag_docker_image
 from .requirements import build_requirements_hash
 from .requirements import download_requirements
 from .shell import ShellCmd
 from .utils import log_timing_context
 
 
-class Ci(ExitStacked):
+class Ci(AsyncExitStacked):
     FILE_NAME_HASH_LEN = 16
 
     @dc.dataclass(frozen=True)
@@ -45,6 +47,9 @@ class Ci(ExitStacked):
         requirements_txts: ta.Optional[ta.Sequence[str]] = None
 
         always_pull: bool = False
+        always_build: bool = False
+
+        no_dependencies: bool = False
 
         def __post_init__(self) -> None:
             check.not_isinstance(self.requirements_txts, str)
@@ -64,7 +69,7 @@ class Ci(ExitStacked):
 
     #
 
-    def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+    async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
         if self._shell_cache is None:
             return None
 
@@ -74,9 +79,9 @@ class Ci(ExitStacked):
 
         get_cache_cmd = dc.replace(get_cache_cmd, s=f'{get_cache_cmd.s} | zstd -cd --long')  # noqa
 
-        return load_docker_tar_cmd(get_cache_cmd)
+        return await load_docker_tar_cmd(get_cache_cmd)
 
-    def _save_cache_docker_image(self, key: str, image: str) -> None:
+    async def _save_cache_docker_image(self, key: str, image: str) -> None:
         if self._shell_cache is None:
             return
 
@@ -85,12 +90,12 @@ class Ci(ExitStacked):
 
             put_cache_cmd = dc.replace(put_cache_cmd, s=f'zstd | {put_cache_cmd.s}')
 
-            save_docker_tar_cmd(image, put_cache_cmd)
+            await save_docker_tar_cmd(image, put_cache_cmd)
 
     #
 
-    def _load_docker_image(self, image: str) -> None:
-        if not self._cfg.always_pull and is_docker_image_present(image):
+    async def _load_docker_image(self, image: str) -> None:
+        if not self._cfg.always_pull and (await is_docker_image_present(image)):
             return
 
         dep_suffix = image
@@ -98,63 +103,79 @@ class Ci(ExitStacked):
             dep_suffix = dep_suffix.replace(c, '-')
 
         cache_key = f'docker-{dep_suffix}'
-        if self._load_cache_docker_image(cache_key) is not None:
+        if (await self._load_cache_docker_image(cache_key)) is not None:
             return
 
-        pull_docker_image(image)
+        await pull_docker_image(image)
 
-        self._save_cache_docker_image(cache_key, image)
+        await self._save_cache_docker_image(cache_key, image)
 
-    def load_docker_image(self, image: str) -> None:
+    async def load_docker_image(self, image: str) -> None:
         with log_timing_context(f'Load docker image: {image}'):
-            self._load_docker_image(image)
+            await self._load_docker_image(image)
 
-    @cached_nullary
-    def load_compose_service_dependencies(self) -> None:
+    @async_cached_nullary
+    async def load_compose_service_dependencies(self) -> None:
         deps = get_compose_service_dependencies(
             self._cfg.compose_file,
             self._cfg.service,
         )
 
         for dep_image in deps.values():
-            self.load_docker_image(dep_image)
+            await self.load_docker_image(dep_image)
 
     #
 
-    def _resolve_ci_image(self) -> str:
-        docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+    @cached_nullary
+    def docker_file_hash(self) -> str:
+        return build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
+
+    async def _resolve_ci_image(self) -> str:
+        cache_key = f'ci-{self.docker_file_hash()}'
+        image_tag = f'{self._cfg.service}:{cache_key}'
+
+        if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
+            return image_tag
 
-        cache_key = f'ci-{docker_file_hash}'
-        if (cache_image_id := self._load_cache_docker_image(cache_key)) is not None:
-            return cache_image_id
+        if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
+            await tag_docker_image(
+                cache_image_id,
+                image_tag,
+            )
+            return image_tag
 
-        image_id = build_docker_image(
+        image_id = await build_docker_image(
             self._cfg.docker_file,
+            tag=image_tag,
             cwd=self._cfg.project_dir,
         )
 
-        self._save_cache_docker_image(cache_key, image_id)
+        await self._save_cache_docker_image(cache_key, image_id)
 
-        return image_id
+        return image_tag
 
-    @cached_nullary
-    def resolve_ci_image(self) -> str:
+    @async_cached_nullary
+    async def resolve_ci_image(self) -> str:
         with log_timing_context('Resolve ci image') as ltc:
-            image_id = self._resolve_ci_image()
+            image_id = await self._resolve_ci_image()
             ltc.set_description(f'Resolve ci image: {image_id}')
             return image_id
 
     #
 
-    def _resolve_requirements_dir(self) -> str:
-        requirements_txts = [
+    @cached_nullary
+    def requirements_txts(self) -> ta.Sequence[str]:
+        return [
             os.path.join(self._cfg.project_dir, rf)
             for rf in check.not_none(self._cfg.requirements_txts)
         ]
 
-        requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
+    @cached_nullary
+    def requirements_hash(self) -> str:
+        return build_requirements_hash(self.requirements_txts())[:self.FILE_NAME_HASH_LEN]
 
-        tar_file_key = f'requirements-{requirements_hash}'
+    async def _resolve_requirements_dir(self) -> str:
+        tar_file_key = f'requirements-{self.docker_file_hash()}-{self.requirements_hash()}'
         tar_file_name = f'{tar_file_key}.tar'
 
         temp_dir = tempfile.mkdtemp()
@@ -170,9 +191,9 @@ class Ci(ExitStacked):
         os.makedirs(temp_requirements_dir)
 
         download_requirements(
-            self.resolve_ci_image(),
+            await self.resolve_ci_image(),
             temp_requirements_dir,
-            requirements_txts,
+            self.requirements_txts(),
         )
 
         if self._file_cache is not None:
@@ -189,16 +210,16 @@ class Ci(ExitStacked):
 
         return temp_requirements_dir
 
-    @cached_nullary
-    def resolve_requirements_dir(self) -> str:
+    @async_cached_nullary
+    async def resolve_requirements_dir(self) -> str:
         with log_timing_context('Resolve requirements dir') as ltc:
-            requirements_dir = self._resolve_requirements_dir()
+            requirements_dir = await self._resolve_requirements_dir()
             ltc.set_description(f'Resolve requirements dir: {requirements_dir}')
             return requirements_dir
 
     #
 
-    def _run_compose_(self) -> None:
+    async def _run_compose_(self) -> None:
         setup_cmds = [
             'pip install --root-user-action ignore --find-links /requirements --no-index uv',
             (
@@ -216,34 +237,36 @@ class Ci(ExitStacked):
 
         #
 
-        with DockerComposeRun(DockerComposeRun.Config(
+        async with DockerComposeRun(DockerComposeRun.Config(
             compose_file=self._cfg.compose_file,
             service=self._cfg.service,
 
-            image=self.resolve_ci_image(),
+            image=await self.resolve_ci_image(),
 
             cmd=ci_cmd,
 
             run_options=[
                 '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
-                '-v', f'{os.path.abspath(self.resolve_requirements_dir())}:/requirements',
+                '-v', f'{os.path.abspath(await self.resolve_requirements_dir())}:/requirements',
             ],
 
             cwd=self._cfg.project_dir,
+
+            no_dependencies=self._cfg.no_dependencies,
         )) as ci_compose_run:
-            ci_compose_run.run()
+            await ci_compose_run.run()
 
-    def _run_compose(self) -> None:
+    async def _run_compose(self) -> None:
         with log_timing_context('Run compose'):
-            self._run_compose_()
+            await self._run_compose_()
 
     #
 
-    def run(self) -> None:
-        self.load_compose_service_dependencies()
+    async def run(self) -> None:
+        await self.load_compose_service_dependencies()
 
-        self.resolve_ci_image()
+        await self.resolve_ci_image()
 
-        self.resolve_requirements_dir()
+        await self.resolve_requirements_dir()
 
-        self._run_compose()
+        await self._run_compose()

omdev/ci/cli.py

@@ -76,18 +76,21 @@ class CiCli(ArgparseCli):
         argparse_arg('--docker-file'),
         argparse_arg('--compose-file'),
         argparse_arg('-r', '--requirements-txt', action='append'),
+
         argparse_arg('--github-cache', action='store_true'),
         argparse_arg('--cache-dir'),
+
         argparse_arg('--always-pull', action='store_true'),
+        argparse_arg('--always-build', action='store_true'),
+
+        argparse_arg('--no-dependencies', action='store_true'),
     )
     async def run(self) -> None:
         project_dir = self.args.project_dir
         docker_file = self.args.docker_file
         compose_file = self.args.compose_file
-        service = self.args.service
         requirements_txts = self.args.requirements_txt
         cache_dir = self.args.cache_dir
-        always_pull = self.args.always_pull
 
         #
 
@@ -157,14 +160,14 @@ class CiCli(ArgparseCli):
 
         #
 
-        with Ci(
+        async with Ci(
                 Ci.Config(
                     project_dir=project_dir,
 
                     docker_file=docker_file,
 
                     compose_file=compose_file,
-                    service=service,
+                    service=self.args.service,
 
                     requirements_txts=requirements_txts,
 
@@ -173,12 +176,15 @@ class CiCli(ArgparseCli):
                         'python3 -m pytest -svv test.py',
                     ])),
 
-                    always_pull=always_pull,
+                    always_pull=self.args.always_pull,
+                    always_build=self.args.always_build,
+
+                    no_dependencies=self.args.no_dependencies,
                 ),
                 file_cache=file_cache,
                 shell_cache=shell_cache,
         ) as ci:
-            ci.run()
+            await ci.run()
 
 
 async def _async_main() -> ta.Optional[int]:

omdev/ci/compose.py

@@ -11,12 +11,13 @@ import os.path
 import shlex
 import typing as ta
 
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
-from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.lite.contextmanagers import adefer
 from omlish.lite.contextmanagers import defer
 from omlish.lite.json import json_dumps_pretty
-from omlish.subprocesses import subprocesses
 
 from .shell import ShellCmd
 from .utils import make_temp_file
@@ -46,7 +47,7 @@ def get_compose_service_dependencies(
 ##
 
 
-class DockerComposeRun(ExitStacked):
+class DockerComposeRun(AsyncExitStacked):
     @dc.dataclass(frozen=True)
     class Config:
         compose_file: str
@@ -64,6 +65,7 @@ class DockerComposeRun(ExitStacked):
 
         #
 
+        no_dependencies: bool = False
         no_dependency_cleanup: bool = False
 
         #
@@ -82,40 +84,6 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    @property
-    def image_tag(self) -> str:
-        pfx = 'sha256:'
-        if (image := self._cfg.image).startswith(pfx):
-            image = image[len(pfx):]
-
-        return f'{self._cfg.service}:{image}'
-
-    @cached_nullary
-    def tag_image(self) -> str:
-        image_tag = self.image_tag
-
-        subprocesses.check_call(
-            'docker',
-            'tag',
-            self._cfg.image,
-            image_tag,
-            **self._subprocess_kwargs,
-        )
-
-        def delete_tag() -> None:
-            subprocesses.check_call(
-                'docker',
-                'rmi',
-                image_tag,
-                **self._subprocess_kwargs,
-            )
-
-        self._enter_context(defer(delete_tag))  # noqa
-
-        return image_tag
-
-    #
-
     def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
         out = dict(in_dct)
 
@@ -129,7 +97,7 @@ class DockerComposeRun(ExitStacked):
         in_service: dict = in_services[self._cfg.service]
         out_services[self._cfg.service] = out_service = dict(in_service)
 
-        out_service['image'] = self.image_tag
+        out_service['image'] = self._cfg.image
 
         for k in ['build', 'platform']:
             if k in out_service:
@@ -142,16 +110,21 @@ class DockerComposeRun(ExitStacked):
 
         #
 
-        depends_on = in_service.get('depends_on', [])
+        if not self._cfg.no_dependencies:
+            depends_on = in_service.get('depends_on', [])
 
-        for dep_service, in_dep_service_dct in list(in_services.items()):
-            if dep_service not in depends_on:
-                continue
+            for dep_service, in_dep_service_dct in list(in_services.items()):
+                if dep_service not in depends_on:
+                    continue
 
-            out_dep_service: dict = dict(in_dep_service_dct)
-            out_services[dep_service] = out_dep_service
+                out_dep_service: dict = dict(in_dep_service_dct)
+                out_services[dep_service] = out_dep_service
 
-            out_dep_service['ports'] = []
+                out_dep_service['ports'] = []
+
+        else:
+            out_service['depends_on'] = []
+            out_service['links'] = []
 
         #
 
@@ -177,22 +150,20 @@ class DockerComposeRun(ExitStacked):
 
     #
 
-    def _cleanup_dependencies(self) -> None:
-        subprocesses.check_call(
+    async def _cleanup_dependencies(self) -> None:
+        await asyncio_subprocesses.check_call(
             'docker',
             'compose',
             '-f', self.rewrite_compose_file(),
             'down',
         )
 
-    def run(self) -> None:
-        self.tag_image()
-
+    async def run(self) -> None:
         compose_file = self.rewrite_compose_file()
 
-        with contextlib.ExitStack() as es:
-            if not self._cfg.no_dependency_cleanup:
-                es.enter_context(defer(self._cleanup_dependencies))  # noqa
+        async with contextlib.AsyncExitStack() as es:
+            if not (self._cfg.no_dependencies or self._cfg.no_dependency_cleanup):
+                await es.enter_async_context(adefer(self._cleanup_dependencies))  # noqa
 
             sh_cmd = ' '.join([
                 'docker',
@@ -211,7 +182,7 @@ class DockerComposeRun(ExitStacked):
 
             run_cmd = dc.replace(self._cfg.cmd, s=sh_cmd)
 
-            run_cmd.run(
-                subprocesses.check_call,
+            await run_cmd.run(
+                asyncio_subprocesses.check_call,
                 **self._subprocess_kwargs,
             )