omdev 0.0.0.dev212__py3-none-any.whl → 0.0.0.dev214__py3-none-any.whl

omdev/.manifests.json

@@ -267,7 +267,7 @@
     "module": ".tools.docker",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/docker.py",
-    "line": 258,
+    "line": 264,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "docker",

omdev/cc/cdeps.py

@@ -5,6 +5,7 @@ import typing as ta
 from omlish import cached
 from omlish import lang
 from omlish import marshal as msh
+from omlish.configs import all as configs
 
 
 @dc.dataclass(frozen=True)
@@ -20,19 +21,51 @@ class Cdep:
 
     #
 
+    sources: ta.Sequence[str] | None = None
     include: ta.Sequence[str] | None = None
 
     #
 
     @dc.dataclass(frozen=True)
     class Cmake:
-        fetch_content_url: str | None = None
+        @dc.dataclass(frozen=True)
+        class FetchContent:
+            url: str | None = None
+
+            @dc.dataclass(frozen=True)
+            class Git:
+                repository: str | None = None
+                tag: str | None = None
+
+            git: Git | None = None
+
+        fetch_content: FetchContent | None = None
 
     cmake: Cmake | None = None
 
 
+def process_marshaled_cdep(obj: ta.Any) -> ta.Any:
+    obj = configs.processing.matched_rewrite(
+        lambda s: s if isinstance(s, str) else ''.join(s),
+        obj,
+        ('sources', None),
+        ('include', None),
+    )
+
+    return obj
+
+
 @cached.function
 def load_cdeps() -> ta.Mapping[str, Cdep]:
     src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
     dct = tomllib.loads(src)
+
+    dct = {
+        **dct,
+        'deps': {
+            k: process_marshaled_cdep(v)
+            for k, v in dct.get('deps', {}).items()
+        },
+    }
+
     return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore

omdev/cc/cdeps.toml

@@ -1,3 +1,18 @@
+[deps.httplib]
+include = ['.']
+
+[deps.httplib.git]
+url = 'https://github.com/yhirose/cpp-httplib'
+rev = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+subtrees = ['httplib.h']
+
+[deps.httplib.cmake.fetch_content.git]
+repository = 'https://github.com/yhirose/cpp-httplib.git'
+tag = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+
+
+#
+
 [deps.json]
 include = ['single_include']
 
@@ -6,8 +21,9 @@ url = 'https://github.com/nlohmann/json'
 rev = '9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03'
 subtrees = ['single_include']
 
-[deps.json.cmake]
-fetch_content_url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+[deps.json.cmake.fetch_content]
+url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+
 
 #
 
@@ -19,6 +35,7 @@ url = 'https://github.com/pybind/pybind11'
 rev = '945e251a6ce7273058b36214d94415e9c9530b8e'
 subtrees = ['include']
 
+
 #
 
 [deps.ut]

omdev/cc/cli.py

@@ -18,6 +18,8 @@ Freestanding options:
 TODO:
  - cext interop
  - gen cmake
+ - fix CFLAGS/CCFLAGS/CPPFLAGS/CXXFLAGS
+ - jit-gen cmake mode? multi-src builds
 """
 import os
 import shlex
@@ -35,6 +37,7 @@ from .. import magic
 from ..cache import data as dcache
 from .cdeps import Cdep
 from .cdeps import load_cdeps
+from .cdeps import process_marshaled_cdep
 
 
 class Cli(ap.Cli):
@@ -64,16 +67,22 @@ class Cli(ap.Cli):
             if src_magic.key == '@omlish-cdeps':
                 for dep in check.isinstance(src_magic.prepared, ta.Sequence):
                     if isinstance(dep, ta.Mapping):
-                        dep = msh.unmarshal(dep, Cdep)  # type: ignore
+                        dep = process_marshaled_cdep(dep)
+                        dep = msh.unmarshal(dep, Cdep)
                     else:
                         dep = load_cdeps()[check.isinstance(dep, str)]
 
+                    if dep.sources:
+                        # TODO
+                        raise NotImplementedError
+
                     dep_spec = dcache.GitSpec(
                         url=dep.git.url,
                         rev=dep.git.rev,
                         subtrees=dep.git.subtrees,
                     )
                     dep_dir = dcache.default().get(dep_spec)
+
                     for dep_inc in dep.include or []:
                         inc_dir = os.path.join(dep_dir, dep_inc)
                         check.state(os.path.isdir(inc_dir))
@@ -96,6 +105,9 @@ class Cli(ap.Cli):
         if cflags := os.environ.get('CFLAGS'):
             sh_parts.append(cflags)  # Explicitly shell-unquoted
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            sh_parts.append(ldflags)  # Explicitly shell-unquoted
+
         sh_parts.extend([
             '-std=c++20',
             shlex.quote(os.path.abspath(src_file)),

omdev/ci/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite

omdev/ci/cache.py

@@ -1,12 +1,14 @@
 # ruff: noqa: UP006 UP007
-# @omlish-lite
 import abc
 import os.path
-import shlex
 import shutil
 import typing as ta
 
-from .shell import ShellCmd
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.logs import log
+
+from .consts import CI_CACHE_VERSION
 
 
 ##
@@ -14,12 +16,35 @@ from .shell import ShellCmd
 
 @abc.abstractmethod
 class FileCache(abc.ABC):
+    def __init__(
+            self,
+            *,
+            version: int = CI_CACHE_VERSION,
+    ) -> None:
+        super().__init__()
+
+        check.isinstance(version, int)
+        check.arg(version >= 0)
+        self._version = version
+
+    @property
+    def version(self) -> int:
+        return self._version
+
+    #
+
     @abc.abstractmethod
-    def get_file(self, key: str) -> ta.Optional[str]:
+    def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
+    def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> ta.Awaitable[str]:
         raise NotImplementedError
 
 
@@ -27,142 +52,96 @@ class FileCache(abc.ABC):
 
 
 class DirectoryFileCache(FileCache):
-    def __init__(self, dir: str) -> None:  # noqa
-        super().__init__()
-
-        self._dir = dir
-
-    #
-
-    def get_cache_file_path(
+    def __init__(
             self,
-            key: str,
+            dir: str,  # noqa
             *,
-            make_dirs: bool = False,
-    ) -> str:
-        if make_dirs:
-            os.makedirs(self._dir, exist_ok=True)
-        return os.path.join(self._dir, key)
+            no_create: bool = False,
+            no_purge: bool = False,
+            **kwargs: ta.Any,
+    ) -> None:  # noqa
+        super().__init__(**kwargs)
 
-    def format_incomplete_file(self, f: str) -> str:
-        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+        self._dir = dir
+        self._no_create = no_create
+        self._no_purge = no_purge
 
     #
 
-    def get_file(self, key: str) -> ta.Optional[str]:
-        cache_file_path = self.get_cache_file_path(key)
-        if not os.path.exists(cache_file_path):
-            return None
-        return cache_file_path
-
-    def put_file(self, key: str, file_path: str) -> None:
-        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
-        shutil.copyfile(file_path, cache_file_path)
-
-
-##
-
-
-class ShellCache(abc.ABC):
-    @abc.abstractmethod
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        raise NotImplementedError
-
-    class PutFileCmdContext(abc.ABC):
-        def __init__(self) -> None:
-            super().__init__()
-
-            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
-
-        @property
-        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
-            return self._state
+    VERSION_FILE_NAME = '.ci-cache-version'
 
-        #
+    @cached_nullary
+    def setup_dir(self) -> None:
+        version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
 
-        @property
-        @abc.abstractmethod
-        def cmd(self) -> ShellCmd:
-            raise NotImplementedError
+        if self._no_create:
+            check.state(os.path.isdir(self._dir))
 
-        #
+        elif not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+            with open(version_file, 'w') as f:
+                f.write(str(self._version))
+            return
 
-        def __enter__(self):
-            return self
+        with open(version_file) as f:
+            dir_version = int(f.read().strip())
 
-        def __exit__(self, exc_type, exc_val, exc_tb):
-            if exc_val is None:
-                self.commit()
-            else:
-                self.abort()
+        if dir_version == self._version:
+            return
 
-        #
+        if self._no_purge:
+            raise RuntimeError(f'{dir_version=} != {self._version=}')
 
-        @abc.abstractmethod
-        def _commit(self) -> None:
-            raise NotImplementedError
+        dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
+        if dirs:
+            raise RuntimeError(
+                f'Refusing to remove stale cache dir {self._dir!r} '
+                f'due to present directories: {", ".join(dirs)}',
+            )
 
-        def commit(self) -> None:
-            if self._state == 'committed':
-                return
-            elif self._state == 'open':
-                self._commit()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
+        for n in sorted(os.listdir(self._dir)):
+            if n.startswith('.'):
+                continue
+            fp = os.path.join(self._dir, n)
+            check.state(os.path.isfile(fp))
+            log.debug('Purging stale cache file: %s', fp)
+            os.unlink(fp)
 
-        #
-
-        @abc.abstractmethod
-        def _abort(self) -> None:
-            raise NotImplementedError
-
-        def abort(self) -> None:
-            if self._state == 'aborted':
-                return
-            elif self._state == 'open':
-                self._abort()
-                self._state = 'committed'
-            else:
-                raise RuntimeError(self._state)
-
-    @abc.abstractmethod
-    def put_file_cmd(self, key: str) -> PutFileCmdContext:
-        raise NotImplementedError
+        os.unlink(version_file)
 
+        with open(version_file, 'w') as f:
+            f.write(str(self._version))
 
-#
+    #
 
+    def get_cache_file_path(
+            self,
+            key: str,
+    ) -> str:
+        self.setup_dir()
+        return os.path.join(self._dir, key)
 
-class DirectoryShellCache(ShellCache):
-    def __init__(self, dfc: DirectoryFileCache) -> None:
-        super().__init__()
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
 
-        self._dfc = dfc
+    #
 
-    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
-        f = self._dfc.get_file(key)
-        if f is None:
+    async def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
             return None
-        return ShellCmd(f'cat {shlex.quote(f)}')
-
-    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
-        def __init__(self, tf: str, f: str) -> None:
-            super().__init__()
-
-            self._tf = tf
-            self._f = f
-
-        @property
-        def cmd(self) -> ShellCmd:
-            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
-
-        def _commit(self) -> None:
-            os.replace(self._tf, self._f)
-
-        def _abort(self) -> None:
-            os.unlink(self._tf)
+        return cache_file_path
 
-    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
-        f = self._dfc.get_cache_file_path(key, make_dirs=True)
-        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)
+    async def put_file(
+            self,
+            key: str,
+            file_path: str,
+            *,
+            steal: bool = False,
+    ) -> str:
+        cache_file_path = self.get_cache_file_path(key)
+        if steal:
+            shutil.move(file_path, cache_file_path)
+        else:
+            shutil.copyfile(file_path, cache_file_path)
+        return cache_file_path