omdev 0.0.0.dev240__py3-none-any.whl → 0.0.0.dev241__py3-none-any.whl

omdev/.manifests.json

@@ -267,7 +267,7 @@
     "module": ".tools.docker",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/docker.py",
-    "line": 267,
+    "line": 251,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "docker",

omdev/ci/cache.py

@@ -118,7 +118,7 @@ class DirectoryFileCache(FileCache):
         elif not os.path.isdir(self.dir):
             os.makedirs(self.dir)
             with open(version_file, 'w') as f:
-                f.write(str(self._version))
+                f.write(f'{self._version}\n')
             return
 
         # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.

omdev/ci/ci.py

@@ -1,8 +1,10 @@
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
+import functools
 import os.path
 import typing as ta
 
+from omlish.asyncs.asyncio.asyncio import asyncio_wait_maybe_concurrent
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
@@ -13,6 +15,7 @@ from omlish.os.temp import temp_file_context
 from .compose import DockerComposeRun
 from .compose import get_compose_service_dependencies
 from .docker.buildcaching import DockerBuildCaching
+from .docker.cache import DockerCacheKey
 from .docker.cmds import build_docker_image
 from .docker.imagepulling import DockerImagePulling
 from .docker.utils import build_docker_file_hash
@@ -44,6 +47,8 @@ class Ci(AsyncExitStacked):
         always_pull: bool = False
         always_build: bool = False
 
+        setup_concurrency: ta.Optional[int] = None
+
         no_dependencies: bool = False
 
         run_options: ta.Optional[ta.Sequence[str]] = None
@@ -74,8 +79,8 @@ class Ci(AsyncExitStacked):
         return build_docker_file_hash(self._config.docker_file)[:self.KEY_HASH_LEN]
 
     @cached_nullary
-    def ci_base_image_cache_key(self) -> str:
-        return f'ci-base-{self.docker_file_hash()}'
+    def ci_base_image_cache_key(self) -> DockerCacheKey:
+        return DockerCacheKey(['ci-base'], self.docker_file_hash())
 
     async def _resolve_ci_base_image(self) -> str:
         async def build_and_tag(image_tag: str) -> str:
@@ -111,8 +116,8 @@ class Ci(AsyncExitStacked):
         return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
 
     @cached_nullary
-    def ci_image_cache_key(self) -> str:
-        return f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
+    def ci_image_cache_key(self) -> DockerCacheKey:
+        return DockerCacheKey(['ci'], f'{self.docker_file_hash()}-{self.requirements_hash()}')
 
     async def _resolve_ci_image(self) -> str:
         async def build_and_tag(image_tag: str) -> str:
@@ -168,15 +173,37 @@ class Ci(AsyncExitStacked):
 
     #
 
-    @async_cached_nullary
-    async def pull_dependencies(self) -> None:
+    @cached_nullary
+    def pull_dependencies_funcs(self) -> ta.Sequence[ta.Callable[[], ta.Awaitable]]:
         deps = get_compose_service_dependencies(
             self._config.compose_file,
             self._config.service,
         )
 
-        for dep_image in deps.values():
-            await self._docker_image_pulling.pull_docker_image(dep_image)
+        return [
+            async_cached_nullary(functools.partial(
+                self._docker_image_pulling.pull_docker_image,
+                dep_image,
+            ))
+            for dep_image in deps.values()
+        ]
+
+    #
+
+    @cached_nullary
+    def setup_funcs(self) -> ta.Sequence[ta.Callable[[], ta.Awaitable]]:
+        return [
+            self.resolve_ci_image,
+
+            *(self.pull_dependencies_funcs() if not self._config.no_dependencies else []),
+        ]
+
+    @async_cached_nullary
+    async def setup(self) -> None:
+        await asyncio_wait_maybe_concurrent(
+            [fn() for fn in self.setup_funcs()],
+            self._config.setup_concurrency,
+        )
 
     #
 
@@ -207,8 +234,6 @@ class Ci(AsyncExitStacked):
     #
 
     async def run(self) -> None:
-        await self.resolve_ci_image()
-
-        await self.pull_dependencies()
+        await self.setup()
 
         await self._run_compose()

omdev/ci/cli.py

@@ -88,6 +88,10 @@ class CiCli(ArgparseCli):
         argparse_arg('--github', action='store_true'),
         argparse_arg('--github-detect', action='store_true'),
 
+        argparse_arg('--cache-served-docker', action='store_true'),
+
+        argparse_arg('--setup-concurrency', type=int),
+
         argparse_arg('--always-pull', action='store_true'),
         argparse_arg('--always-build', action='store_true'),
 
@@ -203,6 +207,8 @@ class CiCli(ArgparseCli):
             always_pull=self.args.always_pull,
             always_build=self.args.always_build,
 
+            setup_concurrency=self.args.setup_concurrency,
+
             no_dependencies=self.args.no_dependencies,
 
             run_options=run_options,
@@ -225,6 +231,8 @@ class CiCli(ArgparseCli):
             directory_file_cache_config=directory_file_cache_config,
 
             github=github,
+
+            cache_served_docker=self.args.cache_served_docker,
         ))
 
         async with injector[Ci] as ci:

omdev/ci/consts.py

@@ -1 +1 @@
-CI_CACHE_VERSION = 1
+CI_CACHE_VERSION = 2

omdev/ci/docker/buildcaching.py

@@ -4,6 +4,7 @@ import dataclasses as dc
 import typing as ta
 
 from .cache import DockerCache
+from .cache import DockerCacheKey
 from .cmds import is_docker_image_present
 from .cmds import tag_docker_image
 
@@ -15,7 +16,7 @@ class DockerBuildCaching(abc.ABC):
     @abc.abstractmethod
     def cached_build_docker_image(
             self,
-            cache_key: str,
+            cache_key: DockerCacheKey,
             build_and_tag: ta.Callable[[str], ta.Awaitable[str]],  # image_tag -> image_id
     ) -> ta.Awaitable[str]:
         raise NotImplementedError
@@ -43,10 +44,10 @@ class DockerBuildCachingImpl(DockerBuildCaching):
 
     async def cached_build_docker_image(
             self,
-            cache_key: str,
+            cache_key: DockerCacheKey,
             build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
     ) -> str:
-        image_tag = f'{self._config.service}:{cache_key}'
+        image_tag = f'{self._config.service}:{cache_key!s}'
 
         if not self._config.always_build and (await is_docker_image_present(image_tag)):
             return image_tag

omdev/ci/docker/cache.py

@@ -1,7 +1,9 @@
 # ruff: noqa: UP006 UP007
 import abc
+import dataclasses as dc
 import typing as ta
 
+from omlish.lite.check import check
 from omlish.os.temp import temp_file_context
 
 from ..cache import FileCache
@@ -13,13 +15,33 @@ from .cmds import save_docker_tar_cmd
 ##
 
 
+@dc.dataclass(frozen=True)
+class DockerCacheKey:
+    prefixes: ta.Sequence[str]
+    content: str
+
+    def __post_init__(self) -> None:
+        check.not_isinstance(self.prefixes, str)
+
+    def append_prefix(self, *prefixes: str) -> 'DockerCacheKey':
+        return dc.replace(self, prefixes=(*self.prefixes, *prefixes))
+
+    SEPARATOR: ta.ClassVar[str] = '--'
+
+    def __str__(self) -> str:
+        return self.SEPARATOR.join([*self.prefixes, self.content])
+
+
+##
+
+
 class DockerCache(abc.ABC):
     @abc.abstractmethod
-    def load_cache_docker_image(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
+    def load_cache_docker_image(self, key: DockerCacheKey) -> ta.Awaitable[ta.Optional[str]]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def save_cache_docker_image(self, key: str, image: str) -> ta.Awaitable[None]:
+    def save_cache_docker_image(self, key: DockerCacheKey, image: str) -> ta.Awaitable[None]:
         raise NotImplementedError
 
 
@@ -33,11 +55,11 @@ class DockerCacheImpl(DockerCache):
 
         self._file_cache = file_cache
 
-    async def load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+    async def load_cache_docker_image(self, key: DockerCacheKey) -> ta.Optional[str]:
         if self._file_cache is None:
             return None
 
-        cache_file = await self._file_cache.get_file(key)
+        cache_file = await self._file_cache.get_file(str(key))
         if cache_file is None:
             return None
 
@@ -45,7 +67,7 @@ class DockerCacheImpl(DockerCache):
 
         return await load_docker_tar_cmd(get_cache_cmd)
 
-    async def save_cache_docker_image(self, key: str, image: str) -> None:
+    async def save_cache_docker_image(self, key: DockerCacheKey, image: str) -> None:
         if self._file_cache is None:
             return
 
@@ -54,4 +76,4 @@ class DockerCacheImpl(DockerCache):
 
             await save_docker_tar_cmd(image, write_tmp_cmd)
 
-            await self._file_cache.put_file(key, tmp_file, steal=True)
+            await self._file_cache.put_file(str(key), tmp_file, steal=True)

omdev/ci/docker/cacheserved/cache.py

@@ -0,0 +1,209 @@
+# ruff: noqa: UP006 UP007
+import asyncio
+import contextlib
+import dataclasses as dc
+import json
+import os.path
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_compact
+from omlish.lite.logs import log
+from omlish.lite.marshal import marshal_obj
+from omlish.lite.marshal import unmarshal_obj
+from omlish.lite.timeouts import Timeout
+from omlish.lite.timeouts import TimeoutLike
+
+from ....dataserver.server import DataServer
+from ....dataserver.targets import DataServerTarget
+from ....oci.building import build_oci_index_repository
+from ....oci.data import get_single_leaf_oci_image_index
+from ....oci.dataserver import build_oci_repository_data_server_routes
+from ....oci.loading import read_oci_repository_root_index
+from ....oci.pack.repositories import OciPackedRepositoryBuilder
+from ....oci.repositories import OciRepository
+from ...cache import DataCache
+from ...cache import read_data_cache_data
+from ..cache import DockerCache
+from ..cache import DockerCacheKey
+from ..dataserver import DockerDataServer
+from ..repositories import DockerImageRepositoryOpener
+from .manifests import CacheServedDockerImageManifest
+from .manifests import build_cache_served_docker_image_data_server_routes
+from .manifests import build_cache_served_docker_image_manifest
+
+
+##
+
+
+class CacheServedDockerCache(DockerCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        port: int = 5021
+
+        repack: bool = True
+
+        key_prefix: ta.Optional[str] = 'cs'
+
+        #
+
+        pull_run_cmd: ta.Optional[str] = 'true'
+
+        #
+
+        server_start_timeout: TimeoutLike = 5.
+        server_start_sleep: float = .1
+
+    def __init__(
+            self,
+            *,
+            config: Config = Config(),
+
+            image_repo_opener: DockerImageRepositoryOpener,
+            data_cache: DataCache,
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self._image_repo_opener = image_repo_opener
+        self._data_cache = data_cache
+
+    async def load_cache_docker_image(self, key: DockerCacheKey) -> ta.Optional[str]:
+        if (kp := self._config.key_prefix) is not None:
+            key = key.append_prefix(kp)
+
+        if (manifest_data := await self._data_cache.get_data(str(key))) is None:
+            return None
+
+        manifest_bytes = await read_data_cache_data(manifest_data)
+
+        manifest: CacheServedDockerImageManifest = unmarshal_obj(
+            json.loads(manifest_bytes.decode('utf-8')),
+            CacheServedDockerImageManifest,
+        )
+
+        async def make_cache_key_target(target_cache_key: str, **target_kwargs: ta.Any) -> DataServerTarget:  # noqa
+            cache_data = check.not_none(await self._data_cache.get_data(target_cache_key))
+
+            if isinstance(cache_data, DataCache.BytesData):
+                return DataServerTarget.of(
+                    cache_data.data,
+                    **target_kwargs,
+                )
+
+            elif isinstance(cache_data, DataCache.FileData):
+                return DataServerTarget.of(
+                    file_path=cache_data.file_path,
+                    **target_kwargs,
+                )
+
+            elif isinstance(cache_data, DataCache.UrlData):
+                return DataServerTarget.of(
+                    url=cache_data.url,
+                    methods=['GET'],
+                    **target_kwargs,
+                )
+
+            else:
+                raise TypeError(cache_data)
+
+        data_server_routes = await build_cache_served_docker_image_data_server_routes(
+            manifest,
+            make_cache_key_target,
+        )
+
+        data_server = DataServer(DataServer.HandlerRoute.of_(*data_server_routes))
+
+        image_url = f'localhost:{self._config.port}/{key!s}'
+
+        async with DockerDataServer(
+                self._config.port,
+                data_server,
+                handler_log=log,
+        ) as dds:
+            dds_run_task = asyncio.create_task(dds.run())
+            try:
+                timeout = Timeout.of(self._config.server_start_timeout)
+                while True:
+                    timeout()
+                    try:
+                        reader, writer = await asyncio.open_connection('localhost', self._config.port)
+                    except Exception as e:  # noqa
+                        log.exception('Failed to connect to cache server - will try again')
+                    else:
+                        writer.close()
+                        await asyncio.wait_for(writer.wait_closed(), timeout=timeout.remaining())
+                        break
+                    await asyncio.sleep(self._config.server_start_sleep)
+
+                if (prc := self._config.pull_run_cmd) is not None:
+                    pull_cmd = [
+                        'run',
+                        '--rm',
+                        image_url,
+                        prc,
+                    ]
+                else:
+                    pull_cmd = [
+                        'pull',
+                        image_url,
+                    ]
+
+                await asyncio_subprocesses.check_call(
+                    'docker',
+                    *pull_cmd,
+                )
+
+            finally:
+                dds.stop_event.set()
+                await dds_run_task
+
+        return image_url
+
+    async def save_cache_docker_image(self, key: DockerCacheKey, image: str) -> None:
+        if (kp := self._config.key_prefix) is not None:
+            key = key.append_prefix(kp)
+
+        async with contextlib.AsyncExitStack() as es:
+            image_repo: OciRepository = await es.enter_async_context(
+                self._image_repo_opener.open_docker_image_repository(image),
+            )
+
+            root_image_index = read_oci_repository_root_index(image_repo)
+            image_index = get_single_leaf_oci_image_index(root_image_index)
+
+            if self._config.repack:
+                prb: OciPackedRepositoryBuilder = es.enter_context(OciPackedRepositoryBuilder(
+                    image_repo,
+                ))
+                built_repo = await asyncio.get_running_loop().run_in_executor(None, prb.build)  # noqa
+
+            else:
+                built_repo = build_oci_index_repository(image_index)
+
+            data_server_routes = build_oci_repository_data_server_routes(
+                str(key),
+                built_repo,
+            )
+
+            async def make_file_cache_key(file_path: str) -> str:
+                target_cache_key = f'{key!s}--{os.path.basename(file_path).split(".")[0]}'
+                await self._data_cache.put_data(
+                    target_cache_key,
+                    DataCache.FileData(file_path),
+                )
+                return target_cache_key
+
+            cache_served_manifest = await build_cache_served_docker_image_manifest(
+                data_server_routes,
+                make_file_cache_key,
+            )
+
+        manifest_data = json_dumps_compact(marshal_obj(cache_served_manifest)).encode('utf-8')
+
+        await self._data_cache.put_data(
+            str(key),
+            DataCache.BytesData(manifest_data),
+        )

omdev/ci/docker/cacheserved/manifests.py

@@ -0,0 +1,122 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish.lite.check import check
+
+from ....dataserver.routes import DataServerRoute
+from ....dataserver.targets import BytesDataServerTarget
+from ....dataserver.targets import DataServerTarget
+from ....dataserver.targets import FileDataServerTarget
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheServedDockerImageManifest:
+    @dc.dataclass(frozen=True)
+    class Route:
+        paths: ta.Sequence[str]
+
+        content_type: str
+        content_length: int
+
+        @dc.dataclass(frozen=True)
+        class Target(abc.ABC):  # noqa
+            pass
+
+        @dc.dataclass(frozen=True)
+        class BytesTarget(Target):
+            data: bytes
+
+        @dc.dataclass(frozen=True)
+        class CacheKeyTarget(Target):
+            key: str
+
+        target: Target
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.paths, str)
+
+    routes: ta.Sequence[Route]
+
+
+#
+
+
+async def build_cache_served_docker_image_manifest(
+        data_server_routes: ta.Iterable[DataServerRoute],
+        make_file_cache_key: ta.Callable[[str], ta.Awaitable[str]],
+) -> CacheServedDockerImageManifest:
+    routes: ta.List[CacheServedDockerImageManifest.Route] = []
+
+    for data_server_route in data_server_routes:
+        content_length: int
+
+        data_server_target = data_server_route.target
+        target: CacheServedDockerImageManifest.Route.Target
+        if isinstance(data_server_target, BytesDataServerTarget):
+            bytes_data = check.isinstance(data_server_target.data, bytes)
+            content_length = len(bytes_data)
+            target = CacheServedDockerImageManifest.Route.BytesTarget(bytes_data)
+
+        elif isinstance(data_server_target, FileDataServerTarget):
+            file_path = check.non_empty_str(data_server_target.file_path)
+            content_length = os.path.getsize(file_path)
+            cache_key = await make_file_cache_key(file_path)
+            target = CacheServedDockerImageManifest.Route.CacheKeyTarget(cache_key)
+
+        else:
+            raise TypeError(data_server_target)
+
+        routes.append(CacheServedDockerImageManifest.Route(
+            paths=data_server_route.paths,
+
+            content_type=check.non_empty_str(data_server_target.content_type),
+            content_length=content_length,
+
+            target=target,
+        ))
+
+    return CacheServedDockerImageManifest(
+        routes=routes,
+    )
+
+
+#
+
+
+async def build_cache_served_docker_image_data_server_routes(
+        manifest: CacheServedDockerImageManifest,
+        make_cache_key_target: ta.Callable[..., ta.Awaitable[DataServerTarget]],
+) -> ta.List[DataServerRoute]:
+    routes: ta.List[DataServerRoute] = []
+
+    for manifest_route in manifest.routes:
+        manifest_target = manifest_route.target
+
+        target_kwargs: dict = dict(
+            content_type=manifest_route.content_type,
+            content_length=manifest_route.content_length,
+        )
+
+        target: DataServerTarget
+
+        if isinstance(manifest_target, CacheServedDockerImageManifest.Route.BytesTarget):
+            target = DataServerTarget.of(manifest_target.data, **target_kwargs)
+
+        elif isinstance(manifest_target, CacheServedDockerImageManifest.Route.CacheKeyTarget):
+            target = await make_cache_key_target(manifest_target.key, **target_kwargs)
+
+        else:
+            raise TypeError(manifest_target)
+
+        routes.append(DataServerRoute(
+            paths=manifest_route.paths,
+            target=target,
+        ))
+
+    return routes

omdev/ci/docker/dataserver.py

@@ -7,7 +7,7 @@ import sys
 import threading
 import typing as ta
 
-from omlish.docker.portrelay import DockerPortRelay
+from omlish.docker.ports import DockerPortRelay
 from omlish.http.coro.simple import make_simple_http_server
 from omlish.http.handlers import HttpHandler
 from omlish.http.handlers import LoggingHttpHandler
@@ -93,6 +93,7 @@ class AsyncioManagedSimpleHttpServer(AsyncExitStacked):
                 self._port,
                 self._handler,
                 ssl_context=self._ssl_context(),
+                ignore_ssl_errors=True,
                 use_threads=True,
         ) as server:
             yield server
@@ -163,6 +164,13 @@ class DockerDataServer(AsyncExitStacked):
         return self._stop_event
 
     async def run(self) -> None:
+        # FIXME:
+        #  - shared single server with updatable routes
+        #  - get docker used ports with ns1
+        #  - discover server port with get_available_port
+        #  - discover relay port pair with get_available_ports
+        # relay_port: ta.Optional[ta.Tuple[int, int]] = None
+
         relay_port: ta.Optional[int] = None
         if sys.platform == 'darwin':
             relay_port = self._port

omdev/ci/docker/imagepulling.py

@@ -4,9 +4,11 @@ import dataclasses as dc
 import typing as ta
 
 from omlish.lite.timing import log_timing_context
+from omlish.text.mangle import StringMangler
 
 from ..cache import FileCache
 from .cache import DockerCache
+from .cache import DockerCacheKey
 from .cmds import is_docker_image_present
 from .cmds import pull_docker_image
 
@@ -44,11 +46,9 @@ class DockerImagePullingImpl(DockerImagePulling):
         if not self._config.always_pull and (await is_docker_image_present(image)):
             return
 
-        dep_suffix = image
-        for c in '/:.-_':
-            dep_suffix = dep_suffix.replace(c, '-')
+        key_content = StringMangler.of('-', '/:._').mangle(image)
 
-        cache_key = f'docker-{dep_suffix}'
+        cache_key = DockerCacheKey(['docker'], key_content)
         if (
                 self._docker_cache is not None and
                 (await self._docker_cache.load_cache_docker_image(cache_key)) is not None