omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/ci/docker/cache.py

@@ -0,0 +1,57 @@
+# ruff: noqa: UP006 UP007
+import abc
+import typing as ta
+
+from omlish.os.temp import temp_file_context
+
+from ..cache import FileCache
+from ..shell import ShellCmd
+from .cmds import load_docker_tar_cmd
+from .cmds import save_docker_tar_cmd
+
+
+##
+
+
+class DockerCache(abc.ABC):
+    @abc.abstractmethod
+    def load_cache_docker_image(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def save_cache_docker_image(self, key: str, image: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+class DockerCacheImpl(DockerCache):
+    def __init__(
+            self,
+            *,
+            file_cache: ta.Optional[FileCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._file_cache = file_cache
+
+    async def load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if self._file_cache is None:
+            return None
+
+        cache_file = await self._file_cache.get_file(key)
+        if cache_file is None:
+            return None
+
+        get_cache_cmd = ShellCmd(f'cat {cache_file} | zstd -cd --long')
+
+        return await load_docker_tar_cmd(get_cache_cmd)
+
+    async def save_cache_docker_image(self, key: str, image: str) -> None:
+        if self._file_cache is None:
+            return
+
+        with temp_file_context() as tmp_file:
+            write_tmp_cmd = ShellCmd(f'zstd > {tmp_file}')
+
+            await save_docker_tar_cmd(image, write_tmp_cmd)
+
+            await self._file_cache.put_file(key, tmp_file, steal=True)

omdev/ci/docker/cacheserved.py

@@ -0,0 +1,262 @@
+# ruff: noqa: UP006 UP007
+import abc
+import asyncio
+import contextlib
+import dataclasses as dc
+import json
+import os.path
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_compact
+from omlish.lite.logs import log
+from omlish.lite.marshal import marshal_obj
+from omlish.lite.marshal import unmarshal_obj
+
+from ...dataserver.routes import DataServerRoute
+from ...dataserver.server import DataServer
+from ...dataserver.targets import BytesDataServerTarget
+from ...dataserver.targets import DataServerTarget
+from ...dataserver.targets import FileDataServerTarget
+from ...oci.building import build_oci_index_repository
+from ...oci.data import get_single_leaf_oci_image_index
+from ...oci.dataserver import build_oci_repository_data_server_routes
+from ...oci.loading import read_oci_repository_root_index
+from ...oci.pack.repositories import OciPackedRepositoryBuilder
+from ...oci.repositories import OciRepository
+from ..cache import DataCache
+from ..cache import read_data_cache_data
+from .cache import DockerCache
+from .dataserver import DockerDataServer
+from .repositories import DockerImageRepositoryOpener
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheServedDockerImageManifest:
+    @dc.dataclass(frozen=True)
+    class Route:
+        paths: ta.Sequence[str]
+
+        content_type: str
+        content_length: int
+
+        @dc.dataclass(frozen=True)
+        class Target(abc.ABC):  # noqa
+            pass
+
+        @dc.dataclass(frozen=True)
+        class BytesTarget(Target):
+            data: bytes
+
+        @dc.dataclass(frozen=True)
+        class CacheKeyTarget(Target):
+            key: str
+
+        target: Target
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.paths, str)
+
+    routes: ta.Sequence[Route]
+
+
+#
+
+
+async def build_cache_served_docker_image_manifest(
+        data_server_routes: ta.Iterable[DataServerRoute],
+        make_file_cache_key: ta.Callable[[str], ta.Awaitable[str]],
+) -> CacheServedDockerImageManifest:
+    routes: ta.List[CacheServedDockerImageManifest.Route] = []
+
+    for data_server_route in data_server_routes:
+        content_length: int
+
+        data_server_target = data_server_route.target
+        target: CacheServedDockerImageManifest.Route.Target
+        if isinstance(data_server_target, BytesDataServerTarget):
+            bytes_data = check.isinstance(data_server_target.data, bytes)
+            content_length = len(bytes_data)
+            target = CacheServedDockerImageManifest.Route.BytesTarget(bytes_data)
+
+        elif isinstance(data_server_target, FileDataServerTarget):
+            file_path = check.non_empty_str(data_server_target.file_path)
+            content_length = os.path.getsize(file_path)
+            cache_key = await make_file_cache_key(file_path)
+            target = CacheServedDockerImageManifest.Route.CacheKeyTarget(cache_key)
+
+        else:
+            raise TypeError(data_server_target)
+
+        routes.append(CacheServedDockerImageManifest.Route(
+            paths=data_server_route.paths,
+
+            content_type=check.non_empty_str(data_server_target.content_type),
+            content_length=content_length,
+
+            target=target,
+        ))
+
+    return CacheServedDockerImageManifest(
+        routes=routes,
+    )
+
+
+#
+
+
+async def build_cache_served_docker_image_data_server_routes(
+        manifest: CacheServedDockerImageManifest,
+        make_cache_key_target: ta.Callable[..., ta.Awaitable[DataServerTarget]],
+) -> ta.List[DataServerRoute]:
+    routes: ta.List[DataServerRoute] = []
+
+    for manifest_route in manifest.routes:
+        manifest_target = manifest_route.target
+
+        target_kwargs: dict = dict(
+            content_type=manifest_route.content_type,
+            content_length=manifest_route.content_length,
+        )
+
+        target: DataServerTarget
+
+        if isinstance(manifest_target, CacheServedDockerImageManifest.Route.BytesTarget):
+            target = DataServerTarget.of(manifest_target.data, **target_kwargs)
+
+        elif isinstance(manifest_target, CacheServedDockerImageManifest.Route.CacheKeyTarget):
+            target = await make_cache_key_target(manifest_target.key, **target_kwargs)
+
+        else:
+            raise TypeError(manifest_target)
+
+        routes.append(DataServerRoute(
+            paths=manifest_route.paths,
+            target=target,
+        ))
+
+    return routes
+
+
+##
+
+
+class CacheServedDockerCache(DockerCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        port: int = 5021
+
+        repack: bool = True
+
+    def __init__(
+            self,
+            *,
+            config: Config = Config(),
+
+            image_repo_opener: DockerImageRepositoryOpener,
+            data_cache: DataCache,
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self._image_repo_opener = image_repo_opener
+        self._data_cache = data_cache
+
+    async def load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if (manifest_data := await self._data_cache.get_data(key)) is None:
+            return None
+
+        manifest_bytes = await read_data_cache_data(manifest_data)
+
+        manifest: CacheServedDockerImageManifest = unmarshal_obj(
+            json.loads(manifest_bytes.decode('utf-8')),
+            CacheServedDockerImageManifest,
+        )
+
+        async def make_cache_key_target(target_cache_key: str, **target_kwargs: ta.Any) -> DataServerTarget:  # noqa
+            # FIXME: url
+            cache_data = check.not_none(await self._data_cache.get_data(target_cache_key))
+            file_path = check.isinstance(cache_data, DataCache.FileData).file_path
+            return DataServerTarget.of(
+                file_path=file_path,
+                **target_kwargs,
+            )
+
+        data_server_routes = await build_cache_served_docker_image_data_server_routes(
+            manifest,
+            make_cache_key_target,
+        )
+
+        data_server = DataServer(DataServer.HandlerRoute.of_(*data_server_routes))
+
+        image_url = f'localhost:{self._config.port}/{key}'
+
+        async with DockerDataServer(
+                self._config.port,
+                data_server,
+                handler_log=log,
+        ) as dds:
+            dds_run_task = asyncio.create_task(dds.run())
+            try:
+                # FIXME: lol
+                await asyncio.sleep(3.)
+
+                await asyncio_subprocesses.check_call(
+                    'docker',
+                    'pull',
+                    image_url,
+                )
+
+            finally:
+                dds.stop_event.set()
+                await dds_run_task
+
+        return image_url
+
+    async def save_cache_docker_image(self, key: str, image: str) -> None:
+        async with contextlib.AsyncExitStack() as es:
+            image_repo: OciRepository = await es.enter_async_context(
+                self._image_repo_opener.open_docker_image_repository(image),
+            )
+
+            root_image_index = read_oci_repository_root_index(image_repo)
+            image_index = get_single_leaf_oci_image_index(root_image_index)
+
+            if self._config.repack:
+                prb: OciPackedRepositoryBuilder = es.enter_context(OciPackedRepositoryBuilder(
+                    image_repo,
+                ))
+                built_repo = await asyncio.get_running_loop().run_in_executor(None, prb.build)
+
+            else:
+                built_repo = build_oci_index_repository(image_index)
+
+            data_server_routes = build_oci_repository_data_server_routes(
+                key,
+                built_repo,
+            )
+
+            async def make_file_cache_key(file_path: str) -> str:
+                target_cache_key = f'{key}--{os.path.basename(file_path).split(".")[0]}'
+                await self._data_cache.put_data(
+                    target_cache_key,
+                    DataCache.FileData(file_path),
+                )
+                return target_cache_key
+
+            cache_served_manifest = await build_cache_served_docker_image_manifest(
+                data_server_routes,
+                make_file_cache_key,
+            )
+
+        manifest_data = json_dumps_compact(marshal_obj(cache_served_manifest)).encode('utf-8')
+
+        await self._data_cache.put_data(
+            key,
+            DataCache.BytesData(manifest_data),
+        )

omdev/ci/{docker.py → docker/cmds.py}

@@ -1,58 +1,15 @@
 # ruff: noqa: UP006 UP007
-"""
-TODO:
- - some less stupid Dockerfile hash
-  - doesn't change too much though
-"""
-import contextlib
 import dataclasses as dc
 import json
 import os.path
 import shlex
-import tarfile
 import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.check import check
 from omlish.os.temp import temp_file_context
 
-from .shell import ShellCmd
-from .utils import sha256_str
-
-
-##
-
-
-def build_docker_file_hash(docker_file: str) -> str:
-    with open(docker_file) as f:
-        contents = f.read()
-
-    return sha256_str(contents)
-
-
-##
-
-
-def read_docker_tar_image_tag(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
-            m = mf.read()
-
-    manifests = json.loads(m.decode('utf-8'))
-    manifest = check.single(manifests)
-    tag = check.non_empty_str(check.single(manifest['RepoTags']))
-    return tag
-
-
-def read_docker_tar_image_id(tar_file: str) -> str:
-    with tarfile.open(tar_file) as tf:
-        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
-            i = mf.read()
-
-    index = json.loads(i.decode('utf-8'))
-    manifest = check.single(index['manifests'])
-    image_id = check.non_empty_str(manifest['digest'])
-    return image_id
+from ..shell import ShellCmd
 
 
 ##

omdev/ci/docker/dataserver.py

@@ -0,0 +1,204 @@
+# ruff: noqa: UP006 UP007
+import asyncio
+import contextlib
+import logging
+import ssl
+import sys
+import threading
+import typing as ta
+
+from omlish.docker.portrelay import DockerPortRelay
+from omlish.http.coro.simple import make_simple_http_server
+from omlish.http.handlers import HttpHandler
+from omlish.http.handlers import LoggingHttpHandler
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.secrets.tempssl import generate_temp_localhost_ssl_cert
+from omlish.sockets.server.server import SocketServer
+
+from ...dataserver.http import DataServerHttpHandler
+from ...dataserver.server import DataServer
+
+
+##
+
+
+@contextlib.asynccontextmanager
+async def start_docker_port_relay(
+        docker_port: int,
+        host_port: int,
+        **kwargs: ta.Any,
+) -> ta.AsyncGenerator[None, None]:
+    proc = await asyncio.create_subprocess_exec(*DockerPortRelay(
+        docker_port,
+        host_port,
+        **kwargs,
+    ).run_cmd())
+
+    try:
+        yield
+
+    finally:
+        try:
+            proc.kill()
+        except ProcessLookupError:
+            pass
+        await proc.wait()
+
+
+##
+
+
+class AsyncioManagedSimpleHttpServer(AsyncExitStacked):
+    def __init__(
+            self,
+            port: int,
+            handler: HttpHandler,
+            *,
+            temp_ssl: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._port = port
+        self._handler = handler
+
+        self._temp_ssl = temp_ssl
+
+        self._lock = threading.RLock()
+
+        self._loop: ta.Optional[asyncio.AbstractEventLoop] = None
+        self._thread: ta.Optional[threading.Thread] = None
+        self._thread_exit_event = asyncio.Event()
+        self._server: ta.Optional[SocketServer] = None
+
+    @cached_nullary
+    def _ssl_context(self) -> ta.Optional['ssl.SSLContext']:
+        if not self._temp_ssl:
+            return None
+
+        ssl_cert = generate_temp_localhost_ssl_cert().cert  # FIXME: async blocking
+
+        ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        ssl_context.load_cert_chain(
+            keyfile=ssl_cert.key_file,
+            certfile=ssl_cert.cert_file,
+        )
+
+        return ssl_context
+
+    @contextlib.contextmanager
+    def _make_server(self) -> ta.Iterator[SocketServer]:
+        with make_simple_http_server(
+                self._port,
+                self._handler,
+                ssl_context=self._ssl_context(),
+                use_threads=True,
+        ) as server:
+            yield server
+
+    def _thread_main(self) -> None:
+        try:
+            check.none(self._server)
+            with self._make_server() as server:
+                self._server = server
+
+                server.run()
+
+        finally:
+            check.not_none(self._loop).call_soon_threadsafe(self._thread_exit_event.set)
+
+    def is_running(self) -> bool:
+        return self._server is not None
+
+    def shutdown(self) -> None:
+        if (server := self._server) is not None:
+            server.shutdown(block=False)
+
+    async def run(self) -> None:
+        with self._lock:
+            check.none(self._loop)
+            check.none(self._thread)
+            check.state(not self._thread_exit_event.is_set())
+
+            if self._temp_ssl:
+                # Hit the ExitStack from this thread
+                self._ssl_context()
+
+            self._loop = check.not_none(asyncio.get_running_loop())
+            self._thread = threading.Thread(
+                target=self._thread_main,
+                daemon=True,
+            )
+            self._thread.start()
+
+        await self._thread_exit_event.wait()
+
+
+##
+
+
+class DockerDataServer(AsyncExitStacked):
+    def __init__(
+            self,
+            port: int,
+            data_server: DataServer,
+            *,
+            handler_log: ta.Optional[logging.Logger] = None,
+            stop_event: ta.Optional[asyncio.Event] = None,
+    ) -> None:
+        super().__init__()
+
+        self._port = port
+        self._data_server = data_server
+
+        self._handler_log = handler_log
+
+        if stop_event is None:
+            stop_event = asyncio.Event()
+        self._stop_event = stop_event
+
+    @property
+    def stop_event(self) -> asyncio.Event:
+        return self._stop_event
+
+    async def run(self) -> None:
+        relay_port: ta.Optional[int] = None
+        if sys.platform == 'darwin':
+            relay_port = self._port
+            server_port = self._port + 1
+        else:
+            server_port = self._port
+
+        #
+
+        handler: HttpHandler = DataServerHttpHandler(self._data_server)
+
+        if self._handler_log is not None:
+            handler = LoggingHttpHandler(
+                handler,
+                self._handler_log,
+            )
+
+        #
+
+        async with contextlib.AsyncExitStack() as es:
+            if relay_port is not None:
+                await es.enter_async_context(start_docker_port_relay(  # noqa
+                    relay_port,
+                    server_port,
+                    intermediate_port=server_port + 1,
+                ))
+
+            async with AsyncioManagedSimpleHttpServer(
+                    server_port,
+                    handler,
+                    temp_ssl=True,
+            ) as server:
+                server_run_task = asyncio.create_task(server.run())
+                try:
+                    await self._stop_event.wait()
+
+                finally:
+                    server.shutdown()
+                    await server_run_task

omdev/ci/docker/imagepulling.py

@@ -0,0 +1,65 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.timing import log_timing_context
+
+from ..cache import FileCache
+from .cache import DockerCache
+from .cmds import is_docker_image_present
+from .cmds import pull_docker_image
+
+
+##
+
+
+class DockerImagePulling(abc.ABC):
+    @abc.abstractmethod
+    def pull_docker_image(self, image: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+class DockerImagePullingImpl(DockerImagePulling):
+    @dc.dataclass(frozen=True)
+    class Config:
+        always_pull: bool = False
+
+    def __init__(
+            self,
+            *,
+            config: Config = Config(),
+
+            file_cache: ta.Optional[FileCache] = None,
+            docker_cache: ta.Optional[DockerCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self._file_cache = file_cache
+        self._docker_cache = docker_cache
+
+    async def _pull_docker_image(self, image: str) -> None:
+        if not self._config.always_pull and (await is_docker_image_present(image)):
+            return
+
+        dep_suffix = image
+        for c in '/:.-_':
+            dep_suffix = dep_suffix.replace(c, '-')
+
+        cache_key = f'docker-{dep_suffix}'
+        if (
+                self._docker_cache is not None and
+                (await self._docker_cache.load_cache_docker_image(cache_key)) is not None
+        ):
+            return
+
+        await pull_docker_image(image)
+
+        if self._docker_cache is not None:
+            await self._docker_cache.save_cache_docker_image(cache_key, image)
+
+    async def pull_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            await self._pull_docker_image(image)

omdev/ci/docker/inject.py

@@ -0,0 +1,37 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from .buildcaching import DockerBuildCaching
+from .buildcaching import DockerBuildCachingImpl
+from .cache import DockerCache
+from .cache import DockerCacheImpl
+from .imagepulling import DockerImagePulling
+from .imagepulling import DockerImagePullingImpl
+
+
+##
+
+
+def bind_docker(
+    *,
+    build_caching_config: DockerBuildCachingImpl.Config,
+    image_pulling_config: DockerImagePullingImpl.Config = DockerImagePullingImpl.Config(),
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(build_caching_config),
+        inj.bind(DockerBuildCachingImpl, singleton=True),
+        inj.bind(DockerBuildCaching, to_key=DockerBuildCachingImpl),
+
+        inj.bind(DockerCacheImpl, singleton=True),
+        inj.bind(DockerCache, to_key=DockerCacheImpl),
+
+        inj.bind(image_pulling_config),
+        inj.bind(DockerImagePullingImpl, singleton=True),
+        inj.bind(DockerImagePulling, to_key=DockerImagePullingImpl),
+    ]
+
+    return inj.as_bindings(*lst)