omdev 0.0.0.dev223__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/ci/cache.py

@@ -1,13 +1,17 @@
 # ruff: noqa: UP006 UP007
 import abc
+import asyncio
 import dataclasses as dc
+import functools
 import os.path
 import shutil
 import typing as ta
+import urllib.request
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.logs import log
+from omlish.os.temp import make_temp_file
 
 from .consts import CI_CACHE_VERSION
 
@@ -162,3 +166,107 @@ class DirectoryFileCache(FileCache):
         else:
             shutil.copyfile(file_path, cache_file_path)
         return cache_file_path
+
+
+##
+
+
+class DataCache:
+    @dc.dataclass(frozen=True)
+    class Data(abc.ABC):  # noqa
+        pass
+
+    @dc.dataclass(frozen=True)
+    class BytesData(Data):
+        data: bytes
+
+    @dc.dataclass(frozen=True)
+    class FileData(Data):
+        file_path: str
+
+    @dc.dataclass(frozen=True)
+    class UrlData(Data):
+        url: str
+
+    #
+
+    @abc.abstractmethod
+    def get_data(self, key: str) -> ta.Awaitable[ta.Optional[Data]]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def put_data(self, key: str, data: Data) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+#
+
+
+@functools.singledispatch
+async def read_data_cache_data(data: DataCache.Data) -> bytes:
+    raise TypeError(data)
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.BytesData) -> bytes:
+    return data.data
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.FileData) -> bytes:
+    with open(data.file_path, 'rb') as f:  # noqa
+        return f.read()
+
+
+@read_data_cache_data.register
+async def _(data: DataCache.UrlData) -> bytes:
+    def inner() -> bytes:
+        with urllib.request.urlopen(urllib.request.Request(  # noqa
+            data.url,
+        )) as resp:
+            return resp.read()
+
+    return await asyncio.get_running_loop().run_in_executor(None, inner)
+
+
+#
+
+
+class FileCacheDataCache(DataCache):
+    def __init__(
+            self,
+            file_cache: FileCache,
+    ) -> None:
+        super().__init__()
+
+        self._file_cache = file_cache
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        if (file_path := await self._file_cache.get_file(key)) is None:
+            return None
+
+        return DataCache.FileData(file_path)
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        steal = False
+
+        if isinstance(data, DataCache.BytesData):
+            file_path = make_temp_file()
+            with open(file_path, 'wb') as f:  # noqa
+                f.write(data.data)
+            steal = True
+
+        elif isinstance(data, DataCache.FileData):
+            file_path = data.file_path
+
+        elif isinstance(data, DataCache.UrlData):
+            raise NotImplementedError
+
+        else:
+            raise TypeError(data)
+
+        await self._file_cache.put_file(
+            key,
+            file_path,
+            steal=steal,
+        )

omdev/ci/ci.py

@@ -7,6 +7,7 @@ from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
 from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.lite.timing import log_timing_context
 from omlish.os.temp import temp_file_context
 
 from .compose import DockerComposeRun
@@ -17,7 +18,6 @@ from .docker.imagepulling import DockerImagePulling
 from .docker.utils import build_docker_file_hash
 from .requirements import build_requirements_hash
 from .shell import ShellCmd
-from .utils import log_timing_context
 
 
 ##

omdev/ci/docker/cacheserved.py

@@ -0,0 +1,262 @@
+# ruff: noqa: UP006 UP007
+import abc
+import asyncio
+import contextlib
+import dataclasses as dc
+import json
+import os.path
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_compact
+from omlish.lite.logs import log
+from omlish.lite.marshal import marshal_obj
+from omlish.lite.marshal import unmarshal_obj
+
+from ...dataserver.routes import DataServerRoute
+from ...dataserver.server import DataServer
+from ...dataserver.targets import BytesDataServerTarget
+from ...dataserver.targets import DataServerTarget
+from ...dataserver.targets import FileDataServerTarget
+from ...oci.building import build_oci_index_repository
+from ...oci.data import get_single_leaf_oci_image_index
+from ...oci.dataserver import build_oci_repository_data_server_routes
+from ...oci.loading import read_oci_repository_root_index
+from ...oci.pack.repositories import OciPackedRepositoryBuilder
+from ...oci.repositories import OciRepository
+from ..cache import DataCache
+from ..cache import read_data_cache_data
+from .cache import DockerCache
+from .dataserver import DockerDataServer
+from .repositories import DockerImageRepositoryOpener
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheServedDockerImageManifest:
+    @dc.dataclass(frozen=True)
+    class Route:
+        paths: ta.Sequence[str]
+
+        content_type: str
+        content_length: int
+
+        @dc.dataclass(frozen=True)
+        class Target(abc.ABC):  # noqa
+            pass
+
+        @dc.dataclass(frozen=True)
+        class BytesTarget(Target):
+            data: bytes
+
+        @dc.dataclass(frozen=True)
+        class CacheKeyTarget(Target):
+            key: str
+
+        target: Target
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.paths, str)
+
+    routes: ta.Sequence[Route]
+
+
+#
+
+
+async def build_cache_served_docker_image_manifest(
+        data_server_routes: ta.Iterable[DataServerRoute],
+        make_file_cache_key: ta.Callable[[str], ta.Awaitable[str]],
+) -> CacheServedDockerImageManifest:
+    routes: ta.List[CacheServedDockerImageManifest.Route] = []
+
+    for data_server_route in data_server_routes:
+        content_length: int
+
+        data_server_target = data_server_route.target
+        target: CacheServedDockerImageManifest.Route.Target
+        if isinstance(data_server_target, BytesDataServerTarget):
+            bytes_data = check.isinstance(data_server_target.data, bytes)
+            content_length = len(bytes_data)
+            target = CacheServedDockerImageManifest.Route.BytesTarget(bytes_data)
+
+        elif isinstance(data_server_target, FileDataServerTarget):
+            file_path = check.non_empty_str(data_server_target.file_path)
+            content_length = os.path.getsize(file_path)
+            cache_key = await make_file_cache_key(file_path)
+            target = CacheServedDockerImageManifest.Route.CacheKeyTarget(cache_key)
+
+        else:
+            raise TypeError(data_server_target)
+
+        routes.append(CacheServedDockerImageManifest.Route(
+            paths=data_server_route.paths,
+
+            content_type=check.non_empty_str(data_server_target.content_type),
+            content_length=content_length,
+
+            target=target,
+        ))
+
+    return CacheServedDockerImageManifest(
+        routes=routes,
+    )
+
+
+#
+
+
+async def build_cache_served_docker_image_data_server_routes(
+        manifest: CacheServedDockerImageManifest,
+        make_cache_key_target: ta.Callable[..., ta.Awaitable[DataServerTarget]],
+) -> ta.List[DataServerRoute]:
+    routes: ta.List[DataServerRoute] = []
+
+    for manifest_route in manifest.routes:
+        manifest_target = manifest_route.target
+
+        target_kwargs: dict = dict(
+            content_type=manifest_route.content_type,
+            content_length=manifest_route.content_length,
+        )
+
+        target: DataServerTarget
+
+        if isinstance(manifest_target, CacheServedDockerImageManifest.Route.BytesTarget):
+            target = DataServerTarget.of(manifest_target.data, **target_kwargs)
+
+        elif isinstance(manifest_target, CacheServedDockerImageManifest.Route.CacheKeyTarget):
+            target = await make_cache_key_target(manifest_target.key, **target_kwargs)
+
+        else:
+            raise TypeError(manifest_target)
+
+        routes.append(DataServerRoute(
+            paths=manifest_route.paths,
+            target=target,
+        ))
+
+    return routes
+
+
+##
+
+
+class CacheServedDockerCache(DockerCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        port: int = 5021
+
+        repack: bool = True
+
+    def __init__(
+            self,
+            *,
+            config: Config = Config(),
+
+            image_repo_opener: DockerImageRepositoryOpener,
+            data_cache: DataCache,
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self._image_repo_opener = image_repo_opener
+        self._data_cache = data_cache
+
+    async def load_cache_docker_image(self, key: str) -> ta.Optional[str]:
+        if (manifest_data := await self._data_cache.get_data(key)) is None:
+            return None
+
+        manifest_bytes = await read_data_cache_data(manifest_data)
+
+        manifest: CacheServedDockerImageManifest = unmarshal_obj(
+            json.loads(manifest_bytes.decode('utf-8')),
+            CacheServedDockerImageManifest,
+        )
+
+        async def make_cache_key_target(target_cache_key: str, **target_kwargs: ta.Any) -> DataServerTarget:  # noqa
+            # FIXME: url
+            cache_data = check.not_none(await self._data_cache.get_data(target_cache_key))
+            file_path = check.isinstance(cache_data, DataCache.FileData).file_path
+            return DataServerTarget.of(
+                file_path=file_path,
+                **target_kwargs,
+            )
+
+        data_server_routes = await build_cache_served_docker_image_data_server_routes(
+            manifest,
+            make_cache_key_target,
+        )
+
+        data_server = DataServer(DataServer.HandlerRoute.of_(*data_server_routes))
+
+        image_url = f'localhost:{self._config.port}/{key}'
+
+        async with DockerDataServer(
+                self._config.port,
+                data_server,
+                handler_log=log,
+        ) as dds:
+            dds_run_task = asyncio.create_task(dds.run())
+            try:
+                # FIXME: lol
+                await asyncio.sleep(3.)
+
+                await asyncio_subprocesses.check_call(
+                    'docker',
+                    'pull',
+                    image_url,
+                )
+
+            finally:
+                dds.stop_event.set()
+                await dds_run_task
+
+        return image_url
+
+    async def save_cache_docker_image(self, key: str, image: str) -> None:
+        async with contextlib.AsyncExitStack() as es:
+            image_repo: OciRepository = await es.enter_async_context(
+                self._image_repo_opener.open_docker_image_repository(image),
+            )
+
+            root_image_index = read_oci_repository_root_index(image_repo)
+            image_index = get_single_leaf_oci_image_index(root_image_index)
+
+            if self._config.repack:
+                prb: OciPackedRepositoryBuilder = es.enter_context(OciPackedRepositoryBuilder(
+                    image_repo,
+                ))
+                built_repo = await asyncio.get_running_loop().run_in_executor(None, prb.build)
+
+            else:
+                built_repo = build_oci_index_repository(image_index)
+
+            data_server_routes = build_oci_repository_data_server_routes(
+                key,
+                built_repo,
+            )
+
+            async def make_file_cache_key(file_path: str) -> str:
+                target_cache_key = f'{key}--{os.path.basename(file_path).split(".")[0]}'
+                await self._data_cache.put_data(
+                    target_cache_key,
+                    DataCache.FileData(file_path),
+                )
+                return target_cache_key
+
+            cache_served_manifest = await build_cache_served_docker_image_manifest(
+                data_server_routes,
+                make_file_cache_key,
+            )
+
+        manifest_data = json_dumps_compact(marshal_obj(cache_served_manifest)).encode('utf-8')
+
+        await self._data_cache.put_data(
+            key,
+            DataCache.BytesData(manifest_data),
+        )

omdev/ci/docker/dataserver.py

@@ -0,0 +1,204 @@
+# ruff: noqa: UP006 UP007
+import asyncio
+import contextlib
+import logging
+import ssl
+import sys
+import threading
+import typing as ta
+
+from omlish.docker.portrelay import DockerPortRelay
+from omlish.http.coro.simple import make_simple_http_server
+from omlish.http.handlers import HttpHandler
+from omlish.http.handlers import LoggingHttpHandler
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import AsyncExitStacked
+from omlish.secrets.tempssl import generate_temp_localhost_ssl_cert
+from omlish.sockets.server.server import SocketServer
+
+from ...dataserver.http import DataServerHttpHandler
+from ...dataserver.server import DataServer
+
+
+##
+
+
+@contextlib.asynccontextmanager
+async def start_docker_port_relay(
+        docker_port: int,
+        host_port: int,
+        **kwargs: ta.Any,
+) -> ta.AsyncGenerator[None, None]:
+    proc = await asyncio.create_subprocess_exec(*DockerPortRelay(
+        docker_port,
+        host_port,
+        **kwargs,
+    ).run_cmd())
+
+    try:
+        yield
+
+    finally:
+        try:
+            proc.kill()
+        except ProcessLookupError:
+            pass
+        await proc.wait()
+
+
+##
+
+
+class AsyncioManagedSimpleHttpServer(AsyncExitStacked):
+    def __init__(
+            self,
+            port: int,
+            handler: HttpHandler,
+            *,
+            temp_ssl: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._port = port
+        self._handler = handler
+
+        self._temp_ssl = temp_ssl
+
+        self._lock = threading.RLock()
+
+        self._loop: ta.Optional[asyncio.AbstractEventLoop] = None
+        self._thread: ta.Optional[threading.Thread] = None
+        self._thread_exit_event = asyncio.Event()
+        self._server: ta.Optional[SocketServer] = None
+
+    @cached_nullary
+    def _ssl_context(self) -> ta.Optional['ssl.SSLContext']:
+        if not self._temp_ssl:
+            return None
+
+        ssl_cert = generate_temp_localhost_ssl_cert().cert  # FIXME: async blocking
+
+        ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        ssl_context.load_cert_chain(
+            keyfile=ssl_cert.key_file,
+            certfile=ssl_cert.cert_file,
+        )
+
+        return ssl_context
+
+    @contextlib.contextmanager
+    def _make_server(self) -> ta.Iterator[SocketServer]:
+        with make_simple_http_server(
+                self._port,
+                self._handler,
+                ssl_context=self._ssl_context(),
+                use_threads=True,
+        ) as server:
+            yield server
+
+    def _thread_main(self) -> None:
+        try:
+            check.none(self._server)
+            with self._make_server() as server:
+                self._server = server
+
+                server.run()
+
+        finally:
+            check.not_none(self._loop).call_soon_threadsafe(self._thread_exit_event.set)
+
+    def is_running(self) -> bool:
+        return self._server is not None
+
+    def shutdown(self) -> None:
+        if (server := self._server) is not None:
+            server.shutdown(block=False)
+
+    async def run(self) -> None:
+        with self._lock:
+            check.none(self._loop)
+            check.none(self._thread)
+            check.state(not self._thread_exit_event.is_set())
+
+            if self._temp_ssl:
+                # Hit the ExitStack from this thread
+                self._ssl_context()
+
+            self._loop = check.not_none(asyncio.get_running_loop())
+            self._thread = threading.Thread(
+                target=self._thread_main,
+                daemon=True,
+            )
+            self._thread.start()
+
+        await self._thread_exit_event.wait()
+
+
+##
+
+
+class DockerDataServer(AsyncExitStacked):
+    def __init__(
+            self,
+            port: int,
+            data_server: DataServer,
+            *,
+            handler_log: ta.Optional[logging.Logger] = None,
+            stop_event: ta.Optional[asyncio.Event] = None,
+    ) -> None:
+        super().__init__()
+
+        self._port = port
+        self._data_server = data_server
+
+        self._handler_log = handler_log
+
+        if stop_event is None:
+            stop_event = asyncio.Event()
+        self._stop_event = stop_event
+
+    @property
+    def stop_event(self) -> asyncio.Event:
+        return self._stop_event
+
+    async def run(self) -> None:
+        relay_port: ta.Optional[int] = None
+        if sys.platform == 'darwin':
+            relay_port = self._port
+            server_port = self._port + 1
+        else:
+            server_port = self._port
+
+        #
+
+        handler: HttpHandler = DataServerHttpHandler(self._data_server)
+
+        if self._handler_log is not None:
+            handler = LoggingHttpHandler(
+                handler,
+                self._handler_log,
+            )
+
+        #
+
+        async with contextlib.AsyncExitStack() as es:
+            if relay_port is not None:
+                await es.enter_async_context(start_docker_port_relay(  # noqa
+                    relay_port,
+                    server_port,
+                    intermediate_port=server_port + 1,
+                ))
+
+            async with AsyncioManagedSimpleHttpServer(
+                    server_port,
+                    handler,
+                    temp_ssl=True,
+            ) as server:
+                server_run_task = asyncio.create_task(server.run())
+                try:
+                    await self._stop_event.wait()
+
+                finally:
+                    server.shutdown()
+                    await server_run_task

omdev/ci/docker/imagepulling.py

@@ -3,8 +3,9 @@ import abc
 import dataclasses as dc
 import typing as ta
 
+from omlish.lite.timing import log_timing_context
+
 from ..cache import FileCache
-from ..utils import log_timing_context
 from .cache import DockerCache
 from .cmds import is_docker_image_present
 from .cmds import pull_docker_image

omdev/ci/docker/packing.py

@@ -0,0 +1,72 @@
+# ruff: noqa: UP006 UP007
+import asyncio
+import os.path
+import shlex
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.cached import cached_nullary
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.logs.timing import log_timing_context
+from omlish.os.temp import temp_dir_context
+
+from ...oci.building import BuiltOciImageIndexRepository
+from ...oci.pack.repositories import OciPackedRepositoryBuilder
+from ...oci.repositories import DirectoryOciRepository
+
+
+##
+
+
+class PackedDockerImageIndexRepositoryBuilder(ExitStacked):
+    def __init__(
+            self,
+            *,
+            image_id: str,
+
+            temp_dir: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        self._image_id = image_id
+
+        self._given_temp_dir = temp_dir
+
+    @cached_nullary
+    def _temp_dir(self) -> str:
+        if (given := self._given_temp_dir) is not None:
+            return given
+        else:
+            return self._enter_context(temp_dir_context())  # noqa
+
+    #
+
+    @async_cached_nullary
+    async def _save_to_dir(self) -> str:
+        save_dir = os.path.join(self._temp_dir(), 'built-image')
+        os.mkdir(save_dir)
+
+        with log_timing_context(f'Saving docker image {self._image_id}'):
+            await asyncio_subprocesses.check_call(
+                ' | '.join([
+                    f'docker save {shlex.quote(self._image_id)}',
+                    f'tar x -C {shlex.quote(save_dir)}',
+                ]),
+                shell=True,
+            )
+
+        return save_dir
+
+    #
+
+    @async_cached_nullary
+    async def build(self) -> BuiltOciImageIndexRepository:
+        saved_dir = await self._save_to_dir()
+
+        with OciPackedRepositoryBuilder(
+                DirectoryOciRepository(saved_dir),
+
+                temp_dir=self._temp_dir(),
+        ) as prb:
+            return await asyncio.get_running_loop().run_in_executor(None, prb.build)