omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/ci/docker/packing.py

@@ -0,0 +1,72 @@
+# ruff: noqa: UP006 UP007
+import asyncio
+import os.path
+import shlex
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.cached import cached_nullary
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.logs.timing import log_timing_context
+from omlish.os.temp import temp_dir_context
+
+from ...oci.building import BuiltOciImageIndexRepository
+from ...oci.pack.repositories import OciPackedRepositoryBuilder
+from ...oci.repositories import DirectoryOciRepository
+
+
+##
+
+
+class PackedDockerImageIndexRepositoryBuilder(ExitStacked):
+    def __init__(
+            self,
+            *,
+            image_id: str,
+
+            temp_dir: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+
+        self._image_id = image_id
+
+        self._given_temp_dir = temp_dir
+
+    @cached_nullary
+    def _temp_dir(self) -> str:
+        if (given := self._given_temp_dir) is not None:
+            return given
+        else:
+            return self._enter_context(temp_dir_context())  # noqa
+
+    #
+
+    @async_cached_nullary
+    async def _save_to_dir(self) -> str:
+        save_dir = os.path.join(self._temp_dir(), 'built-image')
+        os.mkdir(save_dir)
+
+        with log_timing_context(f'Saving docker image {self._image_id}'):
+            await asyncio_subprocesses.check_call(
+                ' | '.join([
+                    f'docker save {shlex.quote(self._image_id)}',
+                    f'tar x -C {shlex.quote(save_dir)}',
+                ]),
+                shell=True,
+            )
+
+        return save_dir
+
+    #
+
+    @async_cached_nullary
+    async def build(self) -> BuiltOciImageIndexRepository:
+        saved_dir = await self._save_to_dir()
+
+        with OciPackedRepositoryBuilder(
+                DirectoryOciRepository(saved_dir),
+
+                temp_dir=self._temp_dir(),
+        ) as prb:
+            return await asyncio.get_running_loop().run_in_executor(None, prb.build)

omdev/ci/docker/repositories.py

@@ -0,0 +1,40 @@
+# ruff: noqa: UP006 UP007
+import abc
+import contextlib
+import shlex
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.timing import log_timing_context
+from omlish.os.temp import temp_dir_context
+
+from ...oci.repositories import DirectoryOciRepository
+from ...oci.repositories import OciRepository
+
+
+##
+
+
+class DockerImageRepositoryOpener(abc.ABC):
+    @abc.abstractmethod
+    def open_docker_image_repository(self, image: str) -> ta.AsyncContextManager[OciRepository]:
+        raise NotImplementedError
+
+
+#
+
+
+class DockerImageRepositoryOpenerImpl(DockerImageRepositoryOpener):
+    @contextlib.asynccontextmanager
+    async def open_docker_image_repository(self, image: str) -> ta.AsyncGenerator[OciRepository, None]:
+        with temp_dir_context() as save_dir:
+            with log_timing_context(f'Saving docker image {image}'):
+                await asyncio_subprocesses.check_call(
+                    ' | '.join([
+                        f'docker save {shlex.quote(image)}',
+                        f'tar x -C {shlex.quote(save_dir)}',
+                    ]),
+                    shell=True,
+                )
+
+            yield DirectoryOciRepository(save_dir)

omdev/ci/docker/utils.py

@@ -0,0 +1,48 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - some less stupid Dockerfile hash
+  - doesn't change too much though
+"""
+import contextlib
+import json
+import tarfile
+
+from omlish.lite.check import check
+
+from ..utils import sha256_str
+
+
+##
+
+
+def build_docker_file_hash(docker_file: str) -> str:
+    with open(docker_file) as f:
+        contents = f.read()
+
+    return sha256_str(contents)
+
+
+##
+
+
+def read_docker_tar_image_tag(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
+            m = mf.read()
+
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
+
+
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
+
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id

omdev/ci/github/cache.py

@@ -1,12 +1,16 @@
 # ruff: noqa: UP006 UP007
+import dataclasses as dc
 import os.path
 import typing as ta
 
 from omlish.lite.check import check
 from omlish.os.files import unlinking_if_exists
 
+from ..cache import CacheVersion
+from ..cache import DataCache
 from ..cache import DirectoryFileCache
 from ..cache import FileCache
+from ..cache import FileCacheDataCache
 from .client import GithubCacheClient
 from .client import GithubCacheServiceV1Client
 
@@ -14,17 +18,23 @@ from .client import GithubCacheServiceV1Client
 ##
 
 
-class GithubFileCache(FileCache):
+class GithubCache(FileCache, DataCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        dir: str
+
     def __init__(
             self,
-            dir: str,  # noqa
+            config: Config,
             *,
             client: ta.Optional[GithubCacheClient] = None,
-            **kwargs: ta.Any,
+            version: ta.Optional[CacheVersion] = None,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__(
+            version=version,
+        )
 
-        self._dir = check.not_none(dir)
+        self._config = config
 
         if client is None:
             client = GithubCacheServiceV1Client(
@@ -33,10 +43,14 @@ class GithubFileCache(FileCache):
         self._client: GithubCacheClient = client
 
         self._local = DirectoryFileCache(
-            self._dir,
+            DirectoryFileCache.Config(
+                dir=check.non_empty_str(config.dir),
+            ),
             version=self._version,
         )
 
+    #
+
     async def get_file(self, key: str) -> ta.Optional[str]:
         local_file = self._local.get_cache_file_path(key)
         if os.path.exists(local_file):
@@ -69,3 +83,18 @@ class GithubFileCache(FileCache):
         await self._client.upload_file(key, cache_file_path)
 
         return cache_file_path
+
+    #
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return DataCache.FileData(local_file)
+
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
+
+        return DataCache.UrlData(check.non_empty_str(self._client.get_entry_url(entry)))
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        await FileCacheDataCache(self).put_data(key, data)

omdev/ci/github/client.py

@@ -13,9 +13,9 @@ from omlish.asyncs.asyncio.asyncio import asyncio_wait_concurrent
 from omlish.lite.check import check
 from omlish.lite.json import json_dumps_compact
 from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
 
 from ..consts import CI_CACHE_VERSION
-from ..utils import log_timing_context
 from .api import GithubCacheServiceV1
 from .env import register_github_env_var
 
@@ -31,6 +31,9 @@ class GithubCacheClient(abc.ABC):
     def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
         raise NotImplementedError
 
+    def get_entry_url(self, entry: Entry) -> ta.Optional[str]:
+        return None
+
     @abc.abstractmethod
     def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
         raise NotImplementedError
@@ -97,7 +100,7 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     def _get_loop(self) -> asyncio.AbstractEventLoop:
         if (loop := self._given_loop) is not None:
             return loop
-        return asyncio.get_event_loop()
+        return asyncio.get_running_loop()
 
     #
 
@@ -225,6 +228,10 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     class Entry(GithubCacheClient.Entry):
         artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry1 = check.isinstance(entry, self.Entry)
+        return entry1.artifact.cache_key
+
     #
 
     def build_get_entry_url_path(self, *keys: str) -> str:

omdev/ci/github/inject.py

@@ -0,0 +1,30 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from ..cache import FileCache
+from .cache import GithubCache
+
+
+##
+
+
+def bind_github(
+        *,
+        cache_dir: ta.Optional[str] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = []
+
+    if cache_dir is not None:
+        lst.extend([
+            inj.bind(GithubCache.Config(
+                dir=cache_dir,
+            )),
+            inj.bind(GithubCache, singleton=True),
+            inj.bind(FileCache, to_key=GithubCache),
+        ])
+
+    return inj.as_bindings(*lst)

omdev/ci/inject.py

@@ -0,0 +1,61 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from .cache import DirectoryFileCache
+from .cache import FileCache
+from .ci import Ci
+from .docker.buildcaching import DockerBuildCachingImpl
+from .docker.imagepulling import DockerImagePullingImpl
+from .docker.inject import bind_docker
+from .github.inject import bind_github
+
+
+##
+
+
+def bind_ci(
+        *,
+        config: Ci.Config,
+
+        github: bool = False,
+
+        cache_dir: ta.Optional[str] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [  # noqa
+        inj.bind(config),
+        inj.bind(Ci, singleton=True),
+    ]
+
+    lst.append(bind_docker(
+        build_caching_config=DockerBuildCachingImpl.Config(
+            service=config.service,
+
+            always_build=config.always_build,
+        ),
+
+        image_pulling_config=DockerImagePullingImpl.Config(
+            always_pull=config.always_pull,
+        ),
+    ))
+
+    if cache_dir is not None:
+        if github:
+            lst.append(bind_github(
+                cache_dir=cache_dir,
+            ))
+
+        else:
+            lst.extend([
+                inj.bind(DirectoryFileCache.Config(
+                    dir=cache_dir,
+                )),
+                inj.bind(DirectoryFileCache, singleton=True),
+                inj.bind(FileCache, to_key=DirectoryFileCache),
+
+            ])
+
+    return inj.as_bindings(*lst)

omdev/ci/utils.py

@@ -1,11 +1,7 @@
 # ruff: noqa: UP006 UP007
 import hashlib
-import logging
-import time
 import typing as ta
 
-from omlish.lite.logs import log
-
 
 ##
 
@@ -22,48 +18,3 @@ def read_yaml_file(yaml_file: str) -> ta.Any:
 
 def sha256_str(s: str) -> str:
     return hashlib.sha256(s.encode('utf-8')).hexdigest()
-
-
-##
-
-
-class LogTimingContext:
-    DEFAULT_LOG: ta.ClassVar[logging.Logger] = log
-
-    def __init__(
-            self,
-            description: str,
-            *,
-            log: ta.Optional[logging.Logger] = None,  # noqa
-            level: int = logging.DEBUG,
-    ) -> None:
-        super().__init__()
-
-        self._description = description
-        self._log = log if log is not None else self.DEFAULT_LOG
-        self._level = level
-
-    def set_description(self, description: str) -> 'LogTimingContext':
-        self._description = description
-        return self
-
-    _begin_time: float
-    _end_time: float
-
-    def __enter__(self) -> 'LogTimingContext':
-        self._begin_time = time.time()
-
-        self._log.log(self._level, f'Begin : {self._description}')  # noqa
-
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self._end_time = time.time()
-
-        self._log.log(
-            self._level,
-            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
-        )
-
-
-log_timing_context = LogTimingContext

omdev/dataserver/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite

omdev/dataserver/handlers.py

@@ -0,0 +1,198 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import http.client
+import io
+import os
+import typing as ta
+import urllib.request
+
+from omlish.lite.check import check
+
+from .targets import BytesDataServerTarget
+from .targets import DataServerTarget
+from .targets import FileDataServerTarget
+from .targets import UrlDataServerTarget
+
+
+DataServerTargetT = ta.TypeVar('DataServerTargetT', bound='DataServerTarget')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DataServerRequest:
+    method: str
+    path: str
+
+
+@dc.dataclass(frozen=True)
+class DataServerResponse:
+    status: int
+    headers: ta.Optional[ta.Mapping[str, str]] = None
+    body: ta.Optional[io.IOBase] = None
+
+    #
+
+    def close(self) -> None:
+        if (body := self.body) is not None:
+            body.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
+class DataServerError(Exception):
+    pass
+
+
+class DataServerHandler(abc.ABC):
+    @abc.abstractmethod
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        raise NotImplementedError
+
+
+##
+
+
+class DataServerTargetHandler(DataServerHandler, abc.ABC, ta.Generic[DataServerTargetT]):
+    def __init__(self, target: DataServerTargetT) -> None:
+        super().__init__()
+
+        self._target = target
+
+    #
+
+    @classmethod
+    def for_target(cls, tgt: DataServerTarget, **kwargs: ta.Any) -> 'DataServerTargetHandler':
+        try:
+            hc = _DATA_SERVER_TARGET_HANDLERS[type(tgt)]
+        except KeyError:
+            raise TypeError(tgt)  # noqa
+        else:
+            return hc(tgt, **kwargs)
+
+    #
+
+    def _make_headers(self) -> ta.Dict[str, str]:
+        dct = {}
+        if (ct := self._target.content_type) is not None:
+            dct['Content-Type'] = ct
+        if (cl := self._target.content_length) is not None:
+            dct['Content-Length'] = str(cl)
+        return dct
+
+
+#
+
+
+_DATA_SERVER_TARGET_HANDLERS: ta.Dict[ta.Type[DataServerTarget], ta.Type[DataServerTargetHandler]] = {}
+
+
+def _register_data_server_target_handler(*tcs):
+    def inner(hc):
+        check.issubclass(hc, DataServerTargetHandler)
+        for tc in tcs:
+            check.issubclass(tc, DataServerTarget)
+            check.not_in(tc, _DATA_SERVER_TARGET_HANDLERS)
+            _DATA_SERVER_TARGET_HANDLERS[tc] = hc
+        return hc
+    return inner
+
+
+#
+
+
+@_register_data_server_target_handler(BytesDataServerTarget)
+class BytesDataServerTargetHandler(DataServerTargetHandler[BytesDataServerTarget]):
+    def _make_headers(self) -> ta.Dict[str, str]:
+        dct = super()._make_headers()
+        if 'Content-Length' not in dct and self._target.data is not None:
+            dct['Content-Length'] = str(len(self._target.data))
+        return dct
+
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        if req.method not in ('GET', 'HEAD'):
+            return DataServerResponse(http.HTTPStatus.METHOD_NOT_ALLOWED)
+
+        return DataServerResponse(
+            http.HTTPStatus.OK,
+            headers=self._make_headers(),
+            body=io.BytesIO(self._target.data) if self._target.data is not None and req.method == 'GET' else None,
+        )
+
+
+#
+
+
+@_register_data_server_target_handler(FileDataServerTarget)
+class FileDataServerTargetHandler(DataServerTargetHandler[FileDataServerTarget]):
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        if req.method == 'HEAD':
+            try:
+                st = os.stat(check.not_none(self._target.file_path))
+            except FileNotFoundError:
+                return DataServerResponse(http.HTTPStatus.NOT_FOUND)
+
+            return DataServerResponse(
+                http.HTTPStatus.OK,
+                headers={
+                    'Content-Length': str(st.st_size),
+                    **self._make_headers(),
+                },
+            )
+
+        elif req.method == 'GET':
+            try:
+                f = open(check.not_none(self._target.file_path), 'rb')  # noqa
+            except FileNotFoundError:
+                return DataServerResponse(http.HTTPStatus.NOT_FOUND)
+
+            try:
+                sz = os.fstat(f.fileno())
+
+                return DataServerResponse(
+                    http.HTTPStatus.OK,
+                    headers={
+                        'Content-Length': str(sz.st_size),
+                        **self._make_headers(),
+                    },
+                    body=f,  # noqa
+                )
+
+            except Exception:  # noqa
+                f.close()
+                raise
+
+        else:
+            return DataServerResponse(http.HTTPStatus.METHOD_NOT_ALLOWED)
+
+
+#
+
+
+@_register_data_server_target_handler(UrlDataServerTarget)
+class UrlDataServerTargetHandler(DataServerTargetHandler[UrlDataServerTarget]):
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        if req.method not in check.not_none(self._target.methods):
+            return DataServerResponse(http.HTTPStatus.METHOD_NOT_ALLOWED)
+
+        resp: http.client.HTTPResponse = urllib.request.urlopen(urllib.request.Request(  # noqa
+            method=req.method,
+            url=check.not_none(self._target.url),
+        ))
+
+        try:
+            return DataServerResponse(
+                resp.status,
+                headers=dict(resp.headers.items()),
+                body=resp,
+            )
+
+        except Exception:  # noqa
+            resp.close()
+            raise

omdev/dataserver/http.py

@@ -0,0 +1,69 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - asyncio
+ - chunked transfer - both output and urllib input
+"""
+import typing as ta
+
+from omlish.http.handlers import HttpHandler_
+from omlish.http.handlers import HttpHandlerRequest
+from omlish.http.handlers import HttpHandlerResponse
+from omlish.http.handlers import HttpHandlerResponseStreamedData
+
+from .handlers import DataServerRequest
+from .server import DataServer
+
+
+##
+
+
+class DataServerHttpHandler(HttpHandler_):
+    DEFAULT_READ_CHUNK_SIZE = 0x10000
+
+    def __init__(
+            self,
+            ps: DataServer,
+            *,
+            read_chunk_size: int = DEFAULT_READ_CHUNK_SIZE,
+    ) -> None:
+        super().__init__()
+
+        self._ps = ps
+        self._read_chunk_size = read_chunk_size
+
+    def __call__(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
+        p_req = DataServerRequest(
+            req.method,
+            req.path,
+        )
+
+        p_resp = self._ps.handle(p_req)
+        try:
+            data: ta.Any
+            if (p_body := p_resp.body) is not None:
+                def stream_data():
+                    try:
+                        while (b := p_body.read(self._read_chunk_size)):
+                            yield b
+                    finally:
+                        p_body.close()
+
+                data = HttpHandlerResponseStreamedData(stream_data())
+
+            else:
+                data = None
+
+            resp = HttpHandlerResponse(
+                status=p_resp.status,
+                headers=p_resp.headers,
+                data=data,
+                close_connection=True,
+            )
+
+            return resp
+
+        except Exception:  # noqa
+            p_resp.close()
+
+            raise