omdev 0.0.0.dev221__py3-none-any.whl → 0.0.0.dev223__py3-none-any.whl

omdev/ci/docker/imagepulling.py

@@ -0,0 +1,64 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import typing as ta
+
+from ..cache import FileCache
+from ..utils import log_timing_context
+from .cache import DockerCache
+from .cmds import is_docker_image_present
+from .cmds import pull_docker_image
+
+
+##
+
+
+class DockerImagePulling(abc.ABC):
+    @abc.abstractmethod
+    def pull_docker_image(self, image: str) -> ta.Awaitable[None]:
+        raise NotImplementedError
+
+
+class DockerImagePullingImpl(DockerImagePulling):
+    @dc.dataclass(frozen=True)
+    class Config:
+        always_pull: bool = False
+
+    def __init__(
+            self,
+            *,
+            config: Config = Config(),
+
+            file_cache: ta.Optional[FileCache] = None,
+            docker_cache: ta.Optional[DockerCache] = None,
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self._file_cache = file_cache
+        self._docker_cache = docker_cache
+
+    async def _pull_docker_image(self, image: str) -> None:
+        if not self._config.always_pull and (await is_docker_image_present(image)):
+            return
+
+        dep_suffix = image
+        for c in '/:.-_':
+            dep_suffix = dep_suffix.replace(c, '-')
+
+        cache_key = f'docker-{dep_suffix}'
+        if (
+                self._docker_cache is not None and
+                (await self._docker_cache.load_cache_docker_image(cache_key)) is not None
+        ):
+            return
+
+        await pull_docker_image(image)
+
+        if self._docker_cache is not None:
+            await self._docker_cache.save_cache_docker_image(cache_key, image)
+
+    async def pull_docker_image(self, image: str) -> None:
+        with log_timing_context(f'Load docker image: {image}'):
+            await self._pull_docker_image(image)

omdev/ci/docker/inject.py

@@ -0,0 +1,37 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from .buildcaching import DockerBuildCaching
+from .buildcaching import DockerBuildCachingImpl
+from .cache import DockerCache
+from .cache import DockerCacheImpl
+from .imagepulling import DockerImagePulling
+from .imagepulling import DockerImagePullingImpl
+
+
+##
+
+
+def bind_docker(
+    *,
+    build_caching_config: DockerBuildCachingImpl.Config,
+    image_pulling_config: DockerImagePullingImpl.Config = DockerImagePullingImpl.Config(),
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [
+        inj.bind(build_caching_config),
+        inj.bind(DockerBuildCachingImpl, singleton=True),
+        inj.bind(DockerBuildCaching, to_key=DockerBuildCachingImpl),
+
+        inj.bind(DockerCacheImpl, singleton=True),
+        inj.bind(DockerCache, to_key=DockerCacheImpl),
+
+        inj.bind(image_pulling_config),
+        inj.bind(DockerImagePullingImpl, singleton=True),
+        inj.bind(DockerImagePulling, to_key=DockerImagePullingImpl),
+    ]
+
+    return inj.as_bindings(*lst)

omdev/ci/docker/utils.py

@@ -0,0 +1,48 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - some less stupid Dockerfile hash
+  - doesn't change too much though
+"""
+import contextlib
+import json
+import tarfile
+
+from omlish.lite.check import check
+
+from ..utils import sha256_str
+
+
+##
+
+
+def build_docker_file_hash(docker_file: str) -> str:
+    with open(docker_file) as f:
+        contents = f.read()
+
+    return sha256_str(contents)
+
+
+##
+
+
+def read_docker_tar_image_tag(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
+            m = mf.read()
+
+    manifests = json.loads(m.decode('utf-8'))
+    manifest = check.single(manifests)
+    tag = check.non_empty_str(check.single(manifest['RepoTags']))
+    return tag
+
+
+def read_docker_tar_image_id(tar_file: str) -> str:
+    with tarfile.open(tar_file) as tf:
+        with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
+            i = mf.read()
+
+    index = json.loads(i.decode('utf-8'))
+    manifest = check.single(index['manifests'])
+    image_id = check.non_empty_str(manifest['digest'])
+    return image_id

omdev/ci/github/cache.py

@@ -1,10 +1,12 @@
 # ruff: noqa: UP006 UP007
+import dataclasses as dc
 import os.path
 import typing as ta
 
 from omlish.lite.check import check
 from omlish.os.files import unlinking_if_exists
 
+from ..cache import CacheVersion
 from ..cache import DirectoryFileCache
 from ..cache import FileCache
 from .client import GithubCacheClient
@@ -15,16 +17,22 @@ from .client import GithubCacheServiceV1Client
 
 
 class GithubFileCache(FileCache):
+    @dc.dataclass(frozen=True)
+    class Config:
+        dir: str
+
     def __init__(
             self,
-            dir: str,  # noqa
+            config: Config,
             *,
             client: ta.Optional[GithubCacheClient] = None,
-            **kwargs: ta.Any,
+            version: ta.Optional[CacheVersion] = None,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__(
+            version=version,
+        )
 
-        self._dir = check.not_none(dir)
+        self._config = config
 
         if client is None:
             client = GithubCacheServiceV1Client(
@@ -33,7 +41,9 @@ class GithubFileCache(FileCache):
         self._client: GithubCacheClient = client
 
         self._local = DirectoryFileCache(
-            self._dir,
+            DirectoryFileCache.Config(
+                dir=check.non_empty_str(config.dir),
+            ),
             version=self._version,
         )
 

omdev/ci/github/inject.py

@@ -0,0 +1,30 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from ..cache import FileCache
+from .cache import GithubFileCache
+
+
+##
+
+
+def bind_github(
+        *,
+        cache_dir: ta.Optional[str] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = []
+
+    if cache_dir is not None:
+        lst.extend([
+            inj.bind(GithubFileCache.Config(
+                dir=cache_dir,
+            )),
+            inj.bind(GithubFileCache, singleton=True),
+            inj.bind(FileCache, to_key=GithubFileCache),
+        ])
+
+    return inj.as_bindings(*lst)

omdev/ci/inject.py

@@ -0,0 +1,61 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+from .cache import DirectoryFileCache
+from .cache import FileCache
+from .ci import Ci
+from .docker.buildcaching import DockerBuildCachingImpl
+from .docker.imagepulling import DockerImagePullingImpl
+from .docker.inject import bind_docker
+from .github.inject import bind_github
+
+
+##
+
+
+def bind_ci(
+        *,
+        config: Ci.Config,
+
+        github: bool = False,
+
+        cache_dir: ta.Optional[str] = None,
+) -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = [  # noqa
+        inj.bind(config),
+        inj.bind(Ci, singleton=True),
+    ]
+
+    lst.append(bind_docker(
+        build_caching_config=DockerBuildCachingImpl.Config(
+            service=config.service,
+
+            always_build=config.always_build,
+        ),
+
+        image_pulling_config=DockerImagePullingImpl.Config(
+            always_pull=config.always_pull,
+        ),
+    ))
+
+    if cache_dir is not None:
+        if github:
+            lst.append(bind_github(
+                cache_dir=cache_dir,
+            ))
+
+        else:
+            lst.extend([
+                inj.bind(DirectoryFileCache.Config(
+                    dir=cache_dir,
+                )),
+                inj.bind(DirectoryFileCache, singleton=True),
+                inj.bind(FileCache, to_key=DirectoryFileCache),
+
+            ])
+
+    return inj.as_bindings(*lst)

omdev/dataserver/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite

omdev/dataserver/handlers.py

@@ -0,0 +1,198 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import http.client
+import io
+import os
+import typing as ta
+import urllib.request
+
+from omlish.lite.check import check
+
+from .targets import BytesDataServerTarget
+from .targets import DataServerTarget
+from .targets import FileDataServerTarget
+from .targets import UrlDataServerTarget
+
+
+DataServerTargetT = ta.TypeVar('DataServerTargetT', bound='DataServerTarget')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DataServerRequest:
+    method: str
+    path: str
+
+
+@dc.dataclass(frozen=True)
+class DataServerResponse:
+    status: int
+    headers: ta.Optional[ta.Mapping[str, str]] = None
+    body: ta.Optional[io.IOBase] = None
+
+    #
+
+    def close(self) -> None:
+        if (body := self.body) is not None:
+            body.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
+class DataServerError(Exception):
+    pass
+
+
+class DataServerHandler(abc.ABC):
+    @abc.abstractmethod
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        raise NotImplementedError
+
+
+##
+
+
+class DataServerTargetHandler(DataServerHandler, abc.ABC, ta.Generic[DataServerTargetT]):
+    def __init__(self, target: DataServerTargetT) -> None:
+        super().__init__()
+
+        self._target = target
+
+    #
+
+    @classmethod
+    def for_target(cls, tgt: DataServerTarget, **kwargs: ta.Any) -> 'DataServerTargetHandler':
+        try:
+            hc = _DATA_SERVER_TARGET_HANDLERS[type(tgt)]
+        except KeyError:
+            raise TypeError(tgt)  # noqa
+        else:
+            return hc(tgt, **kwargs)
+
+    #
+
+    def _make_headers(self) -> ta.Dict[str, str]:
+        dct = {}
+        if (ct := self._target.content_type) is not None:
+            dct['Content-Type'] = ct
+        if (cl := self._target.content_length) is not None:
+            dct['Content-Length'] = str(cl)
+        return dct
+
+
+#
+
+
+_DATA_SERVER_TARGET_HANDLERS: ta.Dict[ta.Type[DataServerTarget], ta.Type[DataServerTargetHandler]] = {}
+
+
+def _register_data_server_target_handler(*tcs):
+    def inner(hc):
+        check.issubclass(hc, DataServerTargetHandler)
+        for tc in tcs:
+            check.issubclass(tc, DataServerTarget)
+            check.not_in(tc, _DATA_SERVER_TARGET_HANDLERS)
+            _DATA_SERVER_TARGET_HANDLERS[tc] = hc
+        return hc
+    return inner
+
+
+#
+
+
+@_register_data_server_target_handler(BytesDataServerTarget)
+class BytesDataServerTargetHandler(DataServerTargetHandler[BytesDataServerTarget]):
+    def _make_headers(self) -> ta.Dict[str, str]:
+        dct = super()._make_headers()
+        if 'Content-Length' not in dct and self._target.data is not None:
+            dct['Content-Length'] = str(len(self._target.data))
+        return dct
+
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        if req.method not in ('GET', 'HEAD'):
+            return DataServerResponse(http.HTTPStatus.METHOD_NOT_ALLOWED)
+
+        return DataServerResponse(
+            http.HTTPStatus.OK,
+            headers=self._make_headers(),
+            body=io.BytesIO(self._target.data) if self._target.data is not None and req.method == 'GET' else None,
+        )
+
+
+#
+
+
+@_register_data_server_target_handler(FileDataServerTarget)
+class FileDataServerTargetHandler(DataServerTargetHandler[FileDataServerTarget]):
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        if req.method == 'HEAD':
+            try:
+                st = os.stat(check.not_none(self._target.file_path))
+            except FileNotFoundError:
+                return DataServerResponse(http.HTTPStatus.NOT_FOUND)
+
+            return DataServerResponse(
+                http.HTTPStatus.OK,
+                headers={
+                    'Content-Length': str(st.st_size),
+                    **self._make_headers(),
+                },
+            )
+
+        elif req.method == 'GET':
+            try:
+                f = open(check.not_none(self._target.file_path), 'rb')  # noqa
+            except FileNotFoundError:
+                return DataServerResponse(http.HTTPStatus.NOT_FOUND)
+
+            try:
+                sz = os.fstat(f.fileno())
+
+                return DataServerResponse(
+                    http.HTTPStatus.OK,
+                    headers={
+                        'Content-Length': str(sz.st_size),
+                        **self._make_headers(),
+                    },
+                    body=f,  # noqa
+                )
+
+            except Exception:  # noqa
+                f.close()
+                raise
+
+        else:
+            return DataServerResponse(http.HTTPStatus.METHOD_NOT_ALLOWED)
+
+
+#
+
+
+@_register_data_server_target_handler(UrlDataServerTarget)
+class UrlDataServerTargetHandler(DataServerTargetHandler[UrlDataServerTarget]):
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        if req.method not in check.not_none(self._target.methods):
+            return DataServerResponse(http.HTTPStatus.METHOD_NOT_ALLOWED)
+
+        resp: http.client.HTTPResponse = urllib.request.urlopen(urllib.request.Request(  # noqa
+            method=req.method,
+            url=check.not_none(self._target.url),
+        ))
+
+        try:
+            return DataServerResponse(
+                resp.status,
+                headers=dict(resp.headers.items()),
+                body=resp,
+            )
+
+        except Exception:  # noqa
+            resp.close()
+            raise

omdev/dataserver/http.py

@@ -0,0 +1,69 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - asyncio
+ - chunked transfer - both output and urllib input
+"""
+import typing as ta
+
+from omlish.http.handlers import HttpHandler_
+from omlish.http.handlers import HttpHandlerRequest
+from omlish.http.handlers import HttpHandlerResponse
+from omlish.http.handlers import HttpHandlerResponseStreamedData
+
+from .handlers import DataServerRequest
+from .server import DataServer
+
+
+##
+
+
+class DataServerHttpHandler(HttpHandler_):
+    DEFAULT_READ_CHUNK_SIZE = 0x10000
+
+    def __init__(
+            self,
+            ps: DataServer,
+            *,
+            read_chunk_size: int = DEFAULT_READ_CHUNK_SIZE,
+    ) -> None:
+        super().__init__()
+
+        self._ps = ps
+        self._read_chunk_size = read_chunk_size
+
+    def __call__(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
+        p_req = DataServerRequest(
+            req.method,
+            req.path,
+        )
+
+        p_resp = self._ps.handle(p_req)
+        try:
+            data: ta.Any
+            if (p_body := p_resp.body) is not None:
+                def stream_data():
+                    try:
+                        while (b := p_body.read(self._read_chunk_size)):
+                            yield b
+                    finally:
+                        p_body.close()
+
+                data = HttpHandlerResponseStreamedData(stream_data())
+
+            else:
+                data = None
+
+            resp = HttpHandlerResponse(
+                status=p_resp.status,
+                headers=p_resp.headers,
+                data=data,
+                close_connection=True,
+            )
+
+            return resp
+
+        except Exception:  # noqa
+            p_resp.close()
+
+            raise

omdev/dataserver/routes.py

@@ -0,0 +1,49 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - generate to nginx config
+"""
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+
+from .targets import DataServerTarget
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DataServerRoute:
+    paths: ta.Sequence[str]
+    target: DataServerTarget
+
+    @classmethod
+    def of(cls, obj: ta.Union[
+        'DataServerRoute',
+        ta.Tuple[
+            ta.Union[str, ta.Iterable[str]],
+            DataServerTarget,
+        ],
+    ]) -> 'DataServerRoute':
+        if isinstance(obj, cls):
+            return obj
+
+        elif isinstance(obj, tuple):
+            p, t = obj
+
+            if isinstance(p, str):
+                p = [p]
+
+            return cls(
+                paths=tuple(p),
+                target=check.isinstance(t, DataServerTarget),
+            )
+
+        else:
+            raise TypeError(obj)
+
+    @classmethod
+    def of_(cls, *objs: ta.Any) -> ta.List['DataServerRoute']:
+        return [cls.of(obj) for obj in objs]

omdev/dataserver/server.py

@@ -0,0 +1,90 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import http.client
+import typing as ta
+
+from omlish.lite.check import check
+
+from .handlers import DataServerHandler
+from .handlers import DataServerRequest
+from .handlers import DataServerResponse
+from .handlers import DataServerTargetHandler
+from .routes import DataServerRoute
+
+
+##
+
+
+class DataServer:
+    @dc.dataclass(frozen=True)
+    class HandlerRoute:
+        paths: ta.Sequence[str]
+        handler: DataServerHandler
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.paths, str)
+            for p in self.paths:
+                check.non_empty_str(p)
+            check.isinstance(self.handler, DataServerHandler)
+
+        @classmethod
+        def of(cls, obj: ta.Union[
+            'DataServer.HandlerRoute',
+            DataServerRoute,
+        ]) -> 'DataServer.HandlerRoute':
+            if isinstance(obj, cls):
+                return obj
+
+            elif isinstance(obj, DataServerRoute):
+                return cls(
+                    paths=obj.paths,
+                    handler=DataServerTargetHandler.for_target(obj.target),
+                )
+
+            else:
+                raise TypeError(obj)
+
+        @classmethod
+        def of_(cls, *objs: ta.Any) -> ta.List['DataServer.HandlerRoute']:
+            return [cls.of(obj) for obj in objs]
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Config:
+        pass
+
+    def __init__(
+            self,
+            routes: ta.Optional[ta.Iterable[HandlerRoute]] = None,
+            config: Config = Config(),
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self.set_routes(routes)
+
+    #
+
+    _routes_by_path: ta.Dict[str, HandlerRoute]
+
+    def set_routes(self, routes: ta.Optional[ta.Iterable[HandlerRoute]]) -> None:
+        routes_by_path: ta.Dict[str, DataServer.HandlerRoute] = {}
+
+        for r in routes or []:
+            for p in r.paths:
+                check.not_in(p, routes_by_path)
+                routes_by_path[p] = r
+
+        self._routes_by_path = routes_by_path
+
+    #
+
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        try:
+            rt = self._routes_by_path[req.path]
+        except KeyError:
+            return DataServerResponse(http.HTTPStatus.NOT_FOUND)
+
+        return rt.handler.handle(req)