omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/dataserver/routes.py

@@ -0,0 +1,49 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - generate to nginx config
+"""
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+
+from .targets import DataServerTarget
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DataServerRoute:
+    paths: ta.Sequence[str]
+    target: DataServerTarget
+
+    @classmethod
+    def of(cls, obj: ta.Union[
+        'DataServerRoute',
+        ta.Tuple[
+            ta.Union[str, ta.Iterable[str]],
+            DataServerTarget,
+        ],
+    ]) -> 'DataServerRoute':
+        if isinstance(obj, cls):
+            return obj
+
+        elif isinstance(obj, tuple):
+            p, t = obj
+
+            if isinstance(p, str):
+                p = [p]
+
+            return cls(
+                paths=tuple(p),
+                target=check.isinstance(t, DataServerTarget),
+            )
+
+        else:
+            raise TypeError(obj)
+
+    @classmethod
+    def of_(cls, *objs: ta.Any) -> ta.List['DataServerRoute']:
+        return [cls.of(obj) for obj in objs]

omdev/dataserver/server.py

@@ -0,0 +1,90 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import http.client
+import typing as ta
+
+from omlish.lite.check import check
+
+from .handlers import DataServerHandler
+from .handlers import DataServerRequest
+from .handlers import DataServerResponse
+from .handlers import DataServerTargetHandler
+from .routes import DataServerRoute
+
+
+##
+
+
+class DataServer:
+    @dc.dataclass(frozen=True)
+    class HandlerRoute:
+        paths: ta.Sequence[str]
+        handler: DataServerHandler
+
+        def __post_init__(self) -> None:
+            check.not_isinstance(self.paths, str)
+            for p in self.paths:
+                check.non_empty_str(p)
+            check.isinstance(self.handler, DataServerHandler)
+
+        @classmethod
+        def of(cls, obj: ta.Union[
+            'DataServer.HandlerRoute',
+            DataServerRoute,
+        ]) -> 'DataServer.HandlerRoute':
+            if isinstance(obj, cls):
+                return obj
+
+            elif isinstance(obj, DataServerRoute):
+                return cls(
+                    paths=obj.paths,
+                    handler=DataServerTargetHandler.for_target(obj.target),
+                )
+
+            else:
+                raise TypeError(obj)
+
+        @classmethod
+        def of_(cls, *objs: ta.Any) -> ta.List['DataServer.HandlerRoute']:
+            return [cls.of(obj) for obj in objs]
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Config:
+        pass
+
+    def __init__(
+            self,
+            routes: ta.Optional[ta.Iterable[HandlerRoute]] = None,
+            config: Config = Config(),
+    ) -> None:
+        super().__init__()
+
+        self._config = config
+
+        self.set_routes(routes)
+
+    #
+
+    _routes_by_path: ta.Dict[str, HandlerRoute]
+
+    def set_routes(self, routes: ta.Optional[ta.Iterable[HandlerRoute]]) -> None:
+        routes_by_path: ta.Dict[str, DataServer.HandlerRoute] = {}
+
+        for r in routes or []:
+            for p in r.paths:
+                check.not_in(p, routes_by_path)
+                routes_by_path[p] = r
+
+        self._routes_by_path = routes_by_path
+
+    #
+
+    def handle(self, req: DataServerRequest) -> DataServerResponse:
+        try:
+            rt = self._routes_by_path[req.path]
+        except KeyError:
+            return DataServerResponse(http.HTTPStatus.NOT_FOUND)
+
+        return rt.handler.handle(req)

omdev/dataserver/targets.py

@@ -0,0 +1,121 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.dataclasses import dataclass_maybe_post_init
+from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DataServerTarget(abc.ABC):  # noqa
+    content_type: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    content_length: ta.Optional[int] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    #
+
+    @classmethod
+    def of(
+            cls,
+            obj: ta.Union[
+                'DataServerTarget',
+                bytes,
+                None,
+            ] = None,
+            *,
+
+            file_path: ta.Optional[str] = None,
+            url: ta.Optional[str] = None,
+
+            **kwargs: ta.Any,
+    ) -> 'DataServerTarget':
+        if isinstance(obj, DataServerTarget):
+            check.none(file_path)
+            check.none(url)
+            check.empty(kwargs)
+            return obj
+
+        elif isinstance(obj, bytes):
+            return BytesDataServerTarget(
+                data=obj,
+                **kwargs,
+            )
+
+        elif file_path is not None:
+            check.none(obj)
+            check.none(url)
+            return FileDataServerTarget(
+                file_path=file_path,
+                **kwargs,
+            )
+
+        elif url is not None:
+            check.none(obj)
+            check.none(file_path)
+            return UrlDataServerTarget(
+                url=url,
+                **kwargs,
+            )
+
+        else:
+            raise TypeError('No target type provided')
+
+    #
+
+    @classmethod
+    def of_bytes(cls, data: bytes) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data,
+            content_type='application/octet-stream',
+        )
+
+    @classmethod
+    def of_text(cls, data: str) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data.encode('utf-8'),
+            content_type='text/plain; charset=utf-8',
+        )
+
+    @classmethod
+    def of_json(cls, data: str) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data.encode('utf-8'),
+            content_type='application/json; charset=utf-8',
+        )
+
+    @classmethod
+    def of_html(cls, data: str) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data.encode('utf-8'),
+            content_type='text/html; charset=utf-8',
+        )
+
+
+@dc.dataclass(frozen=True)
+class BytesDataServerTarget(DataServerTarget):
+    data: ta.Optional[bytes] = None  # required
+
+
+@dc.dataclass(frozen=True)
+class FileDataServerTarget(DataServerTarget):
+    file_path: ta.Optional[str] = None  # required
+
+    def __post_init__(self) -> None:
+        dataclass_maybe_post_init(super())
+        check.non_empty_str(self.file_path)
+
+
+@dc.dataclass(frozen=True)
+class UrlDataServerTarget(DataServerTarget):
+    url: ta.Optional[str] = None  # required
+    methods: ta.Optional[ta.Sequence[str]] = None  # required
+
+    def __post_init__(self) -> None:
+        dataclass_maybe_post_init(super())
+        check.non_empty_str(self.url)
+        check.not_none(self.methods)
+        check.not_isinstance(self.methods, str)

omdev/oci/building.py

@@ -1,6 +1,7 @@
 # ruff: noqa: UP006 UP007
 # @omlish-lite
 import dataclasses as dc
+import json
 import typing as ta
 
 from omlish.lite.check import check
@@ -15,36 +16,90 @@ from .data import OciImageManifest
 from .datarefs import BytesOciDataRef
 from .datarefs import OciDataRef
 from .datarefs import OciDataRefInfo
+from .datarefs import open_oci_data_ref
 from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES
 from .media import OciMediaDataclass
 from .media import OciMediaDescriptor
 from .media import OciMediaImageConfig
 from .media import OciMediaImageIndex
 from .media import OciMediaImageManifest
+from .media import unmarshal_oci_media_dataclass
+
+
+OciMediaDataclassT = ta.TypeVar('OciMediaDataclassT', bound='OciMediaDataclass')
 
 
 ##
 
 
 class OciRepositoryBuilder:
+    @dc.dataclass(frozen=True)
+    class Blob:
+        digest: str
+
+        data: OciDataRef
+        info: OciDataRefInfo
+
+        media_type: ta.Optional[str] = None
+
+        #
+
+        def read(self) -> bytes:
+            with open_oci_data_ref(self.data) as f:
+                return f.read()
+
+        def read_json(self) -> ta.Any:
+            return json.loads(self.read().decode('utf-8'))
+
+        def read_media(
+                self,
+                cls: ta.Type[OciMediaDataclassT] = OciMediaDataclass,  # type: ignore[assignment]
+        ) -> OciMediaDataclassT:
+            mt = check.non_empty_str(self.media_type)
+            dct = self.read_json()
+            obj = unmarshal_oci_media_dataclass(
+                dct,
+                media_type=mt,
+            )
+            return check.isinstance(obj, cls)
+
     def __init__(self) -> None:
         super().__init__()
 
-        self._blobs: ta.Dict[str, OciDataRef] = {}
+        self._blobs: ta.Dict[str, OciRepositoryBuilder.Blob] = {}
 
-    def get_blobs(self) -> ta.Dict[str, OciDataRef]:
+    #
+
+    def get_blobs(self) -> ta.Dict[str, Blob]:
         return dict(self._blobs)
 
     def add_blob(
             self,
             r: OciDataRef,
             ri: ta.Optional[OciDataRefInfo] = None,
-    ) -> None:
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> Blob:
         if ri is None:
             ri = OciDataRefInfo(r)
-        if ri.digest() in self._blobs:
+
+        if (dg := ri.digest()) in self._blobs:
             raise KeyError(ri.digest())
-        self._blobs[ri.digest()] = r
+
+        blob = self.Blob(
+            digest=dg,
+
+            data=r,
+            info=ri,
+
+            media_type=media_type,
+        )
+
+        self._blobs[dg] = blob
+
+        return blob
+
+    #
 
     def marshal_media(self, obj: OciMediaDataclass) -> bytes:
         check.isinstance(obj, OciMediaDataclass)
@@ -58,14 +113,20 @@ class OciRepositoryBuilder:
 
         r = BytesOciDataRef(b)
         ri = OciDataRefInfo(r)
-        self.add_blob(r, ri)
+        self.add_blob(
+            r,
+            ri,
+            media_type=obj.media_type,
+        )
 
         return OciMediaDescriptor(
-            media_type=getattr(obj, 'media_type'),
+            media_type=obj.media_type,
             digest=ri.digest(),
             size=ri.size(),
         )
 
+    #
+
     def to_media(self, obj: OciDataclass) -> ta.Union[OciMediaDataclass, OciMediaDescriptor]:
         def make_kw(*exclude):
             return {
@@ -97,9 +158,14 @@ class OciRepositoryBuilder:
 
         elif isinstance(obj, OciImageLayer):
             ri = OciDataRefInfo(obj.data)
-            self.add_blob(obj.data, ri)
+            mt = OCI_IMAGE_LAYER_KIND_MEDIA_TYPES[obj.kind]
+            self.add_blob(
+                obj.data,
+                ri,
+                media_type=mt,
+            )
             return OciMediaDescriptor(
-                media_type=OCI_IMAGE_LAYER_KIND_MEDIA_TYPES[obj.kind],
+                media_type=mt,
                 digest=ri.digest(),
                 size=ri.size(),
             )
@@ -121,3 +187,35 @@ class OciRepositoryBuilder:
 
         else:
             raise TypeError(ret)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class BuiltOciImageIndexRepository:
+    index: OciImageIndex
+
+    media_index_descriptor: OciMediaDescriptor
+    media_index: OciMediaImageIndex
+
+    blobs: ta.Mapping[str, OciRepositoryBuilder.Blob]
+
+
+def build_oci_index_repository(index: OciImageIndex) -> BuiltOciImageIndexRepository:
+    builder = OciRepositoryBuilder()
+
+    media_index_descriptor = builder.add_data(index)
+
+    blobs = builder.get_blobs()
+
+    media_index = blobs[media_index_descriptor.digest].read_media(OciMediaImageIndex)
+
+    return BuiltOciImageIndexRepository(
+        index=index,
+
+        media_index_descriptor=media_index_descriptor,
+        media_index=media_index,
+
+        blobs=blobs,
+    )

omdev/oci/compression.py

@@ -0,0 +1,8 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import enum
+
+
+class OciCompression(enum.Enum):
+    GZIP = enum.auto()
+    ZSTD = enum.auto()

omdev/oci/data.py

@@ -5,9 +5,11 @@ import dataclasses as dc
 import enum
 import typing as ta
 
+from omlish.lite.check import check
 from omlish.lite.marshal import OBJ_MARSHALER_FIELD_KEY
 from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
 
+from .compression import OciCompression
 from .datarefs import OciDataRef
 
 
@@ -40,6 +42,7 @@ class OciImageManifest(OciDataclass):
 
     annotations: ta.Optional[ta.Dict[str, str]] = None
 
+
 #
 
 
@@ -50,6 +53,28 @@ class OciImageLayer(OciDataclass):
         TAR_GZIP = enum.auto()
         TAR_ZSTD = enum.auto()
 
+        @property
+        def compression(self) -> ta.Optional[OciCompression]:
+            if self is self.TAR:
+                return None
+            elif self is self.TAR_GZIP:
+                return OciCompression.GZIP
+            elif self is self.TAR_ZSTD:
+                return OciCompression.ZSTD
+            else:
+                raise ValueError(self)
+
+        @classmethod
+        def from_compression(cls, compression: ta.Optional[OciCompression]) -> 'OciImageLayer.Kind':
+            if compression is None:
+                return cls.TAR
+            elif compression == OciCompression.GZIP:
+                return cls.TAR_GZIP
+            elif compression == OciCompression.ZSTD:
+                return cls.TAR_ZSTD
+            else:
+                raise ValueError(compression)
+
     kind: Kind
 
     data: OciDataRef
@@ -125,3 +150,21 @@ def is_empty_oci_dataclass(obj: OciDataclass) -> bool:
 
     else:
         return False
+
+
+##
+
+
+def get_single_leaf_oci_image_index(image_index: OciImageIndex) -> OciImageIndex:
+    while True:
+        child_manifest = check.single(image_index.manifests)
+        if isinstance(child_manifest, OciImageManifest):
+            break
+        image_index = check.isinstance(child_manifest, OciImageIndex)
+
+    return image_index
+
+
+def get_single_oci_image_manifest(image_index: OciImageIndex) -> OciImageManifest:
+    child_index = check.single(image_index.manifests)
+    return check.isinstance(child_index, OciImageManifest)

omdev/oci/datarefs.py

@@ -2,13 +2,16 @@
 # @omlish-lite
 import abc
 import dataclasses as dc
+import functools
 import hashlib
 import io
 import os.path
 import shutil
+import tarfile
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
 
 
 ##
@@ -29,70 +32,107 @@ class FileOciDataRef(OciDataRef):
     path: str
 
 
+@dc.dataclass(frozen=True)
+class TarFileOciDataRef(OciDataRef):
+    tar_file: tarfile.TarFile
+    tar_info: tarfile.TarInfo
+
+
 ##
 
 
-@dc.dataclass(frozen=True)
-class OciDataRefInfo:
-    data: OciDataRef
+@functools.singledispatch
+def write_oci_data_ref_to_file(
+        src_data: OciDataRef,
+        dst_file: str,
+        *,
+        symlink: bool = False,  # noqa
+        chunk_size: int = 1024 * 1024,
+) -> None:
+    with open_oci_data_ref(src_data) as f_src:
+        with open(dst_file, 'wb') as f_dst:
+            shutil.copyfileobj(f_src, f_dst, length=chunk_size)  # noqa
 
-    @cached_nullary
-    def sha256(self) -> str:
-        if isinstance(self.data, FileOciDataRef):
-            with open(self.data.path, 'rb') as f:
-                return hashlib.file_digest(f, 'sha256').hexdigest()  # noqa
 
-        elif isinstance(self.data, BytesOciDataRef):
-            return hashlib.sha256(self.data.data).hexdigest()
+@write_oci_data_ref_to_file.register
+def _(
+        src_data: FileOciDataRef,
+        dst_file: str,
+        *,
+        symlink: bool = False,
+        **kwargs: ta.Any,
+) -> None:
+    if symlink:
+        os.symlink(
+            os.path.relpath(src_data.path, os.path.dirname(dst_file)),
+            dst_file,
+        )
+    else:
+        shutil.copyfile(src_data.path, dst_file)
 
-        else:
-            raise TypeError(self.data)
 
-    @cached_nullary
-    def digest(self) -> str:
-        return f'sha256:{self.sha256()}'
+#
 
-    @cached_nullary
-    def size(self) -> int:
-        if isinstance(self.data, FileOciDataRef):
-            return os.path.getsize(self.data.path)
 
-        elif isinstance(self.data, BytesOciDataRef):
-            return len(self.data.data)
+@functools.singledispatch
+def open_oci_data_ref(data: OciDataRef) -> ta.BinaryIO:
+    raise TypeError(data)
 
-        else:
-            raise TypeError(self.data)
 
+@open_oci_data_ref.register
+def _(data: FileOciDataRef) -> ta.BinaryIO:
+    return open(data.path, 'rb')
 
-def write_oci_data_ref_to_file(
-        data: OciDataRef,
-        dst: str,
-        *,
-        symlink: bool = False,
-) -> None:
-    if isinstance(data, FileOciDataRef):
-        if symlink:
-            os.symlink(
-                os.path.relpath(data.path, os.path.dirname(dst)),
-                dst,
-            )
-        else:
-            shutil.copyfile(data.path, dst)
-
-    elif isinstance(data, BytesOciDataRef):
-        with open(dst, 'wb') as f:
-            f.write(data.data)
 
-    else:
-        raise TypeError(data)
+@open_oci_data_ref.register
+def _(data: BytesOciDataRef) -> ta.BinaryIO:
+    return io.BytesIO(data.data)
 
 
-def open_oci_data_ref(data: OciDataRef) -> ta.BinaryIO:
-    if isinstance(data, FileOciDataRef):
-        return open(data.path, 'rb')
+@open_oci_data_ref.register
+def _(data: TarFileOciDataRef) -> ta.BinaryIO:
+    return check.not_none(data.tar_file.extractfile(data.tar_info))  # type: ignore[return-value]
 
-    elif isinstance(data, BytesOciDataRef):
-        return io.BytesIO(data.data)
 
-    else:
-        raise TypeError(data)
+#
+
+
+@functools.singledispatch
+def get_oci_data_ref_size(data: OciDataRef) -> int:
+    raise TypeError(data)
+
+
+@get_oci_data_ref_size.register
+def _(data: FileOciDataRef) -> int:
+    return os.path.getsize(data.path)
+
+
+@get_oci_data_ref_size.register
+def _(data: BytesOciDataRef) -> int:
+    return len(data.data)
+
+
+@get_oci_data_ref_size.register
+def _(data: TarFileOciDataRef) -> int:
+    return data.tar_info.size
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRefInfo:
+    data: OciDataRef
+
+    @cached_nullary
+    def sha256(self) -> str:
+        with open_oci_data_ref(self.data) as f:
+            return hashlib.file_digest(f, 'sha256').hexdigest()  # type: ignore[arg-type]
+
+    @cached_nullary
+    def digest(self) -> str:
+        return f'sha256:{self.sha256()}'
+
+    @cached_nullary
+    def size(self) -> int:
+        return get_oci_data_ref_size(self.data)