omdev 0.0.0.dev221__py3-none-any.whl → 0.0.0.dev223__py3-none-any.whl

omdev/dataserver/targets.py

@@ -0,0 +1,89 @@
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.dataclasses import dataclass_maybe_post_init
+from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DataServerTarget(abc.ABC):  # noqa
+    content_type: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    content_length: ta.Optional[int] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    @classmethod
+    def of(
+            cls,
+            obj: ta.Union[
+                'DataServerTarget',
+                bytes,
+                None,
+            ] = None,
+            *,
+
+            file_path: ta.Optional[str] = None,
+            url: ta.Optional[str] = None,
+
+            **kwargs: ta.Any,
+    ) -> 'DataServerTarget':
+        if isinstance(obj, DataServerTarget):
+            check.none(file_path)
+            check.none(url)
+            check.empty(kwargs)
+            return obj
+
+        elif isinstance(obj, bytes):
+            return BytesDataServerTarget(
+                data=obj,
+                **kwargs,
+            )
+
+        elif file_path is not None:
+            check.none(obj)
+            check.none(url)
+            return FileDataServerTarget(
+                file_path=file_path,
+                **kwargs,
+            )
+
+        elif url is not None:
+            check.none(obj)
+            check.none(file_path)
+            return UrlDataServerTarget(
+                url=url,
+                **kwargs,
+            )
+
+        else:
+            raise TypeError('No target type provided')
+
+
+@dc.dataclass(frozen=True)
+class BytesDataServerTarget(DataServerTarget):
+    data: ta.Optional[bytes] = None  # required
+
+
+@dc.dataclass(frozen=True)
+class FileDataServerTarget(DataServerTarget):
+    file_path: ta.Optional[str] = None  # required
+
+    def __post_init__(self) -> None:
+        dataclass_maybe_post_init(super())
+        check.non_empty_str(self.file_path)
+
+
+@dc.dataclass(frozen=True)
+class UrlDataServerTarget(DataServerTarget):
+    url: ta.Optional[str] = None  # required
+    methods: ta.Optional[ta.Sequence[str]] = None  # required
+
+    def __post_init__(self) -> None:
+        dataclass_maybe_post_init(super())
+        check.non_empty_str(self.url)
+        check.not_none(self.methods)
+        check.not_isinstance(self.methods, str)

omdev/oci/building.py

@@ -0,0 +1,221 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_compact
+from omlish.lite.marshal import marshal_obj
+
+from .data import OciDataclass
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .datarefs import BytesOciDataRef
+from .datarefs import OciDataRef
+from .datarefs import OciDataRefInfo
+from .datarefs import open_oci_data_ref
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES
+from .media import OciMediaDataclass
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+from .media import unmarshal_oci_media_dataclass
+
+
+OciMediaDataclassT = ta.TypeVar('OciMediaDataclassT', bound='OciMediaDataclass')
+
+
+##
+
+
+class OciRepositoryBuilder:
+    @dc.dataclass(frozen=True)
+    class Blob:
+        digest: str
+
+        data: OciDataRef
+        info: OciDataRefInfo
+
+        media_type: ta.Optional[str] = None
+
+        #
+
+        def read(self) -> bytes:
+            with open_oci_data_ref(self.data) as f:
+                return f.read()
+
+        def read_json(self) -> ta.Any:
+            return json.loads(self.read().decode('utf-8'))
+
+        def read_media(
+                self,
+                cls: ta.Type[OciMediaDataclassT] = OciMediaDataclass,  # type: ignore[assignment]
+        ) -> OciMediaDataclassT:
+            mt = check.non_empty_str(self.media_type)
+            dct = self.read_json()
+            obj = unmarshal_oci_media_dataclass(
+                dct,
+                media_type=mt,
+            )
+            return check.isinstance(obj, cls)
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._blobs: ta.Dict[str, OciRepositoryBuilder.Blob] = {}
+
+    #
+
+    def get_blobs(self) -> ta.Dict[str, Blob]:
+        return dict(self._blobs)
+
+    def add_blob(
+            self,
+            r: OciDataRef,
+            ri: ta.Optional[OciDataRefInfo] = None,
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> Blob:
+        if ri is None:
+            ri = OciDataRefInfo(r)
+
+        if (dg := ri.digest()) in self._blobs:
+            raise KeyError(ri.digest())
+
+        blob = self.Blob(
+            digest=dg,
+
+            data=r,
+            info=ri,
+
+            media_type=media_type,
+        )
+
+        self._blobs[dg] = blob
+
+        return blob
+
+    #
+
+    def marshal_media(self, obj: OciMediaDataclass) -> bytes:
+        check.isinstance(obj, OciMediaDataclass)
+        m = marshal_obj(obj)
+        j = json_dumps_compact(m)
+        b = j.encode('utf-8')
+        return b
+
+    def add_media(self, obj: OciMediaDataclass) -> OciMediaDescriptor:
+        b = self.marshal_media(obj)
+
+        r = BytesOciDataRef(b)
+        ri = OciDataRefInfo(r)
+        self.add_blob(
+            r,
+            ri,
+            media_type=obj.media_type,
+        )
+
+        return OciMediaDescriptor(
+            media_type=obj.media_type,
+            digest=ri.digest(),
+            size=ri.size(),
+        )
+
+    #
+
+    def to_media(self, obj: OciDataclass) -> ta.Union[OciMediaDataclass, OciMediaDescriptor]:
+        def make_kw(*exclude):
+            return {
+                a: v
+                for f in dc.fields(obj)
+                if (a := f.name) not in exclude
+                for v in [getattr(obj, a)]
+                if v is not None
+            }
+
+        if isinstance(obj, OciImageIndex):
+            return OciMediaImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    self.add_data(m)
+                    for m in obj.manifests
+                ],
+            )
+
+        elif isinstance(obj, OciImageManifest):
+            return OciMediaImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.add_data(obj.config),
+                layers=[
+                    self.add_data(l)
+                    for l in obj.layers
+                ],
+            )
+
+        elif isinstance(obj, OciImageLayer):
+            ri = OciDataRefInfo(obj.data)
+            mt = OCI_IMAGE_LAYER_KIND_MEDIA_TYPES[obj.kind]
+            self.add_blob(
+                obj.data,
+                ri,
+                media_type=mt,
+            )
+            return OciMediaDescriptor(
+                media_type=mt,
+                digest=ri.digest(),
+                size=ri.size(),
+            )
+
+        elif isinstance(obj, OciImageConfig):
+            return OciMediaImageConfig(**make_kw())
+
+        else:
+            raise TypeError(obj)
+
+    def add_data(self, obj: OciDataclass) -> OciMediaDescriptor:
+        ret = self.to_media(obj)
+
+        if isinstance(ret, OciMediaDataclass):
+            return self.add_media(ret)
+
+        elif isinstance(ret, OciMediaDescriptor):
+            return ret
+
+        else:
+            raise TypeError(ret)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class BuiltOciImageIndexRepository:
+    index: OciImageIndex
+
+    media_index_descriptor: OciMediaDescriptor
+    media_index: OciMediaImageIndex
+
+    blobs: ta.Mapping[str, OciRepositoryBuilder.Blob]
+
+
+def build_oci_index_repository(index: OciImageIndex) -> BuiltOciImageIndexRepository:
+    builder = OciRepositoryBuilder()
+
+    media_index_descriptor = builder.add_data(index)
+
+    blobs = builder.get_blobs()
+
+    media_index = blobs[media_index_descriptor.digest].read_media(OciMediaImageIndex)
+
+    return BuiltOciImageIndexRepository(
+        index=index,
+
+        media_index_descriptor=media_index_descriptor,
+        media_index=media_index,
+
+        blobs=blobs,
+    )

omdev/oci/compression.py

@@ -0,0 +1,8 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import enum
+
+
+class OciCompression(enum.Enum):
+    GZIP = enum.auto()
+    ZSTD = enum.auto()

omdev/oci/data.py

@@ -0,0 +1,151 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import enum
+import typing as ta
+
+from omlish.lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+
+from .compression import OciCompression
+from .datarefs import OciDataRef
+
+
+##
+
+
+@dc.dataclass()
+class OciDataclass(abc.ABC):  # noqa
+    pass
+
+
+##
+
+
+@dc.dataclass()
+class OciImageIndex(OciDataclass):
+    manifests: ta.List[ta.Union['OciImageIndex', 'OciImageManifest']]
+
+    annotations: ta.Optional[ta.Dict[str, str]] = None
+
+
+#
+
+
+@dc.dataclass()
+class OciImageManifest(OciDataclass):
+    config: 'OciImageConfig'
+
+    layers: ta.List['OciImageLayer']
+
+    annotations: ta.Optional[ta.Dict[str, str]] = None
+
+
+#
+
+
+@dc.dataclass()
+class OciImageLayer(OciDataclass):
+    class Kind(enum.Enum):
+        TAR = enum.auto()
+        TAR_GZIP = enum.auto()
+        TAR_ZSTD = enum.auto()
+
+        @property
+        def compression(self) -> ta.Optional[OciCompression]:
+            if self is self.TAR:
+                return None
+            elif self is self.TAR_GZIP:
+                return OciCompression.GZIP
+            elif self is self.TAR_ZSTD:
+                return OciCompression.ZSTD
+            else:
+                raise ValueError(self)
+
+        @classmethod
+        def from_compression(cls, compression: ta.Optional[OciCompression]) -> 'OciImageLayer.Kind':
+            if compression is None:
+                return cls.TAR
+            elif compression == OciCompression.GZIP:
+                return cls.TAR_GZIP
+            elif compression == OciCompression.ZSTD:
+                return cls.TAR_ZSTD
+            else:
+                raise ValueError(compression)
+
+    kind: Kind
+
+    data: OciDataRef
+
+
+#
+
+
+@dc.dataclass()
+class OciImageConfig(OciDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/config.md"""
+
+    architecture: str
+    os: str
+
+    @dc.dataclass()
+    class RootFs:
+        type: str
+        diff_ids: ta.List[str]
+
+    rootfs: RootFs
+
+    #
+
+    created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    os_version: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.version', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    os_features: ta.Optional[ta.List[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    variant: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    """
+    config object, OPTIONAL
+        User string, OPTIONAL
+        ExposedPorts object, OPTIONAL
+        Env array of strings, OPTIONAL
+        Entrypoint array of strings, OPTIONAL
+        Cmd array of strings, OPTIONAL
+        Volumes object, OPTIONAL
+        WorkingDir string, OPTIONAL
+        Labels object, OPTIONAL
+        StopSignal string, OPTIONAL
+        ArgsEscaped boolean, OPTIONAL
+        Memory integer, OPTIONAL
+        MemorySwap integer, OPTIONAL
+        CpuShares integer, OPTIONAL
+        Healthcheck object, OPTIONAL
+    """
+    config: ta.Optional[ta.Dict[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    @dc.dataclass()
+    class History:
+        created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        created_by: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        comment: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        empty_layer: ta.Optional[bool] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    history: ta.Optional[ta.List[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+
+##
+
+
+def is_empty_oci_dataclass(obj: OciDataclass) -> bool:
+    if not isinstance(obj, OciDataclass):
+        raise TypeError(obj)
+
+    elif isinstance(obj, OciImageIndex):
+        return not obj.manifests
+
+    elif isinstance(obj, OciImageManifest):
+        return not obj.layers
+
+    else:
+        return False

omdev/oci/datarefs.py

@@ -0,0 +1,138 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import functools
+import hashlib
+import io
+import os.path
+import shutil
+import tarfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRef(abc.ABC):  # noqa
+    pass
+
+
+@dc.dataclass(frozen=True)
+class BytesOciDataRef(OciDataRef):
+    data: bytes
+
+
+@dc.dataclass(frozen=True)
+class FileOciDataRef(OciDataRef):
+    path: str
+
+
+@dc.dataclass(frozen=True)
+class TarFileOciDataRef(OciDataRef):
+    tar_file: tarfile.TarFile
+    tar_info: tarfile.TarInfo
+
+
+##
+
+
+@functools.singledispatch
+def write_oci_data_ref_to_file(
+        src_data: OciDataRef,
+        dst_file: str,
+        *,
+        symlink: bool = False,  # noqa
+        chunk_size: int = 1024 * 1024,
+) -> None:
+    with open_oci_data_ref(src_data) as f_src:
+        with open(dst_file, 'wb') as f_dst:
+            shutil.copyfileobj(f_src, f_dst, length=chunk_size)  # noqa
+
+
+@write_oci_data_ref_to_file.register
+def _(
+        src_data: FileOciDataRef,
+        dst_file: str,
+        *,
+        symlink: bool = False,
+        **kwargs: ta.Any,
+) -> None:
+    if symlink:
+        os.symlink(
+            os.path.relpath(src_data.path, os.path.dirname(dst_file)),
+            dst_file,
+        )
+    else:
+        shutil.copyfile(src_data.path, dst_file)
+
+
+#
+
+
+@functools.singledispatch
+def open_oci_data_ref(data: OciDataRef) -> ta.BinaryIO:
+    raise TypeError(data)
+
+
+@open_oci_data_ref.register
+def _(data: FileOciDataRef) -> ta.BinaryIO:
+    return open(data.path, 'rb')
+
+
+@open_oci_data_ref.register
+def _(data: BytesOciDataRef) -> ta.BinaryIO:
+    return io.BytesIO(data.data)
+
+
+@open_oci_data_ref.register
+def _(data: TarFileOciDataRef) -> ta.BinaryIO:
+    return check.not_none(data.tar_file.extractfile(data.tar_info))  # type: ignore[return-value]
+
+
+#
+
+
+@functools.singledispatch
+def get_oci_data_ref_size(data: OciDataRef) -> int:
+    raise TypeError(data)
+
+
+@get_oci_data_ref_size.register
+def _(data: FileOciDataRef) -> int:
+    return os.path.getsize(data.path)
+
+
+@get_oci_data_ref_size.register
+def _(data: BytesOciDataRef) -> int:
+    return len(data.data)
+
+
+@get_oci_data_ref_size.register
+def _(data: TarFileOciDataRef) -> int:
+    return data.tar_info.size
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRefInfo:
+    data: OciDataRef
+
+    @cached_nullary
+    def sha256(self) -> str:
+        with open_oci_data_ref(self.data) as f:
+            return hashlib.file_digest(f, 'sha256').hexdigest()  # type: ignore[arg-type]
+
+    @cached_nullary
+    def digest(self) -> str:
+        return f'sha256:{self.sha256()}'
+
+    @cached_nullary
+    def size(self) -> int:
+        return get_oci_data_ref_size(self.data)

omdev/oci/dataserver.py

@@ -0,0 +1,61 @@
+# ruff: noqa: PT009 UP006 UP007
+import typing as ta
+
+from omlish.lite.check import check
+
+from ..dataserver.routes import DataServerRoute
+from ..dataserver.targets import DataServerTarget
+from .building import BuiltOciImageIndexRepository
+from .building import OciRepositoryBuilder
+from .datarefs import BytesOciDataRef
+from .datarefs import FileOciDataRef
+from .datarefs import open_oci_data_ref
+from .media import OCI_MANIFEST_MEDIA_TYPES
+
+
+def build_oci_repository_data_server_routes(
+        repo_name: str,
+        built_repo: BuiltOciImageIndexRepository,
+) -> ta.List[DataServerRoute]:
+    base_url_path = f'/v2/{repo_name}'
+
+    repo_contents: ta.Dict[str, OciRepositoryBuilder.Blob] = {}
+
+    repo_contents[f'{base_url_path}/manifests/latest'] = built_repo.blobs[built_repo.media_index_descriptor.digest]
+
+    for blob in built_repo.blobs.values():
+        repo_contents['/'.join([
+            base_url_path,
+            'manifests' if blob.media_type in OCI_MANIFEST_MEDIA_TYPES else 'blobs',
+            blob.digest,
+        ])] = blob
+
+    #
+
+    def build_blob_target(blob: OciRepositoryBuilder.Blob) -> ta.Optional[DataServerTarget]:  # noqa
+        kw: dict = dict(
+            content_type=check.non_empty_str(blob.media_type),
+        )
+
+        if isinstance(blob.data, BytesOciDataRef):
+            return DataServerTarget.of(blob.data.data, **kw)
+
+        elif isinstance(blob.data, FileOciDataRef):
+            return DataServerTarget.of(file_path=blob.data.path, **kw)
+
+        else:
+            with open_oci_data_ref(blob.data) as f:
+                data = f.read()
+
+            return DataServerTarget.of(data, **kw)
+
+    #
+
+    return [
+        DataServerRoute(
+            paths=[path],
+            target=target,
+        )
+        for path, blob in repo_contents.items()
+        if (target := build_blob_target(blob)) is not None
+    ]