omlish 0.0.0.dev219__py3-none-any.whl → 0.0.0.dev221__py3-none-any.whl

omlish/dataclasses/impl/simple.py

@@ -1,53 +1,11 @@
 import inspect
 
-from ... import lang
-from .fields import field_assign
 from .init import get_init_fields
-from .params import get_field_extras
 from .processing import Processor
 from .utils import create_fn
 from .utils import set_new_attribute
 
 
-class OverridesProcessor(Processor):
-    def _process(self) -> None:
-        for f in self._info.instance_fields:
-            fx = get_field_extras(f)
-            if not fx.override:
-                continue
-
-            if self._info.params.slots:
-                raise TypeError
-
-            self_name = '__dataclass_self__' if 'self' in self._info.fields else 'self'
-
-            getter = create_fn(
-                f.name,
-                (self_name,),
-                [f'return {self_name}.__dict__[{f.name!r}]'],
-                globals=self._info.globals,
-                return_type=lang.just(f.type),
-            )
-            prop = property(getter)
-
-            if not self._info.params.frozen:
-                setter = create_fn(
-                    f.name,
-                    (self_name, f'{f.name}: __dataclass_type_{f.name}__'),
-                    [field_assign(self._info.params.frozen, f.name, f.name, self_name, fx.override)],
-                    globals=self._info.globals,
-                    locals={f'__dataclass_type_{f.name}__': f.type},
-                    return_type=lang.just(None),
-                )
-                prop = prop.setter(setter)
-
-            set_new_attribute(
-                self._cls,
-                f.name,
-                prop,
-            )
-
-
 class EqProcessor(Processor):
     def _process(self) -> None:
         if not self._info.params.eq:

omlish/docker/oci/building.py

@@ -0,0 +1,122 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import typing as ta
+
+from ...lite.check import check
+from ...lite.json import json_dumps_compact
+from ...lite.marshal import marshal_obj
+from .data import OciDataclass
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .datarefs import BytesOciDataRef
+from .datarefs import OciDataRef
+from .datarefs import OciDataRefInfo
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES
+from .media import OciMediaDataclass
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+
+
+##
+
+
+class OciRepositoryBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._blobs: ta.Dict[str, OciDataRef] = {}
+
+    def get_blobs(self) -> ta.Dict[str, OciDataRef]:
+        return dict(self._blobs)
+
+    def add_blob(
+            self,
+            r: OciDataRef,
+            ri: ta.Optional[OciDataRefInfo] = None,
+    ) -> None:
+        if ri is None:
+            ri = OciDataRefInfo(r)
+        if ri.digest() in self._blobs:
+            raise KeyError(ri.digest())
+        self._blobs[ri.digest()] = r
+
+    def marshal_media(self, obj: OciMediaDataclass) -> bytes:
+        check.isinstance(obj, OciMediaDataclass)
+        m = marshal_obj(obj)
+        j = json_dumps_compact(m)
+        b = j.encode('utf-8')
+        return b
+
+    def add_media(self, obj: OciMediaDataclass) -> OciMediaDescriptor:
+        b = self.marshal_media(obj)
+
+        r = BytesOciDataRef(b)
+        ri = OciDataRefInfo(r)
+        self.add_blob(r, ri)
+
+        return OciMediaDescriptor(
+            media_type=getattr(obj, 'media_type'),
+            digest=ri.digest(),
+            size=ri.size(),
+        )
+
+    def to_media(self, obj: OciDataclass) -> ta.Union[OciMediaDataclass, OciMediaDescriptor]:
+        def make_kw(*exclude):
+            return {
+                a: v
+                for f in dc.fields(obj)
+                if (a := f.name) not in exclude
+                for v in [getattr(obj, a)]
+                if v is not None
+            }
+
+        if isinstance(obj, OciImageIndex):
+            return OciMediaImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    self.add_data(m)
+                    for m in obj.manifests
+                ],
+            )
+
+        elif isinstance(obj, OciImageManifest):
+            return OciMediaImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.add_data(obj.config),
+                layers=[
+                    self.add_data(l)
+                    for l in obj.layers
+                ],
+            )
+
+        elif isinstance(obj, OciImageLayer):
+            ri = OciDataRefInfo(obj.data)
+            self.add_blob(obj.data, ri)
+            return OciMediaDescriptor(
+                media_type=OCI_IMAGE_LAYER_KIND_MEDIA_TYPES[obj.kind],
+                digest=ri.digest(),
+                size=ri.size(),
+            )
+
+        elif isinstance(obj, OciImageConfig):
+            return OciMediaImageConfig(**make_kw())
+
+        else:
+            raise TypeError(obj)
+
+    def add_data(self, obj: OciDataclass) -> OciMediaDescriptor:
+        ret = self.to_media(obj)
+
+        if isinstance(ret, OciMediaDataclass):
+            return self.add_media(ret)
+
+        elif isinstance(ret, OciMediaDescriptor):
+            return ret
+
+        else:
+            raise TypeError(ret)

omlish/docker/oci/data.py

@@ -2,16 +2,18 @@
 # @omlish-lite
 import abc
 import dataclasses as dc
+import enum
 import typing as ta
 
 from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
 from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+from .datarefs import OciDataRef
 
 
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass()
 class OciDataclass(abc.ABC):  # noqa
     pass
 
@@ -19,17 +21,52 @@ class OciDataclass(abc.ABC):  # noqa
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass()
+class OciImageIndex(OciDataclass):
+    manifests: ta.List[ta.Union['OciImageIndex', 'OciImageManifest']]
+
+    annotations: ta.Optional[ta.Dict[str, str]] = None
+
+
+#
+
+
+@dc.dataclass()
+class OciImageManifest(OciDataclass):
+    config: 'OciImageConfig'
+
+    layers: ta.List['OciImageLayer']
+
+
+#
+
+
+@dc.dataclass()
+class OciImageLayer(OciDataclass):
+    class Kind(enum.Enum):
+        TAR = enum.auto()
+        TAR_GZIP = enum.auto()
+        TAR_ZSTD = enum.auto()
+
+    kind: Kind
+
+    data: OciDataRef
+
+
+#
+
+
+@dc.dataclass()
 class OciImageConfig(OciDataclass):
     """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/config.md"""
 
     architecture: str
     os: str
 
-    @dc.dataclass(frozen=True)
+    @dc.dataclass()
     class RootFs:
         type: str
-        diff_ids: ta.Sequence[str]
+        diff_ids: ta.List[str]
 
     rootfs: RootFs
 
@@ -38,7 +75,7 @@ class OciImageConfig(OciDataclass):
     created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
     author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
     os_version: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.version', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
-    os_features: ta.Optional[ta.Sequence[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    os_features: ta.Optional[ta.List[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
     variant: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
     """
@@ -58,9 +95,9 @@ class OciImageConfig(OciDataclass):
         CpuShares integer, OPTIONAL
         Healthcheck object, OPTIONAL
     """
-    config: ta.Optional[ta.Mapping[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    config: ta.Optional[ta.Dict[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
-    @dc.dataclass(frozen=True)
+    @dc.dataclass()
     class History:
         created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
         author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
@@ -68,4 +105,21 @@ class OciImageConfig(OciDataclass):
         comment: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
         empty_layer: ta.Optional[bool] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
-    history: ta.Optional[ta.Sequence[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    history: ta.Optional[ta.List[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+
+##
+
+
+def is_empty_oci_dataclass(obj: OciDataclass) -> bool:
+    if not isinstance(obj, OciDataclass):
+        raise TypeError(obj)
+
+    elif isinstance(obj, OciImageIndex):
+        return not obj.manifests
+
+    elif isinstance(obj, OciImageManifest):
+        return not obj.layers
+
+    else:
+        return False

omlish/docker/oci/datarefs.py

@@ -0,0 +1,98 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import hashlib
+import io
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRef(abc.ABC):  # noqa
+    pass
+
+
+@dc.dataclass(frozen=True)
+class BytesOciDataRef(OciDataRef):
+    data: bytes
+
+
+@dc.dataclass(frozen=True)
+class FileOciDataRef(OciDataRef):
+    path: str
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRefInfo:
+    data: OciDataRef
+
+    @cached_nullary
+    def sha256(self) -> str:
+        if isinstance(self.data, FileOciDataRef):
+            with open(self.data.path, 'rb') as f:
+                return hashlib.file_digest(f, 'sha256').hexdigest()  # noqa
+
+        elif isinstance(self.data, BytesOciDataRef):
+            return hashlib.sha256(self.data.data).hexdigest()
+
+        else:
+            raise TypeError(self.data)
+
+    @cached_nullary
+    def digest(self) -> str:
+        return f'sha256:{self.sha256()}'
+
+    @cached_nullary
+    def size(self) -> int:
+        if isinstance(self.data, FileOciDataRef):
+            return os.path.getsize(self.data.path)
+
+        elif isinstance(self.data, BytesOciDataRef):
+            return len(self.data.data)
+
+        else:
+            raise TypeError(self.data)
+
+
+def write_oci_data_ref_to_file(
+        data: OciDataRef,
+        dst: str,
+        *,
+        symlink: bool = False,
+) -> None:
+    if isinstance(data, FileOciDataRef):
+        if symlink:
+            os.symlink(
+                os.path.relpath(data.path, os.path.dirname(dst)),
+                dst,
+            )
+        else:
+            shutil.copyfile(data.path, dst)
+
+    elif isinstance(data, BytesOciDataRef):
+        with open(dst, 'wb') as f:
+            f.write(data.data)
+
+    else:
+        raise TypeError(data)
+
+
+def open_oci_data_ref(data: OciDataRef) -> ta.BinaryIO:
+    if isinstance(data, FileOciDataRef):
+        return open(data.path, 'rb')
+
+    elif isinstance(data, BytesOciDataRef):
+        return io.BytesIO(data.data)
+
+    else:
+        raise TypeError(data)

omlish/docker/oci/loading.py

@@ -0,0 +1,120 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from ...lite.check import check
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .data import is_empty_oci_dataclass
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_
+from .media import OCI_MEDIA_FIELDS
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+from .media import unmarshal_oci_media_dataclass
+from .repositories import OciRepository
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class OciRepositoryLoader:
+    def __init__(
+            self,
+            repo: OciRepository,
+    ) -> None:
+        super().__init__()
+
+        self._repo = repo
+
+    #
+
+    def load_object(
+            self,
+            data: bytes,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> T:
+        text = data.decode('utf-8')
+        dct = json.loads(text)
+        obj = unmarshal_oci_media_dataclass(
+            dct,
+            media_type=media_type,
+        )
+        return check.isinstance(obj, cls)
+
+    def read_object(
+            self,
+            digest: str,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> T:
+        data = self._repo.read_blob(digest)
+        return self.load_object(
+            data,
+            cls,
+            media_type=media_type,
+        )
+
+    def read_descriptor(
+            self,
+            desc: OciMediaDescriptor,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+    ) -> ta.Any:
+        return self.read_object(
+            desc.digest,
+            cls,
+            media_type=desc.media_type,
+        )
+
+    #
+
+    def from_media(self, obj: ta.Any) -> ta.Any:
+        def make_kw(*exclude):
+            return {
+                a: getattr(obj, a)
+                for f in dc.fields(obj)
+                if (a := f.name) not in OCI_MEDIA_FIELDS
+                and a not in exclude
+            }
+
+        if isinstance(obj, OciMediaImageConfig):
+            return OciImageConfig(**make_kw())
+
+        elif isinstance(obj, OciMediaImageManifest):
+            return OciImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.from_media(self.read_descriptor(obj.config)),
+                layers=[
+                    OciImageLayer(
+                        kind=lk,
+                        data=self._repo.ref_blob(l.digest),
+                    )
+                    for l in obj.layers
+                    if (lk := OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_.get(l.media_type)) is not None
+                ],
+            )
+
+        elif isinstance(obj, OciMediaImageIndex):
+            return OciImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    fm
+                    for m in obj.manifests
+                    for fm in [self.from_media(self.read_descriptor(m))]
+                    if not is_empty_oci_dataclass(fm)
+                ],
+            )
+
+        else:
+            raise TypeError(obj)

omlish/docker/oci/media.py

@@ -8,18 +8,29 @@ from ...lite.check import check
 from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
 from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
 from ...lite.marshal import unmarshal_obj
-from .data import OciDataclass
 from .data import OciImageConfig
+from .data import OciImageLayer
 
 
 ##
 
 
-@dc.dataclass(frozen=True)
-class OciMediaDataclass(OciDataclass, abc.ABC):  # noqa
+OCI_MEDIA_FIELDS: ta.Collection[str] = frozenset([
+    'schema_version',
+    'media_type',
+])
+
+
+@dc.dataclass()
+class OciMediaDataclass(abc.ABC):  # noqa
     SCHEMA_VERSION: ta.ClassVar[int]
     MEDIA_TYPE: ta.ClassVar[str]
 
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+        for a in OCI_MEDIA_FIELDS:
+            check.in_(a, cls.__dict__)
+
 
 _REGISTERED_OCI_MEDIA_DATACLASSES: ta.Dict[str, ta.Type[OciMediaDataclass]] = {}
 
@@ -48,11 +59,11 @@ def unmarshal_oci_media_dataclass(
     return unmarshal_obj(dct, cls)
 
 
-#
+##
 
 
-@dc.dataclass(frozen=True)
-class OciMediaDescriptor(OciDataclass):
+@dc.dataclass()
+class OciMediaDescriptor:
     """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/descriptor.md#properties"""  # noqa
 
     media_type: str = dc.field(metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
@@ -71,8 +82,11 @@ class OciMediaDescriptor(OciDataclass):
     platform: ta.Optional[ta.Mapping[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
 
 
+##
+
+
 @_register_oci_media_dataclass
-@dc.dataclass(frozen=True)
+@dc.dataclass()
 class OciMediaImageIndex(OciMediaDataclass):
     """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/image-index.md"""
 
@@ -91,18 +105,16 @@ class OciMediaImageIndex(OciMediaDataclass):
     media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
 
 
+#
+
+
 @_register_oci_media_dataclass
-@dc.dataclass(frozen=True)
+@dc.dataclass()
 class OciMediaImageManifest(OciMediaDataclass):
     """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/manifest.md"""
 
     config: OciMediaDescriptor  # -> OciMediaImageConfig
 
-    # MEDIA_TYPES: ta.ClassVar[ta.Mapping[str, str]] = {
-    #     'TAR': 'application/vnd.oci.image.layer.v1.tar',
-    #     'TAR_GZIP': 'application/vnd.oci.image.layer.v1.tar+gzip',
-    #     'TAR_ZSTD': 'application/vnd.oci.image.layer.v1.tar+zstd',
-    # }
     layers: ta.Sequence[OciMediaDescriptor]
 
     #
@@ -114,8 +126,26 @@ class OciMediaImageManifest(OciMediaDataclass):
     media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
 
 
+#
+
+
+OCI_IMAGE_LAYER_KIND_MEDIA_TYPES: ta.Mapping[OciImageLayer.Kind, str] = {
+    OciImageLayer.Kind.TAR: 'application/vnd.oci.image.layer.v1.tar',
+    OciImageLayer.Kind.TAR_GZIP: 'application/vnd.oci.image.layer.v1.tar+gzip',
+    OciImageLayer.Kind.TAR_ZSTD: 'application/vnd.oci.image.layer.v1.tar+zstd',
+}
+
+OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_: ta.Mapping[str, OciImageLayer.Kind] = {
+    v: k
+    for k, v in OCI_IMAGE_LAYER_KIND_MEDIA_TYPES.items()
+}
+
+
+#
+
+
 @_register_oci_media_dataclass
-@dc.dataclass(frozen=True)
+@dc.dataclass()
 class OciMediaImageConfig(OciImageConfig, OciMediaDataclass):
     SCHEMA_VERSION: ta.ClassVar[int] = 2
     schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})

omlish/docker/oci/repositories.py

@@ -0,0 +1,72 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import os.path
+import typing as ta
+
+from ...lite.check import check
+from ...os.paths import is_path_in_dir
+from .datarefs import BytesOciDataRef
+from .datarefs import FileOciDataRef
+from .datarefs import OciDataRef
+
+
+##
+
+
+class OciRepository(abc.ABC):
+    @abc.abstractmethod
+    def read_blob(self, digest: str) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def ref_blob(self, digest: str) -> OciDataRef:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryOciRepository(OciRepository):
+    def __init__(self, data_dir: str) -> None:
+        super().__init__()
+
+        self._data_dir = check.non_empty_str(data_dir)
+
+    def read_file(self, path: str) -> bytes:
+        full_path = os.path.join(self._data_dir, path)
+        check.arg(is_path_in_dir(self._data_dir, full_path))
+        with open(full_path, 'rb') as f:
+            return f.read()
+
+    def blob_path(self, digest: str) -> str:
+        scheme, value = digest.split(':')
+        return os.path.join('blobs', scheme, value)
+
+    def blob_full_path(self, digest: str) -> str:
+        path = self.blob_path(digest)
+        full_path = os.path.join(self._data_dir, path)
+        check.arg(is_path_in_dir(self._data_dir, full_path))
+        return full_path
+
+    def read_blob(self, digest: str) -> bytes:
+        return self.read_file(self.blob_path(digest))
+
+    def ref_blob(self, digest: str) -> OciDataRef:
+        return FileOciDataRef(self.blob_full_path(digest))
+
+
+#
+
+
+class DictionaryOciRepository(OciRepository):
+    def __init__(self, blobs: ta.Mapping[str, bytes]) -> None:
+        super().__init__()
+
+        self._blobs = blobs
+
+    def read_blob(self, digest: str) -> bytes:
+        return self._blobs[digest]
+
+    def ref_blob(self, digest: str) -> OciDataRef:
+        return BytesOciDataRef(self._blobs[digest])