omdev 0.0.0.dev220__tar.gz → 0.0.0.dev222__tar.gz

{omdev-0.0.0.dev220/omdev.egg-info → omdev-0.0.0.dev222}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev220
+Version: 0.0.0.dev222
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev220
+Requires-Dist: omlish==0.0.0.dev222
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev220 → omdev-0.0.0.dev222}/omdev/amalg/gen.py

@@ -4,8 +4,8 @@ import textwrap
 import typing as ta
 
 from omlish import cached
-from omlish import collections as col
 from omlish import lang
+from omlish.algorithm import all as alg
 from omlish.lite.runtime import LITE_REQUIRED_PYTHON_VERSION
 
 from ..tokens import all as tks
@@ -152,7 +152,7 @@ class AmalgGenerator:
 
         ##
 
-        ts = list(col.toposort({  # noqa
+        ts = list(alg.toposort({  # noqa
             f.path: {mp for i in f.imports if (mp := i.mod_path) is not None}
             for f in src_files.values()
         }))

omdev-0.0.0.dev222/omdev/oci/building.py

@@ -0,0 +1,123 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_compact
+from omlish.lite.marshal import marshal_obj
+
+from .data import OciDataclass
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .datarefs import BytesOciDataRef
+from .datarefs import OciDataRef
+from .datarefs import OciDataRefInfo
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES
+from .media import OciMediaDataclass
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+
+
+##
+
+
+class OciRepositoryBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._blobs: ta.Dict[str, OciDataRef] = {}
+
+    def get_blobs(self) -> ta.Dict[str, OciDataRef]:
+        return dict(self._blobs)
+
+    def add_blob(
+            self,
+            r: OciDataRef,
+            ri: ta.Optional[OciDataRefInfo] = None,
+    ) -> None:
+        if ri is None:
+            ri = OciDataRefInfo(r)
+        if ri.digest() in self._blobs:
+            raise KeyError(ri.digest())
+        self._blobs[ri.digest()] = r
+
+    def marshal_media(self, obj: OciMediaDataclass) -> bytes:
+        check.isinstance(obj, OciMediaDataclass)
+        m = marshal_obj(obj)
+        j = json_dumps_compact(m)
+        b = j.encode('utf-8')
+        return b
+
+    def add_media(self, obj: OciMediaDataclass) -> OciMediaDescriptor:
+        b = self.marshal_media(obj)
+
+        r = BytesOciDataRef(b)
+        ri = OciDataRefInfo(r)
+        self.add_blob(r, ri)
+
+        return OciMediaDescriptor(
+            media_type=getattr(obj, 'media_type'),
+            digest=ri.digest(),
+            size=ri.size(),
+        )
+
+    def to_media(self, obj: OciDataclass) -> ta.Union[OciMediaDataclass, OciMediaDescriptor]:
+        def make_kw(*exclude):
+            return {
+                a: v
+                for f in dc.fields(obj)
+                if (a := f.name) not in exclude
+                for v in [getattr(obj, a)]
+                if v is not None
+            }
+
+        if isinstance(obj, OciImageIndex):
+            return OciMediaImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    self.add_data(m)
+                    for m in obj.manifests
+                ],
+            )
+
+        elif isinstance(obj, OciImageManifest):
+            return OciMediaImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.add_data(obj.config),
+                layers=[
+                    self.add_data(l)
+                    for l in obj.layers
+                ],
+            )
+
+        elif isinstance(obj, OciImageLayer):
+            ri = OciDataRefInfo(obj.data)
+            self.add_blob(obj.data, ri)
+            return OciMediaDescriptor(
+                media_type=OCI_IMAGE_LAYER_KIND_MEDIA_TYPES[obj.kind],
+                digest=ri.digest(),
+                size=ri.size(),
+            )
+
+        elif isinstance(obj, OciImageConfig):
+            return OciMediaImageConfig(**make_kw())
+
+        else:
+            raise TypeError(obj)
+
+    def add_data(self, obj: OciDataclass) -> OciMediaDescriptor:
+        ret = self.to_media(obj)
+
+        if isinstance(ret, OciMediaDataclass):
+            return self.add_media(ret)
+
+        elif isinstance(ret, OciMediaDescriptor):
+            return ret
+
+        else:
+            raise TypeError(ret)

omdev-0.0.0.dev222/omdev/oci/data.py

@@ -0,0 +1,127 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import enum
+import typing as ta
+
+from omlish.lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+
+from .datarefs import OciDataRef
+
+
+##
+
+
+@dc.dataclass()
+class OciDataclass(abc.ABC):  # noqa
+    pass
+
+
+##
+
+
+@dc.dataclass()
+class OciImageIndex(OciDataclass):
+    manifests: ta.List[ta.Union['OciImageIndex', 'OciImageManifest']]
+
+    annotations: ta.Optional[ta.Dict[str, str]] = None
+
+
+#
+
+
+@dc.dataclass()
+class OciImageManifest(OciDataclass):
+    config: 'OciImageConfig'
+
+    layers: ta.List['OciImageLayer']
+
+    annotations: ta.Optional[ta.Dict[str, str]] = None
+
+#
+
+
+@dc.dataclass()
+class OciImageLayer(OciDataclass):
+    class Kind(enum.Enum):
+        TAR = enum.auto()
+        TAR_GZIP = enum.auto()
+        TAR_ZSTD = enum.auto()
+
+    kind: Kind
+
+    data: OciDataRef
+
+
+#
+
+
+@dc.dataclass()
+class OciImageConfig(OciDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/config.md"""
+
+    architecture: str
+    os: str
+
+    @dc.dataclass()
+    class RootFs:
+        type: str
+        diff_ids: ta.List[str]
+
+    rootfs: RootFs
+
+    #
+
+    created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    os_version: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.version', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    os_features: ta.Optional[ta.List[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    variant: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    """
+    config object, OPTIONAL
+        User string, OPTIONAL
+        ExposedPorts object, OPTIONAL
+        Env array of strings, OPTIONAL
+        Entrypoint array of strings, OPTIONAL
+        Cmd array of strings, OPTIONAL
+        Volumes object, OPTIONAL
+        WorkingDir string, OPTIONAL
+        Labels object, OPTIONAL
+        StopSignal string, OPTIONAL
+        ArgsEscaped boolean, OPTIONAL
+        Memory integer, OPTIONAL
+        MemorySwap integer, OPTIONAL
+        CpuShares integer, OPTIONAL
+        Healthcheck object, OPTIONAL
+    """
+    config: ta.Optional[ta.Dict[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    @dc.dataclass()
+    class History:
+        created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        created_by: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        comment: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+        empty_layer: ta.Optional[bool] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+    history: ta.Optional[ta.List[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+
+##
+
+
+def is_empty_oci_dataclass(obj: OciDataclass) -> bool:
+    if not isinstance(obj, OciDataclass):
+        raise TypeError(obj)
+
+    elif isinstance(obj, OciImageIndex):
+        return not obj.manifests
+
+    elif isinstance(obj, OciImageManifest):
+        return not obj.layers
+
+    else:
+        return False

omdev-0.0.0.dev222/omdev/oci/datarefs.py

@@ -0,0 +1,98 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import hashlib
+import io
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRef(abc.ABC):  # noqa
+    pass
+
+
+@dc.dataclass(frozen=True)
+class BytesOciDataRef(OciDataRef):
+    data: bytes
+
+
+@dc.dataclass(frozen=True)
+class FileOciDataRef(OciDataRef):
+    path: str
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRefInfo:
+    data: OciDataRef
+
+    @cached_nullary
+    def sha256(self) -> str:
+        if isinstance(self.data, FileOciDataRef):
+            with open(self.data.path, 'rb') as f:
+                return hashlib.file_digest(f, 'sha256').hexdigest()  # noqa
+
+        elif isinstance(self.data, BytesOciDataRef):
+            return hashlib.sha256(self.data.data).hexdigest()
+
+        else:
+            raise TypeError(self.data)
+
+    @cached_nullary
+    def digest(self) -> str:
+        return f'sha256:{self.sha256()}'
+
+    @cached_nullary
+    def size(self) -> int:
+        if isinstance(self.data, FileOciDataRef):
+            return os.path.getsize(self.data.path)
+
+        elif isinstance(self.data, BytesOciDataRef):
+            return len(self.data.data)
+
+        else:
+            raise TypeError(self.data)
+
+
+def write_oci_data_ref_to_file(
+        data: OciDataRef,
+        dst: str,
+        *,
+        symlink: bool = False,
+) -> None:
+    if isinstance(data, FileOciDataRef):
+        if symlink:
+            os.symlink(
+                os.path.relpath(data.path, os.path.dirname(dst)),
+                dst,
+            )
+        else:
+            shutil.copyfile(data.path, dst)
+
+    elif isinstance(data, BytesOciDataRef):
+        with open(dst, 'wb') as f:
+            f.write(data.data)
+
+    else:
+        raise TypeError(data)
+
+
+def open_oci_data_ref(data: OciDataRef) -> ta.BinaryIO:
+    if isinstance(data, FileOciDataRef):
+        return open(data.path, 'rb')
+
+    elif isinstance(data, BytesOciDataRef):
+        return io.BytesIO(data.data)
+
+    else:
+        raise TypeError(data)

omdev-0.0.0.dev222/omdev/oci/loading.py

@@ -0,0 +1,122 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check
+
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .data import is_empty_oci_dataclass
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_
+from .media import OCI_MEDIA_FIELDS
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+from .media import unmarshal_oci_media_dataclass
+from .repositories import OciRepository
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class OciRepositoryLoader:
+    def __init__(
+            self,
+            repo: OciRepository,
+    ) -> None:
+        super().__init__()
+
+        self._repo = repo
+
+    #
+
+    def load_object(
+            self,
+            data: bytes,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> T:
+        text = data.decode('utf-8')
+        dct = json.loads(text)
+        obj = unmarshal_oci_media_dataclass(
+            dct,
+            media_type=media_type,
+        )
+        return check.isinstance(obj, cls)
+
+    def read_object(
+            self,
+            digest: str,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> T:
+        data = self._repo.read_blob(digest)
+        return self.load_object(
+            data,
+            cls,
+            media_type=media_type,
+        )
+
+    def read_descriptor(
+            self,
+            desc: OciMediaDescriptor,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+    ) -> ta.Any:
+        return self.read_object(
+            desc.digest,
+            cls,
+            media_type=desc.media_type,
+        )
+
+    #
+
+    def from_media(self, obj: ta.Any) -> ta.Any:
+        def make_kw(*exclude):
+            return {
+                a: getattr(obj, a)
+                for f in dc.fields(obj)
+                if (a := f.name) not in OCI_MEDIA_FIELDS
+                and a not in exclude
+            }
+
+        if isinstance(obj, OciMediaImageConfig):
+            return OciImageConfig(**make_kw())
+
+        elif isinstance(obj, OciMediaImageManifest):
+            return OciImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.from_media(self.read_descriptor(obj.config)),
+                layers=[
+                    OciImageLayer(
+                        kind=lk,
+                        data=self._repo.ref_blob(l.digest),
+                    )
+                    for l in obj.layers
+                    if (lk := OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_.get(l.media_type)) is not None
+                ],
+            )
+
+        elif isinstance(obj, OciMediaImageIndex):
+            return OciImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    fm
+                    for m in obj.manifests
+                    if self._repo.contains_blob(m.digest)
+                    for fm in [self.from_media(self.read_descriptor(m))]
+                    if not is_empty_oci_dataclass(fm)
+                ],
+            )
+
+        else:
+            raise TypeError(obj)

omdev-0.0.0.dev222/omdev/oci/media.py

@@ -0,0 +1,159 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+from omlish.lite.marshal import unmarshal_obj
+
+from .data import OciImageConfig
+from .data import OciImageLayer
+
+
+##
+
+
+OCI_MEDIA_FIELDS: ta.Collection[str] = frozenset([
+    'schema_version',
+    'media_type',
+])
+
+
+@dc.dataclass()
+class OciMediaDataclass(abc.ABC):  # noqa
+    SCHEMA_VERSION: ta.ClassVar[int]
+    MEDIA_TYPE: ta.ClassVar[str]
+
+    def __init_subclass__(cls, **kwargs: ta.Any) -> None:
+        super().__init_subclass__(**kwargs)
+        for a in OCI_MEDIA_FIELDS:
+            check.in_(a, cls.__dict__)
+
+
+_REGISTERED_OCI_MEDIA_DATACLASSES: ta.Dict[str, ta.Type[OciMediaDataclass]] = {}
+
+
+def _register_oci_media_dataclass(cls):
+    check.issubclass(cls, OciMediaDataclass)
+    check.arg(dc.is_dataclass(cls))
+    mt = check.non_empty_str(cls.__dict__['MEDIA_TYPE'])
+    check.not_in(mt, _REGISTERED_OCI_MEDIA_DATACLASSES)
+    _REGISTERED_OCI_MEDIA_DATACLASSES[mt] = cls
+    return cls
+
+
+def get_registered_oci_media_dataclass(media_type: str) -> ta.Optional[ta.Type[OciMediaDataclass]]:
+    return _REGISTERED_OCI_MEDIA_DATACLASSES.get(media_type)
+
+
+def unmarshal_oci_media_dataclass(
+        dct: ta.Mapping[str, ta.Any],
+        *,
+        media_type: ta.Optional[str] = None,
+) -> ta.Any:
+    if media_type is None:
+        media_type = check.non_empty_str(dct['mediaType'])
+    cls = _REGISTERED_OCI_MEDIA_DATACLASSES[media_type]
+    return unmarshal_obj(dct, cls)
+
+
+##
+
+
+@dc.dataclass()
+class OciMediaDescriptor:
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/descriptor.md#properties"""  # noqa
+
+    media_type: str = dc.field(metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+    digest: str
+    size: int
+
+    #
+
+    urls: ta.Optional[ta.Sequence[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    annotations: ta.Optional[ta.Mapping[str, str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    data: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    artifact_type: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'artifactType', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+    #
+
+    platform: ta.Optional[ta.Mapping[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+
+##
+
+
+@_register_oci_media_dataclass
+@dc.dataclass()
+class OciMediaImageIndex(OciMediaDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/image-index.md"""
+
+    manifests: ta.Sequence[OciMediaDescriptor]  # -> OciMediaImageIndex | OciMediaImageManifest
+
+    #
+
+    annotations: ta.Optional[ta.Mapping[str, str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+    #
+
+    SCHEMA_VERSION: ta.ClassVar[int] = 2
+    schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})
+
+    MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.index.v1+json'
+    media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+
+
+#
+
+
+@_register_oci_media_dataclass
+@dc.dataclass()
+class OciMediaImageManifest(OciMediaDataclass):
+    """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/manifest.md"""
+
+    config: OciMediaDescriptor  # -> OciMediaImageConfig
+
+    layers: ta.Sequence[OciMediaDescriptor]
+
+    #
+
+    annotations: ta.Optional[ta.Mapping[str, str]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+
+    #
+
+    SCHEMA_VERSION: ta.ClassVar[int] = 2
+    schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})
+
+    MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.manifest.v1+json'
+    media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+
+
+#
+
+
+OCI_IMAGE_LAYER_KIND_MEDIA_TYPES: ta.Mapping[OciImageLayer.Kind, str] = {
+    OciImageLayer.Kind.TAR: 'application/vnd.oci.image.layer.v1.tar',
+    OciImageLayer.Kind.TAR_GZIP: 'application/vnd.oci.image.layer.v1.tar+gzip',
+    OciImageLayer.Kind.TAR_ZSTD: 'application/vnd.oci.image.layer.v1.tar+zstd',
+}
+
+OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_: ta.Mapping[str, OciImageLayer.Kind] = {
+    v: k
+    for k, v in OCI_IMAGE_LAYER_KIND_MEDIA_TYPES.items()
+}
+
+
+#
+
+
+@_register_oci_media_dataclass
+@dc.dataclass()
+class OciMediaImageConfig(OciImageConfig, OciMediaDataclass):
+    SCHEMA_VERSION: ta.ClassVar[int] = 2
+    schema_version: int = dc.field(default=SCHEMA_VERSION, metadata={OBJ_MARSHALER_FIELD_KEY: 'schemaVersion'})
+
+    MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.config.v1+json'
+    media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})