omlish 0.0.0.dev220__tar.gz → 0.0.0.dev221__tar.gz

{omlish-0.0.0.dev220/omlish.egg-info → omlish-0.0.0.dev221}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omlish
-Version: 0.0.0.dev220
+Version: 0.0.0.dev221
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev220 → omlish-0.0.0.dev221}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev220'
-__revision__ = '3af8855e48e09d340dd21835f45fe8a21a262fa9'
+__version__ = '0.0.0.dev221'
+__revision__ = '2c4c1480e414176972a21bd34529721b39818ad8'
 
 
 #

omlish-0.0.0.dev221/omlish/algorithm/all.py

@@ -0,0 +1,13 @@
+from .distribute import (  # noqa
+    distribute_evenly,
+)
+
+from .toposort import (  # noqa
+    mut_toposort,
+    toposort,
+)
+
+from .unify import (  # noqa
+    mut_unify_sets,
+    unify_sets,
+)

omlish-0.0.0.dev221/omlish/algorithm/distribute.py

@@ -0,0 +1,46 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import collections
+import heapq
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def distribute_evenly(
+        items: ta.Iterable[ta.Tuple[T, float]],
+        n_bins: int,
+) -> ta.List[ta.List[ta.Tuple[T, float]]]:
+    """
+    Distribute items into n bins as evenly as possible in terms of total size.
+     - Sorting ensures larger items are placed first, preventing large leftover gaps in bins.
+     - A min-heap efficiently finds the least loaded bin in O(log n), keeping the distribution balanced.
+     - Each item is placed in the lightest bin, preventing a few bins from getting overloaded early.
+
+    :param items: List of tuples (item, size).
+    :param n_bins: Number of bins.
+    :return: List of n_bins lists, each containing items assigned to that bin.
+    """
+
+    # Sort items by size in descending order
+    items_sorted = sorted(items, key=lambda x: x[1], reverse=True)
+
+    # Min-heap to track bin loads (size, index)
+    bins = [(0, i) for i in range(n_bins)]  # (current size, bin index)
+    heapq.heapify(bins)
+
+    # Allocate items to bins
+    bin_contents = collections.defaultdict(list)
+
+    for item, size in items_sorted:
+        # Get the least loaded bin
+        bin_size, bin_index = heapq.heappop(bins)
+
+        # Assign item to this bin
+        bin_contents[bin_index].append((item, size))
+
+        # Update bin load and push back to heap
+        heapq.heappush(bins, (bin_size + size, bin_index))  # type: ignore
+
+    return [bin_contents[i] for i in range(n_bins)]

omlish-0.0.0.dev221/omlish/algorithm/toposort.py

@@ -0,0 +1,26 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import functools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def mut_toposort(data: ta.Dict[T, ta.Set[T]]) -> ta.Iterator[ta.Set[T]]:
+    for k, v in data.items():
+        v.discard(k)
+    extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
+    data.update({item: set() for item in extra_items_in_deps})
+    while True:
+        ordered = {item for item, dep in data.items() if not dep}
+        if not ordered:
+            break
+        yield ordered
+        data = {item: (dep - ordered) for item, dep in data.items() if item not in ordered}
+    if data:
+        raise ValueError('Cyclic dependencies exist among these items: ' + ' '.join(repr(x) for x in data.items()))
+
+
+def toposort(data: ta.Mapping[T, ta.AbstractSet[T]]) -> ta.Iterator[ta.Set[T]]:
+    return mut_toposort({k: set(v) for k, v in data.items()})

omlish-0.0.0.dev221/omlish/algorithm/unify.py

@@ -0,0 +1,31 @@
+import itertools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def mut_unify_sets(sets: ta.Iterable[set[T]]) -> list[set[T]]:
+    rem: list[set[T]] = list(sets)
+    ret: list[set[T]] = []
+    while rem:
+        cur = rem.pop()
+        while True:
+            moved = False
+            for i in range(len(rem) - 1, -1, -1):
+                if any(e in cur for e in rem[i]):
+                    cur.update(rem.pop(i))
+                    moved = True
+            if not moved:
+                break
+        ret.append(cur)
+    if ret:
+        all_ = set(itertools.chain.from_iterable(ret))
+        num = sum(map(len, ret))
+        if len(all_) != num:
+            raise ValueError('Length mismatch')
+    return ret
+
+
+def unify_sets(sets: ta.Iterable[ta.AbstractSet[T]]) -> list[set[T]]:
+    return mut_unify_sets([set(s) for s in sets])

{omlish-0.0.0.dev220 → omlish-0.0.0.dev221}/omlish/collections/__init__.py

@@ -126,8 +126,6 @@ from .utils import (  # noqa
     make_map_by,
     multi_map,
     multi_map_by,
-    mut_toposort,
     partition,
-    toposort,
     unique,
 )

{omlish-0.0.0.dev220 → omlish-0.0.0.dev221}/omlish/collections/utils.py

@@ -1,8 +1,5 @@
-import functools
-import itertools
 import typing as ta
 
-from .. import check
 from .. import lang
 from .exceptions import DuplicateKeyError
 from .identity import IdentityKeyDict
@@ -17,28 +14,6 @@ V = ta.TypeVar('V')
 ##
 
 
-def mut_toposort(data: dict[T, set[T]]) -> ta.Iterator[set[T]]:
-    for k, v in data.items():
-        v.discard(k)
-    extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
-    data.update({item: set() for item in extra_items_in_deps})
-    while True:
-        ordered = {item for item, dep in data.items() if not dep}
-        if not ordered:
-            break
-        yield ordered
-        data = {item: (dep - ordered) for item, dep in data.items() if item not in ordered}
-    if data:
-        raise ValueError('Cyclic dependencies exist among these items: ' + ' '.join(repr(x) for x in data.items()))
-
-
-def toposort(data: ta.Mapping[T, ta.AbstractSet[T]]) -> ta.Iterator[set[T]]:
-    return mut_toposort({k: set(v) for k, v in data.items()})
-
-
-##
-
-
 class PartitionResult(ta.NamedTuple, ta.Generic[T]):
     t: list[T]
     f: list[T]
@@ -153,24 +128,3 @@ def key_cmp(fn: ta.Callable[[K, K], int]) -> ta.Callable[[tuple[K, V], tuple[K,
 
 def indexes(it: ta.Iterable[T]) -> dict[T, int]:
     return {e: i for i, e in enumerate(it)}
-
-
-def mut_unify_sets(sets: ta.Iterable[set[T]]) -> list[set[T]]:
-    rem: list[set[T]] = list(sets)
-    ret: list[set[T]] = []
-    while rem:
-        cur = rem.pop()
-        while True:
-            moved = False
-            for i in range(len(rem) - 1, -1, -1):
-                if any(e in cur for e in rem[i]):
-                    cur.update(rem.pop(i))
-                    moved = True
-            if not moved:
-                break
-        ret.append(cur)
-    if ret:
-        all_ = set(itertools.chain.from_iterable(ret))
-        num = sum(map(len, ret))
-        check.equal(len(all_), num)
-    return ret

omlish-0.0.0.dev221/omlish/docker/oci/building.py

@@ -0,0 +1,122 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import typing as ta
+
+from ...lite.check import check
+from ...lite.json import json_dumps_compact
+from ...lite.marshal import marshal_obj
+from .data import OciDataclass
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .datarefs import BytesOciDataRef
+from .datarefs import OciDataRef
+from .datarefs import OciDataRefInfo
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES
+from .media import OciMediaDataclass
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+
+
+##
+
+
+class OciRepositoryBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._blobs: ta.Dict[str, OciDataRef] = {}
+
+    def get_blobs(self) -> ta.Dict[str, OciDataRef]:
+        return dict(self._blobs)
+
+    def add_blob(
+            self,
+            r: OciDataRef,
+            ri: ta.Optional[OciDataRefInfo] = None,
+    ) -> None:
+        if ri is None:
+            ri = OciDataRefInfo(r)
+        if ri.digest() in self._blobs:
+            raise KeyError(ri.digest())
+        self._blobs[ri.digest()] = r
+
+    def marshal_media(self, obj: OciMediaDataclass) -> bytes:
+        check.isinstance(obj, OciMediaDataclass)
+        m = marshal_obj(obj)
+        j = json_dumps_compact(m)
+        b = j.encode('utf-8')
+        return b
+
+    def add_media(self, obj: OciMediaDataclass) -> OciMediaDescriptor:
+        b = self.marshal_media(obj)
+
+        r = BytesOciDataRef(b)
+        ri = OciDataRefInfo(r)
+        self.add_blob(r, ri)
+
+        return OciMediaDescriptor(
+            media_type=getattr(obj, 'media_type'),
+            digest=ri.digest(),
+            size=ri.size(),
+        )
+
+    def to_media(self, obj: OciDataclass) -> ta.Union[OciMediaDataclass, OciMediaDescriptor]:
+        def make_kw(*exclude):
+            return {
+                a: v
+                for f in dc.fields(obj)
+                if (a := f.name) not in exclude
+                for v in [getattr(obj, a)]
+                if v is not None
+            }
+
+        if isinstance(obj, OciImageIndex):
+            return OciMediaImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    self.add_data(m)
+                    for m in obj.manifests
+                ],
+            )
+
+        elif isinstance(obj, OciImageManifest):
+            return OciMediaImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.add_data(obj.config),
+                layers=[
+                    self.add_data(l)
+                    for l in obj.layers
+                ],
+            )
+
+        elif isinstance(obj, OciImageLayer):
+            ri = OciDataRefInfo(obj.data)
+            self.add_blob(obj.data, ri)
+            return OciMediaDescriptor(
+                media_type=OCI_IMAGE_LAYER_KIND_MEDIA_TYPES[obj.kind],
+                digest=ri.digest(),
+                size=ri.size(),
+            )
+
+        elif isinstance(obj, OciImageConfig):
+            return OciMediaImageConfig(**make_kw())
+
+        else:
+            raise TypeError(obj)
+
+    def add_data(self, obj: OciDataclass) -> OciMediaDescriptor:
+        ret = self.to_media(obj)
+
+        if isinstance(ret, OciMediaDataclass):
+            return self.add_media(ret)
+
+        elif isinstance(ret, OciMediaDescriptor):
+            return ret
+
+        else:
+            raise TypeError(ret)

{omlish-0.0.0.dev220 → omlish-0.0.0.dev221}/omlish/docker/oci/data.py

@@ -2,16 +2,18 @@
 # @omlish-lite
 import abc
 import dataclasses as dc
+import enum
 import typing as ta
 
 from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
 from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+from .datarefs import OciDataRef
 
 
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass()
 class OciDataclass(abc.ABC):  # noqa
     pass
 
@@ -19,17 +21,52 @@ class OciDataclass(abc.ABC):  # noqa
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass()
+class OciImageIndex(OciDataclass):
+    manifests: ta.List[ta.Union['OciImageIndex', 'OciImageManifest']]
+
+    annotations: ta.Optional[ta.Dict[str, str]] = None
+
+
+#
+
+
+@dc.dataclass()
+class OciImageManifest(OciDataclass):
+    config: 'OciImageConfig'
+
+    layers: ta.List['OciImageLayer']
+
+
+#
+
+
+@dc.dataclass()
+class OciImageLayer(OciDataclass):
+    class Kind(enum.Enum):
+        TAR = enum.auto()
+        TAR_GZIP = enum.auto()
+        TAR_ZSTD = enum.auto()
+
+    kind: Kind
+
+    data: OciDataRef
+
+
+#
+
+
+@dc.dataclass()
 class OciImageConfig(OciDataclass):
     """https://github.com/opencontainers/image-spec/blob/92353b0bee778725c617e7d57317b568a7796bd0/config.md"""
 
     architecture: str
     os: str
 
-    @dc.dataclass(frozen=True)
+    @dc.dataclass()
     class RootFs:
         type: str
-        diff_ids: ta.Sequence[str]
+        diff_ids: ta.List[str]
 
     rootfs: RootFs
 
@@ -38,7 +75,7 @@ class OciImageConfig(OciDataclass):
     created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
     author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
     os_version: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.version', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
-    os_features: ta.Optional[ta.Sequence[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
+    os_features: ta.Optional[ta.List[str]] = dc.field(default=None, metadata={OBJ_MARSHALER_FIELD_KEY: 'os.features', OBJ_MARSHALER_OMIT_IF_NONE: True})  # noqa
     variant: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
     """
@@ -58,9 +95,9 @@ class OciImageConfig(OciDataclass):
         CpuShares integer, OPTIONAL
         Healthcheck object, OPTIONAL
     """
-    config: ta.Optional[ta.Mapping[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    config: ta.Optional[ta.Dict[str, ta.Any]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
-    @dc.dataclass(frozen=True)
+    @dc.dataclass()
     class History:
         created: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
         author: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
@@ -68,4 +105,21 @@ class OciImageConfig(OciDataclass):
         comment: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
         empty_layer: ta.Optional[bool] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
-    history: ta.Optional[ta.Sequence[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+    history: ta.Optional[ta.List[History]] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
+
+
+##
+
+
+def is_empty_oci_dataclass(obj: OciDataclass) -> bool:
+    if not isinstance(obj, OciDataclass):
+        raise TypeError(obj)
+
+    elif isinstance(obj, OciImageIndex):
+        return not obj.manifests
+
+    elif isinstance(obj, OciImageManifest):
+        return not obj.layers
+
+    else:
+        return False

omlish-0.0.0.dev221/omlish/docker/oci/datarefs.py

@@ -0,0 +1,98 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import dataclasses as dc
+import hashlib
+import io
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRef(abc.ABC):  # noqa
+    pass
+
+
+@dc.dataclass(frozen=True)
+class BytesOciDataRef(OciDataRef):
+    data: bytes
+
+
+@dc.dataclass(frozen=True)
+class FileOciDataRef(OciDataRef):
+    path: str
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class OciDataRefInfo:
+    data: OciDataRef
+
+    @cached_nullary
+    def sha256(self) -> str:
+        if isinstance(self.data, FileOciDataRef):
+            with open(self.data.path, 'rb') as f:
+                return hashlib.file_digest(f, 'sha256').hexdigest()  # noqa
+
+        elif isinstance(self.data, BytesOciDataRef):
+            return hashlib.sha256(self.data.data).hexdigest()
+
+        else:
+            raise TypeError(self.data)
+
+    @cached_nullary
+    def digest(self) -> str:
+        return f'sha256:{self.sha256()}'
+
+    @cached_nullary
+    def size(self) -> int:
+        if isinstance(self.data, FileOciDataRef):
+            return os.path.getsize(self.data.path)
+
+        elif isinstance(self.data, BytesOciDataRef):
+            return len(self.data.data)
+
+        else:
+            raise TypeError(self.data)
+
+
+def write_oci_data_ref_to_file(
+        data: OciDataRef,
+        dst: str,
+        *,
+        symlink: bool = False,
+) -> None:
+    if isinstance(data, FileOciDataRef):
+        if symlink:
+            os.symlink(
+                os.path.relpath(data.path, os.path.dirname(dst)),
+                dst,
+            )
+        else:
+            shutil.copyfile(data.path, dst)
+
+    elif isinstance(data, BytesOciDataRef):
+        with open(dst, 'wb') as f:
+            f.write(data.data)
+
+    else:
+        raise TypeError(data)
+
+
+def open_oci_data_ref(data: OciDataRef) -> ta.BinaryIO:
+    if isinstance(data, FileOciDataRef):
+        return open(data.path, 'rb')
+
+    elif isinstance(data, BytesOciDataRef):
+        return io.BytesIO(data.data)
+
+    else:
+        raise TypeError(data)

omlish-0.0.0.dev221/omlish/docker/oci/loading.py

@@ -0,0 +1,120 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from ...lite.check import check
+from .data import OciImageConfig
+from .data import OciImageIndex
+from .data import OciImageLayer
+from .data import OciImageManifest
+from .data import is_empty_oci_dataclass
+from .media import OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_
+from .media import OCI_MEDIA_FIELDS
+from .media import OciMediaDescriptor
+from .media import OciMediaImageConfig
+from .media import OciMediaImageIndex
+from .media import OciMediaImageManifest
+from .media import unmarshal_oci_media_dataclass
+from .repositories import OciRepository
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+class OciRepositoryLoader:
+    def __init__(
+            self,
+            repo: OciRepository,
+    ) -> None:
+        super().__init__()
+
+        self._repo = repo
+
+    #
+
+    def load_object(
+            self,
+            data: bytes,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> T:
+        text = data.decode('utf-8')
+        dct = json.loads(text)
+        obj = unmarshal_oci_media_dataclass(
+            dct,
+            media_type=media_type,
+        )
+        return check.isinstance(obj, cls)
+
+    def read_object(
+            self,
+            digest: str,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+            *,
+            media_type: ta.Optional[str] = None,
+    ) -> T:
+        data = self._repo.read_blob(digest)
+        return self.load_object(
+            data,
+            cls,
+            media_type=media_type,
+        )
+
+    def read_descriptor(
+            self,
+            desc: OciMediaDescriptor,
+            cls: ta.Type[T] = object,  # type: ignore[assignment]
+    ) -> ta.Any:
+        return self.read_object(
+            desc.digest,
+            cls,
+            media_type=desc.media_type,
+        )
+
+    #
+
+    def from_media(self, obj: ta.Any) -> ta.Any:
+        def make_kw(*exclude):
+            return {
+                a: getattr(obj, a)
+                for f in dc.fields(obj)
+                if (a := f.name) not in OCI_MEDIA_FIELDS
+                and a not in exclude
+            }
+
+        if isinstance(obj, OciMediaImageConfig):
+            return OciImageConfig(**make_kw())
+
+        elif isinstance(obj, OciMediaImageManifest):
+            return OciImageManifest(
+                **make_kw('config', 'layers'),
+                config=self.from_media(self.read_descriptor(obj.config)),
+                layers=[
+                    OciImageLayer(
+                        kind=lk,
+                        data=self._repo.ref_blob(l.digest),
+                    )
+                    for l in obj.layers
+                    if (lk := OCI_IMAGE_LAYER_KIND_MEDIA_TYPES_.get(l.media_type)) is not None
+                ],
+            )
+
+        elif isinstance(obj, OciMediaImageIndex):
+            return OciImageIndex(
+                **make_kw('manifests'),
+                manifests=[
+                    fm
+                    for m in obj.manifests
+                    for fm in [self.from_media(self.read_descriptor(m))]
+                    if not is_empty_oci_dataclass(fm)
+                ],
+            )
+
+        else:
+            raise TypeError(obj)