omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/oci/dataserver.py

@@ -0,0 +1,64 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.check import check
+
+from ..dataserver.routes import DataServerRoute
+from ..dataserver.targets import DataServerTarget
+from .building import BuiltOciImageIndexRepository
+from .building import OciRepositoryBuilder
+from .datarefs import BytesOciDataRef
+from .datarefs import FileOciDataRef
+from .datarefs import open_oci_data_ref
+from .media import OCI_MANIFEST_MEDIA_TYPES
+
+
+##
+
+
+def build_oci_repository_data_server_routes(
+        repo_name: str,
+        built_repo: BuiltOciImageIndexRepository,
+) -> ta.List[DataServerRoute]:
+    base_url_path = f'/v2/{repo_name}'
+
+    repo_contents: ta.Dict[str, OciRepositoryBuilder.Blob] = {}
+
+    repo_contents[f'{base_url_path}/manifests/latest'] = built_repo.blobs[built_repo.media_index_descriptor.digest]
+
+    for blob in built_repo.blobs.values():
+        repo_contents['/'.join([
+            base_url_path,
+            'manifests' if blob.media_type in OCI_MANIFEST_MEDIA_TYPES else 'blobs',
+            blob.digest,
+        ])] = blob
+
+    #
+
+    def build_blob_target(blob: OciRepositoryBuilder.Blob) -> ta.Optional[DataServerTarget]:  # noqa
+        kw: dict = dict(
+            content_type=check.non_empty_str(blob.media_type),
+        )
+
+        if isinstance(blob.data, BytesOciDataRef):
+            return DataServerTarget.of(blob.data.data, **kw)
+
+        elif isinstance(blob.data, FileOciDataRef):
+            return DataServerTarget.of(file_path=blob.data.path, **kw)
+
+        else:
+            with open_oci_data_ref(blob.data) as f:
+                data = f.read()
+
+            return DataServerTarget.of(data, **kw)
+
+    #
+
+    return [
+        DataServerRoute(
+            paths=[path],
+            target=target,
+        )
+        for path, blob in repo_contents.items()
+        if (target := build_blob_target(blob)) is not None
+    ]

omdev/oci/loading.py

@@ -18,6 +18,7 @@ from .media import OciMediaImageConfig
 from .media import OciMediaImageIndex
 from .media import OciMediaImageManifest
 from .media import unmarshal_oci_media_dataclass
+from .repositories import FileOciRepository
 from .repositories import OciRepository
 
 
@@ -120,3 +121,22 @@ class OciRepositoryLoader:
 
         else:
             raise TypeError(obj)
+
+
+##
+
+
+def read_oci_repository_root_index(
+        obj: ta.Any,
+        *,
+        file_name: str = 'index.json',
+) -> OciImageIndex:
+    file_repo = check.isinstance(OciRepository.of(obj), FileOciRepository)
+
+    repo_ldr = OciRepositoryLoader(file_repo)
+
+    media_image_idx = repo_ldr.load_object(file_repo.read_file(file_name), OciMediaImageIndex)
+
+    image_idx = repo_ldr.from_media(media_image_idx)
+
+    return check.isinstance(image_idx, OciImageIndex)

omdev/oci/media.py

@@ -25,8 +25,19 @@ OCI_MEDIA_FIELDS: ta.Collection[str] = frozenset([
 @dc.dataclass()
 class OciMediaDataclass(abc.ABC):  # noqa
     SCHEMA_VERSION: ta.ClassVar[int]
+
+    @property
+    def schema_version(self) -> int:
+        raise TypeError
+
     MEDIA_TYPE: ta.ClassVar[str]
 
+    @property
+    def media_type(self) -> str:
+        raise TypeError
+
+    #
+
     def __init_subclass__(cls, **kwargs: ta.Any) -> None:
         super().__init_subclass__(**kwargs)
         for a in OCI_MEDIA_FIELDS:
@@ -157,3 +168,12 @@ class OciMediaImageConfig(OciImageConfig, OciMediaDataclass):
 
     MEDIA_TYPE: ta.ClassVar[str] = 'application/vnd.oci.image.config.v1+json'
     media_type: str = dc.field(default=MEDIA_TYPE, metadata={OBJ_MARSHALER_FIELD_KEY: 'mediaType'})
+
+
+##
+
+
+OCI_MANIFEST_MEDIA_TYPES: ta.AbstractSet[str] = frozenset([
+    OciMediaImageIndex.MEDIA_TYPE,
+    OciMediaImageManifest.MEDIA_TYPE,
+])

omdev/oci/pack/packing.py

@@ -0,0 +1,185 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import heapq
+import tarfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+
+from ..compression import OciCompression
+from ..tars import OciDataTarWriter
+from ..tars import WrittenOciDataTarFileInfo
+
+
+##
+
+
+class OciLayerPacker(ExitStacked):
+    def __init__(
+            self,
+            input_file_path: str,
+            output_file_paths: ta.Sequence[str],
+            *,
+            compression: ta.Optional[OciCompression] = None,
+    ) -> None:
+        super().__init__()
+
+        self._input_file_path = input_file_path
+        self._output_file_paths = list(output_file_paths)
+        self._compression = compression
+
+        self._output_file_indexes_by_name: ta.Dict[str, int] = {}
+
+    #
+
+    @cached_nullary
+    def _input_tar_file(self) -> tarfile.TarFile:
+        # FIXME: check uncompressed
+        return self._enter_context(tarfile.open(self._input_file_path))
+
+    #
+
+    @cached_nullary
+    def _entries_by_name(self) -> ta.Mapping[str, tarfile.TarInfo]:
+        return {
+            info.name: info
+            for info in self._input_tar_file().getmembers()
+        }
+
+    #
+
+    class _CategorizedEntries(ta.NamedTuple):
+        files_by_name: ta.Mapping[str, tarfile.TarInfo]
+        non_files_by_name: ta.Mapping[str, tarfile.TarInfo]
+        links_by_name: ta.Mapping[str, tarfile.TarInfo]
+
+    @cached_nullary
+    def _categorized_entries(self) -> _CategorizedEntries:
+        files_by_name: ta.Dict[str, tarfile.TarInfo] = {}
+        non_files_by_name: ta.Dict[str, tarfile.TarInfo] = {}
+        links_by_name: ta.Dict[str, tarfile.TarInfo] = {}
+
+        for name, info in self._entries_by_name().items():
+            if info.type in tarfile.REGULAR_TYPES:
+                files_by_name[name] = info
+            elif info.type in (tarfile.LNKTYPE, tarfile.GNUTYPE_LONGLINK):
+                links_by_name[name] = info
+            else:
+                non_files_by_name[name] = info
+
+        return self._CategorizedEntries(
+            files_by_name=files_by_name,
+            non_files_by_name=non_files_by_name,
+            links_by_name=links_by_name,
+        )
+
+    #
+
+    @cached_nullary
+    def _non_files_sorted_by_name(self) -> ta.Sequence[tarfile.TarInfo]:
+        return sorted(
+            self._categorized_entries().non_files_by_name.values(),
+            key=lambda info: info.name,
+        )
+
+    @cached_nullary
+    def _files_descending_by_size(self) -> ta.Sequence[tarfile.TarInfo]:
+        return sorted(
+            self._categorized_entries().files_by_name.values(),
+            key=lambda info: -check.isinstance(info.size, int),
+        )
+
+    #
+
+    @cached_nullary
+    def _output_files(self) -> ta.Sequence[ta.BinaryIO]:
+        return [
+            self._enter_context(open(output_file_path, 'wb'))
+            for output_file_path in self._output_file_paths
+        ]
+
+    @cached_nullary
+    def _output_tar_writers(self) -> ta.Sequence[OciDataTarWriter]:
+        return [
+            self._enter_context(
+                OciDataTarWriter(
+                    output_file,
+                    compression=self._compression,
+                ),
+            )
+            for output_file in self._output_files()
+        ]
+
+    #
+
+    def _write_entry(
+            self,
+            info: tarfile.TarInfo,
+            output_file_idx: int,
+    ) -> None:
+        check.not_in(info.name, self._output_file_indexes_by_name)
+
+        writer = self._output_tar_writers()[output_file_idx]
+
+        if info.type in tarfile.REGULAR_TYPES:
+            with check.not_none(self._input_tar_file().extractfile(info)) as f:
+                writer.add_file(info, f)  # type: ignore
+
+        else:
+            writer.add_file(info)
+
+        self._output_file_indexes_by_name[info.name] = output_file_idx
+
+    @cached_nullary
+    def _write_non_files(self) -> None:
+        for non_file in self._non_files_sorted_by_name():
+            self._write_entry(non_file, 0)
+
+    @cached_nullary
+    def _write_files(self) -> None:
+        writers = self._output_tar_writers()
+
+        bins = [
+            (writer.info().compressed_sz, i)
+            for i, writer in enumerate(writers)
+        ]
+
+        heapq.heapify(bins)
+
+        for file in self._files_descending_by_size():
+            _, bin_index = heapq.heappop(bins)
+
+            writer = writers[bin_index]
+
+            self._write_entry(file, bin_index)
+
+            bin_size = writer.info().compressed_sz
+
+            heapq.heappush(bins, (bin_size, bin_index))
+
+    @cached_nullary
+    def _write_links(self) -> None:
+        for link in self._categorized_entries().links_by_name.values():
+            link_name = check.non_empty_str(link.linkname)
+
+            output_file_idx = self._output_file_indexes_by_name[link_name]
+
+            self._write_entry(link, output_file_idx)
+
+    @cached_nullary
+    def write(self) -> ta.Mapping[str, WrittenOciDataTarFileInfo]:
+        writers = self._output_tar_writers()
+
+        self._write_non_files()
+        self._write_files()
+        self._write_links()
+
+        for output_tar_writer in writers:
+            output_tar_writer.tar_file().close()
+
+        return {
+            output_file_path: output_tar_writer.info()
+            for output_file_path, output_tar_writer in zip(self._output_file_paths, writers)
+        }

omdev/oci/pack/repositories.py

@@ -0,0 +1,162 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import copy
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.timing import log_timing_context
+from omlish.os.temp import temp_dir_context
+
+from ...oci.tars import WrittenOciDataTarFileInfo
+from ..building import BuiltOciImageIndexRepository
+from ..building import build_oci_index_repository
+from ..compression import OciCompression
+from ..data import OciImageIndex
+from ..data import OciImageLayer
+from ..data import OciImageManifest
+from ..data import get_single_leaf_oci_image_index
+from ..data import get_single_oci_image_manifest
+from ..datarefs import FileOciDataRef
+from ..datarefs import open_oci_data_ref
+from ..loading import read_oci_repository_root_index
+from ..repositories import OciRepository
+from .packing import OciLayerPacker
+from .unpacking import OciLayerUnpacker
+
+
+##
+
+
+class OciPackedRepositoryBuilder(ExitStacked):
+    def __init__(
+            self,
+            source_repo: OciRepository,
+            *,
+            temp_dir: ta.Optional[str] = None,
+
+            num_packed_files: int = 3,  # GH actions have this set to 3, the default
+            packed_compression: ta.Optional[OciCompression] = OciCompression.ZSTD,
+    ) -> None:
+        super().__init__()
+
+        self._source_repo = source_repo
+
+        self._given_temp_dir = temp_dir
+
+        check.arg(num_packed_files > 0)
+        self._num_packed_files = num_packed_files
+
+        self._packed_compression = packed_compression
+
+    @cached_nullary
+    def _temp_dir(self) -> str:
+        if (given := self._given_temp_dir) is not None:
+            return given
+        else:
+            return self._enter_context(temp_dir_context())  # noqa
+
+    #
+
+    @cached_nullary
+    def _source_image_index(self) -> OciImageIndex:
+        image_index = read_oci_repository_root_index(self._source_repo)
+        return get_single_leaf_oci_image_index(image_index)
+
+    @cached_nullary
+    def _source_image_manifest(self) -> OciImageManifest:
+        return get_single_oci_image_manifest(self._source_image_index())
+
+    #
+
+    @cached_nullary
+    def _extracted_layer_tar_files(self) -> ta.List[str]:
+        image = self._source_image_manifest()
+
+        layer_tar_files = []
+
+        for i, layer in enumerate(image.layers):
+            if isinstance(layer.data, FileOciDataRef):
+                input_file_path = layer.data.path
+
+            else:
+                input_file_path = os.path.join(self._temp_dir(), f'save-layer-{i}.tar')
+                with open(input_file_path, 'wb') as input_file:  # noqa
+                    with open_oci_data_ref(layer.data) as layer_file:
+                        shutil.copyfileobj(layer_file, input_file, length=1024 * 1024)  # noqa
+
+            layer_tar_files.append(input_file_path)
+
+        return layer_tar_files
+
+    #
+
+    @cached_nullary
+    def _unpacked_tar_file(self) -> str:
+        layer_tar_files = self._extracted_layer_tar_files()
+        unpacked_file = os.path.join(self._temp_dir(), 'unpacked.tar')
+
+        with log_timing_context(f'Unpacking docker image {self._source_repo}'):
+            with OciLayerUnpacker(
+                    layer_tar_files,
+                    unpacked_file,
+            ) as lu:
+                lu.write()
+
+        return unpacked_file
+
+    #
+
+    @cached_nullary
+    def _packed_tar_files(self) -> ta.Mapping[str, WrittenOciDataTarFileInfo]:
+        unpacked_tar_file = self._unpacked_tar_file()
+
+        packed_tar_files = [
+            os.path.join(self._temp_dir(), f'packed-{i}.tar')
+            for i in range(self._num_packed_files)
+        ]
+
+        with log_timing_context(f'Packing docker image {self._source_repo}'):
+            with OciLayerPacker(
+                    unpacked_tar_file,
+                    packed_tar_files,
+                    compression=self._packed_compression,
+            ) as lp:
+                return lp.write()
+
+    #
+
+    @cached_nullary
+    def _packed_image_index(self) -> OciImageIndex:
+        image_index = copy.deepcopy(self._source_image_index())
+
+        image = get_single_oci_image_manifest(image_index)
+
+        image.config.history = None
+
+        written = self._packed_tar_files()
+
+        # FIXME: use prebuilt sha256
+        image.layers = [
+            OciImageLayer(
+                kind=OciImageLayer.Kind.from_compression(self._packed_compression),
+                data=FileOciDataRef(output_file),
+            )
+            for output_file, output_file_info in written.items()
+        ]
+
+        image.config.rootfs.diff_ids = [
+            f'sha256:{output_file_info.tar_sha256}'
+            for output_file_info in written.values()
+        ]
+
+        return image_index
+
+    #
+
+    @cached_nullary
+    def build(self) -> BuiltOciImageIndexRepository:
+        return build_oci_index_repository(self._packed_image_index())

omdev/oci/pack/unpacking.py

@@ -0,0 +1,204 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import contextlib
+import os.path
+import tarfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+
+
+##
+
+
+class OciLayerUnpacker(ExitStacked):
+    def __init__(
+            self,
+            input_files: ta.Sequence[ta.Union[str, tarfile.TarFile]],
+            output_file_path: str,
+    ) -> None:
+        super().__init__()
+
+        self._input_files = list(input_files)
+        self._output_file_path = output_file_path
+
+    #
+
+    @contextlib.contextmanager
+    def _open_input_file(self, input_file: ta.Union[str, tarfile.TarFile]) -> ta.Iterator[tarfile.TarFile]:
+        if isinstance(input_file, tarfile.TarFile):
+            yield input_file
+
+        elif isinstance(input_file, str):
+            with tarfile.open(input_file) as tar_file:
+                yield tar_file
+
+        else:
+            raise TypeError(input_file)
+
+    #
+
+    class _Entry(ta.NamedTuple):
+        file: ta.Union[str, tarfile.TarFile]
+        info: tarfile.TarInfo
+
+    def _build_input_file_sorted_entries(self, input_file: ta.Union[str, tarfile.TarFile]) -> ta.Sequence[_Entry]:
+        dct: ta.Dict[str, OciLayerUnpacker._Entry] = {}
+
+        with self._open_input_file(input_file) as input_tar_file:
+            for info in input_tar_file.getmembers():
+                check.not_in(info.name, dct)
+                dct[info.name] = self._Entry(
+                    file=input_file,
+                    info=info,
+                )
+
+        return sorted(dct.values(), key=lambda entry: entry.info.name)
+
+    @cached_nullary
+    def _entries_by_name(self) -> ta.Mapping[str, _Entry]:
+        root: dict = {}
+
+        def find_dir(dir_name: str) -> dict:  # noqa
+            if dir_name:
+                dir_parts = dir_name.split('/')
+            else:
+                dir_parts = []
+
+            cur = root  # noqa
+            for dir_part in dir_parts:
+                cur = cur[dir_part]  # noqa
+
+            return check.isinstance(cur, dict)
+
+        #
+
+        for input_file in self._input_files:
+            sorted_entries = self._build_input_file_sorted_entries(input_file)
+
+            wh_names = set()
+            wh_opaques = set()
+
+            #
+
+            for entry in sorted_entries:
+                info = entry.info
+                name = check.non_empty_str(info.name)
+                base_name = os.path.basename(name)
+                dir_name = os.path.dirname(name)
+
+                if base_name == '.wh..wh..opq':
+                    wh_opaques.add(dir_name)
+                    continue
+
+                if base_name.startswith('.wh.'):
+                    wh_base_name = os.path.basename(base_name[4:])
+                    wh_name = os.path.join(dir_name, wh_base_name)
+                    wh_names.add(wh_name)
+                    continue
+
+                cur = find_dir(dir_name)
+
+                if info.type == tarfile.DIRTYPE:
+                    try:
+                        ex = cur[base_name]
+                    except KeyError:
+                        cur[base_name] = {'': entry}
+                    else:
+                        ex[''] = entry
+
+                else:
+                    cur[base_name] = entry
+
+            #
+
+            for wh_name in reversed(sorted(wh_names)):  # noqa
+                wh_dir_name = os.path.dirname(wh_name)
+                wh_base_name = os.path.basename(wh_name)
+
+                cur = find_dir(wh_dir_name)
+                rm = cur[wh_base_name]
+
+                if isinstance(rm, dict):
+                    # Whiteouts wipe out whole directory:
+                    # https://github.com/containerd/containerd/blob/59c8cf6ea5f4175ad512914dd5ce554942bf144f/pkg/archive/tar_test.go#L648
+                    # check.equal(set(rm), '')
+                    del cur[wh_base_name]
+
+                elif isinstance(rm, self._Entry):
+                    del cur[wh_base_name]
+
+                else:
+                    raise TypeError(rm)
+
+            if wh_opaques:
+                raise NotImplementedError
+
+        #
+
+        out: ta.Dict[str, OciLayerUnpacker._Entry] = {}
+
+        def rec(cur):  # noqa
+            for _, child in sorted(cur.items(), key=lambda t: t[0]):
+                if isinstance(child, dict):
+                    rec(child)
+
+                elif isinstance(child, self._Entry):
+                    check.not_in(child.info.name, out)
+                    out[child.info.name] = child
+
+                else:
+                    raise TypeError(child)
+
+        rec(root)
+
+        return out
+
+    #
+
+    @cached_nullary
+    def _output_tar_file(self) -> tarfile.TarFile:
+        return self._enter_context(tarfile.open(self._output_file_path, 'w'))
+
+    #
+
+    def _add_unpacked_entry(
+            self,
+            input_tar_file: tarfile.TarFile,
+            info: tarfile.TarInfo,
+    ) -> None:
+        base_name = os.path.basename(info.name)
+        check.state(not base_name.startswith('.wh.'))
+
+        if info.type in tarfile.REGULAR_TYPES:
+            with check.not_none(input_tar_file.extractfile(info)) as f:
+                self._output_tar_file().addfile(info, f)
+
+        else:
+            self._output_tar_file().addfile(info)
+
+    def _unpack_file(
+            self,
+            input_file: ta.Union[str, tarfile.TarFile],
+    ) -> None:
+        entries_by_name = self._entries_by_name()
+
+        with self._open_input_file(input_file) as input_tar_file:
+            info: tarfile.TarInfo
+            for info in input_tar_file.getmembers():
+                try:
+                    entry = entries_by_name[info.name]
+                except KeyError:
+                    continue
+
+                if entry.file != input_file:
+                    continue
+
+                self._add_unpacked_entry(input_tar_file, info)
+
+    @cached_nullary
+    def write(self) -> None:
+        for input_file in self._input_files:
+            self._unpack_file(input_file)