omdev 0.0.0.dev223__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

omdev/ci/docker/repositories.py

@@ -0,0 +1,40 @@
+# ruff: noqa: UP006 UP007
+import abc
+import contextlib
+import shlex
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.timing import log_timing_context
+from omlish.os.temp import temp_dir_context
+
+from ...oci.repositories import DirectoryOciRepository
+from ...oci.repositories import OciRepository
+
+
+##
+
+
+class DockerImageRepositoryOpener(abc.ABC):
+    @abc.abstractmethod
+    def open_docker_image_repository(self, image: str) -> ta.AsyncContextManager[OciRepository]:
+        raise NotImplementedError
+
+
+#
+
+
+class DockerImageRepositoryOpenerImpl(DockerImageRepositoryOpener):
+    @contextlib.asynccontextmanager
+    async def open_docker_image_repository(self, image: str) -> ta.AsyncGenerator[OciRepository, None]:
+        with temp_dir_context() as save_dir:
+            with log_timing_context(f'Saving docker image {image}'):
+                await asyncio_subprocesses.check_call(
+                    ' | '.join([
+                        f'docker save {shlex.quote(image)}',
+                        f'tar x -C {shlex.quote(save_dir)}',
+                    ]),
+                    shell=True,
+                )
+
+            yield DirectoryOciRepository(save_dir)

omdev/ci/github/cache.py

@@ -7,8 +7,10 @@ from omlish.lite.check import check
 from omlish.os.files import unlinking_if_exists
 
 from ..cache import CacheVersion
+from ..cache import DataCache
 from ..cache import DirectoryFileCache
 from ..cache import FileCache
+from ..cache import FileCacheDataCache
 from .client import GithubCacheClient
 from .client import GithubCacheServiceV1Client
 
@@ -16,7 +18,7 @@ from .client import GithubCacheServiceV1Client
 ##
 
 
-class GithubFileCache(FileCache):
+class GithubCache(FileCache, DataCache):
     @dc.dataclass(frozen=True)
     class Config:
         dir: str
@@ -47,6 +49,8 @@ class GithubFileCache(FileCache):
             version=self._version,
         )
 
+    #
+
     async def get_file(self, key: str) -> ta.Optional[str]:
         local_file = self._local.get_cache_file_path(key)
         if os.path.exists(local_file):
@@ -79,3 +83,18 @@ class GithubFileCache(FileCache):
         await self._client.upload_file(key, cache_file_path)
 
         return cache_file_path
+
+    #
+
+    async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
+        local_file = self._local.get_cache_file_path(key)
+        if os.path.exists(local_file):
+            return DataCache.FileData(local_file)
+
+        if (entry := await self._client.get_entry(key)) is None:
+            return None
+
+        return DataCache.UrlData(check.non_empty_str(self._client.get_entry_url(entry)))
+
+    async def put_data(self, key: str, data: DataCache.Data) -> None:
+        await FileCacheDataCache(self).put_data(key, data)

omdev/ci/github/client.py

@@ -13,9 +13,9 @@ from omlish.asyncs.asyncio.asyncio import asyncio_wait_concurrent
 from omlish.lite.check import check
 from omlish.lite.json import json_dumps_compact
 from omlish.lite.logs import log
+from omlish.lite.timing import log_timing_context
 
 from ..consts import CI_CACHE_VERSION
-from ..utils import log_timing_context
 from .api import GithubCacheServiceV1
 from .env import register_github_env_var
 
@@ -31,6 +31,9 @@ class GithubCacheClient(abc.ABC):
     def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
         raise NotImplementedError
 
+    def get_entry_url(self, entry: Entry) -> ta.Optional[str]:
+        return None
+
     @abc.abstractmethod
     def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
         raise NotImplementedError
@@ -97,7 +100,7 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     def _get_loop(self) -> asyncio.AbstractEventLoop:
         if (loop := self._given_loop) is not None:
             return loop
-        return asyncio.get_event_loop()
+        return asyncio.get_running_loop()
 
     #
 
@@ -225,6 +228,10 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
     class Entry(GithubCacheClient.Entry):
         artifact: GithubCacheServiceV1.ArtifactCacheEntry
 
+    def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
+        entry1 = check.isinstance(entry, self.Entry)
+        return entry1.artifact.cache_key
+
     #
 
     def build_get_entry_url_path(self, *keys: str) -> str:

omdev/ci/github/inject.py

@@ -6,7 +6,7 @@ from omlish.lite.inject import InjectorBindings
 from omlish.lite.inject import inj
 
 from ..cache import FileCache
-from .cache import GithubFileCache
+from .cache import GithubCache
 
 
 ##
@@ -20,11 +20,11 @@ def bind_github(
 
     if cache_dir is not None:
         lst.extend([
-            inj.bind(GithubFileCache.Config(
+            inj.bind(GithubCache.Config(
                 dir=cache_dir,
             )),
-            inj.bind(GithubFileCache, singleton=True),
-            inj.bind(FileCache, to_key=GithubFileCache),
+            inj.bind(GithubCache, singleton=True),
+            inj.bind(FileCache, to_key=GithubCache),
         ])
 
     return inj.as_bindings(*lst)

omdev/ci/utils.py

@@ -1,11 +1,7 @@
 # ruff: noqa: UP006 UP007
 import hashlib
-import logging
-import time
 import typing as ta
 
-from omlish.lite.logs import log
-
 
 ##
 
@@ -22,48 +18,3 @@ def read_yaml_file(yaml_file: str) -> ta.Any:
 
 def sha256_str(s: str) -> str:
     return hashlib.sha256(s.encode('utf-8')).hexdigest()
-
-
-##
-
-
-class LogTimingContext:
-    DEFAULT_LOG: ta.ClassVar[logging.Logger] = log
-
-    def __init__(
-            self,
-            description: str,
-            *,
-            log: ta.Optional[logging.Logger] = None,  # noqa
-            level: int = logging.DEBUG,
-    ) -> None:
-        super().__init__()
-
-        self._description = description
-        self._log = log if log is not None else self.DEFAULT_LOG
-        self._level = level
-
-    def set_description(self, description: str) -> 'LogTimingContext':
-        self._description = description
-        return self
-
-    _begin_time: float
-    _end_time: float
-
-    def __enter__(self) -> 'LogTimingContext':
-        self._begin_time = time.time()
-
-        self._log.log(self._level, f'Begin : {self._description}')  # noqa
-
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self._end_time = time.time()
-
-        self._log.log(
-            self._level,
-            f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
-        )
-
-
-log_timing_context = LogTimingContext

omdev/dataserver/targets.py

@@ -16,6 +16,8 @@ class DataServerTarget(abc.ABC):  # noqa
     content_type: ta.Optional[str] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
     content_length: ta.Optional[int] = dc.field(default=None, metadata={OBJ_MARSHALER_OMIT_IF_NONE: True})
 
+    #
+
     @classmethod
     def of(
             cls,
@@ -62,6 +64,36 @@ class DataServerTarget(abc.ABC):  # noqa
         else:
             raise TypeError('No target type provided')
 
+    #
+
+    @classmethod
+    def of_bytes(cls, data: bytes) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data,
+            content_type='application/octet-stream',
+        )
+
+    @classmethod
+    def of_text(cls, data: str) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data.encode('utf-8'),
+            content_type='text/plain; charset=utf-8',
+        )
+
+    @classmethod
+    def of_json(cls, data: str) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data.encode('utf-8'),
+            content_type='application/json; charset=utf-8',
+        )
+
+    @classmethod
+    def of_html(cls, data: str) -> 'BytesDataServerTarget':
+        return BytesDataServerTarget(
+            data=data.encode('utf-8'),
+            content_type='text/html; charset=utf-8',
+        )
+
 
 @dc.dataclass(frozen=True)
 class BytesDataServerTarget(DataServerTarget):

omdev/oci/data.py

@@ -5,6 +5,7 @@ import dataclasses as dc
 import enum
 import typing as ta
 
+from omlish.lite.check import check
 from omlish.lite.marshal import OBJ_MARSHALER_FIELD_KEY
 from omlish.lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
 
@@ -149,3 +150,21 @@ def is_empty_oci_dataclass(obj: OciDataclass) -> bool:
 
     else:
         return False
+
+
+##
+
+
+def get_single_leaf_oci_image_index(image_index: OciImageIndex) -> OciImageIndex:
+    while True:
+        child_manifest = check.single(image_index.manifests)
+        if isinstance(child_manifest, OciImageManifest):
+            break
+        image_index = check.isinstance(child_manifest, OciImageIndex)
+
+    return image_index
+
+
+def get_single_oci_image_manifest(image_index: OciImageIndex) -> OciImageManifest:
+    child_index = check.single(image_index.manifests)
+    return check.isinstance(child_index, OciImageManifest)

omdev/oci/dataserver.py

@@ -1,4 +1,4 @@
-# ruff: noqa: PT009 UP006 UP007
+# ruff: noqa: UP006 UP007
 import typing as ta
 
 from omlish.lite.check import check
@@ -13,6 +13,9 @@ from .datarefs import open_oci_data_ref
 from .media import OCI_MANIFEST_MEDIA_TYPES
 
 
+##
+
+
 def build_oci_repository_data_server_routes(
         repo_name: str,
         built_repo: BuiltOciImageIndexRepository,

omdev/oci/pack/packing.py

@@ -0,0 +1,185 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import heapq
+import tarfile
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+
+from ..compression import OciCompression
+from ..tars import OciDataTarWriter
+from ..tars import WrittenOciDataTarFileInfo
+
+
+##
+
+
+class OciLayerPacker(ExitStacked):
+    def __init__(
+            self,
+            input_file_path: str,
+            output_file_paths: ta.Sequence[str],
+            *,
+            compression: ta.Optional[OciCompression] = None,
+    ) -> None:
+        super().__init__()
+
+        self._input_file_path = input_file_path
+        self._output_file_paths = list(output_file_paths)
+        self._compression = compression
+
+        self._output_file_indexes_by_name: ta.Dict[str, int] = {}
+
+    #
+
+    @cached_nullary
+    def _input_tar_file(self) -> tarfile.TarFile:
+        # FIXME: check uncompressed
+        return self._enter_context(tarfile.open(self._input_file_path))
+
+    #
+
+    @cached_nullary
+    def _entries_by_name(self) -> ta.Mapping[str, tarfile.TarInfo]:
+        return {
+            info.name: info
+            for info in self._input_tar_file().getmembers()
+        }
+
+    #
+
+    class _CategorizedEntries(ta.NamedTuple):
+        files_by_name: ta.Mapping[str, tarfile.TarInfo]
+        non_files_by_name: ta.Mapping[str, tarfile.TarInfo]
+        links_by_name: ta.Mapping[str, tarfile.TarInfo]
+
+    @cached_nullary
+    def _categorized_entries(self) -> _CategorizedEntries:
+        files_by_name: ta.Dict[str, tarfile.TarInfo] = {}
+        non_files_by_name: ta.Dict[str, tarfile.TarInfo] = {}
+        links_by_name: ta.Dict[str, tarfile.TarInfo] = {}
+
+        for name, info in self._entries_by_name().items():
+            if info.type in tarfile.REGULAR_TYPES:
+                files_by_name[name] = info
+            elif info.type in (tarfile.LNKTYPE, tarfile.GNUTYPE_LONGLINK):
+                links_by_name[name] = info
+            else:
+                non_files_by_name[name] = info
+
+        return self._CategorizedEntries(
+            files_by_name=files_by_name,
+            non_files_by_name=non_files_by_name,
+            links_by_name=links_by_name,
+        )
+
+    #
+
+    @cached_nullary
+    def _non_files_sorted_by_name(self) -> ta.Sequence[tarfile.TarInfo]:
+        return sorted(
+            self._categorized_entries().non_files_by_name.values(),
+            key=lambda info: info.name,
+        )
+
+    @cached_nullary
+    def _files_descending_by_size(self) -> ta.Sequence[tarfile.TarInfo]:
+        return sorted(
+            self._categorized_entries().files_by_name.values(),
+            key=lambda info: -check.isinstance(info.size, int),
+        )
+
+    #
+
+    @cached_nullary
+    def _output_files(self) -> ta.Sequence[ta.BinaryIO]:
+        return [
+            self._enter_context(open(output_file_path, 'wb'))
+            for output_file_path in self._output_file_paths
+        ]
+
+    @cached_nullary
+    def _output_tar_writers(self) -> ta.Sequence[OciDataTarWriter]:
+        return [
+            self._enter_context(
+                OciDataTarWriter(
+                    output_file,
+                    compression=self._compression,
+                ),
+            )
+            for output_file in self._output_files()
+        ]
+
+    #
+
+    def _write_entry(
+            self,
+            info: tarfile.TarInfo,
+            output_file_idx: int,
+    ) -> None:
+        check.not_in(info.name, self._output_file_indexes_by_name)
+
+        writer = self._output_tar_writers()[output_file_idx]
+
+        if info.type in tarfile.REGULAR_TYPES:
+            with check.not_none(self._input_tar_file().extractfile(info)) as f:
+                writer.add_file(info, f)  # type: ignore
+
+        else:
+            writer.add_file(info)
+
+        self._output_file_indexes_by_name[info.name] = output_file_idx
+
+    @cached_nullary
+    def _write_non_files(self) -> None:
+        for non_file in self._non_files_sorted_by_name():
+            self._write_entry(non_file, 0)
+
+    @cached_nullary
+    def _write_files(self) -> None:
+        writers = self._output_tar_writers()
+
+        bins = [
+            (writer.info().compressed_sz, i)
+            for i, writer in enumerate(writers)
+        ]
+
+        heapq.heapify(bins)
+
+        for file in self._files_descending_by_size():
+            _, bin_index = heapq.heappop(bins)
+
+            writer = writers[bin_index]
+
+            self._write_entry(file, bin_index)
+
+            bin_size = writer.info().compressed_sz
+
+            heapq.heappush(bins, (bin_size, bin_index))
+
+    @cached_nullary
+    def _write_links(self) -> None:
+        for link in self._categorized_entries().links_by_name.values():
+            link_name = check.non_empty_str(link.linkname)
+
+            output_file_idx = self._output_file_indexes_by_name[link_name]
+
+            self._write_entry(link, output_file_idx)
+
+    @cached_nullary
+    def write(self) -> ta.Mapping[str, WrittenOciDataTarFileInfo]:
+        writers = self._output_tar_writers()
+
+        self._write_non_files()
+        self._write_files()
+        self._write_links()
+
+        for output_tar_writer in writers:
+            output_tar_writer.tar_file().close()
+
+        return {
+            output_file_path: output_tar_writer.info()
+            for output_file_path, output_tar_writer in zip(self._output_file_paths, writers)
+        }

omdev/oci/pack/repositories.py

@@ -0,0 +1,162 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import copy
+import os.path
+import shutil
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.contextmanagers import ExitStacked
+from omlish.lite.timing import log_timing_context
+from omlish.os.temp import temp_dir_context
+
+from ...oci.tars import WrittenOciDataTarFileInfo
+from ..building import BuiltOciImageIndexRepository
+from ..building import build_oci_index_repository
+from ..compression import OciCompression
+from ..data import OciImageIndex
+from ..data import OciImageLayer
+from ..data import OciImageManifest
+from ..data import get_single_leaf_oci_image_index
+from ..data import get_single_oci_image_manifest
+from ..datarefs import FileOciDataRef
+from ..datarefs import open_oci_data_ref
+from ..loading import read_oci_repository_root_index
+from ..repositories import OciRepository
+from .packing import OciLayerPacker
+from .unpacking import OciLayerUnpacker
+
+
+##
+
+
+class OciPackedRepositoryBuilder(ExitStacked):
+    def __init__(
+            self,
+            source_repo: OciRepository,
+            *,
+            temp_dir: ta.Optional[str] = None,
+
+            num_packed_files: int = 3,  # GH actions have this set to 3, the default
+            packed_compression: ta.Optional[OciCompression] = OciCompression.ZSTD,
+    ) -> None:
+        super().__init__()
+
+        self._source_repo = source_repo
+
+        self._given_temp_dir = temp_dir
+
+        check.arg(num_packed_files > 0)
+        self._num_packed_files = num_packed_files
+
+        self._packed_compression = packed_compression
+
+    @cached_nullary
+    def _temp_dir(self) -> str:
+        if (given := self._given_temp_dir) is not None:
+            return given
+        else:
+            return self._enter_context(temp_dir_context())  # noqa
+
+    #
+
+    @cached_nullary
+    def _source_image_index(self) -> OciImageIndex:
+        image_index = read_oci_repository_root_index(self._source_repo)
+        return get_single_leaf_oci_image_index(image_index)
+
+    @cached_nullary
+    def _source_image_manifest(self) -> OciImageManifest:
+        return get_single_oci_image_manifest(self._source_image_index())
+
+    #
+
+    @cached_nullary
+    def _extracted_layer_tar_files(self) -> ta.List[str]:
+        image = self._source_image_manifest()
+
+        layer_tar_files = []
+
+        for i, layer in enumerate(image.layers):
+            if isinstance(layer.data, FileOciDataRef):
+                input_file_path = layer.data.path
+
+            else:
+                input_file_path = os.path.join(self._temp_dir(), f'save-layer-{i}.tar')
+                with open(input_file_path, 'wb') as input_file:  # noqa
+                    with open_oci_data_ref(layer.data) as layer_file:
+                        shutil.copyfileobj(layer_file, input_file, length=1024 * 1024)  # noqa
+
+            layer_tar_files.append(input_file_path)
+
+        return layer_tar_files
+
+    #
+
+    @cached_nullary
+    def _unpacked_tar_file(self) -> str:
+        layer_tar_files = self._extracted_layer_tar_files()
+        unpacked_file = os.path.join(self._temp_dir(), 'unpacked.tar')
+
+        with log_timing_context(f'Unpacking docker image {self._source_repo}'):
+            with OciLayerUnpacker(
+                    layer_tar_files,
+                    unpacked_file,
+            ) as lu:
+                lu.write()
+
+        return unpacked_file
+
+    #
+
+    @cached_nullary
+    def _packed_tar_files(self) -> ta.Mapping[str, WrittenOciDataTarFileInfo]:
+        unpacked_tar_file = self._unpacked_tar_file()
+
+        packed_tar_files = [
+            os.path.join(self._temp_dir(), f'packed-{i}.tar')
+            for i in range(self._num_packed_files)
+        ]
+
+        with log_timing_context(f'Packing docker image {self._source_repo}'):
+            with OciLayerPacker(
+                    unpacked_tar_file,
+                    packed_tar_files,
+                    compression=self._packed_compression,
+            ) as lp:
+                return lp.write()
+
+    #
+
+    @cached_nullary
+    def _packed_image_index(self) -> OciImageIndex:
+        image_index = copy.deepcopy(self._source_image_index())
+
+        image = get_single_oci_image_manifest(image_index)
+
+        image.config.history = None
+
+        written = self._packed_tar_files()
+
+        # FIXME: use prebuilt sha256
+        image.layers = [
+            OciImageLayer(
+                kind=OciImageLayer.Kind.from_compression(self._packed_compression),
+                data=FileOciDataRef(output_file),
+            )
+            for output_file, output_file_info in written.items()
+        ]
+
+        image.config.rootfs.diff_ids = [
+            f'sha256:{output_file_info.tar_sha256}'
+            for output_file_info in written.values()
+        ]
+
+        return image_index
+
+    #
+
+    @cached_nullary
+    def build(self) -> BuiltOciImageIndexRepository:
+        return build_oci_index_repository(self._packed_image_index())