omdev 0.0.0.dev221__py3-none-any.whl → 0.0.0.dev223__py3-none-any.whl

omdev/oci/repositories.py

@@ -0,0 +1,159 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import abc
+import os.path
+import tarfile
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.os.paths import is_path_in_dir
+
+from .datarefs import BytesOciDataRef
+from .datarefs import FileOciDataRef
+from .datarefs import OciDataRef
+from .datarefs import TarFileOciDataRef
+
+
+##
+
+
+class OciRepository(abc.ABC):
+    @abc.abstractmethod
+    def contains_blob(self, digest: str) -> bool:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def read_blob(self, digest: str) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def ref_blob(self, digest: str) -> OciDataRef:
+        raise NotImplementedError
+
+    @classmethod
+    def of(
+            cls,
+            obj: ta.Union[
+                'OciRepository',
+                str,
+                tarfile.TarFile,
+                ta.Mapping[str, bytes],
+            ],
+    ) -> 'OciRepository':
+        if isinstance(obj, OciRepository):
+            return obj
+
+        elif isinstance(obj, str):
+            check.arg(os.path.isdir(obj))
+            return DirectoryOciRepository(obj)
+
+        elif isinstance(obj, tarfile.TarFile):
+            return TarFileOciRepository(obj)
+
+        elif isinstance(obj, ta.Mapping):
+            return DictOciRepository(obj)
+
+        else:
+            raise TypeError(obj)
+
+
+class FileOciRepository(OciRepository, abc.ABC):
+    @abc.abstractmethod
+    def read_file(self, path: str) -> bytes:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryOciRepository(FileOciRepository):
+    def __init__(self, data_dir: str) -> None:
+        super().__init__()
+
+        self._data_dir = check.non_empty_str(data_dir)
+
+    def read_file(self, path: str) -> bytes:
+        full_path = os.path.join(self._data_dir, path)
+        check.arg(is_path_in_dir(self._data_dir, full_path))
+        with open(full_path, 'rb') as f:
+            return f.read()
+
+    def blob_path(self, digest: str) -> str:
+        scheme, value = digest.split(':')
+        return os.path.join('blobs', scheme, value)
+
+    def blob_full_path(self, digest: str) -> str:
+        path = self.blob_path(digest)
+        full_path = os.path.join(self._data_dir, path)
+        check.arg(is_path_in_dir(self._data_dir, full_path))
+        return full_path
+
+    def contains_blob(self, digest: str) -> bool:
+        return os.path.isfile(self.blob_full_path(digest))
+
+    def read_blob(self, digest: str) -> bytes:
+        return self.read_file(self.blob_path(digest))
+
+    def ref_blob(self, digest: str) -> OciDataRef:
+        return FileOciDataRef(self.blob_full_path(digest))
+
+
+#
+
+
+class TarFileOciRepository(FileOciRepository):
+    def __init__(self, tar_file: tarfile.TarFile) -> None:
+        super().__init__()
+
+        check.arg('r' in tar_file.mode)
+
+        self._tar_file = tar_file
+
+    def read_file(self, path: str) -> bytes:
+        if (ti := self._tar_file.getmember(path)) is None:
+            raise FileNotFoundError(path)
+        with check.not_none(self._tar_file.extractfile(ti)) as f:
+            return f.read()
+
+    def blob_name(self, digest: str) -> str:
+        scheme, value = digest.split(':')
+        return os.path.join('blobs', scheme, value)
+
+    def contains_blob(self, digest: str) -> bool:
+        try:
+            self._tar_file.getmember(self.blob_name(digest))
+        except KeyError:
+            return False
+        else:
+            return True
+
+    def read_blob(self, digest: str) -> bytes:
+        if (ti := self._tar_file.getmember(self.blob_name(digest))) is None:
+            raise KeyError(digest)
+        with check.not_none(self._tar_file.extractfile(ti)) as f:
+            return f.read()
+
+    def ref_blob(self, digest: str) -> OciDataRef:
+        return TarFileOciDataRef(
+            tar_file=self._tar_file,
+            tar_info=self._tar_file.getmember(self.blob_name(digest)),
+        )
+
+
+#
+
+
+class DictOciRepository(OciRepository):
+    def __init__(self, blobs: ta.Mapping[str, bytes]) -> None:
+        super().__init__()
+
+        self._blobs = blobs
+
+    def contains_blob(self, digest: str) -> bool:
+        return digest in self._blobs
+
+    def read_blob(self, digest: str) -> bytes:
+        return self._blobs[digest]
+
+    def ref_blob(self, digest: str) -> OciDataRef:
+        return BytesOciDataRef(self._blobs[digest])

omdev/oci/tars.py

@@ -0,0 +1,144 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import gzip
+import hashlib
+import tarfile
+import typing as ta
+
+from omlish.lite.contextmanagers import ExitStacked
+
+from .compression import OciCompression
+from .datarefs import OciDataRef
+from .datarefs import OciDataRefInfo
+from .datarefs import open_oci_data_ref
+
+
+##
+
+
+class WrittenOciDataTarFileInfo(ta.NamedTuple):
+    compressed_sz: int
+    compressed_sha256: str
+
+    tar_sz: int
+    tar_sha256: str
+
+
+class OciDataTarWriter(ExitStacked):
+    def __init__(
+            self,
+            f: ta.BinaryIO,
+            compression: ta.Optional[OciCompression] = None,
+            *,
+            gzip_level: int = 1,
+            zstd_level: int = 10,
+    ) -> None:
+        super().__init__()
+
+        self._f = f
+        self._compression = compression
+
+        self._gzip_level = gzip_level
+        self._zstd_level = zstd_level
+
+    class _FileWrapper:
+        def __init__(self, f):
+            super().__init__()
+
+            self._f = f
+            self._c = 0
+            self._h = hashlib.sha256()
+
+        @property
+        def size(self) -> int:
+            return self._c
+
+        def sha256(self) -> str:
+            return self._h.hexdigest()
+
+        def write(self, d):
+            self._c += len(d)
+            self._h.update(d)
+            self._f.write(d)
+
+        def tell(self) -> int:
+            return self._f.tell()
+
+    _cw: _FileWrapper
+    _cf: ta.BinaryIO
+
+    _tw: _FileWrapper
+    _tf: tarfile.TarFile
+
+    def info(self) -> WrittenOciDataTarFileInfo:
+        return WrittenOciDataTarFileInfo(
+            compressed_sz=self._cw.size,
+            compressed_sha256=self._cw.sha256(),
+
+            tar_sz=self._tw.size,
+            tar_sha256=self._tw.sha256(),
+        )
+
+    def __enter__(self) -> 'OciDataTarWriter':
+        super().__enter__()
+
+        #
+
+        self._cw = self._FileWrapper(self._f)
+
+        if self._compression is OciCompression.GZIP:
+            self._cf = self._enter_context(
+                gzip.GzipFile(  # type: ignore
+                    fileobj=self._cw,
+                    mode='wb',
+                    compresslevel=self._gzip_level,
+                ),
+            )
+
+        elif self._compression is OciCompression.ZSTD:
+            zc = __import__('zstandard').ZstdCompressor(
+                level=self._zstd_level,
+            )
+            self._cf = self._enter_context(zc.stream_writer(self._cw))
+
+        elif self._compression is None:
+            self._cf = self._cw  # type: ignore
+
+        else:
+            raise ValueError(self._compression)
+
+        #
+
+        self._tw = self._FileWrapper(self._cf)
+
+        self._tf = self._enter_context(
+            tarfile.open(  # type: ignore
+                fileobj=self._tw,
+                mode='w',
+            ),
+        )
+
+        #
+
+        return self
+
+    def tar_file(self) -> tarfile.TarFile:
+        return self._tf
+
+    def add_file(self, ti: tarfile.TarInfo, f: ta.Optional[ta.BinaryIO] = None) -> None:
+        self._tf.addfile(ti, f)
+
+
+def write_oci_data_tar_file(
+        f: ta.BinaryIO,
+        data: ta.Mapping[str, OciDataRef],
+) -> WrittenOciDataTarFileInfo:
+    with OciDataTarWriter(f) as tgw:
+        for n, dr in data.items():
+            ti = tarfile.TarInfo(name=n)
+            ri = OciDataRefInfo(dr)
+            ti.size = ri.size()
+            with open_oci_data_ref(dr) as df:
+                tgw.add_file(ti, df)
+
+    return tgw.info()

omdev/pyproject/resources/python.sh

@@ -2,7 +2,7 @@
 set -e
 
 if [ -z "${VENV}" ] ; then
-    if [ $(uname) = "Linux" ] && (cat /proc/mounts | grep -E '^overlay / .*/(docker|desktop-containerd)/' > /dev/null) ; then
+    if [ $(uname) = "Linux" ] && grep -E '^overlay / .*/(docker|desktop-containerd)/' /proc/mounts > /dev/null ; then
         VENV=docker
     else
         VENV=default