objpath 0.1.0__py3-none-any.whl

objpath/_uri_store.py

@@ -0,0 +1,162 @@
+"""`ObjectStoreURIPath`, what `S3URIPath`, `GSURIPath` and `AZURIPath` share: a
+URI path that hands every request to the path of its store that needs no URI,
+and keeps what is about URIs -- the bucket or container the authority and the
+first segment name, the backend the paths of one process share, and the
+paths a callback is given."""
+
+from __future__ import annotations
+
+import typing as _ty
+
+from pathapi.providers import check_follow, copy_error_predicate
+from uripath.schemes import DelegatingURIPath
+
+from ._keys import parse_names
+from ._keys import checked_copy
+
+
+class ObjectStoreURIPath(DelegatingURIPath):
+    """A URI path of an object store, done by the path of the store that
+    needs no URI (`S3Path`, `GSPath`, `AZPath`) on the backend this path
+    carries. Subclasses say which bucket or container a URI names
+    (`_make_core()`) and how a path of the store maps back to a URI
+    (`_from_core()`)."""
+
+    __slots__ = ()
+    _DELEGATED = ("_scandir", "_listdir", "_open", "_mkdir", "unlink", "rmdir")
+
+    def _key_text(self, uri) -> str:
+        """The part of the path of `uri` that is the object key, rooted: the
+        path itself, or what follows the container."""
+        return uri.path or "/"
+
+    @property
+    def key(self) -> str:
+        """The object key this path names: the path without its leading `/`s
+        and without one trailing `/`; an interior empty segment is part of
+        the key."""
+        return "/".join(parse_names(self._key_text(self))).lstrip("/")
+
+    def _core_of(self, target) -> "ObjectStoreURIPath":
+        """The path of this path's core backend that the URI `target` names."""
+        return self._core_path.with_segments(self._key_text(target))
+
+    def _from_core(self, path) -> "ObjectStoreURIPath":
+        """The URI path on this endpoint for a path of the core backend."""
+        raise NotImplementedError
+
+    def _store_copy_possible(self, target) -> bool:
+        """Whether one request of the store can copy this object onto `target`,
+        from what the two URIs say without asking: the same class, reached by
+        one client (one supplied backend, or each path's own default)."""
+        return (
+            type(target) is type(self)
+            and bool(target.key)
+            and self._supplied_backend() is target._supplied_backend()
+        )
+
+    def is_local(self) -> bool:
+        """Whether the store is this machine: never. The host of the URI is a
+        bucket or an account, not a name to resolve."""
+        return False
+
+    def supported_checksums(self) -> "_ty.FrozenSet[str]":
+        with self._delegated():
+            return self._core_path.supported_checksums()
+
+    def checksum(self, algorithm: str = "md5") -> str:
+        with self._delegated():
+            return self._core_path.checksum(algorithm)
+
+    def rename(self, target):
+        # A plain str target is a sibling rename (relative to self's parent): a
+        # literal name, not re-parsed as a URI. A target in another bucket or
+        # container cannot be reached by a native rename (NotImplementedError,
+        # so `move()` copies).
+        target = self._rename_target(target)
+        if isinstance(target, ObjectStoreURIPath) and not self._same_filesystem(target):
+            raise NotImplementedError("rename() onto another backend")
+        destination = self.with_path(target.path)
+        moved = self._core_of(target)
+        with self._delegated((str(moved), str(destination))):
+            self._core_path.rename(moved)
+        # pathlib returns the new path.
+        return destination
+
+    def rm(
+        self,
+        /,
+        recursive=False,
+        missing_ok=False,
+        ignore_error: "bool | _ty.Callable[[Exception, _ty.Self], bool]" = False,
+        *,
+        follow_symlinks=False,
+        follow_binds=False,
+    ):
+        check_follow("follow_symlinks", follow_symlinks)
+        check_follow("follow_binds", follow_binds)
+        on_error = ignore_error
+        if callable(ignore_error):
+
+            def on_error(error, path=None):
+                return ignore_error(
+                    error, self if path is None else self._from_core(path)
+                )
+
+        with self._delegated():
+            return self._core_path.rm(
+                recursive=recursive,
+                missing_ok=missing_ok,
+                ignore_error=on_error,
+                follow_symlinks=follow_symlinks,
+                follow_binds=follow_binds,
+            )
+
+    def copy(
+        self,
+        target,
+        *,
+        overwrite=False,
+        follow_symlinks=True,
+        preserve_metadata=True,
+        recursive=False,
+        ignore_error=None,
+        progress=None,
+        on_error=None,
+    ):
+        # The core path's walk of a prefix is not used: children are URI paths
+        # and are copied through this method, so the refusal of a prefix that
+        # holds keys a walk cannot reach is made here, once.
+        offering = copy_error_predicate(ignore_error, on_error)
+        if isinstance(target, str):
+            target = self._coerce_target(target)
+        with checked_copy(self._core_path, recursive):
+            if self._store_copy_possible(target):
+                shown = progress
+                if progress is not None:
+
+                    def shown(_source, done, total):
+                        progress(self, done, total)
+
+                moved = target._core_path
+                with self._delegated((str(moved), str(target))):
+                    done = self._core_path._copy_in_store(
+                        moved,
+                        overwrite=overwrite,
+                        follow_symlinks=follow_symlinks,
+                        recursive=recursive,
+                        progress=shown,
+                        hint=self._stat_hint,
+                    )
+                if done:
+                    return
+            return super().copy(
+                target,
+                overwrite=overwrite,
+                follow_symlinks=follow_symlinks,
+                preserve_metadata=preserve_metadata,
+                recursive=recursive,
+                ignore_error=ignore_error,
+                progress=progress,
+                **({} if offering is None else {"on_error": offering}),
+            )

objpath/_version.py

@@ -0,0 +1,11 @@
+"""The version of the installed distribution."""
+
+from __future__ import annotations
+
+from importlib import metadata as _metadata
+
+try:
+    __version__ = _metadata.version("objpath")
+except _metadata.PackageNotFoundError:
+    # Run from a source tree that was never installed.
+    __version__ = "0+unknown"

objpath/_writer.py

@@ -0,0 +1,113 @@
+"""The spooled writer of an object stream."""
+
+from __future__ import annotations
+
+import io as _io
+import tempfile as _tempfile
+
+from pathapi import providers as _providers
+
+SPOOL_SIZE = 4 * 1024**2
+"""Bytes a write stream keeps in memory before spilling to a temp file, and
+the largest object a store uploads in one request."""
+
+
+class SpoolUploadStream(_providers.DiscardOnError, _io.BufferedIOBase):
+    """The write stream of a store that uploads an object from a file: it
+    spools the writes (in memory up to `SPOOL_SIZE`, then a temp file) and
+    uploads them on close() with `path._upload_file(file, size, exclusive=)`.
+
+    `exclusive` (`open("x")`) makes the upload a conditional create. With
+    `initial` (`open("r+")`) the spool starts with the object's content at
+    position 0 and is uploaded only if it was modified."""
+
+    #: Bytes kept in memory before the spool moves to a temp file.
+    spool_size = SPOOL_SIZE
+    _dropped = False
+    _dirty = False
+
+    def __init__(self, path, exclusive=False, initial=None):
+        super().__init__()
+        self._path = path
+        self._exclusive = exclusive
+        self._file = _tempfile.SpooledTemporaryFile(max_size=self.spool_size)
+        self._dirty = initial is None
+        if initial is not None:
+            while True:
+                chunk = initial.read(_io.DEFAULT_BUFFER_SIZE)
+                if not chunk:
+                    break
+                self._file.write(chunk)
+            self._file.seek(0)
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+    def _check_open(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+
+    def write(self, data):
+        self._check_open()
+        self._dirty = True
+        return self._file.write(data)
+
+    def read(self, size=-1):
+        self._check_open()
+        return self._file.read(size)
+
+    def read1(self, size=-1):
+        return self.read(size)
+
+    def seek(self, offset, whence=_io.SEEK_SET):
+        self._check_open()
+        self._file.seek(offset, whence)
+        return self._file.tell()
+
+    def tell(self):
+        self._check_open()
+        return self._file.tell()
+
+    def truncate(self, size=None):
+        self._check_open()
+        self._dirty = True
+        if size is None:
+            size = self._file.tell()
+        return self._file.truncate(size)
+
+    def _holds_uncommitted(self):
+        return self._dirty and not self._dropped
+
+    def discard(self):
+        """Drop the spool: `close()` uploads nothing. `open()` calls it when
+        the `with` block over the stream raised, and the collector when the
+        stream is finalized unclosed."""
+        self._dropped = True
+        self._file.close()
+
+    def close(self):
+        if self.closed:
+            return
+        try:
+            if self._dirty and not self._dropped:
+                self._upload()
+        finally:
+            # Closed even when the upload fails, so `IOBase.__del__` does not
+            # retry it (over newer content) at garbage collection.
+            try:
+                self._file.close()
+            finally:
+                super().close()
+
+    def _upload(self):
+        body = self._file
+        body.seek(0, _io.SEEK_END)
+        size = body.tell()
+        body.seek(0)
+        self._path._upload_file(body, size, exclusive=self._exclusive)

objpath/exceptions.py

@@ -0,0 +1,31 @@
+"""The exceptions `objpath` defines.
+
+A failure of a store or of the connection to it is the `OSError` type that
+`pathlib` raises for the same condition (`FileNotFoundError`,
+`PermissionError`, `TimeoutError`, `ConnectionError`, ...); this module holds
+the two that name a cause no builtin does. An exception of a store's SDK is
+never raised across the API. A bad argument is a plain `ValueError` or
+`TypeError`.
+"""
+
+from __future__ import annotations
+
+from pathapi import PathError
+
+__all__ = ["ObjPathError", "ObjectChangedError", "UnreachableKeysError"]
+
+
+class ObjPathError(PathError):
+    """Base class of the exceptions `objpath` defines for a failure at run
+    time that no builtin describes."""
+
+
+class ObjectChangedError(ObjPathError, OSError):
+    """An object was replaced or deleted while a stream was reading it (errno
+    `ESTALE`, else `EIO` where the platform has none): a read is one version
+    of the object and never the bytes of two."""
+
+
+class UnreachableKeysError(ObjPathError, OSError):
+    """A copy or a removal of a prefix was refused because the prefix holds
+    keys a listing does not show (errno `EINVAL`); nothing was changed."""

objpath/uri.py

@@ -0,0 +1,213 @@
+"""The `s3:`, `gs:` and `az:` schemes of `uripath`: `S3URIPath`, `GSURIPath` and
+`AZURIPath`, URI paths that delegate to `S3Path`, `GSPath` and `AZPath`.
+
+Importing this module needs `uripath`: `pip install "objpath[uri]"`. `uripath`
+loads it by itself, through the `uripath.schemes` entry points, the first time
+a path of one of the schemes is used.
+"""
+
+from __future__ import annotations
+
+try:
+    import uripath as _uripath  # noqa: F401
+except ImportError as _error:
+    raise ImportError(
+        'objpath.uri needs the uripath distribution: pip install "objpath[uri]"',
+        name="uripath",
+    ) from _error
+
+import errno as _errno
+import typing as _ty
+
+from pathapi import FileStat
+from pathapi import providers as _providers
+
+from ._az import AZPath
+from ._az_backend import BaseAZBackend
+from ._az_backend import _default_backend as _az_default_backend
+from ._az_errors import translate_errors as _az_translate_errors
+from ._gs import GSPath
+from ._gs_backend import BaseGSBackend
+from ._gs_backend import _default_backend as _gs_default_backend
+from ._s3_backend import BaseS3Backend
+from ._s3_backend import _default_backend as _s3_default_backend
+from ._uri_store import ObjectStoreURIPath
+
+if _ty.TYPE_CHECKING:
+    from ._s3 import S3Path
+
+__all__ = ["AZURIPath", "GSURIPath", "S3URIPath"]
+
+
+class S3URIPath(ObjectStoreURIPath):
+    """`s3:` scheme (`s3://bucket/key/path`): read/write/list via `boto3`,
+    done by an `S3Path` on the client of `backend` (a `BaseS3Backend`; without
+    one, the backend every `s3:` path of the process shares). Needs the `s3`
+    extra. See `S3Path` for the prefix emulation of directories."""
+
+    __SCHEMES = ("s3",)
+    __slots__ = ()
+
+    if _ty.TYPE_CHECKING:
+        backend: BaseS3Backend
+
+    @property
+    def bucket(self) -> str:
+        """The name of the bucket: the host of the URI."""
+        return self.source.host
+
+    def _initbackend(self):
+        return _s3_default_backend()
+
+    def _make_core(self) -> "S3Path":
+        # Imported here: the module reads botocore's exceptions when it is
+        # imported, and importing this one loads no SDK.
+        from ._s3 import S3Path
+
+        if not self.bucket:
+            raise FileNotFoundError(
+                _errno.ENOENT, "an s3: path needs a bucket", str(self)
+            )
+        return S3Path(self.path or "/", bucket=self.bucket, backend=self.backend)
+
+    def _from_core(self, path: "S3Path") -> "S3URIPath":
+        return self.with_path(f"/{path.key}")
+
+
+class GSURIPath(ObjectStoreURIPath):
+    """`gs:` scheme (`gs://bucket/key/path`): read/write/list via
+    `google.cloud.storage`, done by a `GSPath` on the client of `backend` (a
+    `BaseGSBackend`; without one, the backend every `gs:` path of the process
+    shares). Needs the `gs` extra. See `GSPath` for the prefix emulation of
+    directories."""
+
+    __SCHEMES = ("gs",)
+    __slots__ = ()
+
+    if _ty.TYPE_CHECKING:
+        backend: BaseGSBackend
+
+    @property
+    def bucket(self) -> str:
+        """The name of the bucket: the host of the URI."""
+        return self.source.host
+
+    def _initbackend(self):
+        return _gs_default_backend()
+
+    def _make_core(self) -> GSPath:
+        if not self.bucket:
+            raise FileNotFoundError(
+                _errno.ENOENT, "a gs: path needs a bucket", str(self)
+            )
+        return GSPath(self.path or "/", bucket=self.bucket, backend=self.backend)
+
+    def _from_core(self, path: GSPath) -> "GSURIPath":
+        return self.with_path(f"/{path.key}")
+
+
+class AZURIPath(ObjectStoreURIPath):
+    """`az:` scheme (`az://account/container/key/path`): read/write/list via
+    `azure.storage.blob`, done by an `AZPath` on the client of `backend` (a
+    `BaseAZBackend`; without one, the backend every path of the account shares,
+    authenticated by azure-identity's `DefaultAzureCredential`). Needs the `az`
+    extra. The account has the containers as its directories; see `AZPath`
+    for the prefix emulation of directories below them."""
+
+    __SCHEMES = ("az",)
+    __slots__ = ()
+
+    if _ty.TYPE_CHECKING:
+        backend: BaseAZBackend
+
+    @property
+    def account(self) -> str:
+        """The storage account: the host of the URI."""
+        return self.source.host
+
+    @property
+    def container(self) -> str:
+        """The name of the container: the first segment of the path, empty for
+        the account itself."""
+        return self.path.lstrip("/").split("/", 1)[0]
+
+    def _key_text(self, uri) -> str:
+        return "/" + uri.path.lstrip("/").partition("/")[2]
+
+    def _initbackend(self):
+        if not self.account:
+            raise FileNotFoundError(
+                _errno.ENOENT, "an az: path needs a storage account", str(self)
+            )
+        return _az_default_backend(self.account)
+
+    def _make_core(self) -> AZPath:
+        if not self.container:
+            # The account holds containers, not blobs; its few operations are
+            # written below.
+            self.backend
+            raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+        return AZPath(
+            self._key_text(self), container=self.container, backend=self.backend
+        )
+
+    def _from_core(self, path: AZPath) -> "AZURIPath":
+        return self.with_path(f"/{self.container}/{path.key}")
+
+    def _same_location(self, other) -> bool:
+        # The authority is the storage account; a rename is a blob copy within
+        # one container, so the container must match too.
+        if not super()._same_location(other):
+            return False
+        other_container = next((s for s in other.path.split("/") if s), "")
+        return other_container == self.container
+
+    def _store_copy_possible(self, target) -> bool:
+        # The service copies within one container, as rename() does.
+        return super()._store_copy_possible(target) and self._same_location(target)
+
+    def stat(self, *, follow_symlinks=True):
+        if self.container:
+            return super().stat(follow_symlinks=follow_symlinks)
+        self._pop_stat_hint()
+        # The account: ask the server whether it answers, as a container root
+        # does. A one-item listing needs only the permission the rest uses.
+        client = self.backend.client()
+        with _az_translate_errors(self):
+            for _ in client.list_containers(results_per_page=1):
+                break
+        return FileStat(is_dir=True)
+
+    def _scandir(self):
+        if self.container:
+            yield from super()._scandir()
+            return
+        # The account: its containers are its directories.
+        client = self.backend.client()
+        with _az_translate_errors(self):
+            names = [item.name for item in client.list_containers()]
+        for name in names:
+            if _providers.is_safe_child_name(name):
+                yield name, FileStat(is_dir=True)
+
+    def _mkdir(self, mode):
+        if self.container:
+            return super()._mkdir(mode)
+        # The account exists, or stat() says it does not.
+        self.stat()
+        raise FileExistsError(_errno.EEXIST, "File exists", str(self))
+
+    def rmdir(self):
+        if self.container:
+            return super().rmdir()
+        self.stat()
+        raise PermissionError(
+            _errno.EACCES, "removing a container is not supported", str(self)
+        )
+
+    def _open(self, mode="r", buffering=-1):
+        if not self.container:
+            # The backend of an account-less path is the error it raises.
+            self.backend
+            raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+        return super()._open(mode, buffering)