objpath 0.1.0__py3-none-any.whl

objpath/_gs.py

@@ -0,0 +1,447 @@
+from __future__ import annotations
+
+import errno as _errno
+import io as _io
+import typing as _ty
+
+from pathapi import FileStat
+from pathapi import providers as _providers
+
+from . import _failures
+from ._gs_backend import UNSET, BaseGSBackend, GSBackend, _default_backend
+from ._gs_errors import BatchReplyError, http_status, oserror, translate_errors
+from . import _reader
+from ._reader import RangeReader
+from ._digests import digest_hex
+from ._places import check_place
+from ._storepath import ObjectStorePath
+from ._writer import SPOOL_SIZE, SpoolUploadStream
+
+_DELETE_BATCH = 100
+"""Deletes sent in one batch request (the API takes up to 1000)."""
+
+_GSWriteStream = SpoolUploadStream
+
+_UPLOAD_CHUNK = 2 * 1024**2
+"""Bytes of one request of a resumable upload (a multiple of 256 KiB)."""
+
+
+class _GSSnapshot:
+    """The ranged reads of one open stream, as one version of the object. The
+    first response reports the object's generation and every later request
+    requires it (`ifGenerationMatch`), so an object replaced between two of
+    them answers 412 and the stream raises instead of returning the bytes of
+    two objects. Each open has its own, so two opens are two snapshots."""
+
+    def __init__(self, path: "GSPath"):
+        self.object_path = path
+        self.generation = None
+
+    def _failure(self, error: BaseException) -> "OSError | None":
+        path = self.object_path
+        if http_status(error) == 412 and self.generation is not None:
+            # Replaced -- or deleted, which a store may also answer 412.
+            if path._has_object(path.key):
+                return _failures.changed_error(path)
+            return FileNotFoundError(
+                _errno.ENOENT, "No such file or directory", str(path)
+            )
+        return oserror(error, path)
+
+    def fetch(self, start: int, length: int) -> bytes:
+        """The bytes from `start`, at most `length`; none at or past the end (a
+        range there is answered 416, which is also an empty object's answer)."""
+        path = self.object_path
+        # The client is resolved outside the translation, so a missing SDK
+        # raises ImportError instead of FileNotFoundError.
+        blob = path._bucket.blob(path.key)
+        pinned = (
+            {} if self.generation is None else {"if_generation_match": self.generation}
+        )
+        try:
+            # No checksum: the SDK checks a download against the CRC32C or MD5
+            # of the whole object, which a part of it cannot match. The
+            # generation is what ties each range to the object the first read.
+            data = blob.download_as_bytes(
+                start=start,
+                end=start + length - 1,
+                checksum=None,
+                **pinned,
+                **path._options,
+            )
+        except Exception as error:
+            if http_status(error) == 416:
+                return b""
+            failure = self._failure(error)
+            if failure is None:
+                raise
+            raise failure from None
+        if self.generation is None:
+            self.generation = blob.generation
+        return data
+
+    def size(self) -> int:
+        """The size of the version being read."""
+        path = self.object_path
+        if self.generation is None:
+            return path.stat().st_size
+        blob = path._bucket.blob(path.key)
+        try:
+            blob.reload(if_generation_match=self.generation, **path._options)
+        except Exception as error:
+            failure = self._failure(error)
+            if failure is None:
+                raise
+            raise failure from None
+        return blob.size
+
+
+class GSPath(ObjectStorePath):
+    """A path in a Google Cloud Storage bucket, read and written through
+    `google.cloud.storage`. Needs the `gs` extra.
+
+    `GSPath("dir/key", bucket="bkt")` is the key `dir/key` of the bucket
+    `bkt`. The client options (`project=`, `credentials=`,
+    `client_options=`, ...) and `timeout=` / `retry=` build the path's own
+    `GSBackend`; `backend=` shares one client between paths built separately,
+    or plugs in a `BaseGSBackend` of your own. Giving `backend=` together with
+    a client option is a `TypeError`. Without either, paths share one default
+    backend per process. A path joined from another keeps its bucket and its
+    backend, and `bucket=` moves it to another bucket on the same backend.
+    Nothing connects when a path is built.
+
+    GCS has no real directories -- `is_dir()` is prefix emulation (any object
+    key under `"<path>/"`), and `mkdir()` creates a zero-byte `"<path>/"`
+    marker object. `rename()` uses server-side copy + delete (same bucket
+    only) instead of the generic download+upload+delete `move()` fallback; a
+    prefix directory is not renamed (NotImplementedError), so `move()` copies
+    and removes it."""
+
+    __slots__ = ()
+    _LISTING_ERRORS = (Exception,)
+    _CHECKSUMS = frozenset({"md5", "crc32c"})
+
+    if _ty.TYPE_CHECKING:
+        _backend: BaseGSBackend
+
+    def __init__(
+        self,
+        *segments: "str | GSPath",
+        bucket: "str | None" = None,
+        backend: "BaseGSBackend | None" = None,
+        timeout=UNSET,
+        retry=UNSET,
+        **client_kwargs,
+    ):
+        text, donor = self._join(segments)
+        place = check_place("bucket", bucket, donor, "GSPath")
+        options = {
+            name: value
+            for name, value in (("timeout", timeout), ("retry", retry))
+            if value is not UNSET
+        }
+        options.update(client_kwargs)
+        if backend is not None and options:
+            raise TypeError(
+                "pass either backend= or the client options "
+                "(project=, credentials=, timeout=, ...), not both"
+            )
+        if backend is None:
+            if options:
+                backend = GSBackend(**options)
+            elif donor is not None:
+                backend = donor._backend
+            else:
+                backend = _default_backend()
+        self._initialize(text, place, backend)
+
+    def _keywords(self) -> dict:
+        return {"bucket": self._place}
+
+    def _uri_root(self) -> str:
+        return f"gs://{self._place}"
+
+    def __repr__(self) -> str:
+        shown = [repr(self.as_posix()), f"bucket={self._place!r}"]
+        if self._backend is not _default_backend():
+            shown.append(f"backend={self._backend!r}")
+        return f"{type(self).__name__}({', '.join(shown)})"
+
+    @property
+    def bucket(self) -> str:
+        """The name of the bucket."""
+        return self._place
+
+    @property
+    def _client(self):
+        return self._backend.client()
+
+    @property
+    def _bucket(self):
+        return self._client.bucket(self._place)
+
+    @property
+    def _options(self) -> dict:
+        """The backend's `timeout=`/`retry=` for an SDK call."""
+        options = getattr(self._backend, "call_options", None)
+        return options() if callable(options) else {}
+
+    def _reload(self, key: str):
+        """The reloaded blob at `key`, or None if there is no such object.
+        Any other failure raises (as OSError when it is an API error)."""
+        blob = self._bucket.blob(key)
+        try:
+            with translate_errors(self):
+                blob.reload(**self._options)
+        except FileNotFoundError:
+            return None
+        return blob
+
+    def _scandir(self):
+        # Each list_blobs call already carries size/mtime for every object --
+        # reuse it instead of `iterdir()` + a stat call per child.
+        prefix = f"{self.key}/" if self.key else ""
+        seen = set()
+        with translate_errors(self):
+            iterator = self._bucket.list_blobs(
+                prefix=prefix, delimiter="/", **self._options
+            )
+            blobs = list(iterator)
+            prefixes = list(iterator.prefixes)
+        if not blobs and not prefixes and self.key:
+            # Nothing under the prefix: a missing path or a file, which
+            # pathlib's iterdir() refuses. Only an empty listing pays this.
+            if not self.stat().is_dir():
+                raise NotADirectoryError(_errno.ENOTDIR, "Not a directory", str(self))
+        # A key that is both an object and a prefix (`x` and `x/y`) lists as
+        # the object, agreeing with stat()'s exact-object precedence; the
+        # subtree under it is not listed.
+        objects = {blob.name[len(prefix) :] for blob in blobs}
+        # Common prefixes (directories)
+        for common_prefix in prefixes:
+            name = common_prefix[len(prefix) :].rstrip("/")
+            if name in objects:
+                continue
+            # A key segment is whatever its writer chose: "..", "." or an
+            # embedded "/" must not become a child path.
+            if _providers.is_safe_child_name(name) and name not in seen:
+                seen.add(name)
+                yield name, FileStat(is_dir=True)
+        # Blobs (files)
+        for blob in blobs:
+            name = blob.name[len(prefix) :]
+            if name.endswith("/"):
+                continue
+            if _providers.is_safe_child_name(name) and name not in seen:
+                seen.add(name)
+                mtime = int(blob.updated.timestamp()) if blob.updated else 0
+                yield name, FileStat(
+                    st_size=blob.size or 0, st_mtime=mtime, is_dir=False
+                )
+
+    def _open(self, mode="r", buffering=-1):
+        if mode in ("r", "r+"):
+            if not self.key:
+                raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+            snapshot = _GSSnapshot(self)
+            try:
+                first = snapshot.fetch(0, _reader.READ_CHUNK)
+            except FileNotFoundError:
+                if self.is_dir():
+                    raise IsADirectoryError(
+                        _errno.EISDIR, "Is a directory", str(self)
+                    ) from None
+                raise
+            raw = RangeReader(snapshot.fetch, snapshot.size, first)
+            if mode == "r+":
+                # Read-modify-write: what is written is uploaded on close.
+                with raw:
+                    return _GSWriteStream(self, initial=raw)
+            if buffering == 0:
+                return raw
+            size = _io.DEFAULT_BUFFER_SIZE if buffering in (-1, 1) else buffering
+            return _io.BufferedReader(raw, size)
+        if mode not in ("w", "x"):
+            raise NotImplementedError(f"open(mode={mode!r})")
+        if mode == "x" and self.exists():
+            raise FileExistsError(self)
+        if not self.key:
+            raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+        if mode == "w" and self._holds_keys(self.key):
+            raise IsADirectoryError(_errno.EISDIR, "Is a directory", str(self))
+        return _GSWriteStream(self, exclusive=(mode == "x"))
+
+    def _upload_file(self, file, size: int, *, exclusive=False) -> None:
+        """Upload the `size` bytes of `file` as the object: one request while it
+        fits in memory, else a resumable upload in `_UPLOAD_CHUNK` requests."""
+        if size <= SPOOL_SIZE:
+            self._upload(file.read(), exclusive=exclusive)
+            return
+        blob = self._bucket.blob(self.key, chunk_size=_UPLOAD_CHUNK)
+        # Generation 0 matches only a missing object, as in _upload().
+        conditions = {"if_generation_match": 0} if exclusive else {}
+        with translate_errors(self, create=exclusive):
+            blob.upload_from_file(file, size=size, **conditions, **self._options)
+
+    def _upload(self, data: bytes, *, key=None, exclusive=False) -> None:
+        blob = self._bucket.blob(self.key if key is None else key)
+        with translate_errors(self, create=exclusive):
+            if exclusive:
+                # Generation 0 matches only a missing object: a concurrent
+                # creator makes this fail with 412, never overwritten.
+                blob.upload_from_string(data, if_generation_match=0, **self._options)
+            else:
+                blob.upload_from_string(data, **self._options)
+
+    def _flat_entries(self, prefix: str) -> "list[tuple[str, int]]":
+        """`(name, size)` of every object under `prefix`, with no delimiter."""
+        with translate_errors(self):
+            return [
+                (blob.name, blob.size or 0)
+                for blob in self._bucket.list_blobs(prefix=prefix, **self._options)
+            ]
+
+    def _stat_root(self) -> FileStat:
+        # The bucket root: ask the server, as s3: does with HeadBucket, so a mistyped bucket
+        # does not read as an existing directory. A one-item listing needs only the
+        # object-list permission the rest of the path uses.
+        with translate_errors(self):
+            for _ in self._bucket.list_blobs(max_results=1, **self._options):
+                break
+        return FileStat(is_dir=True)
+
+    def _stat_object(self, key: str) -> "FileStat | None":
+        # Only a not-found reply means "no such object": a transient or
+        # permission error read as "missing" let copy() replace an existing
+        # object.
+        blob = self._reload(key)
+        if blob is None:
+            return None
+        return FileStat(
+            st_size=blob.size,
+            st_mtime=int(blob.updated.timestamp()) if blob.updated else 0,
+            is_dir=False,
+        )
+
+    def _stored_digests(self, key: str, algorithm: str) -> "dict[str, str] | None":
+        blob = self._reload(key)
+        if blob is None:
+            return None
+        digests = {}
+        # An object stored with a content encoding is read back decoded, so
+        # the digests of the stored bytes are not those of what a read gives.
+        if getattr(blob, "content_encoding", None) in (None, "", "identity"):
+            for name, value in (("md5", blob.md5_hash), ("crc32c", blob.crc32c)):
+                if value:
+                    digests[name] = digest_hex(value)
+        return digests
+
+    def _has_object(self, key: str) -> bool:
+        return self._reload(key) is not None
+
+    def _has_prefix(self, key: str) -> bool:
+        with translate_errors(self):
+            for _ in self._bucket.list_blobs(
+                prefix=f"{key}/", max_results=1, **self._options
+            ):
+                return True
+        return False
+
+    def _first_keys(self, prefix: str, limit: int) -> "list[str]":
+        with translate_errors(self):
+            return [
+                blob.name
+                for blob in self._bucket.list_blobs(
+                    prefix=prefix, max_results=limit, **self._options
+                )
+            ]
+
+    def _put_marker(self, key: str) -> None:
+        self._upload(b"", key=key, exclusive=True)
+
+    def _delete_object(self, key: str, *, missing_ok: bool) -> None:
+        blob = self._bucket.blob(key)
+        try:
+            with translate_errors(self):
+                blob.delete(**self._options)
+        except FileNotFoundError:
+            # Deleted since the caller looked. Anything else -- a hold, a
+            # permission error -- is not "already gone" and raises.
+            if not missing_ok:
+                raise
+
+    def _delete_each(self, keys, on_error, *, ignore_missing=False) -> None:
+        for key in keys:
+            try:
+                with translate_errors(self._key_path(key)):
+                    self._bucket.blob(key).delete(**self._options)
+            except Exception as error:
+                if ignore_missing and isinstance(error, FileNotFoundError):
+                    # Removed by the batch that was just retried.
+                    continue
+                if not on_error(error, self._key_path(key)):
+                    raise
+
+    def _delete_batch(self, keys: "list[str]"):
+        """Delete `keys` in one batch request and return its replies, one per
+        key in order. None when the batch cannot say what it did to each key:
+        the client has no batches, the request failed, or the replies do not
+        match what was sent."""
+        make_batch = getattr(self._client, "batch", None)
+        if not callable(make_batch):
+            return None
+        bucket = self._bucket
+        try:
+            with make_batch(raise_exception=False) as batch:
+                for key in keys:
+                    bucket.blob(key).delete(**self._options)
+        except Exception:
+            return None
+        replies = getattr(batch, "_responses", None)
+        if isinstance(replies, list) and len(replies) == len(keys):
+            return replies
+        return None
+
+    def _delete_keys(self, keys: "list[str]", on_error) -> None:
+        for index in range(0, len(keys), _DELETE_BATCH):
+            chunk = keys[index : index + _DELETE_BATCH]
+            replies = self._delete_batch(chunk)
+            if replies is None:
+                # A rejected batch deleted none and a broken one an unknown
+                # number: each key is deleted on its own, so one failing blob
+                # leaves no other behind.
+                self._delete_each(chunk, on_error, ignore_missing=True)
+                continue
+            for key, reply in zip(chunk, replies):
+                status = reply.status_code
+                # A blob gone since the listing is what the delete is for.
+                if 200 <= status < 300 or status == 404:
+                    continue
+                where = self._key_path(key)
+                failed = oserror(BatchReplyError(status, reply), where)
+                if not on_error(failed, where):
+                    raise failed
+
+    def _copy_object(self, target: "GSPath", size: int) -> bool:
+        # A target with no bucket is "a path needs a bucket", as for a write.
+        destination = target._bucket
+        try:
+            self._bucket.copy_blob(
+                self._bucket.blob(self.key), destination, target.key, **self._options
+            )
+        except Exception as error:
+            if http_status(error) == 501:
+                return False
+            translated = oserror(error, self)
+            if translated is None:
+                raise
+            raise translated from None
+        return True
+
+    def _rename_source(self, key: str):
+        return self._reload(key)
+
+    def _rename_copy(self, source_blob, dest, dest_key: str) -> None:
+        with translate_errors(self):
+            self._bucket.copy_blob(source_blob, self._bucket, dest_key, **self._options)
+            source_blob.delete(**self._options)

objpath/_gs_backend.py

@@ -0,0 +1,95 @@
+"""The backends of `GSPath`: the contract and the `google-cloud-storage` one,
+and the one every path built without a backend shares."""
+
+from __future__ import annotations
+
+import threading as _threading
+
+from pathapi import providers as _providers
+
+from ._clients import ClientBackend, LazyClientBackend
+from ._optional import import_client
+
+#: "Not given": `None` is a value for `retry=`.
+UNSET = object()
+
+
+class BaseGSBackend(ClientBackend):
+    """Protocol for obtaining a `google.cloud.storage` Client. Subclass this
+    to plug in custom credential/session handling (e.g. tests can override to
+    point at a fake server); `GSBackend` is the real implementation."""
+
+    __slots__ = ()
+
+    @_providers.notimplemented
+    def client(self):
+        """The `google.cloud.storage` client every call of a path goes through."""
+
+    def call_options(self) -> dict:
+        """Keyword arguments (`timeout=`, `retry=`) added to every SDK call
+        a path makes; none by default, which leaves the SDK's own."""
+        return {}
+
+
+class GSBackend(LazyClientBackend, BaseGSBackend):
+    """Lazily creates and caches a `google.cloud.storage` Client. A single
+    client is reused across threads -- it's documented as thread-safe.
+
+    `client_kwargs` go to `storage.Client` unchanged, `client_options`
+    (dict or `ClientOptions`, `api_endpoint` included) among them. A custom
+    endpoint still authenticates; for an emulator or fake server pass
+    `use_auth_w_custom_endpoint=False` (anonymous credentials), or set
+    `STORAGE_EMULATOR_HOST` yourself -- the SDK reads it.
+
+    `timeout` (seconds, or a `(connect, read)` pair) and `retry` (a
+    `google.api_core.retry.Retry`, or None for no retry) are passed to every
+    call a path makes; left out, the SDK's own defaults apply, which keep an
+    unreachable endpoint waiting for about two minutes. A context manager:
+    leaving the block closes the client."""
+
+    __slots__ = ("client_kwargs", "_options")
+
+    def __init__(self, *, timeout=UNSET, retry=UNSET, **client_kwargs):
+        super().__init__()
+        self.client_kwargs = client_kwargs
+        self._options = {
+            name: value
+            for name, value in (("timeout", timeout), ("retry", retry))
+            if value is not UNSET
+        }
+
+    def __repr__(self) -> str:
+        return f"GSBackend({', '.join(sorted({*self.client_kwargs, *self._options}))})"
+
+    def call_options(self) -> dict:
+        return dict(self._options)
+
+    def _build_client(self):
+        storage = import_client("google.cloud.storage", "gs")
+        # Passed through as given: turning a dict `api_endpoint` into a
+        # process-wide `os.environ["STORAGE_EMULATOR_HOST"]` would redirect
+        # every later client in the process, and any subprocess, to it.
+        return storage.Client(**self.client_kwargs)
+
+
+_DEFAULT_BACKEND: "GSBackend | None" = None
+_DEFAULT_BACKEND_LOCK = _threading.Lock()
+
+
+def _default_backend() -> GSBackend:
+    """The one backend, and so the one client, that every `GSPath` built
+    without a backend shares in this process."""
+    global _DEFAULT_BACKEND
+    with _DEFAULT_BACKEND_LOCK:
+        if _DEFAULT_BACKEND is None:
+            _DEFAULT_BACKEND = GSBackend()
+        return _DEFAULT_BACKEND
+
+
+def _reset_default_backend() -> None:
+    """Forget the shared default backend, so the next path builds a new one.
+    The test suite calls it between tests; paths that already hold the old
+    one keep it."""
+    global _DEFAULT_BACKEND
+    with _DEFAULT_BACKEND_LOCK:
+        _DEFAULT_BACKEND = None

objpath/_gs_errors.py

@@ -0,0 +1,112 @@
+"""How a failure of Google Cloud Storage becomes an `OSError`: the status of an
+API error reply, the HTTP client's failures, and the translation of both."""
+
+from __future__ import annotations
+
+import contextlib as _contextlib
+import errno as _errno
+import json as _json
+import sys as _sys
+
+from . import _failures
+
+
+def http_status(error: BaseException) -> "int | None":
+    """The HTTP status of a `google.api_core.exceptions.GoogleAPICallError`
+    (`NotFound.code == 404`, `Forbidden.code == 403`, ...), read from its
+    `code` attribute so the SDK need not be importable to classify it. None
+    for anything that is not an API error reply."""
+    if isinstance(error, OSError):
+        return None
+    code = getattr(error, "code", None)
+    return code if isinstance(code, int) and not isinstance(code, bool) else None
+
+
+def is_http_client_error(error: BaseException) -> bool:
+    """Whether `error` comes from the HTTP client under the SDK (`requests`
+    or `urllib3`), whose exceptions are what a refused connection, a timeout
+    or a body cut short look like."""
+    return any(
+        cls.__module__.split(".")[0] in ("requests", "urllib3")
+        for cls in type(error).__mro__
+    )
+
+
+def transport_kind(error: BaseException) -> str:
+    """Which `_failures.transport_error()` kind a `requests`/`urllib3` failure is."""
+    if _failures.mentions_timeout(error):
+        return _failures.TIMEOUT
+    names = {cls.__name__ for cls in type(error).__mro__}
+    if names & {"ConnectionError", "NewConnectionError", "ProxyError", "SSLError"}:
+        return _failures.UNREACHABLE
+    if names & {
+        "ChunkedEncodingError",
+        "ContentDecodingError",
+        "ProtocolError",
+        "IncompleteRead",
+    }:
+        return _failures.INTERRUPTED
+    return _failures.FAILED
+
+
+def oserror(error: BaseException, path, *, create=False) -> "OSError | None":
+    """The OSError an API error reply or a transport failure means for
+    `path`, or None for any other exception (a missing SDK, bad credentials,
+    a bug), which must propagate as itself rather than read as "no such
+    file". `create`: the reply answers a conditional create, where 412 means
+    the object exists. A request that ran out of time is TimeoutError, an
+    endpoint that cannot be reached ConnectionError. Raise it `from None`."""
+    # An error of this type exists only once the module that defines it is
+    # loaded, so there is nothing to import here.
+    api_exceptions = _sys.modules.get("google.api_core.exceptions")
+    if isinstance(error, getattr(api_exceptions, "RetryError", ())):
+        # The retry deadline passed: what the last attempt failed with is the
+        # reason, and with nothing to say the deadline itself is a timeout.
+        translated = (
+            None if error.cause is None else oserror(error.cause, path, create=create)
+        )
+        return translated or _failures.transport_error(
+            _failures.TIMEOUT, "GCS", path, error
+        )
+    status = http_status(error)
+    if status is None:
+        if is_http_client_error(error):
+            return _failures.transport_error(transport_kind(error), "GCS", path, error)
+        return None
+    if status == 404:
+        return FileNotFoundError(
+            _errno.ENOENT, f"No such file or directory ({error})", str(path)
+        )
+    if status in (401, 403):
+        return PermissionError(_errno.EACCES, str(error), str(path))
+    if create and status == 412:
+        return FileExistsError(_errno.EEXIST, f"File exists ({error})", str(path))
+    if status in (408, 504):
+        return _failures.transport_error(_failures.TIMEOUT, "GCS", path, error)
+    return OSError(_errno.EIO, f"GCS request failed: {error}", str(path))
+
+
+@_contextlib.contextmanager
+def translate_errors(path, *, create=False):
+    try:
+        yield
+    except Exception as error:
+        translated = oserror(error, path, create=create)
+        if translated is None:
+            raise
+        raise translated from None
+
+
+class BatchReplyError(Exception):
+    """A part of a batch reply that is not a success, shaped as the SDK's own
+    API errors are: the HTTP status is `code`."""
+
+    def __init__(self, code: int, reply):
+        text = ""
+        try:
+            error = _json.loads(reply.content.decode("utf-8", "replace"))["error"]
+            text = error.get("message") or ""
+        except (ValueError, KeyError, AttributeError, TypeError):
+            pass
+        super().__init__(f"{code} {text or getattr(reply, 'reason', '') or 'error'}")
+        self.code = code