capo-backup 0.1.0__tar.gz → 0.2.0__tar.gz

{capo_backup-0.1.0/src/capo_backup.egg-info → capo_backup-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: capo-backup
-Version: 0.1.0
+Version: 0.2.0
 Summary: Python SDK for Backup.
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -18,6 +18,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: zapros>=0.13.0
+Provides-Extra: anyio
+Requires-Dist: anyio>=3.0; extra == "anyio"
 Provides-Extra: sso
 Requires-Dist: capo-sso>=0.2.0; extra == "sso"
 Requires-Dist: capo-sso-oidc>=0.2.0; extra == "sso"

{capo_backup-0.1.0 → capo_backup-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "capo-backup"
-version = "0.1.0"
+version = "0.2.0"
 description = "Python SDK for Backup."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -23,6 +23,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+# Async file request bodies via Body.async_from_path
+anyio = ["anyio>=3.0"]
 # IAM Identity Center (SSO), assume-role and web-identity credentials providers
 sso = [
     "capo-sso>=0.2.0",

{capo_backup-0.1.0 → capo_backup-0.2.0}/src/capo_backup/__init__.py

@@ -48,6 +48,7 @@ from ._auth._providers import (
 )
 from ._auth._signers import Signer as Signer
 from ._auth._signers import SigV4Signer as SigV4Signer
+from ._body import Body as Body
 from ._services._pipeline import (
     AsyncOperationOptions as AsyncOperationOptions,
 )

capo_backup-0.2.0/src/capo_backup/_body.py

@@ -0,0 +1,221 @@
+"""Request body abstraction for streaming operations.
+
+Hand-written, not regenerated.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import AsyncIterator, Callable, Iterator
+from contextlib import (
+    AbstractAsyncContextManager,
+    AbstractContextManager,
+    asynccontextmanager,
+    contextmanager,
+)
+from typing import TYPE_CHECKING, Generic, TypeVar, Union, cast
+
+from ._iter import AnyIterator
+
+if TYPE_CHECKING:
+    import anyio
+else:
+    try:
+        import anyio
+    except ImportError:
+        anyio = None
+
+_CHUNK_SIZE = 64 * 1024
+
+TStream = TypeVar("TStream", Iterator[bytes], AsyncIterator[bytes])
+
+# The opener yields the fresh stream together with its length, or ``None`` when
+# no stream could be produced (an exhausted or non-rewindable source, or logic
+# that decides dynamically it cannot reopen). Pairing the length with each open
+# lets it change between rebuilds instead of being fixed for the whole body.
+_ContextManager = Union[
+    AbstractContextManager[Union[tuple[TStream, int], None]],
+    AbstractAsyncContextManager[Union[tuple[TStream, int], None]],
+]
+
+
+class Body(AnyIterator[bytes], Generic[TStream]):
+    """A replayable request body whose length travels with each open.
+
+    A plain ``Iterator[bytes]`` can be sent only once, so a failed request
+    cannot be retried. A ``Body`` instead holds an *opener* — a context manager
+    that opens a fresh stream over the same source — and rebuilds the stream
+    before every attempt, closing whatever was left of the previous one.
+
+    ``TStream`` is ``Iterator[bytes]`` for bodies built with :meth:`from_path`
+    and ``AsyncIterator[bytes]`` for :meth:`async_from_path`; the former use
+    ``rebuild()``/``close()``, the latter ``arebuild()``/``aclose()``.
+    ``build_request`` rebuilds before every attempt.
+
+    Each open produces ``(stream, length)`` together, so the length is not bound
+    to the whole body and may differ from one rebuild to the next. A rebuild is
+    also not guaranteed to succeed: the source may be exhausted or
+    non-rewindable, so the opener can yield ``None`` and :meth:`rebuild` then
+    returns ``None`` instead of a pair. Callers check the return value::
+
+        for attempt in range(max_attempts):
+            opened = body.rebuild()
+            if opened is None:
+                raise RuntimeError("body could not be rebuilt")
+            stream, length = opened
+            send(stream, length)
+
+    ``length`` also holds the current stream's length as an attribute (``None``
+    before the first rebuild). Iterating a ``Body`` reads its current stream.
+    """
+
+    def __init__(self, opener: Callable[[], _ContextManager[TStream]]) -> None:
+        self._opener = opener
+        self._cm: _ContextManager[TStream] | None = None
+        self._stream: TStream | None = None
+        self.length: int | None = None
+
+    @classmethod
+    def from_path(
+        cls, path: str | os.PathLike[str], chunk_size: int = _CHUNK_SIZE
+    ) -> Body[Iterator[bytes]]:
+        """A body that reads ``path`` in ``chunk_size`` chunks, reopening it (and re-measuring it) on every rebuild."""
+
+        @contextmanager
+        def open_() -> Iterator[tuple[Iterator[bytes], int]]:
+            with open(path, "rb") as file:
+                length = os.fstat(file.fileno()).st_size
+                yield iter(lambda: file.read(chunk_size), b""), length
+
+        return Body(open_)
+
+    @classmethod
+    def async_from_path(
+        cls, path: str | os.PathLike[str], chunk_size: int = _CHUNK_SIZE
+    ) -> Body[AsyncIterator[bytes]]:
+        """The async counterpart of :meth:`from_path`; needs ``anyio`` installed."""
+        if anyio is None:
+            dist = (__package__ or "capo").replace("_", "-")
+            raise RuntimeError(
+                "Body.async_from_path() requires the anyio package; "
+                f"reinstall with the anyio feature enabled: {dist}[anyio]"
+            )
+
+        @asynccontextmanager
+        async def open_() -> AsyncIterator[tuple[AsyncIterator[bytes], int]]:
+            async with await anyio.open_file(path, "rb") as file:
+                length = os.fstat(file.wrapped.fileno()).st_size
+
+                async def chunks() -> AsyncIterator[bytes]:
+                    while chunk := await file.read(chunk_size):
+                        yield chunk
+
+                yield chunks(), length
+
+        return Body(open_)
+
+    @property
+    def stream(self) -> TStream | None:
+        """The current open stream, or ``None`` if none is open (before the first
+        rebuild, or after a close).
+
+        ``build_request`` reuses a non-``None`` stream instead of rebuilding, so
+        the retry layer stays in control of when a fresh stream is created.
+        """
+        return self._stream
+
+    def rebuild(self: Body[Iterator[bytes]]) -> tuple[Iterator[bytes], int] | None:
+        """Reopen the stream, closing any previous one, and return ``(stream, length)``.
+
+        ``None`` if the source could not be reopened.
+        """
+        self.close()
+        cm = self._opener()
+        if not isinstance(cm, AbstractContextManager):
+            raise TypeError(
+                "this Body has an async opener; use it with the async client (arebuild/aclose)"
+            )
+        cm = cast(AbstractContextManager["tuple[Iterator[bytes], int] | None"], cm)
+        opened = cm.__enter__()
+        if opened is None:
+            cm.__exit__(None, None, None)
+            return None
+        stream, length = opened
+        self._stream, self.length, self._cm = stream, length, cm
+        return stream, length
+
+    async def arebuild(
+        self: Body[AsyncIterator[bytes]],
+    ) -> tuple[AsyncIterator[bytes], int] | None:
+        """Async :meth:`rebuild`, returning ``(stream, length)`` or ``None``."""
+        await self.aclose()
+        cm = self._opener()
+        if not isinstance(cm, AbstractAsyncContextManager):
+            raise TypeError(
+                "this Body has a sync opener; use it with the sync client (rebuild/close)"
+            )
+        cm = cast(
+            AbstractAsyncContextManager["tuple[AsyncIterator[bytes], int] | None"], cm
+        )
+        opened = await cm.__aenter__()
+        if opened is None:
+            await cm.__aexit__(None, None, None)
+            return None
+        stream, length = opened
+        self._stream, self.length, self._cm = stream, length, cm
+        return stream, length
+
+    def close(self: Body[Iterator[bytes]]) -> None:
+        cm, self._cm, self._stream, self.length = self._cm, None, None, None
+        if cm is not None:
+            cast(AbstractContextManager[object], cm).__exit__(None, None, None)
+
+    async def aclose(self: Body[AsyncIterator[bytes]]) -> None:
+        cm, self._cm, self._stream, self.length = self._cm, None, None, None
+        if cm is not None:
+            await cast(AbstractAsyncContextManager[object], cm).__aexit__(
+                None, None, None
+            )
+
+    def __next__(self) -> bytes:
+        if self._stream is None:
+            raise RuntimeError("body has no open stream; call rebuild() first")
+        return next(cast(Iterator[bytes], self._stream))
+
+    async def __anext__(self) -> bytes:
+        if self._stream is None:
+            raise RuntimeError("body has no open stream; call rebuild() first")
+        return await cast(AsyncIterator[bytes], self._stream).__anext__()
+
+
+@contextmanager
+def closing_bodies(inp: object) -> Iterator[None]:
+    """Close every :class:`Body` in a request input when the operation finishes.
+
+    The operation method wraps its whole run (request build across retries, plus
+    response handling) in this, so a ``Body``'s stream is released once — and only
+    once — the operation is done. Non-``Body`` values (bytes, plain iterators,
+    ``StaticAnyIterator``) are left untouched.
+    """
+    try:
+        yield
+    finally:
+        if isinstance(inp, dict):
+            for value in inp.values():
+                if isinstance(value, Body):
+                    cast("Body[Iterator[bytes]]", value).close()
+
+
+@asynccontextmanager
+async def aclosing_bodies(inp: object) -> AsyncIterator[None]:
+    """Async :func:`closing_bodies`, awaiting :meth:`Body.aclose`."""
+    try:
+        yield
+    finally:
+        if isinstance(inp, dict):
+            for value in inp.values():
+                if isinstance(value, Body):
+                    await cast("Body[AsyncIterator[bytes]]", value).aclose()
+
+
+__all__ = ["Body", "aclosing_bodies", "closing_bodies"]

{capo_backup-0.1.0 → capo_backup-0.2.0}/src/capo_backup/_services/_pipeline.py

@@ -2,20 +2,24 @@ from __future__ import annotations
 
 import random
 import time
-from collections.abc import Sequence
+from collections.abc import AsyncIterator, Iterator, Sequence
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Awaitable, Callable, Generic, TypeVar
+from typing import TYPE_CHECKING, Awaitable, Callable, Generic, TypeVar, cast
 
 from zapros import (
     AsyncClient,
     Client,
     ConnectionError,
+    ConnectTimeoutError,
+    PoolTimeoutError,
     Response,
     SSLError,
     TimeoutError,
 )
 
 from capo_backup._async import anysleep
+from capo_backup._body import Body
+from capo_backup._iter import StaticAnyIterator
 from capo_backup.errors import ServiceError
 
 if TYPE_CHECKING:
@@ -141,6 +145,43 @@ def _is_retryable(exc: Exception) -> bool:
     return False
 
 
+def _is_pre_transmission(exc: Exception) -> bool:
+    # True when the request body was certainly not written before the error,
+    # so the same stream can be reused as-is. A bare ConnectionError is
+    # ambiguous ("cannot be established or is lost"), so it does not qualify.
+    return isinstance(exc, (ConnectTimeoutError, PoolTimeoutError))
+
+
+def _prepare_retry(inp: object) -> bool:
+    if not isinstance(inp, dict):
+        return True
+    for value in inp.values():
+        if isinstance(value, Body):
+            value = cast(Body[Iterator[bytes]], value)
+            if value.rebuild() is None:
+                return False
+        elif isinstance(value, StaticAnyIterator):
+            continue
+        elif isinstance(value, (Iterator, AsyncIterator)):
+            return False
+    return True
+
+
+async def _aprepare_retry(inp: object) -> bool:
+    if not isinstance(inp, dict):
+        return True
+    for value in inp.values():
+        if isinstance(value, Body):
+            value = cast(Body[AsyncIterator[bytes]], value)
+            if await value.arebuild() is None:
+                return False
+        elif isinstance(value, StaticAnyIterator):
+            continue
+        elif isinstance(value, (Iterator, AsyncIterator)):
+            return False
+    return True
+
+
 def _retry_delay(attempt: int, is_throttling: bool) -> float:
     base = 1.0 if is_throttling else 0.5
     delay = base * (2.0 ** min(attempt - 1, 10))
@@ -162,6 +203,10 @@ def retry() -> Interceptor[TInput, TOutput]:
                     raise
                 last_exc = exc
                 if attempt < max_attempts:
+                    if not _is_pre_transmission(exc) and not _prepare_retry(
+                        request.input
+                    ):
+                        raise
                     is_throttling = (
                         isinstance(exc, ServiceError) and exc.is_throttling_error
                     )
@@ -187,6 +232,10 @@ def aretry() -> AsyncInterceptor[TInput, TOutput]:
                     raise
                 last_exc = exc
                 if attempt < max_attempts:
+                    if not _is_pre_transmission(exc) and not await _aprepare_retry(
+                        request.input
+                    ):
+                        raise
                     is_throttling = (
                         isinstance(exc, ServiceError) and exc.is_throttling_error
                     )

{capo_backup-0.1.0 → capo_backup-0.2.0/src/capo_backup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: capo-backup
-Version: 0.1.0
+Version: 0.2.0
 Summary: Python SDK for Backup.
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -18,6 +18,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: zapros>=0.13.0
+Provides-Extra: anyio
+Requires-Dist: anyio>=3.0; extra == "anyio"
 Provides-Extra: sso
 Requires-Dist: capo-sso>=0.2.0; extra == "sso"
 Requires-Dist: capo-sso-oidc>=0.2.0; extra == "sso"

{capo_backup-0.1.0 → capo_backup-0.2.0}/src/capo_backup.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 pyproject.toml
 src/capo_backup/__init__.py
 src/capo_backup/_async.py
+src/capo_backup/_body.py
 src/capo_backup/_iter.py
 src/capo_backup/_pagination.py
 src/capo_backup/py.typed
@@ -563,4 +564,5 @@ src/capo_backup/types/vault_type.py
 src/capo_backup/types/window_minutes.py
 src/capo_backup/types/_prelude/blob.py
 src/capo_backup/types/_prelude/timestamp.py
+tests/test_body.py
 tests/test_sigv4.py

{capo_backup-0.1.0 → capo_backup-0.2.0}/src/capo_backup.egg-info/requires.txt

@@ -1,5 +1,8 @@
 zapros>=0.13.0
 
+[anyio]
+anyio>=3.0
+
 [sso]
 capo-sso>=0.2.0
 capo-sso-oidc>=0.2.0

capo_backup-0.2.0/tests/test_body.py

@@ -0,0 +1,225 @@
+"""Body: replayable streaming request bodies."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager, contextmanager
+from pathlib import Path
+
+import pytest
+
+from capo_backup._body import Body
+from capo_backup._iter import AnyIterator, ensure_async_iterator, ensure_sync_iterator
+
+
+@pytest.fixture
+def data_file(tmp_path: Path) -> Path:
+    path = tmp_path / "data.bin"
+    path.write_bytes(b"0123456789" * 3)
+    return path
+
+
+def test_from_path_reads_in_chunks(data_file: Path) -> None:
+    body = Body.from_path(data_file, chunk_size=10)
+    assert body.length is None  # not known until the first open
+    opened = body.rebuild()
+    assert opened is not None
+    stream, length = opened
+    assert length == 30
+    assert body.length == 30
+    assert list(stream) == [b"0123456789"] * 3
+    body.close()
+    assert body.length is None  # cleared on close
+
+
+def test_rebuild_returns_stream_and_length(data_file: Path) -> None:
+    body = Body.from_path(data_file, chunk_size=10)
+    opened = body.rebuild()
+    assert opened is not None
+    stream, length = opened
+    assert stream is body.stream
+    assert length == 30
+    body.close()
+
+
+def test_length_can_change_between_rebuilds(data_file: Path) -> None:
+    body = Body.from_path(data_file, chunk_size=10)
+    first = body.rebuild()
+    assert first is not None and first[1] == 30
+    data_file.write_bytes(b"shorter")  # source shrinks between attempts
+    second = body.rebuild()
+    assert second is not None and second[1] == 7
+    assert body.length == 7
+    body.close()
+
+
+def test_rebuild_replays_a_consumed_stream(data_file: Path) -> None:
+    body = Body.from_path(data_file, chunk_size=10)
+    first = body.rebuild()
+    assert first is not None
+    assert next(first[0]) == b"0123456789"
+    second = body.rebuild()
+    assert second is not None
+    assert b"".join(second[0]) == data_file.read_bytes()
+    body.close()
+
+
+def test_rebuild_closes_the_previous_stream(data_file: Path) -> None:
+    body = Body.from_path(data_file)
+    opened = body.rebuild()
+    assert opened is not None
+    first = opened[0]
+    again = body.rebuild()
+    assert again is not None and first is not again[0]
+    with pytest.raises(ValueError):  # the file behind the first stream is closed
+        next(first)
+    body.close()
+
+
+def test_stream_is_none_before_rebuild_and_after_close(data_file: Path) -> None:
+    body = Body.from_path(data_file)
+    assert body.stream is None
+    body.rebuild()
+    assert body.stream is not None
+    body.close()
+    assert body.stream is None
+
+
+def test_iterating_an_unopened_body_raises(data_file: Path) -> None:
+    with pytest.raises(RuntimeError):
+        next(Body.from_path(data_file))
+
+
+def test_close_without_stream_is_a_noop(data_file: Path) -> None:
+    Body.from_path(data_file).close()
+
+
+def test_body_is_an_any_iterator_over_its_stream(data_file: Path) -> None:
+    body = Body.from_path(data_file, chunk_size=10)
+    assert isinstance(body, AnyIterator)
+    assert ensure_sync_iterator(body) is body
+    body.rebuild()
+    assert list(body) == [b"0123456789"] * 3
+    body.close()
+
+
+def test_rebuild_returns_none_when_opener_yields_none() -> None:
+    @contextmanager
+    def opener() -> Iterator[tuple[Iterator[bytes], int] | None]:
+        yield None
+
+    body: Body[Iterator[bytes]] = Body(opener)
+    assert body.rebuild() is None
+    assert body.length is None
+    assert body.stream is None  # nothing was opened
+    body.close()
+
+
+def test_async_from_path(data_file: Path) -> None:
+    anyio = pytest.importorskip("anyio")
+
+    async def main() -> None:
+        body = Body.async_from_path(data_file, chunk_size=10)
+        assert body.length is None
+        assert ensure_async_iterator(body) is body
+        opened = await body.arebuild()
+        assert opened is not None
+        stream, length = opened
+        assert length == 30
+        assert body.length == 30
+        assert [chunk async for chunk in stream] == [b"0123456789"] * 3
+        await body.arebuild()
+        assert [chunk async for chunk in body] == [b"0123456789"] * 3
+        await body.aclose()
+        assert body.stream is None
+
+    anyio.run(main)
+
+
+def test_arebuild_returns_none_when_opener_yields_none() -> None:
+    anyio = pytest.importorskip("anyio")
+
+    @asynccontextmanager
+    async def opener() -> AsyncIterator[tuple[AsyncIterator[bytes], int] | None]:
+        yield None
+
+    async def main() -> None:
+        body: Body[AsyncIterator[bytes]] = Body(opener)
+        assert await body.arebuild() is None
+        assert body.length is None
+        assert body.stream is None
+        await body.aclose()
+
+    anyio.run(main)
+
+
+def test_rebuild_on_an_async_opener_raises_a_clear_typeerror() -> None:
+    # The guard is for untyped callers who reach a sync Body method with an
+    # async opener; typed code cannot construct this pairing.
+    @asynccontextmanager
+    async def opener() -> AsyncIterator[tuple[AsyncIterator[bytes], int]]:
+        async def chunks() -> AsyncIterator[bytes]:
+            yield b"x"
+
+        yield chunks(), 1
+
+    body: Body[Iterator[bytes]] = Body(opener)
+    with pytest.raises(TypeError, match="async opener"):
+        body.rebuild()
+
+
+def test_arebuild_on_a_sync_opener_raises_a_clear_typeerror() -> None:
+    anyio = pytest.importorskip("anyio")
+
+    @contextmanager
+    def opener() -> Iterator[tuple[Iterator[bytes], int]]:
+        yield iter([b"x"]), 1
+
+    async def main() -> None:
+        body: Body[AsyncIterator[bytes]] = Body(opener)
+        with pytest.raises(TypeError, match="sync opener"):
+            await body.arebuild()
+
+    anyio.run(main)
+
+
+def test_custom_opener() -> None:
+    opened = 0
+
+    @contextmanager
+    def opener() -> Iterator[tuple[Iterator[bytes], int]]:
+        nonlocal opened
+        opened += 1
+        yield iter([b"ab", b"cd"]), 4
+
+    body: Body[Iterator[bytes]] = Body(opener)
+    body.rebuild()
+    result = body.rebuild()
+    assert opened == 2
+    assert result is not None
+    stream, length = result
+    assert stream is body.stream and length == 4
+    assert b"".join(stream) == b"abcd"
+
+
+def test_custom_async_opener() -> None:
+    anyio = pytest.importorskip("anyio")
+
+    @asynccontextmanager
+    async def opener() -> AsyncIterator[tuple[AsyncIterator[bytes], int]]:
+        async def chunks() -> AsyncIterator[bytes]:
+            yield b"ab"
+            yield b"cd"
+
+        yield chunks(), 4
+
+    async def main() -> None:
+        body: Body[AsyncIterator[bytes]] = Body(opener)
+        opened = await body.arebuild()
+        assert opened is not None
+        stream, length = opened
+        assert length == 4
+        assert b"".join([chunk async for chunk in stream]) == b"abcd"
+        await body.aclose()
+
+    anyio.run(main)