objectstore-client 0.2.10__tar.gz → 0.3.0__tar.gz

{objectstore_client-0.2.10 → objectstore_client-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: objectstore-client
-Version: 0.2.10
+Version: 0.3.0
 Summary: Client SDK for Objectstore, the Sentry object storage platform
 Author: Sentry
 Author-email: Sentry <oss@sentry.io>
@@ -211,9 +211,13 @@ session.put(b"payload", origin="203.0.113.42")
 ### Compression
 
 Uploads are compressed with Zstd by default, and downloads are transparently decompressed.
-Compression can be overridden per-upload:
+The supported algorithms are `"zstd"` and `"gzip"`, or `"none"` to disable compression.
+Compression can be set per-usecase or overridden per-upload:
 
 ```python
+# compress all uploads of this usecase with zstd (the default):
+usecase = Usecase("my-usecase", compression="zstd")
+
 # upload as-is and record no encoding:
 session.put(video_data, compress="none")
 
@@ -235,95 +239,36 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
 session.put(b"payload", metadata={"source": "upload-service"})
 ```
 
-### Multipart Upload API
-
-For large objects, use multipart uploads to upload parts independently and then
-assemble them into a final object.
-
-**Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
-`compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
-records the algorithm, and the caller must pre-compress each part accordingly.
-
-```python
-from concurrent.futures import ThreadPoolExecutor
-
-import zstandard
-
-from objectstore_client.multipart import MultipartCompleteError
-
-upload = session.initiate_multipart_upload(
-    key="my-large-object",
-    compression="zstd",
-    metadata={"source": "upload-service"},
-)
-
-compressor = zstandard.ZstdCompressor()
-chunks = [b"part1", b"part2", b"part3", b"part4"]
-
-def upload_part(part_number: int, data: bytes):
-    compressed = compressor.compress(data)
-    return upload.put_part(
-        compressed, part_number=part_number, content_length=len(compressed)
-    )
-
-with ThreadPoolExecutor(max_workers=4) as executor:
-    futures = [
-        executor.submit(upload_part, i + 1, chunk)
-        for i, chunk in enumerate(chunks)
-    ]
-    parts = [f.result() for f in futures]
-
-try:
-    key = upload.complete(parts)
-except MultipartCompleteError:
-    upload.abort()
-    raise
-```
-
-To resume an in-progress multipart upload after a process restart, persist the
-`key` and `upload_id`, then reconstruct the upload handle later:
-
-```python
-saved_key = upload.key
-saved_upload_id = upload.upload_id
-
-resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
-existing_parts = resumed.list_parts()
-
-# Upload missing parts...
-
-key = resumed.complete(new_parts + existing_parts)
-```
-
 ### Authentication
 
 If your Objectstore instance enforces authorization, you must configure authentication
 via the `token` parameter on `Client`. It accepts either:
 
-- A **`SecretKey`** — for internal services that have access to an EdDSA keypair.
-  The key signs a fresh JWT for each request, scoped to the specific usecase
-  and scope being accessed, and can also sign pre-signed URLs.
+- A **`TokenGenerator`** — for internal services that have access to an EdDSA keypair.
+  It wraps a `SecretKey` and signs a fresh JWT for each request, scoped to the
+  specific usecase and scope being accessed. Its key can also sign pre-signed URLs.
+  The generator's `expiry_seconds` and `permissions` are the defaults for every
+  token it signs; per-token overrides can only narrow them.
 - A **`str`** — a pre-signed JWT, used as-is for every request.
   Use this for external services that receive a token from another source.
 
 ```python
 from objectstore_client import Client, Usecase
-from objectstore_client.auth import SecretKey
+from objectstore_client.auth import SecretKey, TokenGenerator
+
+key = SecretKey(kid="my-service", secret_key="<private key>")
 
 # Option 1: Internal service with a keypair
-client = Client(
-    "http://localhost:8888",
-    token=SecretKey(kid="my-service", secret_key="<private key>"),
-)
+client = Client("http://localhost:8888", token=TokenGenerator(key))
 
 # Option 2: External service with a pre-signed JWT
-# Use SecretKey.token_for_scope() to obtain a static token from an
+# Use TokenGenerator.token_for_scope() to obtain a static token from an
 # internal service, then pass it to the external consumer:
 from objectstore_client.scope import Scope
 
-token = SecretKey(
-    kid="my-service", secret_key="<private key>",
-).token_for_scope("my_app", Scope(org=42, project=1337))
+token = TokenGenerator(key, expiry_seconds=300).token_for_scope(
+    "my_app", Scope(org=42, project=1337)
+)
 
 client = Client("http://localhost:8888", token=token)
 ```
@@ -333,7 +278,7 @@ client = Client("http://localhost:8888", token=token)
 `Session.object_url` returns a GET URL for an object. By default the URL carries
 no auth, so the recipient needs their own key to generate a token for it. Pass
 `token_validity` to embed a read-only token in the URL instead, valid for the
-given duration (requires `SecretKey` authentication - see above section).
+given duration (requires `TokenGenerator` authentication - see above section).
 
 ```python
 from datetime import timedelta
@@ -356,8 +301,8 @@ Pre-signed URLS currently only support `GET` and `HEAD` (a pre-signed URL for
 `GET` authorizes `HEAD` too, and viceversa), so for uploads you should use
 scoped JWTs, as described above.
 
-`Session.presigned_object_url` signs the URL with the session's `SecretKey`,
-so it requires a `SecretKey` and raises `ValueError` otherwise.
+`Session.presigned_object_url` signs the URL with the `SecretKey` of the session's
+`TokenGenerator`, so it requires a `TokenGenerator` and raises `ValueError` otherwise.
 Only `GET` and `HEAD` may be pre-signed, the granted permissions are those
 configured server-side for the signing key, and the
 validity may not exceed one week.
@@ -373,6 +318,22 @@ with urllib.request.urlopen(url) as resp:
     content = resp.read()
 ```
 
+### Download response options
+
+Both `Session.object_url` and `Session.presigned_object_url` accept a
+`response_content_disposition` option:
+
+```python
+url = session.presigned_object_url(
+    "GET",
+    "my-key",
+    response_content_disposition="inline",
+)
+```
+
+Disposition accepts `inline` or `attachment`, defaulting to `attachment` when omitted.
+The stored filename is retained.
+
 ## Configuration
 
 In production, store the `Client` and `Usecase` at module level and reuse them.
@@ -383,7 +344,6 @@ from objectstore_client import Client, Usecase
 
 client = Client(
     "http://localhost:8888",
-    propagate_traces=False,  # default
     retries=3,               # default: 3 connect retries, no read retries
     timeout_ms=None,         # default: no read timeout (connect: 100ms)
     connection_kwargs={},    # default: empty (override urllib3.HTTPConnectionPool kwargs)

{objectstore_client-0.2.10 → objectstore_client-0.3.0}/README.md

@@ -92,9 +92,13 @@ session.put(b"payload", origin="203.0.113.42")
 ### Compression
 
 Uploads are compressed with Zstd by default, and downloads are transparently decompressed.
-Compression can be overridden per-upload:
+The supported algorithms are `"zstd"` and `"gzip"`, or `"none"` to disable compression.
+Compression can be set per-usecase or overridden per-upload:
 
 ```python
+# compress all uploads of this usecase with zstd (the default):
+usecase = Usecase("my-usecase", compression="zstd")
+
 # upload as-is and record no encoding:
 session.put(video_data, compress="none")
 
@@ -116,95 +120,36 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
 session.put(b"payload", metadata={"source": "upload-service"})
 ```
 
-### Multipart Upload API
-
-For large objects, use multipart uploads to upload parts independently and then
-assemble them into a final object.
-
-**Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
-`compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
-records the algorithm, and the caller must pre-compress each part accordingly.
-
-```python
-from concurrent.futures import ThreadPoolExecutor
-
-import zstandard
-
-from objectstore_client.multipart import MultipartCompleteError
-
-upload = session.initiate_multipart_upload(
-    key="my-large-object",
-    compression="zstd",
-    metadata={"source": "upload-service"},
-)
-
-compressor = zstandard.ZstdCompressor()
-chunks = [b"part1", b"part2", b"part3", b"part4"]
-
-def upload_part(part_number: int, data: bytes):
-    compressed = compressor.compress(data)
-    return upload.put_part(
-        compressed, part_number=part_number, content_length=len(compressed)
-    )
-
-with ThreadPoolExecutor(max_workers=4) as executor:
-    futures = [
-        executor.submit(upload_part, i + 1, chunk)
-        for i, chunk in enumerate(chunks)
-    ]
-    parts = [f.result() for f in futures]
-
-try:
-    key = upload.complete(parts)
-except MultipartCompleteError:
-    upload.abort()
-    raise
-```
-
-To resume an in-progress multipart upload after a process restart, persist the
-`key` and `upload_id`, then reconstruct the upload handle later:
-
-```python
-saved_key = upload.key
-saved_upload_id = upload.upload_id
-
-resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
-existing_parts = resumed.list_parts()
-
-# Upload missing parts...
-
-key = resumed.complete(new_parts + existing_parts)
-```
-
 ### Authentication
 
 If your Objectstore instance enforces authorization, you must configure authentication
 via the `token` parameter on `Client`. It accepts either:
 
-- A **`SecretKey`** — for internal services that have access to an EdDSA keypair.
-  The key signs a fresh JWT for each request, scoped to the specific usecase
-  and scope being accessed, and can also sign pre-signed URLs.
+- A **`TokenGenerator`** — for internal services that have access to an EdDSA keypair.
+  It wraps a `SecretKey` and signs a fresh JWT for each request, scoped to the
+  specific usecase and scope being accessed. Its key can also sign pre-signed URLs.
+  The generator's `expiry_seconds` and `permissions` are the defaults for every
+  token it signs; per-token overrides can only narrow them.
 - A **`str`** — a pre-signed JWT, used as-is for every request.
   Use this for external services that receive a token from another source.
 
 ```python
 from objectstore_client import Client, Usecase
-from objectstore_client.auth import SecretKey
+from objectstore_client.auth import SecretKey, TokenGenerator
+
+key = SecretKey(kid="my-service", secret_key="<private key>")
 
 # Option 1: Internal service with a keypair
-client = Client(
-    "http://localhost:8888",
-    token=SecretKey(kid="my-service", secret_key="<private key>"),
-)
+client = Client("http://localhost:8888", token=TokenGenerator(key))
 
 # Option 2: External service with a pre-signed JWT
-# Use SecretKey.token_for_scope() to obtain a static token from an
+# Use TokenGenerator.token_for_scope() to obtain a static token from an
 # internal service, then pass it to the external consumer:
 from objectstore_client.scope import Scope
 
-token = SecretKey(
-    kid="my-service", secret_key="<private key>",
-).token_for_scope("my_app", Scope(org=42, project=1337))
+token = TokenGenerator(key, expiry_seconds=300).token_for_scope(
+    "my_app", Scope(org=42, project=1337)
+)
 
 client = Client("http://localhost:8888", token=token)
 ```
@@ -214,7 +159,7 @@ client = Client("http://localhost:8888", token=token)
 `Session.object_url` returns a GET URL for an object. By default the URL carries
 no auth, so the recipient needs their own key to generate a token for it. Pass
 `token_validity` to embed a read-only token in the URL instead, valid for the
-given duration (requires `SecretKey` authentication - see above section).
+given duration (requires `TokenGenerator` authentication - see above section).
 
 ```python
 from datetime import timedelta
@@ -237,8 +182,8 @@ Pre-signed URLS currently only support `GET` and `HEAD` (a pre-signed URL for
 `GET` authorizes `HEAD` too, and viceversa), so for uploads you should use
 scoped JWTs, as described above.
 
-`Session.presigned_object_url` signs the URL with the session's `SecretKey`,
-so it requires a `SecretKey` and raises `ValueError` otherwise.
+`Session.presigned_object_url` signs the URL with the `SecretKey` of the session's
+`TokenGenerator`, so it requires a `TokenGenerator` and raises `ValueError` otherwise.
 Only `GET` and `HEAD` may be pre-signed, the granted permissions are those
 configured server-side for the signing key, and the
 validity may not exceed one week.
@@ -254,6 +199,22 @@ with urllib.request.urlopen(url) as resp:
     content = resp.read()
 ```
 
+### Download response options
+
+Both `Session.object_url` and `Session.presigned_object_url` accept a
+`response_content_disposition` option:
+
+```python
+url = session.presigned_object_url(
+    "GET",
+    "my-key",
+    response_content_disposition="inline",
+)
+```
+
+Disposition accepts `inline` or `attachment`, defaulting to `attachment` when omitted.
+The stored filename is retained.
+
 ## Configuration
 
 In production, store the `Client` and `Usecase` at module level and reuse them.
@@ -264,7 +225,6 @@ from objectstore_client import Client, Usecase
 
 client = Client(
     "http://localhost:8888",
-    propagate_traces=False,  # default
     retries=3,               # default: 3 connect retries, no read retries
     timeout_ms=None,         # default: no read timeout (connect: 100ms)
     connection_kwargs={},    # default: empty (override urllib3.HTTPConnectionPool kwargs)

{objectstore_client-0.2.10 → objectstore_client-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "objectstore-client"
-version = "0.2.10"
+version = "0.3.0"
 description = "Client SDK for Objectstore, the Sentry object storage platform"
 readme = "README.md"
 authors = [{ name = "Sentry", email = "oss@sentry.io" }]

{objectstore_client-0.2.10 → objectstore_client-0.3.0}/src/objectstore_client/__init__.py

@@ -1,4 +1,10 @@
-from objectstore_client.auth import Permission, SecretKey, TokenGenerator, TokenProvider
+from objectstore_client.auth import (
+    Permission,
+    SecretKey,
+    Token,
+    TokenGenerator,
+    TokenProvider,
+)
 from objectstore_client.client import (
     Client,
     GetResponse,
@@ -35,6 +41,7 @@ __all__ = [
     "TimeToIdle",
     "TimeToLive",
     "TokenProvider",
+    "Token",
     "SecretKey",
     "TokenGenerator",
     "MetricsBackend",

objectstore_client-0.3.0/src/objectstore_client/_compression.py

@@ -0,0 +1,175 @@
+"""Streaming compression and decompression for the supported algorithms."""
+
+from __future__ import annotations
+
+import gzip
+import io
+import shutil
+import zlib
+from typing import IO, Protocol, cast
+
+import zstandard
+
+from objectstore_client.metadata import Compression
+
+SUPPORTED_COMPRESSIONS: tuple[Compression, ...] = ("zstd", "gzip")
+"""Compression algorithms that the client can apply and transparently decode."""
+
+
+class _Compressor(Protocol):
+    """A streaming compression object, such as `zlib.compressobj()`."""
+
+    def compress(self, data: bytes, /) -> bytes: ...
+
+    def flush(self) -> bytes: ...
+
+
+class _CompressionReader(io.RawIOBase):
+    """
+    A readable stream that compresses `source` with `compressor` as it is read.
+
+    `tell()` reports the number of compressed bytes produced so far. `seek(0)` is
+    allowed as long as no data has been read yet, so that urllib3 can record the body
+    position. Closing the reader does not close `source`.
+    """
+
+    _CHUNK_SIZE = 64 * 1024
+
+    def __init__(self, source: IO[bytes], compressor: _Compressor) -> None:
+        super().__init__()
+        self._source = source
+        self._compressor = compressor
+        self._buffer = bytearray()
+        self._finished = False
+        self._position = 0
+
+    def readable(self) -> bool:
+        return True
+
+    def read(self, size: int | None = -1, /) -> bytes:
+        if size is None or size < 0:
+            size = -1
+        while not self._finished and (size < 0 or len(self._buffer) < size):
+            if chunk := self._source.read(self._CHUNK_SIZE):
+                self._buffer += self._compressor.compress(chunk)
+            else:
+                self._buffer += self._compressor.flush()
+                self._finished = True
+
+        if size < 0:
+            size = len(self._buffer)
+        out = bytes(self._buffer[:size])
+        del self._buffer[:size]
+        self._position += len(out)
+        return out
+
+    def readall(self) -> bytes:
+        return self.read(-1)
+
+    def readinto(self, buffer: memoryview | bytearray) -> int:  # type: ignore[override]
+        data = self.read(len(buffer))
+        buffer[: len(data)] = data
+        return len(data)
+
+    def tell(self) -> int:
+        return self._position
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int:
+        if offset == 0 and whence == io.SEEK_SET and self._position == 0:
+            return 0
+        raise OSError("Cannot seek in a compressed stream after reading has started")
+
+
+class _GzipReader(gzip.GzipFile):
+    """Decode gzip from a stream, closing the source with the reader."""
+
+    def __init__(self, source: IO[bytes]) -> None:
+        super().__init__(fileobj=source, mode="rb")
+        self._source = source
+
+    def seekable(self) -> bool:
+        return False
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        raise io.UnsupportedOperation("gzip download stream is not seekable")
+
+    def close(self) -> None:
+        if self.closed:
+            return
+        try:
+            super().close()
+        finally:
+            self._source.close()
+
+
+def _gzip_compressor() -> _Compressor:
+    """Return a zlib compressor that emits the gzip format.
+
+    By default, zlib wraps deflate data in the zlib format. Adding 16 to the window
+    size (`wbits`) selects the gzip header and trailer instead, which is what
+    `Content-Encoding: gzip` requires. See `zlib.compressobj`.
+    """
+    return zlib.compressobj(wbits=16 + zlib.MAX_WBITS)
+
+
+def validate(compression: str | None, *, allow_none: bool = True) -> None:
+    """Raise `ValueError` if `compression` is not a supported algorithm.
+
+    `None` is always accepted. `"none"` is accepted only if `allow_none` is set.
+    """
+    if compression is None or (allow_none and compression == "none"):
+        return
+    if compression not in SUPPORTED_COMPRESSIONS:
+        raise ValueError(f"Invalid compression: {compression}")
+
+
+def compressing_reader(source: IO[bytes], compression: Compression) -> IO[bytes]:
+    """Return a readable stream yielding `source` compressed with `compression`.
+
+    The returned stream compresses lazily as it is read, reports the number of
+    compressed bytes produced via `tell()`, and supports `seek(0)` only until
+    reading starts. Closing it does not close `source`.
+    """
+    if compression == "zstd":
+        compressor = zstandard.ZstdCompressor().compressobj()
+        return cast(IO[bytes], _CompressionReader(source, compressor))
+    if compression == "gzip":
+        return cast(IO[bytes], _CompressionReader(source, _gzip_compressor()))
+    raise ValueError(f"Invalid compression: {compression}")
+
+
+def compress_into(
+    source: IO[bytes], dest: IO[bytes], compression: Compression
+) -> tuple[int, int]:
+    """Compress `source` into `dest`, returning the bytes read and written.
+
+    For gzip, `dest` must support `tell()` to measure the compressed size.
+    """
+    if compression == "zstd":
+        read, written = zstandard.ZstdCompressor().copy_stream(source, dest)
+        return read, written
+    if compression == "gzip":
+        start = dest.tell()
+        with gzip.GzipFile(
+            filename="", fileobj=dest, mode="wb", compresslevel=6, mtime=0
+        ) as writer:
+            shutil.copyfileobj(source, writer)
+            read = writer.tell()
+        return read, dest.tell() - start
+    raise ValueError(f"Invalid compression: {compression}")
+
+
+def decompressing_reader(stream: IO[bytes], compression: str) -> IO[bytes]:
+    """Return a readable stream decoding `stream` compressed with `compression`.
+
+    Concatenated zstd frames and gzip members are decoded as a single payload.
+    The returned reader is non-seekable and closes the source when closed.
+    """
+    if compression == "zstd":
+        dctx = zstandard.ZstdDecompressor()
+        return cast(IO[bytes], dctx.stream_reader(stream, read_across_frames=True))
+    if compression == "gzip":
+        return cast(IO[bytes], _GzipReader(stream))
+    raise NotImplementedError(
+        f"Transparent decoding of `{compression}` is not implemented"
+    )

{objectstore_client-0.2.10 → objectstore_client-0.3.0}/src/objectstore_client/_resumable.py

@@ -12,8 +12,8 @@ from typing import IO, TYPE_CHECKING
 from urllib.parse import urlencode
 
 import urllib3
-import zstandard
 
+from objectstore_client import _compression
 from objectstore_client.errors import RequestError, raise_for_status
 from objectstore_client.metadata import Compression, ExpirationPolicy
 from objectstore_client.metrics import measure_storage_operation
@@ -236,7 +236,7 @@ class ResumableUpload:
 def prepare_body(
     contents: bytes | IO[bytes],
     source_size: int,
-    compress: bool,
+    compression: Compression | None,
     spool_dir: str | PathLike[str] | None,
 ) -> Iterator[tuple[IO[bytes], int, int] | None]:
     """Yield the body, source size, and compressed size (equal if uncompressed).
@@ -248,7 +248,7 @@ def prepare_body(
     with (
         BytesIO(contents) if isinstance(contents, bytes) else nullcontext(contents)
     ) as source:
-        if not compress:
+        if compression is None or compression == "none":
             yield source, source_size, source_size
             return
 
@@ -259,8 +259,8 @@ def prepare_body(
             return
 
         with temporary as body:
-            source_size, compressed_size = zstandard.ZstdCompressor().copy_stream(
-                source, body
+            source_size, compressed_size = _compression.compress_into(
+                source, body, compression
             )
             body.seek(0)
             yield body, source_size, compressed_size