objectstore-client 0.2.10__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/PKG-INFO +42 -77
  2. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/README.md +41 -76
  3. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/pyproject.toml +1 -1
  4. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/__init__.py +8 -1
  5. objectstore_client-0.3.1/src/objectstore_client/_compression.py +175 -0
  6. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/_resumable.py +108 -6
  7. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/auth.py +51 -37
  8. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/client.py +88 -178
  9. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/metadata.py +41 -2
  10. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/utils.py +1 -28
  11. objectstore_client-0.2.10/src/objectstore_client/multipart.py +0 -284
  12. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/LICENSE.md +0 -0
  13. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/errors.py +0 -0
  14. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/metrics.py +0 -0
  15. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/presign.py +0 -0
  16. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/py.typed +0 -0
  17. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/scope.py +0 -0
  18. {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/tracing.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: objectstore-client
3
- Version: 0.2.10
3
+ Version: 0.3.1
4
4
  Summary: Client SDK for Objectstore, the Sentry object storage platform
5
5
  Author: Sentry
6
6
  Author-email: Sentry <oss@sentry.io>
@@ -211,9 +211,13 @@ session.put(b"payload", origin="203.0.113.42")
211
211
  ### Compression
212
212
 
213
213
  Uploads are compressed with Zstd by default, and downloads are transparently decompressed.
214
- Compression can be overridden per-upload:
214
+ The supported algorithms are `"zstd"` and `"gzip"`, or `"none"` to disable compression.
215
+ Compression can be set per-usecase or overridden per-upload:
215
216
 
216
217
  ```python
218
+ # compress all uploads of this usecase with zstd (the default):
219
+ usecase = Usecase("my-usecase", compression="zstd")
220
+
217
221
  # upload as-is and record no encoding:
218
222
  session.put(video_data, compress="none")
219
223
 
@@ -235,95 +239,41 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
235
239
  session.put(b"payload", metadata={"source": "upload-service"})
236
240
  ```
237
241
 
238
- ### Multipart Upload API
239
-
240
- For large objects, use multipart uploads to upload parts independently and then
241
- assemble them into a final object.
242
-
243
- **Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
244
- `compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
245
- records the algorithm, and the caller must pre-compress each part accordingly.
246
-
247
- ```python
248
- from concurrent.futures import ThreadPoolExecutor
249
-
250
- import zstandard
251
-
252
- from objectstore_client.multipart import MultipartCompleteError
253
-
254
- upload = session.initiate_multipart_upload(
255
- key="my-large-object",
256
- compression="zstd",
257
- metadata={"source": "upload-service"},
258
- )
259
-
260
- compressor = zstandard.ZstdCompressor()
261
- chunks = [b"part1", b"part2", b"part3", b"part4"]
262
-
263
- def upload_part(part_number: int, data: bytes):
264
- compressed = compressor.compress(data)
265
- return upload.put_part(
266
- compressed, part_number=part_number, content_length=len(compressed)
267
- )
268
-
269
- with ThreadPoolExecutor(max_workers=4) as executor:
270
- futures = [
271
- executor.submit(upload_part, i + 1, chunk)
272
- for i, chunk in enumerate(chunks)
273
- ]
274
- parts = [f.result() for f in futures]
275
-
276
- try:
277
- key = upload.complete(parts)
278
- except MultipartCompleteError:
279
- upload.abort()
280
- raise
281
- ```
282
-
283
- To resume an in-progress multipart upload after a process restart, persist the
284
- `key` and `upload_id`, then reconstruct the upload handle later:
285
-
286
- ```python
287
- saved_key = upload.key
288
- saved_upload_id = upload.upload_id
289
-
290
- resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
291
- existing_parts = resumed.list_parts()
292
-
293
- # Upload missing parts...
294
-
295
- key = resumed.complete(new_parts + existing_parts)
296
- ```
242
+ Keys are case-insensitive and are lowercased. After lowercasing, a key must be
243
+ non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
244
+ Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
245
+ naming the key as soon as the metadata is passed to `put`, before any request is sent.
297
246
 
298
247
  ### Authentication
299
248
 
300
249
  If your Objectstore instance enforces authorization, you must configure authentication
301
250
  via the `token` parameter on `Client`. It accepts either:
302
251
 
303
- - A **`SecretKey`** — for internal services that have access to an EdDSA keypair.
304
- The key signs a fresh JWT for each request, scoped to the specific usecase
305
- and scope being accessed, and can also sign pre-signed URLs.
252
+ - A **`TokenGenerator`** — for internal services that have access to an EdDSA keypair.
253
+ It wraps a `SecretKey` and signs a fresh JWT for each request, scoped to the
254
+ specific usecase and scope being accessed. Its key can also sign pre-signed URLs.
255
+ The generator's `expiry_seconds` and `permissions` are the defaults for every
256
+ token it signs; per-token overrides can only narrow them.
306
257
  - A **`str`** — a pre-signed JWT, used as-is for every request.
307
258
  Use this for external services that receive a token from another source.
308
259
 
309
260
  ```python
310
261
  from objectstore_client import Client, Usecase
311
- from objectstore_client.auth import SecretKey
262
+ from objectstore_client.auth import SecretKey, TokenGenerator
263
+
264
+ key = SecretKey(kid="my-service", secret_key="<private key>")
312
265
 
313
266
  # Option 1: Internal service with a keypair
314
- client = Client(
315
- "http://localhost:8888",
316
- token=SecretKey(kid="my-service", secret_key="<private key>"),
317
- )
267
+ client = Client("http://localhost:8888", token=TokenGenerator(key))
318
268
 
319
269
  # Option 2: External service with a pre-signed JWT
320
- # Use SecretKey.token_for_scope() to obtain a static token from an
270
+ # Use TokenGenerator.token_for_scope() to obtain a static token from an
321
271
  # internal service, then pass it to the external consumer:
322
272
  from objectstore_client.scope import Scope
323
273
 
324
- token = SecretKey(
325
- kid="my-service", secret_key="<private key>",
326
- ).token_for_scope("my_app", Scope(org=42, project=1337))
274
+ token = TokenGenerator(key, expiry_seconds=300).token_for_scope(
275
+ "my_app", Scope(org=42, project=1337)
276
+ )
327
277
 
328
278
  client = Client("http://localhost:8888", token=token)
329
279
  ```
@@ -333,7 +283,7 @@ client = Client("http://localhost:8888", token=token)
333
283
  `Session.object_url` returns a GET URL for an object. By default the URL carries
334
284
  no auth, so the recipient needs their own key to generate a token for it. Pass
335
285
  `token_validity` to embed a read-only token in the URL instead, valid for the
336
- given duration (requires `SecretKey` authentication - see above section).
286
+ given duration (requires `TokenGenerator` authentication - see above section).
337
287
 
338
288
  ```python
339
289
  from datetime import timedelta
@@ -356,8 +306,8 @@ Pre-signed URLS currently only support `GET` and `HEAD` (a pre-signed URL for
356
306
  `GET` authorizes `HEAD` too, and viceversa), so for uploads you should use
357
307
  scoped JWTs, as described above.
358
308
 
359
- `Session.presigned_object_url` signs the URL with the session's `SecretKey`,
360
- so it requires a `SecretKey` and raises `ValueError` otherwise.
309
+ `Session.presigned_object_url` signs the URL with the `SecretKey` of the session's
310
+ `TokenGenerator`, so it requires a `TokenGenerator` and raises `ValueError` otherwise.
361
311
  Only `GET` and `HEAD` may be pre-signed, the granted permissions are those
362
312
  configured server-side for the signing key, and the
363
313
  validity may not exceed one week.
@@ -373,6 +323,22 @@ with urllib.request.urlopen(url) as resp:
373
323
  content = resp.read()
374
324
  ```
375
325
 
326
+ ### Download response options
327
+
328
+ Both `Session.object_url` and `Session.presigned_object_url` accept a
329
+ `response_content_disposition` option:
330
+
331
+ ```python
332
+ url = session.presigned_object_url(
333
+ "GET",
334
+ "my-key",
335
+ response_content_disposition="inline",
336
+ )
337
+ ```
338
+
339
+ Disposition accepts `inline` or `attachment`, defaulting to `attachment` when omitted.
340
+ The stored filename is retained.
341
+
376
342
  ## Configuration
377
343
 
378
344
  In production, store the `Client` and `Usecase` at module level and reuse them.
@@ -383,7 +349,6 @@ from objectstore_client import Client, Usecase
383
349
 
384
350
  client = Client(
385
351
  "http://localhost:8888",
386
- propagate_traces=False, # default
387
352
  retries=3, # default: 3 connect retries, no read retries
388
353
  timeout_ms=None, # default: no read timeout (connect: 100ms)
389
354
  connection_kwargs={}, # default: empty (override urllib3.HTTPConnectionPool kwargs)
@@ -92,9 +92,13 @@ session.put(b"payload", origin="203.0.113.42")
92
92
  ### Compression
93
93
 
94
94
  Uploads are compressed with Zstd by default, and downloads are transparently decompressed.
95
- Compression can be overridden per-upload:
95
+ The supported algorithms are `"zstd"` and `"gzip"`, or `"none"` to disable compression.
96
+ Compression can be set per-usecase or overridden per-upload:
96
97
 
97
98
  ```python
99
+ # compress all uploads of this usecase with zstd (the default):
100
+ usecase = Usecase("my-usecase", compression="zstd")
101
+
98
102
  # upload as-is and record no encoding:
99
103
  session.put(video_data, compress="none")
100
104
 
@@ -116,95 +120,41 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
116
120
  session.put(b"payload", metadata={"source": "upload-service"})
117
121
  ```
118
122
 
119
- ### Multipart Upload API
120
-
121
- For large objects, use multipart uploads to upload parts independently and then
122
- assemble them into a final object.
123
-
124
- **Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
125
- `compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
126
- records the algorithm, and the caller must pre-compress each part accordingly.
127
-
128
- ```python
129
- from concurrent.futures import ThreadPoolExecutor
130
-
131
- import zstandard
132
-
133
- from objectstore_client.multipart import MultipartCompleteError
134
-
135
- upload = session.initiate_multipart_upload(
136
- key="my-large-object",
137
- compression="zstd",
138
- metadata={"source": "upload-service"},
139
- )
140
-
141
- compressor = zstandard.ZstdCompressor()
142
- chunks = [b"part1", b"part2", b"part3", b"part4"]
143
-
144
- def upload_part(part_number: int, data: bytes):
145
- compressed = compressor.compress(data)
146
- return upload.put_part(
147
- compressed, part_number=part_number, content_length=len(compressed)
148
- )
149
-
150
- with ThreadPoolExecutor(max_workers=4) as executor:
151
- futures = [
152
- executor.submit(upload_part, i + 1, chunk)
153
- for i, chunk in enumerate(chunks)
154
- ]
155
- parts = [f.result() for f in futures]
156
-
157
- try:
158
- key = upload.complete(parts)
159
- except MultipartCompleteError:
160
- upload.abort()
161
- raise
162
- ```
163
-
164
- To resume an in-progress multipart upload after a process restart, persist the
165
- `key` and `upload_id`, then reconstruct the upload handle later:
166
-
167
- ```python
168
- saved_key = upload.key
169
- saved_upload_id = upload.upload_id
170
-
171
- resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
172
- existing_parts = resumed.list_parts()
173
-
174
- # Upload missing parts...
175
-
176
- key = resumed.complete(new_parts + existing_parts)
177
- ```
123
+ Keys are case-insensitive and are lowercased. After lowercasing, a key must be
124
+ non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
125
+ Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
126
+ naming the key as soon as the metadata is passed to `put`, before any request is sent.
178
127
 
179
128
  ### Authentication
180
129
 
181
130
  If your Objectstore instance enforces authorization, you must configure authentication
182
131
  via the `token` parameter on `Client`. It accepts either:
183
132
 
184
- - A **`SecretKey`** — for internal services that have access to an EdDSA keypair.
185
- The key signs a fresh JWT for each request, scoped to the specific usecase
186
- and scope being accessed, and can also sign pre-signed URLs.
133
+ - A **`TokenGenerator`** — for internal services that have access to an EdDSA keypair.
134
+ It wraps a `SecretKey` and signs a fresh JWT for each request, scoped to the
135
+ specific usecase and scope being accessed. Its key can also sign pre-signed URLs.
136
+ The generator's `expiry_seconds` and `permissions` are the defaults for every
137
+ token it signs; per-token overrides can only narrow them.
187
138
  - A **`str`** — a pre-signed JWT, used as-is for every request.
188
139
  Use this for external services that receive a token from another source.
189
140
 
190
141
  ```python
191
142
  from objectstore_client import Client, Usecase
192
- from objectstore_client.auth import SecretKey
143
+ from objectstore_client.auth import SecretKey, TokenGenerator
144
+
145
+ key = SecretKey(kid="my-service", secret_key="<private key>")
193
146
 
194
147
  # Option 1: Internal service with a keypair
195
- client = Client(
196
- "http://localhost:8888",
197
- token=SecretKey(kid="my-service", secret_key="<private key>"),
198
- )
148
+ client = Client("http://localhost:8888", token=TokenGenerator(key))
199
149
 
200
150
  # Option 2: External service with a pre-signed JWT
201
- # Use SecretKey.token_for_scope() to obtain a static token from an
151
+ # Use TokenGenerator.token_for_scope() to obtain a static token from an
202
152
  # internal service, then pass it to the external consumer:
203
153
  from objectstore_client.scope import Scope
204
154
 
205
- token = SecretKey(
206
- kid="my-service", secret_key="<private key>",
207
- ).token_for_scope("my_app", Scope(org=42, project=1337))
155
+ token = TokenGenerator(key, expiry_seconds=300).token_for_scope(
156
+ "my_app", Scope(org=42, project=1337)
157
+ )
208
158
 
209
159
  client = Client("http://localhost:8888", token=token)
210
160
  ```
@@ -214,7 +164,7 @@ client = Client("http://localhost:8888", token=token)
214
164
  `Session.object_url` returns a GET URL for an object. By default the URL carries
215
165
  no auth, so the recipient needs their own key to generate a token for it. Pass
216
166
  `token_validity` to embed a read-only token in the URL instead, valid for the
217
- given duration (requires `SecretKey` authentication - see above section).
167
+ given duration (requires `TokenGenerator` authentication - see above section).
218
168
 
219
169
  ```python
220
170
  from datetime import timedelta
@@ -237,8 +187,8 @@ Pre-signed URLS currently only support `GET` and `HEAD` (a pre-signed URL for
237
187
  `GET` authorizes `HEAD` too, and viceversa), so for uploads you should use
238
188
  scoped JWTs, as described above.
239
189
 
240
- `Session.presigned_object_url` signs the URL with the session's `SecretKey`,
241
- so it requires a `SecretKey` and raises `ValueError` otherwise.
190
+ `Session.presigned_object_url` signs the URL with the `SecretKey` of the session's
191
+ `TokenGenerator`, so it requires a `TokenGenerator` and raises `ValueError` otherwise.
242
192
  Only `GET` and `HEAD` may be pre-signed, the granted permissions are those
243
193
  configured server-side for the signing key, and the
244
194
  validity may not exceed one week.
@@ -254,6 +204,22 @@ with urllib.request.urlopen(url) as resp:
254
204
  content = resp.read()
255
205
  ```
256
206
 
207
+ ### Download response options
208
+
209
+ Both `Session.object_url` and `Session.presigned_object_url` accept a
210
+ `response_content_disposition` option:
211
+
212
+ ```python
213
+ url = session.presigned_object_url(
214
+ "GET",
215
+ "my-key",
216
+ response_content_disposition="inline",
217
+ )
218
+ ```
219
+
220
+ Disposition accepts `inline` or `attachment`, defaulting to `attachment` when omitted.
221
+ The stored filename is retained.
222
+
257
223
  ## Configuration
258
224
 
259
225
  In production, store the `Client` and `Usecase` at module level and reuse them.
@@ -264,7 +230,6 @@ from objectstore_client import Client, Usecase
264
230
 
265
231
  client = Client(
266
232
  "http://localhost:8888",
267
- propagate_traces=False, # default
268
233
  retries=3, # default: 3 connect retries, no read retries
269
234
  timeout_ms=None, # default: no read timeout (connect: 100ms)
270
235
  connection_kwargs={}, # default: empty (override urllib3.HTTPConnectionPool kwargs)
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "objectstore-client"
3
- version = "0.2.10"
3
+ version = "0.3.1"
4
4
  description = "Client SDK for Objectstore, the Sentry object storage platform"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Sentry", email = "oss@sentry.io" }]
@@ -1,4 +1,10 @@
1
- from objectstore_client.auth import Permission, SecretKey, TokenGenerator, TokenProvider
1
+ from objectstore_client.auth import (
2
+ Permission,
3
+ SecretKey,
4
+ Token,
5
+ TokenGenerator,
6
+ TokenProvider,
7
+ )
2
8
  from objectstore_client.client import (
3
9
  Client,
4
10
  GetResponse,
@@ -35,6 +41,7 @@ __all__ = [
35
41
  "TimeToIdle",
36
42
  "TimeToLive",
37
43
  "TokenProvider",
44
+ "Token",
38
45
  "SecretKey",
39
46
  "TokenGenerator",
40
47
  "MetricsBackend",
@@ -0,0 +1,175 @@
1
+ """Streaming compression and decompression for the supported algorithms."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import gzip
6
+ import io
7
+ import shutil
8
+ import zlib
9
+ from typing import IO, Protocol, cast
10
+
11
+ import zstandard
12
+
13
+ from objectstore_client.metadata import Compression
14
+
15
+ SUPPORTED_COMPRESSIONS: tuple[Compression, ...] = ("zstd", "gzip")
16
+ """Compression algorithms that the client can apply and transparently decode."""
17
+
18
+
19
+ class _Compressor(Protocol):
20
+ """A streaming compression object, such as `zlib.compressobj()`."""
21
+
22
+ def compress(self, data: bytes, /) -> bytes: ...
23
+
24
+ def flush(self) -> bytes: ...
25
+
26
+
27
+ class _CompressionReader(io.RawIOBase):
28
+ """
29
+ A readable stream that compresses `source` with `compressor` as it is read.
30
+
31
+ `tell()` reports the number of compressed bytes produced so far. `seek(0)` is
32
+ allowed as long as no data has been read yet, so that urllib3 can record the body
33
+ position. Closing the reader does not close `source`.
34
+ """
35
+
36
+ _CHUNK_SIZE = 64 * 1024
37
+
38
+ def __init__(self, source: IO[bytes], compressor: _Compressor) -> None:
39
+ super().__init__()
40
+ self._source = source
41
+ self._compressor = compressor
42
+ self._buffer = bytearray()
43
+ self._finished = False
44
+ self._position = 0
45
+
46
+ def readable(self) -> bool:
47
+ return True
48
+
49
+ def read(self, size: int | None = -1, /) -> bytes:
50
+ if size is None or size < 0:
51
+ size = -1
52
+ while not self._finished and (size < 0 or len(self._buffer) < size):
53
+ if chunk := self._source.read(self._CHUNK_SIZE):
54
+ self._buffer += self._compressor.compress(chunk)
55
+ else:
56
+ self._buffer += self._compressor.flush()
57
+ self._finished = True
58
+
59
+ if size < 0:
60
+ size = len(self._buffer)
61
+ out = bytes(self._buffer[:size])
62
+ del self._buffer[:size]
63
+ self._position += len(out)
64
+ return out
65
+
66
+ def readall(self) -> bytes:
67
+ return self.read(-1)
68
+
69
+ def readinto(self, buffer: memoryview | bytearray) -> int: # type: ignore[override]
70
+ data = self.read(len(buffer))
71
+ buffer[: len(data)] = data
72
+ return len(data)
73
+
74
+ def tell(self) -> int:
75
+ return self._position
76
+
77
+ def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int:
78
+ if offset == 0 and whence == io.SEEK_SET and self._position == 0:
79
+ return 0
80
+ raise OSError("Cannot seek in a compressed stream after reading has started")
81
+
82
+
83
+ class _GzipReader(gzip.GzipFile):
84
+ """Decode gzip from a stream, closing the source with the reader."""
85
+
86
+ def __init__(self, source: IO[bytes]) -> None:
87
+ super().__init__(fileobj=source, mode="rb")
88
+ self._source = source
89
+
90
+ def seekable(self) -> bool:
91
+ return False
92
+
93
+ def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
94
+ raise io.UnsupportedOperation("gzip download stream is not seekable")
95
+
96
+ def close(self) -> None:
97
+ if self.closed:
98
+ return
99
+ try:
100
+ super().close()
101
+ finally:
102
+ self._source.close()
103
+
104
+
105
+ def _gzip_compressor() -> _Compressor:
106
+ """Return a zlib compressor that emits the gzip format.
107
+
108
+ By default, zlib wraps deflate data in the zlib format. Adding 16 to the window
109
+ size (`wbits`) selects the gzip header and trailer instead, which is what
110
+ `Content-Encoding: gzip` requires. See `zlib.compressobj`.
111
+ """
112
+ return zlib.compressobj(wbits=16 + zlib.MAX_WBITS)
113
+
114
+
115
+ def validate(compression: str | None, *, allow_none: bool = True) -> None:
116
+ """Raise `ValueError` if `compression` is not a supported algorithm.
117
+
118
+ `None` is always accepted. `"none"` is accepted only if `allow_none` is set.
119
+ """
120
+ if compression is None or (allow_none and compression == "none"):
121
+ return
122
+ if compression not in SUPPORTED_COMPRESSIONS:
123
+ raise ValueError(f"Invalid compression: {compression}")
124
+
125
+
126
+ def compressing_reader(source: IO[bytes], compression: Compression) -> IO[bytes]:
127
+ """Return a readable stream yielding `source` compressed with `compression`.
128
+
129
+ The returned stream compresses lazily as it is read, reports the number of
130
+ compressed bytes produced via `tell()`, and supports `seek(0)` only until
131
+ reading starts. Closing it does not close `source`.
132
+ """
133
+ if compression == "zstd":
134
+ compressor = zstandard.ZstdCompressor().compressobj()
135
+ return cast(IO[bytes], _CompressionReader(source, compressor))
136
+ if compression == "gzip":
137
+ return cast(IO[bytes], _CompressionReader(source, _gzip_compressor()))
138
+ raise ValueError(f"Invalid compression: {compression}")
139
+
140
+
141
+ def compress_into(
142
+ source: IO[bytes], dest: IO[bytes], compression: Compression
143
+ ) -> tuple[int, int]:
144
+ """Compress `source` into `dest`, returning the bytes read and written.
145
+
146
+ For gzip, `dest` must support `tell()` to measure the compressed size.
147
+ """
148
+ if compression == "zstd":
149
+ read, written = zstandard.ZstdCompressor().copy_stream(source, dest)
150
+ return read, written
151
+ if compression == "gzip":
152
+ start = dest.tell()
153
+ with gzip.GzipFile(
154
+ filename="", fileobj=dest, mode="wb", compresslevel=6, mtime=0
155
+ ) as writer:
156
+ shutil.copyfileobj(source, writer)
157
+ read = writer.tell()
158
+ return read, dest.tell() - start
159
+ raise ValueError(f"Invalid compression: {compression}")
160
+
161
+
162
+ def decompressing_reader(stream: IO[bytes], compression: str) -> IO[bytes]:
163
+ """Return a readable stream decoding `stream` compressed with `compression`.
164
+
165
+ Concatenated zstd frames and gzip members are decoded as a single payload.
166
+ The returned reader is non-seekable and closes the source when closed.
167
+ """
168
+ if compression == "zstd":
169
+ dctx = zstandard.ZstdDecompressor()
170
+ return cast(IO[bytes], dctx.stream_reader(stream, read_across_frames=True))
171
+ if compression == "gzip":
172
+ return cast(IO[bytes], _GzipReader(stream))
173
+ raise NotImplementedError(
174
+ f"Transparent decoding of `{compression}` is not implemented"
175
+ )