objectstore-client 0.2.10__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/PKG-INFO +42 -77
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/README.md +41 -76
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/pyproject.toml +1 -1
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/__init__.py +8 -1
- objectstore_client-0.3.1/src/objectstore_client/_compression.py +175 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/_resumable.py +108 -6
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/auth.py +51 -37
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/client.py +88 -178
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/metadata.py +41 -2
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/utils.py +1 -28
- objectstore_client-0.2.10/src/objectstore_client/multipart.py +0 -284
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/LICENSE.md +0 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/errors.py +0 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/metrics.py +0 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/presign.py +0 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/py.typed +0 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/scope.py +0 -0
- {objectstore_client-0.2.10 → objectstore_client-0.3.1}/src/objectstore_client/tracing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: objectstore-client
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Client SDK for Objectstore, the Sentry object storage platform
|
|
5
5
|
Author: Sentry
|
|
6
6
|
Author-email: Sentry <oss@sentry.io>
|
|
@@ -211,9 +211,13 @@ session.put(b"payload", origin="203.0.113.42")
|
|
|
211
211
|
### Compression
|
|
212
212
|
|
|
213
213
|
Uploads are compressed with Zstd by default, and downloads are transparently decompressed.
|
|
214
|
-
|
|
214
|
+
The supported algorithms are `"zstd"` and `"gzip"`, or `"none"` to disable compression.
|
|
215
|
+
Compression can be set per-usecase or overridden per-upload:
|
|
215
216
|
|
|
216
217
|
```python
|
|
218
|
+
# compress all uploads of this usecase with zstd (the default):
|
|
219
|
+
usecase = Usecase("my-usecase", compression="zstd")
|
|
220
|
+
|
|
217
221
|
# upload as-is and record no encoding:
|
|
218
222
|
session.put(video_data, compress="none")
|
|
219
223
|
|
|
@@ -235,95 +239,41 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
|
|
|
235
239
|
session.put(b"payload", metadata={"source": "upload-service"})
|
|
236
240
|
```
|
|
237
241
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
**Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
|
|
244
|
-
`compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
|
|
245
|
-
records the algorithm, and the caller must pre-compress each part accordingly.
|
|
246
|
-
|
|
247
|
-
```python
|
|
248
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
249
|
-
|
|
250
|
-
import zstandard
|
|
251
|
-
|
|
252
|
-
from objectstore_client.multipart import MultipartCompleteError
|
|
253
|
-
|
|
254
|
-
upload = session.initiate_multipart_upload(
|
|
255
|
-
key="my-large-object",
|
|
256
|
-
compression="zstd",
|
|
257
|
-
metadata={"source": "upload-service"},
|
|
258
|
-
)
|
|
259
|
-
|
|
260
|
-
compressor = zstandard.ZstdCompressor()
|
|
261
|
-
chunks = [b"part1", b"part2", b"part3", b"part4"]
|
|
262
|
-
|
|
263
|
-
def upload_part(part_number: int, data: bytes):
|
|
264
|
-
compressed = compressor.compress(data)
|
|
265
|
-
return upload.put_part(
|
|
266
|
-
compressed, part_number=part_number, content_length=len(compressed)
|
|
267
|
-
)
|
|
268
|
-
|
|
269
|
-
with ThreadPoolExecutor(max_workers=4) as executor:
|
|
270
|
-
futures = [
|
|
271
|
-
executor.submit(upload_part, i + 1, chunk)
|
|
272
|
-
for i, chunk in enumerate(chunks)
|
|
273
|
-
]
|
|
274
|
-
parts = [f.result() for f in futures]
|
|
275
|
-
|
|
276
|
-
try:
|
|
277
|
-
key = upload.complete(parts)
|
|
278
|
-
except MultipartCompleteError:
|
|
279
|
-
upload.abort()
|
|
280
|
-
raise
|
|
281
|
-
```
|
|
282
|
-
|
|
283
|
-
To resume an in-progress multipart upload after a process restart, persist the
|
|
284
|
-
`key` and `upload_id`, then reconstruct the upload handle later:
|
|
285
|
-
|
|
286
|
-
```python
|
|
287
|
-
saved_key = upload.key
|
|
288
|
-
saved_upload_id = upload.upload_id
|
|
289
|
-
|
|
290
|
-
resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
|
|
291
|
-
existing_parts = resumed.list_parts()
|
|
292
|
-
|
|
293
|
-
# Upload missing parts...
|
|
294
|
-
|
|
295
|
-
key = resumed.complete(new_parts + existing_parts)
|
|
296
|
-
```
|
|
242
|
+
Keys are case-insensitive and are lowercased. After lowercasing, a key must be
|
|
243
|
+
non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
|
|
244
|
+
Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
|
|
245
|
+
naming the key as soon as the metadata is passed to `put`, before any request is sent.
|
|
297
246
|
|
|
298
247
|
### Authentication
|
|
299
248
|
|
|
300
249
|
If your Objectstore instance enforces authorization, you must configure authentication
|
|
301
250
|
via the `token` parameter on `Client`. It accepts either:
|
|
302
251
|
|
|
303
|
-
- A **`
|
|
304
|
-
|
|
305
|
-
and scope being accessed
|
|
252
|
+
- A **`TokenGenerator`** — for internal services that have access to an EdDSA keypair.
|
|
253
|
+
It wraps a `SecretKey` and signs a fresh JWT for each request, scoped to the
|
|
254
|
+
specific usecase and scope being accessed. Its key can also sign pre-signed URLs.
|
|
255
|
+
The generator's `expiry_seconds` and `permissions` are the defaults for every
|
|
256
|
+
token it signs; per-token overrides can only narrow them.
|
|
306
257
|
- A **`str`** — a pre-signed JWT, used as-is for every request.
|
|
307
258
|
Use this for external services that receive a token from another source.
|
|
308
259
|
|
|
309
260
|
```python
|
|
310
261
|
from objectstore_client import Client, Usecase
|
|
311
|
-
from objectstore_client.auth import SecretKey
|
|
262
|
+
from objectstore_client.auth import SecretKey, TokenGenerator
|
|
263
|
+
|
|
264
|
+
key = SecretKey(kid="my-service", secret_key="<private key>")
|
|
312
265
|
|
|
313
266
|
# Option 1: Internal service with a keypair
|
|
314
|
-
client = Client(
|
|
315
|
-
"http://localhost:8888",
|
|
316
|
-
token=SecretKey(kid="my-service", secret_key="<private key>"),
|
|
317
|
-
)
|
|
267
|
+
client = Client("http://localhost:8888", token=TokenGenerator(key))
|
|
318
268
|
|
|
319
269
|
# Option 2: External service with a pre-signed JWT
|
|
320
|
-
# Use
|
|
270
|
+
# Use TokenGenerator.token_for_scope() to obtain a static token from an
|
|
321
271
|
# internal service, then pass it to the external consumer:
|
|
322
272
|
from objectstore_client.scope import Scope
|
|
323
273
|
|
|
324
|
-
token =
|
|
325
|
-
|
|
326
|
-
)
|
|
274
|
+
token = TokenGenerator(key, expiry_seconds=300).token_for_scope(
|
|
275
|
+
"my_app", Scope(org=42, project=1337)
|
|
276
|
+
)
|
|
327
277
|
|
|
328
278
|
client = Client("http://localhost:8888", token=token)
|
|
329
279
|
```
|
|
@@ -333,7 +283,7 @@ client = Client("http://localhost:8888", token=token)
|
|
|
333
283
|
`Session.object_url` returns a GET URL for an object. By default the URL carries
|
|
334
284
|
no auth, so the recipient needs their own key to generate a token for it. Pass
|
|
335
285
|
`token_validity` to embed a read-only token in the URL instead, valid for the
|
|
336
|
-
given duration (requires `
|
|
286
|
+
given duration (requires `TokenGenerator` authentication - see above section).
|
|
337
287
|
|
|
338
288
|
```python
|
|
339
289
|
from datetime import timedelta
|
|
@@ -356,8 +306,8 @@ Pre-signed URLS currently only support `GET` and `HEAD` (a pre-signed URL for
|
|
|
356
306
|
`GET` authorizes `HEAD` too, and viceversa), so for uploads you should use
|
|
357
307
|
scoped JWTs, as described above.
|
|
358
308
|
|
|
359
|
-
`Session.presigned_object_url` signs the URL with the session's
|
|
360
|
-
so it requires a `
|
|
309
|
+
`Session.presigned_object_url` signs the URL with the `SecretKey` of the session's
|
|
310
|
+
`TokenGenerator`, so it requires a `TokenGenerator` and raises `ValueError` otherwise.
|
|
361
311
|
Only `GET` and `HEAD` may be pre-signed, the granted permissions are those
|
|
362
312
|
configured server-side for the signing key, and the
|
|
363
313
|
validity may not exceed one week.
|
|
@@ -373,6 +323,22 @@ with urllib.request.urlopen(url) as resp:
|
|
|
373
323
|
content = resp.read()
|
|
374
324
|
```
|
|
375
325
|
|
|
326
|
+
### Download response options
|
|
327
|
+
|
|
328
|
+
Both `Session.object_url` and `Session.presigned_object_url` accept a
|
|
329
|
+
`response_content_disposition` option:
|
|
330
|
+
|
|
331
|
+
```python
|
|
332
|
+
url = session.presigned_object_url(
|
|
333
|
+
"GET",
|
|
334
|
+
"my-key",
|
|
335
|
+
response_content_disposition="inline",
|
|
336
|
+
)
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
Disposition accepts `inline` or `attachment`, defaulting to `attachment` when omitted.
|
|
340
|
+
The stored filename is retained.
|
|
341
|
+
|
|
376
342
|
## Configuration
|
|
377
343
|
|
|
378
344
|
In production, store the `Client` and `Usecase` at module level and reuse them.
|
|
@@ -383,7 +349,6 @@ from objectstore_client import Client, Usecase
|
|
|
383
349
|
|
|
384
350
|
client = Client(
|
|
385
351
|
"http://localhost:8888",
|
|
386
|
-
propagate_traces=False, # default
|
|
387
352
|
retries=3, # default: 3 connect retries, no read retries
|
|
388
353
|
timeout_ms=None, # default: no read timeout (connect: 100ms)
|
|
389
354
|
connection_kwargs={}, # default: empty (override urllib3.HTTPConnectionPool kwargs)
|
|
@@ -92,9 +92,13 @@ session.put(b"payload", origin="203.0.113.42")
|
|
|
92
92
|
### Compression
|
|
93
93
|
|
|
94
94
|
Uploads are compressed with Zstd by default, and downloads are transparently decompressed.
|
|
95
|
-
|
|
95
|
+
The supported algorithms are `"zstd"` and `"gzip"`, or `"none"` to disable compression.
|
|
96
|
+
Compression can be set per-usecase or overridden per-upload:
|
|
96
97
|
|
|
97
98
|
```python
|
|
99
|
+
# compress all uploads of this usecase with zstd (the default):
|
|
100
|
+
usecase = Usecase("my-usecase", compression="zstd")
|
|
101
|
+
|
|
98
102
|
# upload as-is and record no encoding:
|
|
99
103
|
session.put(video_data, compress="none")
|
|
100
104
|
|
|
@@ -116,95 +120,41 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
|
|
|
116
120
|
session.put(b"payload", metadata={"source": "upload-service"})
|
|
117
121
|
```
|
|
118
122
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
**Important:** unlike single-object uploads, multipart uploads do **not** auto-compress.
|
|
125
|
-
`compression` on `initiate_multipart_upload` behaves like `precompressed` above: it only
|
|
126
|
-
records the algorithm, and the caller must pre-compress each part accordingly.
|
|
127
|
-
|
|
128
|
-
```python
|
|
129
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
130
|
-
|
|
131
|
-
import zstandard
|
|
132
|
-
|
|
133
|
-
from objectstore_client.multipart import MultipartCompleteError
|
|
134
|
-
|
|
135
|
-
upload = session.initiate_multipart_upload(
|
|
136
|
-
key="my-large-object",
|
|
137
|
-
compression="zstd",
|
|
138
|
-
metadata={"source": "upload-service"},
|
|
139
|
-
)
|
|
140
|
-
|
|
141
|
-
compressor = zstandard.ZstdCompressor()
|
|
142
|
-
chunks = [b"part1", b"part2", b"part3", b"part4"]
|
|
143
|
-
|
|
144
|
-
def upload_part(part_number: int, data: bytes):
|
|
145
|
-
compressed = compressor.compress(data)
|
|
146
|
-
return upload.put_part(
|
|
147
|
-
compressed, part_number=part_number, content_length=len(compressed)
|
|
148
|
-
)
|
|
149
|
-
|
|
150
|
-
with ThreadPoolExecutor(max_workers=4) as executor:
|
|
151
|
-
futures = [
|
|
152
|
-
executor.submit(upload_part, i + 1, chunk)
|
|
153
|
-
for i, chunk in enumerate(chunks)
|
|
154
|
-
]
|
|
155
|
-
parts = [f.result() for f in futures]
|
|
156
|
-
|
|
157
|
-
try:
|
|
158
|
-
key = upload.complete(parts)
|
|
159
|
-
except MultipartCompleteError:
|
|
160
|
-
upload.abort()
|
|
161
|
-
raise
|
|
162
|
-
```
|
|
163
|
-
|
|
164
|
-
To resume an in-progress multipart upload after a process restart, persist the
|
|
165
|
-
`key` and `upload_id`, then reconstruct the upload handle later:
|
|
166
|
-
|
|
167
|
-
```python
|
|
168
|
-
saved_key = upload.key
|
|
169
|
-
saved_upload_id = upload.upload_id
|
|
170
|
-
|
|
171
|
-
resumed = session.resume_multipart_upload(saved_key, saved_upload_id)
|
|
172
|
-
existing_parts = resumed.list_parts()
|
|
173
|
-
|
|
174
|
-
# Upload missing parts...
|
|
175
|
-
|
|
176
|
-
key = resumed.complete(new_parts + existing_parts)
|
|
177
|
-
```
|
|
123
|
+
Keys are case-insensitive and are lowercased. After lowercasing, a key must be
|
|
124
|
+
non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
|
|
125
|
+
Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
|
|
126
|
+
naming the key as soon as the metadata is passed to `put`, before any request is sent.
|
|
178
127
|
|
|
179
128
|
### Authentication
|
|
180
129
|
|
|
181
130
|
If your Objectstore instance enforces authorization, you must configure authentication
|
|
182
131
|
via the `token` parameter on `Client`. It accepts either:
|
|
183
132
|
|
|
184
|
-
- A **`
|
|
185
|
-
|
|
186
|
-
and scope being accessed
|
|
133
|
+
- A **`TokenGenerator`** — for internal services that have access to an EdDSA keypair.
|
|
134
|
+
It wraps a `SecretKey` and signs a fresh JWT for each request, scoped to the
|
|
135
|
+
specific usecase and scope being accessed. Its key can also sign pre-signed URLs.
|
|
136
|
+
The generator's `expiry_seconds` and `permissions` are the defaults for every
|
|
137
|
+
token it signs; per-token overrides can only narrow them.
|
|
187
138
|
- A **`str`** — a pre-signed JWT, used as-is for every request.
|
|
188
139
|
Use this for external services that receive a token from another source.
|
|
189
140
|
|
|
190
141
|
```python
|
|
191
142
|
from objectstore_client import Client, Usecase
|
|
192
|
-
from objectstore_client.auth import SecretKey
|
|
143
|
+
from objectstore_client.auth import SecretKey, TokenGenerator
|
|
144
|
+
|
|
145
|
+
key = SecretKey(kid="my-service", secret_key="<private key>")
|
|
193
146
|
|
|
194
147
|
# Option 1: Internal service with a keypair
|
|
195
|
-
client = Client(
|
|
196
|
-
"http://localhost:8888",
|
|
197
|
-
token=SecretKey(kid="my-service", secret_key="<private key>"),
|
|
198
|
-
)
|
|
148
|
+
client = Client("http://localhost:8888", token=TokenGenerator(key))
|
|
199
149
|
|
|
200
150
|
# Option 2: External service with a pre-signed JWT
|
|
201
|
-
# Use
|
|
151
|
+
# Use TokenGenerator.token_for_scope() to obtain a static token from an
|
|
202
152
|
# internal service, then pass it to the external consumer:
|
|
203
153
|
from objectstore_client.scope import Scope
|
|
204
154
|
|
|
205
|
-
token =
|
|
206
|
-
|
|
207
|
-
)
|
|
155
|
+
token = TokenGenerator(key, expiry_seconds=300).token_for_scope(
|
|
156
|
+
"my_app", Scope(org=42, project=1337)
|
|
157
|
+
)
|
|
208
158
|
|
|
209
159
|
client = Client("http://localhost:8888", token=token)
|
|
210
160
|
```
|
|
@@ -214,7 +164,7 @@ client = Client("http://localhost:8888", token=token)
|
|
|
214
164
|
`Session.object_url` returns a GET URL for an object. By default the URL carries
|
|
215
165
|
no auth, so the recipient needs their own key to generate a token for it. Pass
|
|
216
166
|
`token_validity` to embed a read-only token in the URL instead, valid for the
|
|
217
|
-
given duration (requires `
|
|
167
|
+
given duration (requires `TokenGenerator` authentication - see above section).
|
|
218
168
|
|
|
219
169
|
```python
|
|
220
170
|
from datetime import timedelta
|
|
@@ -237,8 +187,8 @@ Pre-signed URLS currently only support `GET` and `HEAD` (a pre-signed URL for
|
|
|
237
187
|
`GET` authorizes `HEAD` too, and viceversa), so for uploads you should use
|
|
238
188
|
scoped JWTs, as described above.
|
|
239
189
|
|
|
240
|
-
`Session.presigned_object_url` signs the URL with the session's
|
|
241
|
-
so it requires a `
|
|
190
|
+
`Session.presigned_object_url` signs the URL with the `SecretKey` of the session's
|
|
191
|
+
`TokenGenerator`, so it requires a `TokenGenerator` and raises `ValueError` otherwise.
|
|
242
192
|
Only `GET` and `HEAD` may be pre-signed, the granted permissions are those
|
|
243
193
|
configured server-side for the signing key, and the
|
|
244
194
|
validity may not exceed one week.
|
|
@@ -254,6 +204,22 @@ with urllib.request.urlopen(url) as resp:
|
|
|
254
204
|
content = resp.read()
|
|
255
205
|
```
|
|
256
206
|
|
|
207
|
+
### Download response options
|
|
208
|
+
|
|
209
|
+
Both `Session.object_url` and `Session.presigned_object_url` accept a
|
|
210
|
+
`response_content_disposition` option:
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
url = session.presigned_object_url(
|
|
214
|
+
"GET",
|
|
215
|
+
"my-key",
|
|
216
|
+
response_content_disposition="inline",
|
|
217
|
+
)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Disposition accepts `inline` or `attachment`, defaulting to `attachment` when omitted.
|
|
221
|
+
The stored filename is retained.
|
|
222
|
+
|
|
257
223
|
## Configuration
|
|
258
224
|
|
|
259
225
|
In production, store the `Client` and `Usecase` at module level and reuse them.
|
|
@@ -264,7 +230,6 @@ from objectstore_client import Client, Usecase
|
|
|
264
230
|
|
|
265
231
|
client = Client(
|
|
266
232
|
"http://localhost:8888",
|
|
267
|
-
propagate_traces=False, # default
|
|
268
233
|
retries=3, # default: 3 connect retries, no read retries
|
|
269
234
|
timeout_ms=None, # default: no read timeout (connect: 100ms)
|
|
270
235
|
connection_kwargs={}, # default: empty (override urllib3.HTTPConnectionPool kwargs)
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
from objectstore_client.auth import
|
|
1
|
+
from objectstore_client.auth import (
|
|
2
|
+
Permission,
|
|
3
|
+
SecretKey,
|
|
4
|
+
Token,
|
|
5
|
+
TokenGenerator,
|
|
6
|
+
TokenProvider,
|
|
7
|
+
)
|
|
2
8
|
from objectstore_client.client import (
|
|
3
9
|
Client,
|
|
4
10
|
GetResponse,
|
|
@@ -35,6 +41,7 @@ __all__ = [
|
|
|
35
41
|
"TimeToIdle",
|
|
36
42
|
"TimeToLive",
|
|
37
43
|
"TokenProvider",
|
|
44
|
+
"Token",
|
|
38
45
|
"SecretKey",
|
|
39
46
|
"TokenGenerator",
|
|
40
47
|
"MetricsBackend",
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Streaming compression and decompression for the supported algorithms."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import gzip
|
|
6
|
+
import io
|
|
7
|
+
import shutil
|
|
8
|
+
import zlib
|
|
9
|
+
from typing import IO, Protocol, cast
|
|
10
|
+
|
|
11
|
+
import zstandard
|
|
12
|
+
|
|
13
|
+
from objectstore_client.metadata import Compression
|
|
14
|
+
|
|
15
|
+
SUPPORTED_COMPRESSIONS: tuple[Compression, ...] = ("zstd", "gzip")
|
|
16
|
+
"""Compression algorithms that the client can apply and transparently decode."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _Compressor(Protocol):
|
|
20
|
+
"""A streaming compression object, such as `zlib.compressobj()`."""
|
|
21
|
+
|
|
22
|
+
def compress(self, data: bytes, /) -> bytes: ...
|
|
23
|
+
|
|
24
|
+
def flush(self) -> bytes: ...
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _CompressionReader(io.RawIOBase):
|
|
28
|
+
"""
|
|
29
|
+
A readable stream that compresses `source` with `compressor` as it is read.
|
|
30
|
+
|
|
31
|
+
`tell()` reports the number of compressed bytes produced so far. `seek(0)` is
|
|
32
|
+
allowed as long as no data has been read yet, so that urllib3 can record the body
|
|
33
|
+
position. Closing the reader does not close `source`.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
_CHUNK_SIZE = 64 * 1024
|
|
37
|
+
|
|
38
|
+
def __init__(self, source: IO[bytes], compressor: _Compressor) -> None:
|
|
39
|
+
super().__init__()
|
|
40
|
+
self._source = source
|
|
41
|
+
self._compressor = compressor
|
|
42
|
+
self._buffer = bytearray()
|
|
43
|
+
self._finished = False
|
|
44
|
+
self._position = 0
|
|
45
|
+
|
|
46
|
+
def readable(self) -> bool:
|
|
47
|
+
return True
|
|
48
|
+
|
|
49
|
+
def read(self, size: int | None = -1, /) -> bytes:
|
|
50
|
+
if size is None or size < 0:
|
|
51
|
+
size = -1
|
|
52
|
+
while not self._finished and (size < 0 or len(self._buffer) < size):
|
|
53
|
+
if chunk := self._source.read(self._CHUNK_SIZE):
|
|
54
|
+
self._buffer += self._compressor.compress(chunk)
|
|
55
|
+
else:
|
|
56
|
+
self._buffer += self._compressor.flush()
|
|
57
|
+
self._finished = True
|
|
58
|
+
|
|
59
|
+
if size < 0:
|
|
60
|
+
size = len(self._buffer)
|
|
61
|
+
out = bytes(self._buffer[:size])
|
|
62
|
+
del self._buffer[:size]
|
|
63
|
+
self._position += len(out)
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
def readall(self) -> bytes:
|
|
67
|
+
return self.read(-1)
|
|
68
|
+
|
|
69
|
+
def readinto(self, buffer: memoryview | bytearray) -> int: # type: ignore[override]
|
|
70
|
+
data = self.read(len(buffer))
|
|
71
|
+
buffer[: len(data)] = data
|
|
72
|
+
return len(data)
|
|
73
|
+
|
|
74
|
+
def tell(self) -> int:
|
|
75
|
+
return self._position
|
|
76
|
+
|
|
77
|
+
def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int:
|
|
78
|
+
if offset == 0 and whence == io.SEEK_SET and self._position == 0:
|
|
79
|
+
return 0
|
|
80
|
+
raise OSError("Cannot seek in a compressed stream after reading has started")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class _GzipReader(gzip.GzipFile):
|
|
84
|
+
"""Decode gzip from a stream, closing the source with the reader."""
|
|
85
|
+
|
|
86
|
+
def __init__(self, source: IO[bytes]) -> None:
|
|
87
|
+
super().__init__(fileobj=source, mode="rb")
|
|
88
|
+
self._source = source
|
|
89
|
+
|
|
90
|
+
def seekable(self) -> bool:
|
|
91
|
+
return False
|
|
92
|
+
|
|
93
|
+
def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
|
|
94
|
+
raise io.UnsupportedOperation("gzip download stream is not seekable")
|
|
95
|
+
|
|
96
|
+
def close(self) -> None:
|
|
97
|
+
if self.closed:
|
|
98
|
+
return
|
|
99
|
+
try:
|
|
100
|
+
super().close()
|
|
101
|
+
finally:
|
|
102
|
+
self._source.close()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _gzip_compressor() -> _Compressor:
|
|
106
|
+
"""Return a zlib compressor that emits the gzip format.
|
|
107
|
+
|
|
108
|
+
By default, zlib wraps deflate data in the zlib format. Adding 16 to the window
|
|
109
|
+
size (`wbits`) selects the gzip header and trailer instead, which is what
|
|
110
|
+
`Content-Encoding: gzip` requires. See `zlib.compressobj`.
|
|
111
|
+
"""
|
|
112
|
+
return zlib.compressobj(wbits=16 + zlib.MAX_WBITS)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def validate(compression: str | None, *, allow_none: bool = True) -> None:
|
|
116
|
+
"""Raise `ValueError` if `compression` is not a supported algorithm.
|
|
117
|
+
|
|
118
|
+
`None` is always accepted. `"none"` is accepted only if `allow_none` is set.
|
|
119
|
+
"""
|
|
120
|
+
if compression is None or (allow_none and compression == "none"):
|
|
121
|
+
return
|
|
122
|
+
if compression not in SUPPORTED_COMPRESSIONS:
|
|
123
|
+
raise ValueError(f"Invalid compression: {compression}")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def compressing_reader(source: IO[bytes], compression: Compression) -> IO[bytes]:
|
|
127
|
+
"""Return a readable stream yielding `source` compressed with `compression`.
|
|
128
|
+
|
|
129
|
+
The returned stream compresses lazily as it is read, reports the number of
|
|
130
|
+
compressed bytes produced via `tell()`, and supports `seek(0)` only until
|
|
131
|
+
reading starts. Closing it does not close `source`.
|
|
132
|
+
"""
|
|
133
|
+
if compression == "zstd":
|
|
134
|
+
compressor = zstandard.ZstdCompressor().compressobj()
|
|
135
|
+
return cast(IO[bytes], _CompressionReader(source, compressor))
|
|
136
|
+
if compression == "gzip":
|
|
137
|
+
return cast(IO[bytes], _CompressionReader(source, _gzip_compressor()))
|
|
138
|
+
raise ValueError(f"Invalid compression: {compression}")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def compress_into(
|
|
142
|
+
source: IO[bytes], dest: IO[bytes], compression: Compression
|
|
143
|
+
) -> tuple[int, int]:
|
|
144
|
+
"""Compress `source` into `dest`, returning the bytes read and written.
|
|
145
|
+
|
|
146
|
+
For gzip, `dest` must support `tell()` to measure the compressed size.
|
|
147
|
+
"""
|
|
148
|
+
if compression == "zstd":
|
|
149
|
+
read, written = zstandard.ZstdCompressor().copy_stream(source, dest)
|
|
150
|
+
return read, written
|
|
151
|
+
if compression == "gzip":
|
|
152
|
+
start = dest.tell()
|
|
153
|
+
with gzip.GzipFile(
|
|
154
|
+
filename="", fileobj=dest, mode="wb", compresslevel=6, mtime=0
|
|
155
|
+
) as writer:
|
|
156
|
+
shutil.copyfileobj(source, writer)
|
|
157
|
+
read = writer.tell()
|
|
158
|
+
return read, dest.tell() - start
|
|
159
|
+
raise ValueError(f"Invalid compression: {compression}")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def decompressing_reader(stream: IO[bytes], compression: str) -> IO[bytes]:
|
|
163
|
+
"""Return a readable stream decoding `stream` compressed with `compression`.
|
|
164
|
+
|
|
165
|
+
Concatenated zstd frames and gzip members are decoded as a single payload.
|
|
166
|
+
The returned reader is non-seekable and closes the source when closed.
|
|
167
|
+
"""
|
|
168
|
+
if compression == "zstd":
|
|
169
|
+
dctx = zstandard.ZstdDecompressor()
|
|
170
|
+
return cast(IO[bytes], dctx.stream_reader(stream, read_across_frames=True))
|
|
171
|
+
if compression == "gzip":
|
|
172
|
+
return cast(IO[bytes], _GzipReader(stream))
|
|
173
|
+
raise NotImplementedError(
|
|
174
|
+
f"Transparent decoding of `{compression}` is not implemented"
|
|
175
|
+
)
|