objectstore-client 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/PKG-INFO +6 -1
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/README.md +5 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/pyproject.toml +1 -1
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/_resumable.py +103 -1
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/client.py +23 -4
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/metadata.py +40 -1
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/LICENSE.md +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/__init__.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/_compression.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/auth.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/errors.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/metrics.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/presign.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/py.typed +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/scope.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/tracing.py +0 -0
- {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: objectstore-client
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Client SDK for Objectstore, the Sentry object storage platform
|
|
5
5
|
Author: Sentry
|
|
6
6
|
Author-email: Sentry <oss@sentry.io>
|
|
@@ -239,6 +239,11 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
|
|
|
239
239
|
session.put(b"payload", metadata={"source": "upload-service"})
|
|
240
240
|
```
|
|
241
241
|
|
|
242
|
+
Keys are case-insensitive and are lowercased. After lowercasing, a key must be
|
|
243
|
+
non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
|
|
244
|
+
Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
|
|
245
|
+
naming the key as soon as the metadata is passed to `put`, before any request is sent.
|
|
246
|
+
|
|
242
247
|
### Authentication
|
|
243
248
|
|
|
244
249
|
If your Objectstore instance enforces authorization, you must configure authentication
|
|
@@ -120,6 +120,11 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
|
|
|
120
120
|
session.put(b"payload", metadata={"source": "upload-service"})
|
|
121
121
|
```
|
|
122
122
|
|
|
123
|
+
Keys are case-insensitive and are lowercased. After lowercasing, a key must be
|
|
124
|
+
non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
|
|
125
|
+
Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
|
|
126
|
+
naming the key as soon as the metadata is passed to `put`, before any request is sent.
|
|
127
|
+
|
|
123
128
|
### Authentication
|
|
124
129
|
|
|
125
130
|
If your Objectstore instance enforces authorization, you must configure authentication
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import io
|
|
3
4
|
import random
|
|
4
5
|
import time
|
|
5
6
|
from collections.abc import Iterator
|
|
@@ -8,7 +9,7 @@ from dataclasses import dataclass
|
|
|
8
9
|
from io import SEEK_END, BytesIO
|
|
9
10
|
from os import PathLike
|
|
10
11
|
from tempfile import TemporaryFile
|
|
11
|
-
from typing import IO, TYPE_CHECKING
|
|
12
|
+
from typing import IO, TYPE_CHECKING, cast
|
|
12
13
|
from urllib.parse import urlencode
|
|
13
14
|
|
|
14
15
|
import urllib3
|
|
@@ -20,6 +21,8 @@ from objectstore_client.metrics import measure_storage_operation
|
|
|
20
21
|
from objectstore_client.tracing import storage_span
|
|
21
22
|
|
|
22
23
|
if TYPE_CHECKING:
|
|
24
|
+
from _typeshed import WriteableBuffer
|
|
25
|
+
|
|
23
26
|
from objectstore_client.client import Session
|
|
24
27
|
|
|
25
28
|
|
|
@@ -50,6 +53,14 @@ class NoUploadProgress(Exception):
|
|
|
50
53
|
super().__init__("Upload made no progress")
|
|
51
54
|
|
|
52
55
|
|
|
56
|
+
class UploadBodyTruncated(OSError):
|
|
57
|
+
"""The upload source ended before its declared length, e.g. a file shrank."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, missing: int) -> None:
|
|
60
|
+
super().__init__(f"Upload source ended {missing} bytes early")
|
|
61
|
+
self.missing = missing
|
|
62
|
+
|
|
63
|
+
|
|
53
64
|
@dataclass(frozen=True)
|
|
54
65
|
class UploadIncomplete:
|
|
55
66
|
"""The upload expects more bytes, starting at the given offset."""
|
|
@@ -156,6 +167,8 @@ class ResumableUpload:
|
|
|
156
167
|
|
|
157
168
|
if offset < 0 or length < 0:
|
|
158
169
|
raise ValueError("Chunk offset and length must not be negative")
|
|
170
|
+
if not isinstance(body, bytes):
|
|
171
|
+
body = bounded_reader(body, length)
|
|
159
172
|
if (
|
|
160
173
|
self._granularity is not None
|
|
161
174
|
and self._total_length is not None
|
|
@@ -292,6 +305,89 @@ def get_size(source: bytes | IO[bytes]) -> int | None:
|
|
|
292
305
|
source.seek(start)
|
|
293
306
|
|
|
294
307
|
|
|
308
|
+
class _BoundedReader(io.RawIOBase):
|
|
309
|
+
"""
|
|
310
|
+
A readable stream over the next `length` bytes of `source`.
|
|
311
|
+
|
|
312
|
+
This keeps a body within its declared `Content-Length`, even if `source` grows
|
|
313
|
+
during the upload. If `source` ends early instead, reading raises
|
|
314
|
+
`UploadBodyTruncated`, which aborts the request rather than leaving the server
|
|
315
|
+
waiting for the missing bytes.
|
|
316
|
+
|
|
317
|
+
Positions are those of `source`, so urllib3 can rewind a seekable body for
|
|
318
|
+
retries and redirects. The stream covers positions from the start of the bound
|
|
319
|
+
to its end; seeking before the start fails. If `source` is not seekable,
|
|
320
|
+
neither is this reader. Closing this reader leaves `source` open.
|
|
321
|
+
"""
|
|
322
|
+
|
|
323
|
+
def __init__(self, source: IO[bytes], length: int) -> None:
|
|
324
|
+
if length < 0:
|
|
325
|
+
raise ValueError("length must not be negative")
|
|
326
|
+
self._source = source
|
|
327
|
+
self._length = length
|
|
328
|
+
self._remaining = length
|
|
329
|
+
self._start: int | None = None
|
|
330
|
+
try:
|
|
331
|
+
if source.seekable():
|
|
332
|
+
self._start = source.tell()
|
|
333
|
+
except (AttributeError, OSError):
|
|
334
|
+
pass
|
|
335
|
+
|
|
336
|
+
def _check_open(self) -> None:
|
|
337
|
+
if self.closed:
|
|
338
|
+
raise ValueError("I/O operation on closed file")
|
|
339
|
+
|
|
340
|
+
def _check_seekable(self) -> int:
|
|
341
|
+
self._check_open()
|
|
342
|
+
if self._start is None:
|
|
343
|
+
raise io.UnsupportedOperation("source is not seekable")
|
|
344
|
+
return self._start
|
|
345
|
+
|
|
346
|
+
def readable(self) -> bool:
|
|
347
|
+
self._check_open()
|
|
348
|
+
return True
|
|
349
|
+
|
|
350
|
+
def readinto(self, buffer: WriteableBuffer, /) -> int:
|
|
351
|
+
self._check_open()
|
|
352
|
+
view = memoryview(buffer).cast("B")
|
|
353
|
+
data = self._source.read(min(len(view), self._remaining))
|
|
354
|
+
if not data and view and self._remaining:
|
|
355
|
+
raise UploadBodyTruncated(self._remaining)
|
|
356
|
+
view[: len(data)] = data
|
|
357
|
+
self._remaining -= len(data)
|
|
358
|
+
return len(data)
|
|
359
|
+
|
|
360
|
+
def seekable(self) -> bool:
|
|
361
|
+
self._check_open()
|
|
362
|
+
return self._start is not None
|
|
363
|
+
|
|
364
|
+
def tell(self) -> int:
|
|
365
|
+
self._check_seekable()
|
|
366
|
+
return self._source.tell()
|
|
367
|
+
|
|
368
|
+
def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int:
|
|
369
|
+
start = self._check_seekable()
|
|
370
|
+
end = start + self._length
|
|
371
|
+
if whence == io.SEEK_SET:
|
|
372
|
+
target = offset
|
|
373
|
+
elif whence == io.SEEK_CUR:
|
|
374
|
+
target = self._source.tell() + offset
|
|
375
|
+
elif whence == io.SEEK_END:
|
|
376
|
+
target = end + offset
|
|
377
|
+
else:
|
|
378
|
+
raise ValueError(f"invalid whence: {whence}")
|
|
379
|
+
if target < start:
|
|
380
|
+
raise OSError("cannot seek before the start of the stream")
|
|
381
|
+
position = self._source.seek(target)
|
|
382
|
+
self._remaining = max(0, end - position)
|
|
383
|
+
return position
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def bounded_reader(source: IO[bytes], length: int) -> IO[bytes]:
|
|
387
|
+
"""Return a stream yielding at most `length` bytes of `source` from its position."""
|
|
388
|
+
return cast(IO[bytes], _BoundedReader(source, length))
|
|
389
|
+
|
|
390
|
+
|
|
295
391
|
def is_transient(error: Exception) -> bool:
|
|
296
392
|
if isinstance(error, urllib3.exceptions.MaxRetryError):
|
|
297
393
|
# Exhausted status retries carry ResponseError rather than the response.
|
|
@@ -300,6 +396,12 @@ def is_transient(error: Exception) -> bool:
|
|
|
300
396
|
)
|
|
301
397
|
if isinstance(error, RequestError):
|
|
302
398
|
return error.status in (408, 429, 502, 503, 504)
|
|
399
|
+
if isinstance(error, urllib3.exceptions.ProtocolError) and any(
|
|
400
|
+
isinstance(arg, UploadBodyTruncated) for arg in error.args
|
|
401
|
+
):
|
|
402
|
+
# urllib3 wraps errors from reading the body. A local source that ended
|
|
403
|
+
# early fails the same way on every attempt, so it is not retried.
|
|
404
|
+
return False
|
|
303
405
|
return isinstance(
|
|
304
406
|
error,
|
|
305
407
|
(
|
|
@@ -30,8 +30,10 @@ from objectstore_client.metadata import (
|
|
|
30
30
|
Compression,
|
|
31
31
|
ExpirationPolicy,
|
|
32
32
|
Metadata,
|
|
33
|
+
encode_metadata_key,
|
|
33
34
|
format_expiration,
|
|
34
35
|
format_timedelta,
|
|
36
|
+
normalize_metadata_key,
|
|
35
37
|
)
|
|
36
38
|
from objectstore_client.metrics import (
|
|
37
39
|
MetricsBackend,
|
|
@@ -349,7 +351,11 @@ class Session:
|
|
|
349
351
|
origin: str | None = None,
|
|
350
352
|
filename: str | None = None,
|
|
351
353
|
) -> dict[str, str]:
|
|
352
|
-
"""Build upload headers with the given compression and usecase expiration.
|
|
354
|
+
"""Build upload headers with the given compression and usecase expiration.
|
|
355
|
+
|
|
356
|
+
Raises `ValueError` if a custom metadata key is invalid (see
|
|
357
|
+
`normalize_metadata_key`), before anything is sent.
|
|
358
|
+
"""
|
|
353
359
|
headers = self._make_headers()
|
|
354
360
|
if compression and compression != "none":
|
|
355
361
|
headers["Content-Encoding"] = compression
|
|
@@ -364,7 +370,8 @@ class Session:
|
|
|
364
370
|
headers[HEADER_FILENAME] = utils.encode_header_value(filename)
|
|
365
371
|
if metadata:
|
|
366
372
|
for k, v in metadata.items():
|
|
367
|
-
|
|
373
|
+
name = encode_metadata_key(normalize_metadata_key(k))
|
|
374
|
+
headers[f"{HEADER_META_PREFIX}{name}"] = utils.encode_header_value(v)
|
|
368
375
|
return headers
|
|
369
376
|
|
|
370
377
|
def _base_url(self) -> str:
|
|
@@ -410,6 +417,10 @@ class Session:
|
|
|
410
417
|
instead. The payload is then uploaded verbatim and the algorithm is recorded
|
|
411
418
|
in the object's metadata, so downloads decompress it transparently.
|
|
412
419
|
|
|
420
|
+
Custom `metadata` keys are lowercased and must consist of `a-z`, `0-9` and
|
|
421
|
+
`_ - | ~ % $ # + . :`; any other key raises `ValueError` before a request is
|
|
422
|
+
sent.
|
|
423
|
+
|
|
413
424
|
You can use the utility function `objectstore_client.utils.guess_mime_type`
|
|
414
425
|
to attempt to guess a `content_type` based on magic bytes.
|
|
415
426
|
|
|
@@ -508,10 +519,14 @@ class Session:
|
|
|
508
519
|
)
|
|
509
520
|
if result_key is None:
|
|
510
521
|
# Session creation failed or resumables are unsupported.
|
|
511
|
-
# Send the prepared body without compressing it again
|
|
522
|
+
# Send the prepared body without compressing it again,
|
|
523
|
+
# capped at its declared length.
|
|
512
524
|
headers["Content-Length"] = str(compressed_size)
|
|
513
525
|
result_key, _, _ = self._put_direct(
|
|
514
|
-
body,
|
|
526
|
+
_resumable.bounded_reader(body, compressed_size),
|
|
527
|
+
key,
|
|
528
|
+
headers,
|
|
529
|
+
compression="none",
|
|
515
530
|
)
|
|
516
531
|
else:
|
|
517
532
|
result_key, source_size, compressed_size = self._put_direct(
|
|
@@ -559,6 +574,10 @@ class Session:
|
|
|
559
574
|
retries = pool_retries.new(read=0)
|
|
560
575
|
elif isinstance(pool_retries, int):
|
|
561
576
|
retries = urllib3.Retry(pool_retries, read=0)
|
|
577
|
+
elif isinstance(contents, bytes):
|
|
578
|
+
# Buffers are sent verbatim and their size cannot change, so they
|
|
579
|
+
# declare it upfront instead of using chunked encoding.
|
|
580
|
+
headers = {**headers, "Content-Length": str(len(contents))}
|
|
562
581
|
|
|
563
582
|
response = self._pool.request(
|
|
564
583
|
"POST" if not key else "PUT",
|
|
@@ -6,6 +6,7 @@ from collections.abc import Iterable, Iterator, Mapping
|
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from datetime import datetime, timedelta
|
|
8
8
|
from typing import Literal, TypeVar, cast
|
|
9
|
+
from urllib.parse import unquote
|
|
9
10
|
|
|
10
11
|
from objectstore_client.utils import decode_header_value
|
|
11
12
|
|
|
@@ -19,6 +20,9 @@ HEADER_FILENAME = "x-sn-filename"
|
|
|
19
20
|
HEADER_SIZE = "x-sn-size"
|
|
20
21
|
HEADER_META_PREFIX = "x-snme-"
|
|
21
22
|
|
|
23
|
+
_METADATA_KEY_PATTERN = re.compile(r"[a-z0-9_\-|~%$#+.:]+")
|
|
24
|
+
_KEY_HEADER_ENCODING = {"%": "%25", ":": "%3a"}
|
|
25
|
+
|
|
22
26
|
|
|
23
27
|
@dataclass
|
|
24
28
|
class TimeToIdle:
|
|
@@ -85,6 +89,9 @@ class Metadata:
|
|
|
85
89
|
"""
|
|
86
90
|
|
|
87
91
|
custom: dict[str, str]
|
|
92
|
+
"""
|
|
93
|
+
User-defined key-value pairs. Keys are lowercase; values are arbitrary strings.
|
|
94
|
+
"""
|
|
88
95
|
|
|
89
96
|
@classmethod
|
|
90
97
|
def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
|
|
@@ -116,7 +123,10 @@ class Metadata:
|
|
|
116
123
|
elif k == HEADER_SIZE:
|
|
117
124
|
size = int(v)
|
|
118
125
|
elif k.startswith(HEADER_META_PREFIX):
|
|
119
|
-
|
|
126
|
+
name = k[len(HEADER_META_PREFIX) :]
|
|
127
|
+
# Key names are decoded leniently: reads never fail on a malformed key.
|
|
128
|
+
key = unquote(name, errors="replace")
|
|
129
|
+
custom_metadata[key] = decode_header_value(v)
|
|
120
130
|
|
|
121
131
|
return Metadata(
|
|
122
132
|
content_type=content_type,
|
|
@@ -131,6 +141,35 @@ class Metadata:
|
|
|
131
141
|
)
|
|
132
142
|
|
|
133
143
|
|
|
144
|
+
def normalize_metadata_key(key: str) -> str:
|
|
145
|
+
"""
|
|
146
|
+
Validates a custom metadata key and returns it in canonical (lowercase) form.
|
|
147
|
+
|
|
148
|
+
ASCII letters are lowercased. The remaining key must be non-empty and consist only
|
|
149
|
+
of `a-z`, `0-9` and `_ - | ~ % $ # + . :`; anything else, including whitespace and
|
|
150
|
+
non-ASCII characters, raises a `ValueError` naming the key.
|
|
151
|
+
"""
|
|
152
|
+
# str.lower() alone would map some non-ASCII characters (e.g. the Kelvin sign) to
|
|
153
|
+
# ASCII letters, so reject non-ASCII input first.
|
|
154
|
+
normalized = key.lower()
|
|
155
|
+
if not key.isascii() or not _METADATA_KEY_PATTERN.fullmatch(normalized):
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"invalid metadata key {key!r}: keys must be non-empty and contain only "
|
|
158
|
+
"ASCII letters, digits and the characters _ - | ~ % $ # + . :"
|
|
159
|
+
)
|
|
160
|
+
return normalized
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def encode_metadata_key(key: str) -> str:
|
|
164
|
+
"""
|
|
165
|
+
Encodes a normalized metadata key for use in an `x-snme-<key>` header name.
|
|
166
|
+
|
|
167
|
+
Only `%` (as `%25`) and `:` (as `%3a`) are escaped; nothing else is.
|
|
168
|
+
Readers decode with full percent-decoding, so any escape spelling is accepted.
|
|
169
|
+
"""
|
|
170
|
+
return "".join(_KEY_HEADER_ENCODING.get(char, char) for char in key)
|
|
171
|
+
|
|
172
|
+
|
|
134
173
|
def format_expiration(expiration_policy: ExpirationPolicy) -> str:
|
|
135
174
|
if isinstance(expiration_policy, TimeToIdle):
|
|
136
175
|
return f"tti:{format_timedelta(expiration_policy.delta)}"
|
|
File without changes
|
|
File without changes
|
{objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/_compression.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|