objectstore-client 0.3.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/PKG-INFO +6 -1
  2. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/README.md +5 -0
  3. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/pyproject.toml +1 -1
  4. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/_resumable.py +103 -1
  5. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/client.py +23 -4
  6. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/metadata.py +40 -1
  7. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/LICENSE.md +0 -0
  8. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/__init__.py +0 -0
  9. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/_compression.py +0 -0
  10. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/auth.py +0 -0
  11. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/errors.py +0 -0
  12. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/metrics.py +0 -0
  13. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/presign.py +0 -0
  14. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/py.typed +0 -0
  15. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/scope.py +0 -0
  16. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/tracing.py +0 -0
  17. {objectstore_client-0.3.0 → objectstore_client-0.3.1}/src/objectstore_client/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: objectstore-client
3
- Version: 0.3.0
3
+ Version: 0.3.1
4
4
  Summary: Client SDK for Objectstore, the Sentry object storage platform
5
5
  Author: Sentry
6
6
  Author-email: Sentry <oss@sentry.io>
@@ -239,6 +239,11 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
239
239
  session.put(b"payload", metadata={"source": "upload-service"})
240
240
  ```
241
241
 
242
+ Keys are case-insensitive and are lowercased. After lowercasing, a key must be
243
+ non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
244
+ Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
245
+ naming the key as soon as the metadata is passed to `put`, before any request is sent.
246
+
242
247
  ### Authentication
243
248
 
244
249
  If your Objectstore instance enforces authorization, you must configure authentication
@@ -120,6 +120,11 @@ Arbitrary key-value pairs can be attached to objects and retrieved on download.
120
120
  session.put(b"payload", metadata={"source": "upload-service"})
121
121
  ```
122
122
 
123
+ Keys are case-insensitive and are lowercased. After lowercasing, a key must be
124
+ non-empty and consist only of `a-z`, `0-9` and the characters `_ - | ~ % $ # + . :`.
125
+ Anything else (spaces, other punctuation, non-ASCII characters) raises a `ValueError`
126
+ naming the key as soon as the metadata is passed to `put`, before any request is sent.
127
+
123
128
  ### Authentication
124
129
 
125
130
  If your Objectstore instance enforces authorization, you must configure authentication
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "objectstore-client"
3
- version = "0.3.0"
3
+ version = "0.3.1"
4
4
  description = "Client SDK for Objectstore, the Sentry object storage platform"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Sentry", email = "oss@sentry.io" }]
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import io
3
4
  import random
4
5
  import time
5
6
  from collections.abc import Iterator
@@ -8,7 +9,7 @@ from dataclasses import dataclass
8
9
  from io import SEEK_END, BytesIO
9
10
  from os import PathLike
10
11
  from tempfile import TemporaryFile
11
- from typing import IO, TYPE_CHECKING
12
+ from typing import IO, TYPE_CHECKING, cast
12
13
  from urllib.parse import urlencode
13
14
 
14
15
  import urllib3
@@ -20,6 +21,8 @@ from objectstore_client.metrics import measure_storage_operation
20
21
  from objectstore_client.tracing import storage_span
21
22
 
22
23
  if TYPE_CHECKING:
24
+ from _typeshed import WriteableBuffer
25
+
23
26
  from objectstore_client.client import Session
24
27
 
25
28
 
@@ -50,6 +53,14 @@ class NoUploadProgress(Exception):
50
53
  super().__init__("Upload made no progress")
51
54
 
52
55
 
56
+ class UploadBodyTruncated(OSError):
57
+ """The upload source ended before its declared length, e.g. a file shrank."""
58
+
59
+ def __init__(self, missing: int) -> None:
60
+ super().__init__(f"Upload source ended {missing} bytes early")
61
+ self.missing = missing
62
+
63
+
53
64
  @dataclass(frozen=True)
54
65
  class UploadIncomplete:
55
66
  """The upload expects more bytes, starting at the given offset."""
@@ -156,6 +167,8 @@ class ResumableUpload:
156
167
 
157
168
  if offset < 0 or length < 0:
158
169
  raise ValueError("Chunk offset and length must not be negative")
170
+ if not isinstance(body, bytes):
171
+ body = bounded_reader(body, length)
159
172
  if (
160
173
  self._granularity is not None
161
174
  and self._total_length is not None
@@ -292,6 +305,89 @@ def get_size(source: bytes | IO[bytes]) -> int | None:
292
305
  source.seek(start)
293
306
 
294
307
 
308
+ class _BoundedReader(io.RawIOBase):
309
+ """
310
+ A readable stream over the next `length` bytes of `source`.
311
+
312
+ This keeps a body within its declared `Content-Length`, even if `source` grows
313
+ during the upload. If `source` ends early instead, reading raises
314
+ `UploadBodyTruncated`, which aborts the request rather than leaving the server
315
+ waiting for the missing bytes.
316
+
317
+ Positions are those of `source`, so urllib3 can rewind a seekable body for
318
+ retries and redirects. The stream covers positions from the start of the bound
319
+ to its end; seeking before the start fails. If `source` is not seekable,
320
+ neither is this reader. Closing this reader leaves `source` open.
321
+ """
322
+
323
+ def __init__(self, source: IO[bytes], length: int) -> None:
324
+ if length < 0:
325
+ raise ValueError("length must not be negative")
326
+ self._source = source
327
+ self._length = length
328
+ self._remaining = length
329
+ self._start: int | None = None
330
+ try:
331
+ if source.seekable():
332
+ self._start = source.tell()
333
+ except (AttributeError, OSError):
334
+ pass
335
+
336
+ def _check_open(self) -> None:
337
+ if self.closed:
338
+ raise ValueError("I/O operation on closed file")
339
+
340
+ def _check_seekable(self) -> int:
341
+ self._check_open()
342
+ if self._start is None:
343
+ raise io.UnsupportedOperation("source is not seekable")
344
+ return self._start
345
+
346
+ def readable(self) -> bool:
347
+ self._check_open()
348
+ return True
349
+
350
+ def readinto(self, buffer: WriteableBuffer, /) -> int:
351
+ self._check_open()
352
+ view = memoryview(buffer).cast("B")
353
+ data = self._source.read(min(len(view), self._remaining))
354
+ if not data and view and self._remaining:
355
+ raise UploadBodyTruncated(self._remaining)
356
+ view[: len(data)] = data
357
+ self._remaining -= len(data)
358
+ return len(data)
359
+
360
+ def seekable(self) -> bool:
361
+ self._check_open()
362
+ return self._start is not None
363
+
364
+ def tell(self) -> int:
365
+ self._check_seekable()
366
+ return self._source.tell()
367
+
368
+ def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int:
369
+ start = self._check_seekable()
370
+ end = start + self._length
371
+ if whence == io.SEEK_SET:
372
+ target = offset
373
+ elif whence == io.SEEK_CUR:
374
+ target = self._source.tell() + offset
375
+ elif whence == io.SEEK_END:
376
+ target = end + offset
377
+ else:
378
+ raise ValueError(f"invalid whence: {whence}")
379
+ if target < start:
380
+ raise OSError("cannot seek before the start of the stream")
381
+ position = self._source.seek(target)
382
+ self._remaining = max(0, end - position)
383
+ return position
384
+
385
+
386
+ def bounded_reader(source: IO[bytes], length: int) -> IO[bytes]:
387
+ """Return a stream yielding at most `length` bytes of `source` from its position."""
388
+ return cast(IO[bytes], _BoundedReader(source, length))
389
+
390
+
295
391
  def is_transient(error: Exception) -> bool:
296
392
  if isinstance(error, urllib3.exceptions.MaxRetryError):
297
393
  # Exhausted status retries carry ResponseError rather than the response.
@@ -300,6 +396,12 @@ def is_transient(error: Exception) -> bool:
300
396
  )
301
397
  if isinstance(error, RequestError):
302
398
  return error.status in (408, 429, 502, 503, 504)
399
+ if isinstance(error, urllib3.exceptions.ProtocolError) and any(
400
+ isinstance(arg, UploadBodyTruncated) for arg in error.args
401
+ ):
402
+ # urllib3 wraps errors from reading the body. A local source that ended
403
+ # early fails the same way on every attempt, so it is not retried.
404
+ return False
303
405
  return isinstance(
304
406
  error,
305
407
  (
@@ -30,8 +30,10 @@ from objectstore_client.metadata import (
30
30
  Compression,
31
31
  ExpirationPolicy,
32
32
  Metadata,
33
+ encode_metadata_key,
33
34
  format_expiration,
34
35
  format_timedelta,
36
+ normalize_metadata_key,
35
37
  )
36
38
  from objectstore_client.metrics import (
37
39
  MetricsBackend,
@@ -349,7 +351,11 @@ class Session:
349
351
  origin: str | None = None,
350
352
  filename: str | None = None,
351
353
  ) -> dict[str, str]:
352
- """Build upload headers with the given compression and usecase expiration."""
354
+ """Build upload headers with the given compression and usecase expiration.
355
+
356
+ Raises `ValueError` if a custom metadata key is invalid (see
357
+ `normalize_metadata_key`), before anything is sent.
358
+ """
353
359
  headers = self._make_headers()
354
360
  if compression and compression != "none":
355
361
  headers["Content-Encoding"] = compression
@@ -364,7 +370,8 @@ class Session:
364
370
  headers[HEADER_FILENAME] = utils.encode_header_value(filename)
365
371
  if metadata:
366
372
  for k, v in metadata.items():
367
- headers[f"{HEADER_META_PREFIX}{k}"] = utils.encode_header_value(v)
373
+ name = encode_metadata_key(normalize_metadata_key(k))
374
+ headers[f"{HEADER_META_PREFIX}{name}"] = utils.encode_header_value(v)
368
375
  return headers
369
376
 
370
377
  def _base_url(self) -> str:
@@ -410,6 +417,10 @@ class Session:
410
417
  instead. The payload is then uploaded verbatim and the algorithm is recorded
411
418
  in the object's metadata, so downloads decompress it transparently.
412
419
 
420
+ Custom `metadata` keys are lowercased and must consist of `a-z`, `0-9` and
421
+ `_ - | ~ % $ # + . :`; any other key raises `ValueError` before a request is
422
+ sent.
423
+
413
424
  You can use the utility function `objectstore_client.utils.guess_mime_type`
414
425
  to attempt to guess a `content_type` based on magic bytes.
415
426
 
@@ -508,10 +519,14 @@ class Session:
508
519
  )
509
520
  if result_key is None:
510
521
  # Session creation failed or resumables are unsupported.
511
- # Send the prepared body without compressing it again.
522
+ # Send the prepared body without compressing it again,
523
+ # capped at its declared length.
512
524
  headers["Content-Length"] = str(compressed_size)
513
525
  result_key, _, _ = self._put_direct(
514
- body, key, headers, compression="none"
526
+ _resumable.bounded_reader(body, compressed_size),
527
+ key,
528
+ headers,
529
+ compression="none",
515
530
  )
516
531
  else:
517
532
  result_key, source_size, compressed_size = self._put_direct(
@@ -559,6 +574,10 @@ class Session:
559
574
  retries = pool_retries.new(read=0)
560
575
  elif isinstance(pool_retries, int):
561
576
  retries = urllib3.Retry(pool_retries, read=0)
577
+ elif isinstance(contents, bytes):
578
+ # Buffers are sent verbatim and their size cannot change, so they
579
+ # declare it upfront instead of using chunked encoding.
580
+ headers = {**headers, "Content-Length": str(len(contents))}
562
581
 
563
582
  response = self._pool.request(
564
583
  "POST" if not key else "PUT",
@@ -6,6 +6,7 @@ from collections.abc import Iterable, Iterator, Mapping
6
6
  from dataclasses import dataclass
7
7
  from datetime import datetime, timedelta
8
8
  from typing import Literal, TypeVar, cast
9
+ from urllib.parse import unquote
9
10
 
10
11
  from objectstore_client.utils import decode_header_value
11
12
 
@@ -19,6 +20,9 @@ HEADER_FILENAME = "x-sn-filename"
19
20
  HEADER_SIZE = "x-sn-size"
20
21
  HEADER_META_PREFIX = "x-snme-"
21
22
 
23
+ _METADATA_KEY_PATTERN = re.compile(r"[a-z0-9_\-|~%$#+.:]+")
24
+ _KEY_HEADER_ENCODING = {"%": "%25", ":": "%3a"}
25
+
22
26
 
23
27
  @dataclass
24
28
  class TimeToIdle:
@@ -85,6 +89,9 @@ class Metadata:
85
89
  """
86
90
 
87
91
  custom: dict[str, str]
92
+ """
93
+ User-defined key-value pairs. Keys are lowercase; values are arbitrary strings.
94
+ """
88
95
 
89
96
  @classmethod
90
97
  def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
@@ -116,7 +123,10 @@ class Metadata:
116
123
  elif k == HEADER_SIZE:
117
124
  size = int(v)
118
125
  elif k.startswith(HEADER_META_PREFIX):
119
- custom_metadata[k[len(HEADER_META_PREFIX) :]] = decode_header_value(v)
126
+ name = k[len(HEADER_META_PREFIX) :]
127
+ # Key names are decoded leniently: reads never fail on a malformed key.
128
+ key = unquote(name, errors="replace")
129
+ custom_metadata[key] = decode_header_value(v)
120
130
 
121
131
  return Metadata(
122
132
  content_type=content_type,
@@ -131,6 +141,35 @@ class Metadata:
131
141
  )
132
142
 
133
143
 
144
+ def normalize_metadata_key(key: str) -> str:
145
+ """
146
+ Validates a custom metadata key and returns it in canonical (lowercase) form.
147
+
148
+ ASCII letters are lowercased. The remaining key must be non-empty and consist only
149
+ of `a-z`, `0-9` and `_ - | ~ % $ # + . :`; anything else, including whitespace and
150
+ non-ASCII characters, raises a `ValueError` naming the key.
151
+ """
152
+ # str.lower() alone would map some non-ASCII characters (e.g. the Kelvin sign) to
153
+ # ASCII letters, so reject non-ASCII input first.
154
+ normalized = key.lower()
155
+ if not key.isascii() or not _METADATA_KEY_PATTERN.fullmatch(normalized):
156
+ raise ValueError(
157
+ f"invalid metadata key {key!r}: keys must be non-empty and contain only "
158
+ "ASCII letters, digits and the characters _ - | ~ % $ # + . :"
159
+ )
160
+ return normalized
161
+
162
+
163
+ def encode_metadata_key(key: str) -> str:
164
+ """
165
+ Encodes a normalized metadata key for use in an `x-snme-<key>` header name.
166
+
167
+ Only `%` (as `%25`) and `:` (as `%3a`) are escaped; nothing else is.
168
+ Readers decode with full percent-decoding, so any escape spelling is accepted.
169
+ """
170
+ return "".join(_KEY_HEADER_ENCODING.get(char, char) for char in key)
171
+
172
+
134
173
  def format_expiration(expiration_policy: ExpirationPolicy) -> str:
135
174
  if isinstance(expiration_policy, TimeToIdle):
136
175
  return f"tti:{format_timedelta(expiration_policy.delta)}"