leblob 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,431 @@
1
+ """Configuration documents: lifecycle rules, CORS, tags and ACLs. Pure.
2
+
3
+ A lifecycle rule deletes data, so anything its parser doesn't understand is refused.
4
+ Skipping an element could widen what a rule applies to.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from datetime import UTC, datetime, time
10
+ from itertools import starmap
11
+ from typing import TYPE_CHECKING
12
+
13
+ from leblob.model import (
14
+ CorsRule,
15
+ LifecycleRule,
16
+ RuleActions,
17
+ RuleScope,
18
+ )
19
+ from leblob.s3 import cors, errors, tagging, xml_base as base
20
+ from leblob.s3.errors import S3Error
21
+ from leblob.s3.xml import iso_date
22
+
23
+ if TYPE_CHECKING:
24
+ from collections.abc import Iterable, Mapping
25
+ from xml.etree.ElementTree import Element
26
+
27
+ _ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers"
28
+
29
+
30
+ MAX_CORS_RULES = 100
31
+
32
+
33
+ _CORS_PARTS = (
34
+ "AllowedHeader",
35
+ "AllowedMethod",
36
+ "AllowedOrigin",
37
+ "ExposeHeader",
38
+ "ID",
39
+ "MaxAgeSeconds",
40
+ )
41
+
42
+
43
+ MAX_LIFECYCLE_RULES = 1000
44
+
45
+
46
+ MAX_RULE_ID = 255
47
+
48
+
49
+ _RULE_PARTS = frozenset({
50
+ "AbortIncompleteMultipartUpload",
51
+ "Expiration",
52
+ "Filter",
53
+ "ID",
54
+ "NoncurrentVersionExpiration",
55
+ "Prefix",
56
+ "Status",
57
+ })
58
+
59
+
60
+ _FILTER_PARTS = frozenset({
61
+ "ObjectSizeGreaterThan",
62
+ "ObjectSizeLessThan",
63
+ "Prefix",
64
+ "Tag",
65
+ })
66
+
67
+
68
+ _TAG_PARTS = frozenset({"Key", "Value"})
69
+
70
+
71
+ _REPEATABLE = frozenset({"Tag"}) # A filter can name several tags, and only those.
72
+
73
+
74
+ _EXPIRATION_PARTS = frozenset({"Date", "Days", "ExpiredObjectDeleteMarker"})
75
+
76
+
77
+ _NONCURRENT_PARTS = frozenset({"NewerNoncurrentVersions", "NoncurrentDays"})
78
+
79
+
80
+ _ABORT_PARTS = frozenset({"DaysAfterInitiation"})
81
+
82
+
83
+ _LIFECYCLE_NOT_BUILT = frozenset({"NoncurrentVersionTransition", "Transition"})
84
+
85
+
86
+ def render_acl(*, public: bool) -> bytes:
87
+ everyone = (
88
+ '<Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
89
+ f'xsi:type="Group"><URI>{_ALL_USERS}</URI></Grantee>'
90
+ "<Permission>READ</Permission></Grant>"
91
+ if public
92
+ else ""
93
+ )
94
+ return base.document(
95
+ "AccessControlPolicy",
96
+ base.OWNER,
97
+ "<AccessControlList><Grant><Grantee "
98
+ 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
99
+ 'xsi:type="CanonicalUser"><ID>leblob</ID><DisplayName>leblob</DisplayName>'
100
+ f"</Grantee><Permission>FULL_CONTROL</Permission></Grant>{everyone}"
101
+ "</AccessControlList>",
102
+ )
103
+
104
+
105
+ def parse_acl(body: bytes) -> bool:
106
+ """Return whether a policy makes something public, if it says no more than that.
107
+
108
+ One owner, and reading by everyone, is all there is. A grant to anyone else, or an
109
+ owner left out, is a promise that wouldn't be kept.
110
+ """
111
+ public = owner = False
112
+ for element in base.parse_document(body).iter():
113
+ if base.local_name(element.tag) != "Grant":
114
+ continue
115
+ grantee_uri = next(
116
+ (
117
+ item.text
118
+ for item in element.iter()
119
+ if base.local_name(item.tag) == "URI"
120
+ ),
121
+ None,
122
+ )
123
+ grantee_id = next(
124
+ (item.text for item in element.iter() if base.local_name(item.tag) == "ID"),
125
+ None,
126
+ )
127
+ permission = base.optional_child_text(element, "Permission")
128
+ if grantee_uri == _ALL_USERS and permission == "READ":
129
+ public = True
130
+ elif grantee_id == base.OWNER_ID and permission == "FULL_CONTROL":
131
+ owner = True
132
+ else:
133
+ raise S3Error(errors.NOT_IMPLEMENTED, "That grant is not implemented.")
134
+ if not owner:
135
+ raise S3Error(errors.NOT_IMPLEMENTED, "The owner keeps full control.")
136
+ return public
137
+
138
+
139
+ def render_cors(rules: Iterable[CorsRule]) -> bytes:
140
+ return base.document("CORSConfiguration", *map(_render_cors_rule, rules))
141
+
142
+
143
+ def _render_cors_rule(rule: CorsRule) -> str:
144
+ inner = (
145
+ base.optional("ID", rule.rule_id)
146
+ + "".join(base.element("AllowedOrigin", item) for item in rule.origins)
147
+ + "".join(base.element("AllowedMethod", item) for item in rule.methods)
148
+ + "".join(base.element("AllowedHeader", item) for item in rule.allowed_headers)
149
+ + "".join(base.element("ExposeHeader", item) for item in rule.expose)
150
+ + base.optional("MaxAgeSeconds", rule.max_age)
151
+ )
152
+ return f"<CORSRule>{inner}</CORSRule>"
153
+
154
+
155
+ def parse_cors(body: bytes) -> tuple[CorsRule, ...]:
156
+ elements = list(base.parse_document(body))
157
+ if not elements or any(
158
+ base.local_name(item.tag) != "CORSRule" for item in elements
159
+ ):
160
+ raise S3Error(errors.MALFORMED_XML, "A configuration is a list of rules.")
161
+ if len(elements) > MAX_CORS_RULES:
162
+ raise S3Error(errors.INVALID_REQUEST, "Too many CORS rules.")
163
+ return tuple(map(_parse_cors_rule, elements))
164
+
165
+
166
+ def _parse_cors_rule(element: Element) -> CorsRule:
167
+ values: dict[str, list[str]] = {name: [] for name in _CORS_PARTS}
168
+ for child in element:
169
+ name = base.local_name(child.tag)
170
+ if name not in values:
171
+ raise S3Error(errors.MALFORMED_XML, f"Unexpected element: {name}.")
172
+ values[name].append(child.text or "")
173
+ if not values["AllowedOrigin"] or not values["AllowedMethod"]:
174
+ raise S3Error(errors.MALFORMED_XML, "A rule needs an origin and a method.")
175
+ _check_cors_values(values)
176
+ return CorsRule(
177
+ tuple(values["AllowedOrigin"]),
178
+ tuple(values["AllowedMethod"]),
179
+ tuple(values["AllowedHeader"]),
180
+ tuple(values["ExposeHeader"]),
181
+ values["ID"][0] if values["ID"] else None,
182
+ int(values["MaxAgeSeconds"][0]) if values["MaxAgeSeconds"] else None,
183
+ )
184
+
185
+
186
+ def _check_cors_values(values: Mapping[str, list[str]]) -> None:
187
+ if len(values["ID"]) > 1 or len(values["MaxAgeSeconds"]) > 1:
188
+ raise S3Error(errors.MALFORMED_XML, "Only one ID and MaxAgeSeconds in a rule.")
189
+ if any(len(rule_id) > MAX_RULE_ID for rule_id in values["ID"]):
190
+ raise S3Error(errors.INVALID_ARGUMENT, "That CORS rule ID is too long.")
191
+ if not all(text.isascii() and text.isdecimal() for text in values["MaxAgeSeconds"]):
192
+ raise S3Error(errors.MALFORMED_XML, "MaxAgeSeconds must be a whole number.")
193
+ if not cors.ALLOWED_METHODS.issuperset(values["AllowedMethod"]):
194
+ raise S3Error(errors.INVALID_REQUEST, "A rule names a method CORS can't allow.")
195
+ for pattern in (*values["AllowedOrigin"], *values["AllowedHeader"]):
196
+ if pattern.count("*") > 1:
197
+ raise S3Error(errors.INVALID_REQUEST, f"Two wildcards in {pattern!r}.")
198
+
199
+
200
+ def render_tagging(tags: Mapping[str, str]) -> bytes:
201
+ inner = "".join(starmap(_render_tag, tags.items()))
202
+ return base.document("Tagging", f"<TagSet>{inner}</TagSet>")
203
+
204
+
205
+ def _render_tag(key: str, value: str) -> str:
206
+ return f"<Tag>{base.element('Key', key)}{base.element('Value', value)}</Tag>"
207
+
208
+
209
+ def parse_tagging(body: bytes) -> dict[str, str]:
210
+ parts = _known_children(base.parse_document(body), frozenset({"TagSet"}))
211
+ if "TagSet" not in parts:
212
+ raise S3Error(errors.MALFORMED_XML, "Missing element: TagSet.")
213
+ if any(base.local_name(child.tag) != "Tag" for child in parts["TagSet"]):
214
+ raise S3Error(errors.MALFORMED_XML, "A TagSet is a list of tags.")
215
+ return tagging.build_tags(map(_parse_tag, parts["TagSet"]))
216
+
217
+
218
+ def _parse_tag(element: Element) -> tuple[str, str]:
219
+ parts = _known_children(element, _TAG_PARTS)
220
+ if parts.keys() != _TAG_PARTS:
221
+ raise S3Error(errors.MALFORMED_XML, "A tag needs a Key and a Value.")
222
+ return parts["Key"].text or "", parts["Value"].text or ""
223
+
224
+
225
+ def render_lifecycle(rules: Iterable[LifecycleRule]) -> bytes:
226
+ return base.document("LifecycleConfiguration", *map(_render_rule, rules))
227
+
228
+
229
+ def _render_rule(rule: LifecycleRule) -> str:
230
+ return (
231
+ f"<Rule>{base.element('ID', rule.rule_id)}{_render_scope(rule.scope)}"
232
+ f"{base.element('Status', 'Enabled' if rule.enabled else 'Disabled')}"
233
+ f"{_render_actions(rule.actions)}</Rule>"
234
+ )
235
+
236
+
237
+ def _render_scope(scope: RuleScope) -> str:
238
+ if scope.legacy_prefix:
239
+ return base.element("Prefix", scope.prefix)
240
+ conditions = [
241
+ base.optional("Prefix", scope.prefix),
242
+ base.optional("ObjectSizeGreaterThan", scope.larger_than),
243
+ base.optional("ObjectSizeLessThan", scope.smaller_than),
244
+ *starmap(_render_tag, scope.tags),
245
+ ]
246
+ inner = "".join(conditions)
247
+ several = sum(bool(condition) for condition in conditions) > 1
248
+ return (
249
+ f"<Filter><And>{inner}</And></Filter>"
250
+ if several
251
+ else f"<Filter>{inner}</Filter>"
252
+ )
253
+
254
+
255
+ def _render_actions(actions: RuleActions) -> str:
256
+ date = None if actions.expire_at is None else iso_date(actions.expire_at)
257
+ markers = "true" if actions.expire_lone_markers else None
258
+ expiration = (
259
+ base.optional("Days", actions.expire_days)
260
+ + base.optional("Date", date)
261
+ + base.optional("ExpiredObjectDeleteMarker", markers)
262
+ )
263
+ noncurrent = base.optional(
264
+ "NoncurrentDays", actions.noncurrent_days
265
+ ) + base.optional("NewerNoncurrentVersions", actions.keep_noncurrent or None)
266
+ abort = base.optional("DaysAfterInitiation", actions.abort_upload_days)
267
+ return (
268
+ _wrap_if_any("Expiration", expiration)
269
+ + _wrap_if_any("NoncurrentVersionExpiration", noncurrent)
270
+ + _wrap_if_any("AbortIncompleteMultipartUpload", abort)
271
+ )
272
+
273
+
274
+ def _wrap_if_any(tag: str, inner: str) -> str:
275
+ return f"<{tag}>{inner}</{tag}>" if inner else ""
276
+
277
+
278
+ def parse_lifecycle(body: bytes) -> tuple[LifecycleRule, ...]:
279
+ """Return the rules a configuration asks for.
280
+
281
+ A rule deletes data, so anything not understood is refused. Skipping an element
282
+ could widen what a rule applies to, as ignoring a <Tag> filter would.
283
+ """
284
+ elements = list(base.parse_document(body))
285
+ if not elements or any(base.local_name(item.tag) != "Rule" for item in elements):
286
+ raise S3Error(errors.MALFORMED_XML, "A configuration is a list of rules.")
287
+ if len(elements) > MAX_LIFECYCLE_RULES:
288
+ raise S3Error(errors.INVALID_ARGUMENT, "Too many lifecycle rules.")
289
+ rules = tuple(
290
+ _parse_rule(element, f"rule-{position}")
291
+ for position, element in enumerate(elements, 1)
292
+ )
293
+ if len({rule.rule_id for rule in rules}) < len(rules):
294
+ raise S3Error(errors.INVALID_ARGUMENT, "Lifecycle rule IDs must be unique.")
295
+ return rules
296
+
297
+
298
+ def _parse_rule(element: Element, default_id: str) -> LifecycleRule:
299
+ parts = _known_children(element, _RULE_PARTS)
300
+ rule_id = _text_of(parts, "ID") or default_id
301
+ if len(rule_id) > MAX_RULE_ID:
302
+ raise S3Error(errors.INVALID_ARGUMENT, "That lifecycle rule ID is too long.")
303
+ status = _text_of(parts, "Status")
304
+ if status not in {"Enabled", "Disabled"}:
305
+ raise S3Error(errors.MALFORMED_XML, f"Bad status: {status!r}.")
306
+ scope = _parse_scope(parts)
307
+ actions = _parse_actions(parts)
308
+ if actions == RuleActions():
309
+ raise S3Error(errors.MALFORMED_XML, "A lifecycle rule needs an action.")
310
+ if actions.abort_upload_days is not None and scope.looks_inside_objects():
311
+ # An upload has no size or tags yet: the rule would apply to more than it says.
312
+ raise S3Error(
313
+ errors.INVALID_REQUEST,
314
+ "A rule that aborts uploads can only filter on a prefix.",
315
+ )
316
+ if actions.expire_lone_markers and scope.tags:
317
+ raise S3Error(
318
+ errors.INVALID_REQUEST, "A delete marker has no tags to filter on."
319
+ )
320
+ return LifecycleRule(rule_id, status == "Enabled", scope, actions)
321
+
322
+
323
+ def _parse_scope(parts: Mapping[str, Element]) -> RuleScope:
324
+ """Require a scope: a rule over a whole bucket says so, with an empty filter."""
325
+ if ("Prefix" in parts) == ("Filter" in parts):
326
+ raise S3Error(errors.MALFORMED_XML, "A rule needs a Prefix or a Filter.")
327
+ if "Prefix" in parts:
328
+ return RuleScope(_text_of(parts, "Prefix") or "", legacy_prefix=True)
329
+ holder = parts["Filter"]
330
+ conditions = _known_children(holder, _FILTER_PARTS | {"And"}, _REPEATABLE)
331
+ if "And" in conditions:
332
+ if len(conditions) > 1:
333
+ raise S3Error(errors.MALFORMED_XML, "An And stands alone in a Filter.")
334
+ holder = conditions["And"]
335
+ conditions = _known_children(holder, _FILTER_PARTS, _REPEATABLE)
336
+ tags = tagging.build_tags(
337
+ _parse_tag(child) for child in holder if base.local_name(child.tag) == "Tag"
338
+ )
339
+ prefix = conditions.get("Prefix")
340
+ return RuleScope(
341
+ None if prefix is None else prefix.text or "",
342
+ larger_than=_whole_number(conditions, "ObjectSizeGreaterThan", minimum=0),
343
+ smaller_than=_whole_number(conditions, "ObjectSizeLessThan", minimum=1),
344
+ tags=tuple(tags.items()),
345
+ )
346
+
347
+
348
+ def _parse_actions(parts: Mapping[str, Element]) -> RuleActions:
349
+ expiration = _known_children_of(parts, "Expiration", _EXPIRATION_PARTS)
350
+ if len(expiration) > 1:
351
+ raise S3Error(errors.MALFORMED_XML, "An expiration takes one condition.")
352
+ noncurrent = _known_children_of(
353
+ parts, "NoncurrentVersionExpiration", _NONCURRENT_PARTS
354
+ )
355
+ if noncurrent and "NoncurrentDays" not in noncurrent:
356
+ raise S3Error(errors.MALFORMED_XML, "Missing element: NoncurrentDays.")
357
+ abort = _known_children_of(parts, "AbortIncompleteMultipartUpload", _ABORT_PARTS)
358
+ return RuleActions(
359
+ expire_days=_whole_number(expiration, "Days", minimum=1),
360
+ expire_at=_parse_rule_date(expiration),
361
+ expire_lone_markers=_parse_marker_flag(expiration),
362
+ noncurrent_days=_whole_number(noncurrent, "NoncurrentDays", minimum=1),
363
+ keep_noncurrent=_whole_number(noncurrent, "NewerNoncurrentVersions", minimum=1)
364
+ or 0,
365
+ abort_upload_days=_whole_number(abort, "DaysAfterInitiation", minimum=1),
366
+ )
367
+
368
+
369
+ def _parse_rule_date(expiration: Mapping[str, Element]) -> int | None:
370
+ if "Date" not in expiration:
371
+ return None
372
+ text = expiration["Date"].text or ""
373
+ try:
374
+ moment = datetime.fromisoformat(text)
375
+ except ValueError:
376
+ raise S3Error(errors.INVALID_ARGUMENT, f"Bad date: {text!r}.") from None
377
+ if moment.tzinfo is None:
378
+ moment = moment.replace(tzinfo=UTC)
379
+ if moment.astimezone(UTC).time() != time():
380
+ raise S3Error(errors.INVALID_ARGUMENT, "A date must be midnight UTC.")
381
+ return int(moment.timestamp()) * 1000
382
+
383
+
384
+ def _parse_marker_flag(expiration: Mapping[str, Element]) -> bool:
385
+ text = _text_of(expiration, "ExpiredObjectDeleteMarker")
386
+ if text not in {None, "true", "false"}:
387
+ raise S3Error(errors.MALFORMED_XML, f"Bad boolean: {text!r}.")
388
+ return text == "true"
389
+
390
+
391
+ def _whole_number(
392
+ parts: Mapping[str, Element], name: str, *, minimum: int
393
+ ) -> int | None:
394
+ if name not in parts:
395
+ return None
396
+ # Present but empty is refused: dropping a size condition would widen the rule.
397
+ text = parts[name].text or ""
398
+ if not (text.isascii() and text.isdecimal()):
399
+ raise S3Error(errors.MALFORMED_XML, f"{name} must be a whole number.")
400
+ value = int(text)
401
+ if value < minimum:
402
+ raise S3Error(errors.INVALID_ARGUMENT, f"{name} must be at least {minimum}.")
403
+ return value
404
+
405
+
406
+ def _known_children_of(
407
+ parts: Mapping[str, Element], name: str, allowed: frozenset[str]
408
+ ) -> dict[str, Element]:
409
+ return _known_children(parts[name], allowed) if name in parts else {}
410
+
411
+
412
+ def _known_children(
413
+ element: Element,
414
+ allowed: frozenset[str],
415
+ repeatable: frozenset[str] = frozenset(),
416
+ ) -> dict[str, Element]:
417
+ """An element's children by name, refusing surprises and repeats."""
418
+ found: dict[str, Element] = {}
419
+ for child in element:
420
+ name = base.local_name(child.tag)
421
+ if name in _LIFECYCLE_NOT_BUILT:
422
+ raise S3Error(errors.NOT_IMPLEMENTED, f"{name} is not implemented.")
423
+ repeated = name in found and name not in repeatable
424
+ if name not in allowed or repeated:
425
+ raise S3Error(errors.MALFORMED_XML, f"Unexpected element: {name}.")
426
+ found[name] = child
427
+ return found
428
+
429
+
430
+ def _text_of(parts: Mapping[str, Element], name: str) -> str | None:
431
+ return parts[name].text if name in parts else None
leblob/streams.py ADDED
@@ -0,0 +1,179 @@
1
+ """An object's bytes on their way in and out: decoding, checking, encrypting, storing.
2
+
3
+ Handlers deal in requests and answers, and leave the bytes to this.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import asyncio
9
+ from contextlib import aclosing
10
+ from typing import TYPE_CHECKING
11
+
12
+ from leblob import blobstore
13
+ from leblob.model import BlobSlice
14
+ from leblob.s3 import conditions, errors, payload, sse
15
+ from leblob.s3.errors import S3Error
16
+
17
+ if TYPE_CHECKING:
18
+ from collections.abc import AsyncGenerator, Callable, Mapping
19
+
20
+ from leblob.handlers import Context, Request
21
+ from leblob.model import Blob, ByteSpan, Checksum, ChecksumPlan, ObjectRecord
22
+
23
+ MAX_OBJECT_BYTES = 5 * 1024**3
24
+
25
+
26
+ class ClientDisconnectedError(Exception):
27
+ """The client went away during an upload; nothing was committed."""
28
+
29
+
30
+ async def decoded_body(
31
+ request: Request, decoder: payload.PlainDecoder | payload.ChunkedDecoder
32
+ ) -> AsyncGenerator[bytes]:
33
+ while True:
34
+ message = await request.receive()
35
+ if message["type"] == "http.disconnect":
36
+ raise ClientDisconnectedError
37
+ data = message.get("body", b"")
38
+ if data:
39
+ yield decoder.feed(data)
40
+ if not message.get("more_body", False):
41
+ return
42
+
43
+
44
+ async def receive_blob(
45
+ ctx: Context,
46
+ request: Request,
47
+ plan: ChecksumPlan | None = None,
48
+ key: sse.CustomerKey | None = None,
49
+ ) -> tuple[Blob, Checksum | None]:
50
+ """Store a request's body as a durable blob, if it keeps its promises."""
51
+ spec = payload.parse_spec(request.headers)
52
+ if plan is not None:
53
+ spec = payload.requiring(spec, plan.algorithm)
54
+ decoder = payload.new_decoder(spec, request.stream_seed)
55
+ return await _write_blob(
56
+ ctx, spec, decoded_body(request, decoder), key, decoder.finish
57
+ )
58
+
59
+
60
+ async def copy_to_new_blob(
61
+ ctx: Context,
62
+ chunks: AsyncGenerator[bytes],
63
+ plan: ChecksumPlan | None = None,
64
+ key: sse.CustomerKey | None = None,
65
+ ) -> tuple[Blob, Checksum | None]:
66
+ """Store bytes already held again, as any new blob is written."""
67
+ spec = payload.parse_spec({})
68
+ if plan is not None:
69
+ spec = payload.requiring(spec, plan.algorithm)
70
+ async with aclosing(chunks) as plain:
71
+ return await _write_blob(ctx, spec, plain, key, dict)
72
+
73
+
74
+ async def _write_blob(
75
+ ctx: Context,
76
+ spec: payload.PayloadSpec,
77
+ chunks: AsyncGenerator[bytes],
78
+ key: sse.CustomerKey | None,
79
+ trailers: Callable[[], Mapping[str, str]],
80
+ ) -> tuple[Blob, Checksum | None]:
81
+ """Make plain bytes a durable blob, once they have kept their promises.
82
+
83
+ Promises are about the plain bytes, and a blob's own MD5 about what is stored, which
84
+ with a key are different bytes, and so are hashed apart.
85
+ """
86
+ if key is None:
87
+ with ctx.blobs.writer(payload.new_hashers(spec)) as writer:
88
+ await blobstore.write_stream(writer, chunks, MAX_OBJECT_BYTES)
89
+ checksum = payload.verify(spec, writer.digests(), writer.size, trailers())
90
+ return await asyncio.to_thread(writer.commit), checksum
91
+
92
+ hashers = payload.new_hashers(spec)
93
+ encryptor = sse.Encryptor(key)
94
+ size = 0
95
+
96
+ async def sealed() -> AsyncGenerator[bytes]:
97
+ nonlocal size
98
+ async for chunk in chunks:
99
+ size += len(chunk)
100
+ for hasher in hashers.values():
101
+ hasher.update(chunk)
102
+ yield encryptor.feed(chunk)
103
+ yield encryptor.finish()
104
+
105
+ with ctx.blobs.writer(payload.new_hashers(payload.parse_spec({}))) as writer:
106
+ await blobstore.write_stream(
107
+ writer, sealed(), sse.stored_size(MAX_OBJECT_BYTES)
108
+ )
109
+ digests = {name: hasher.digest() for name, hasher in hashers.items()}
110
+ checksum = payload.verify(spec, digests, size, trailers())
111
+ return await asyncio.to_thread(writer.commit), checksum
112
+
113
+
114
+ def plain_bytes(
115
+ ctx: Context,
116
+ record: ObjectRecord,
117
+ span: ByteSpan,
118
+ key: sse.CustomerKey | None,
119
+ ) -> AsyncGenerator[bytes]:
120
+ """Stream a range of an object as its client knows it, decrypting if it needs to."""
121
+ if key is None:
122
+ slices = conditions.slice_blobs(record.blobs, span.start, span.length)
123
+ return blobstore.read_slices(ctx.blobs, slices)
124
+ return _decrypted_object(ctx, record, span, key)
125
+
126
+
127
+ async def _decrypted_object(
128
+ ctx: Context,
129
+ record: ObjectRecord,
130
+ span: ByteSpan,
131
+ key: sse.CustomerKey,
132
+ ) -> AsyncGenerator[bytes]:
133
+ start, length = span.start, span.length
134
+ position = 0
135
+ for blob in record.blobs:
136
+ size = sse.plain_size(blob.size)
137
+ first = max(start, position)
138
+ last = min(start + length, position + size)
139
+ if first < last:
140
+ salt = await _read_salt(ctx, blob)
141
+ plan = sse.plan_read(blob.size, first - position, last - first)
142
+ decryptor = sse.Decryptor(key, salt, blob.size, plan)
143
+ piece = BlobSlice(blob, plan.stored_offset, plan.stored_length)
144
+ stored = blobstore.read_slices(ctx.blobs, [piece])
145
+ try:
146
+ async for chunk in stored:
147
+ yield decryptor.feed(chunk)
148
+ finally:
149
+ await stored.aclose()
150
+ yield decryptor.finish()
151
+ position += size
152
+
153
+
154
+ async def _read_salt(ctx: Context, blob: Blob) -> bytes:
155
+ salt = bytearray()
156
+ head = [BlobSlice(blob, 0, sse.SALT_BYTES)]
157
+ async with aclosing(blobstore.read_slices(ctx.blobs, head)) as chunks:
158
+ async for chunk in chunks:
159
+ salt += chunk
160
+ return bytes(salt)
161
+
162
+
163
+ def customer_key(
164
+ ctx: Context, request: Request, prefix: str = sse.HEADER_PREFIX
165
+ ) -> sse.CustomerKey | None:
166
+ """Read a key sent with a request, which over plain HTTP anyone could read too."""
167
+ key = sse.parse_key(request.headers, prefix)
168
+ if key is not None and not request.secure and not ctx.sse_over_http:
169
+ raise S3Error(errors.INVALID_REQUEST, "An encryption key needs HTTPS.")
170
+ return key
171
+
172
+
173
+ def client_size(blob: Blob, key: sse.CustomerKey | None) -> int:
174
+ """How many bytes a client sent, which encrypting them takes more room to store."""
175
+ return blob.size if key is None else sse.plain_size(blob.size)
176
+
177
+
178
+ def key_headers(key: sse.CustomerKey | None) -> tuple[tuple[str, str], ...]:
179
+ return () if key is None else key.response_headers()