leblob 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leblob/__init__.py +1 -0
- leblob/asgi.py +376 -0
- leblob/blobstore.py +594 -0
- leblob/cli.py +499 -0
- leblob/handlers.py +1119 -0
- leblob/maintenance.py +444 -0
- leblob/meta.py +2327 -0
- leblob/metrics.py +226 -0
- leblob/model.py +339 -0
- leblob/s3/__init__.py +1 -0
- leblob/s3/access.py +110 -0
- leblob/s3/chunked.py +175 -0
- leblob/s3/conditions.py +156 -0
- leblob/s3/cors.py +80 -0
- leblob/s3/errors.py +144 -0
- leblob/s3/lifecycle.py +119 -0
- leblob/s3/listing.py +196 -0
- leblob/s3/multipart.py +184 -0
- leblob/s3/payload.py +328 -0
- leblob/s3/post_form.py +344 -0
- leblob/s3/routing.py +443 -0
- leblob/s3/sigv4.py +333 -0
- leblob/s3/sse.py +229 -0
- leblob/s3/tagging.py +48 -0
- leblob/s3/xml.py +436 -0
- leblob/s3/xml_base.py +64 -0
- leblob/s3/xml_config.py +431 -0
- leblob/streams.py +179 -0
- leblob-0.4.0.dev1.dist-info/METADATA +109 -0
- leblob-0.4.0.dev1.dist-info/RECORD +33 -0
- leblob-0.4.0.dev1.dist-info/WHEEL +4 -0
- leblob-0.4.0.dev1.dist-info/entry_points.txt +3 -0
- leblob-0.4.0.dev1.dist-info/licenses/LICENSE +202 -0
leblob/s3/xml_config.py
ADDED
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Configuration documents: lifecycle rules, CORS, tags and ACLs. Pure.
|
|
2
|
+
|
|
3
|
+
A lifecycle rule deletes data, so anything its parser doesn't understand is refused.
|
|
4
|
+
Skipping an element could widen what a rule applies to.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from datetime import UTC, datetime, time
|
|
10
|
+
from itertools import starmap
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from leblob.model import (
|
|
14
|
+
CorsRule,
|
|
15
|
+
LifecycleRule,
|
|
16
|
+
RuleActions,
|
|
17
|
+
RuleScope,
|
|
18
|
+
)
|
|
19
|
+
from leblob.s3 import cors, errors, tagging, xml_base as base
|
|
20
|
+
from leblob.s3.errors import S3Error
|
|
21
|
+
from leblob.s3.xml import iso_date
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from collections.abc import Iterable, Mapping
|
|
25
|
+
from xml.etree.ElementTree import Element
|
|
26
|
+
|
|
27
|
+
_ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
MAX_CORS_RULES = 100
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
_CORS_PARTS = (
|
|
34
|
+
"AllowedHeader",
|
|
35
|
+
"AllowedMethod",
|
|
36
|
+
"AllowedOrigin",
|
|
37
|
+
"ExposeHeader",
|
|
38
|
+
"ID",
|
|
39
|
+
"MaxAgeSeconds",
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
MAX_LIFECYCLE_RULES = 1000
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
MAX_RULE_ID = 255
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
_RULE_PARTS = frozenset({
|
|
50
|
+
"AbortIncompleteMultipartUpload",
|
|
51
|
+
"Expiration",
|
|
52
|
+
"Filter",
|
|
53
|
+
"ID",
|
|
54
|
+
"NoncurrentVersionExpiration",
|
|
55
|
+
"Prefix",
|
|
56
|
+
"Status",
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_FILTER_PARTS = frozenset({
|
|
61
|
+
"ObjectSizeGreaterThan",
|
|
62
|
+
"ObjectSizeLessThan",
|
|
63
|
+
"Prefix",
|
|
64
|
+
"Tag",
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
_TAG_PARTS = frozenset({"Key", "Value"})
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_REPEATABLE = frozenset({"Tag"}) # A filter can name several tags, and only those.
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_EXPIRATION_PARTS = frozenset({"Date", "Days", "ExpiredObjectDeleteMarker"})
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
_NONCURRENT_PARTS = frozenset({"NewerNoncurrentVersions", "NoncurrentDays"})
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_ABORT_PARTS = frozenset({"DaysAfterInitiation"})
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
_LIFECYCLE_NOT_BUILT = frozenset({"NoncurrentVersionTransition", "Transition"})
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def render_acl(*, public: bool) -> bytes:
|
|
87
|
+
everyone = (
|
|
88
|
+
'<Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
|
|
89
|
+
f'xsi:type="Group"><URI>{_ALL_USERS}</URI></Grantee>'
|
|
90
|
+
"<Permission>READ</Permission></Grant>"
|
|
91
|
+
if public
|
|
92
|
+
else ""
|
|
93
|
+
)
|
|
94
|
+
return base.document(
|
|
95
|
+
"AccessControlPolicy",
|
|
96
|
+
base.OWNER,
|
|
97
|
+
"<AccessControlList><Grant><Grantee "
|
|
98
|
+
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
|
|
99
|
+
'xsi:type="CanonicalUser"><ID>leblob</ID><DisplayName>leblob</DisplayName>'
|
|
100
|
+
f"</Grantee><Permission>FULL_CONTROL</Permission></Grant>{everyone}"
|
|
101
|
+
"</AccessControlList>",
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def parse_acl(body: bytes) -> bool:
|
|
106
|
+
"""Return whether a policy makes something public, if it says no more than that.
|
|
107
|
+
|
|
108
|
+
One owner, and reading by everyone, is all there is. A grant to anyone else, or an
|
|
109
|
+
owner left out, is a promise that wouldn't be kept.
|
|
110
|
+
"""
|
|
111
|
+
public = owner = False
|
|
112
|
+
for element in base.parse_document(body).iter():
|
|
113
|
+
if base.local_name(element.tag) != "Grant":
|
|
114
|
+
continue
|
|
115
|
+
grantee_uri = next(
|
|
116
|
+
(
|
|
117
|
+
item.text
|
|
118
|
+
for item in element.iter()
|
|
119
|
+
if base.local_name(item.tag) == "URI"
|
|
120
|
+
),
|
|
121
|
+
None,
|
|
122
|
+
)
|
|
123
|
+
grantee_id = next(
|
|
124
|
+
(item.text for item in element.iter() if base.local_name(item.tag) == "ID"),
|
|
125
|
+
None,
|
|
126
|
+
)
|
|
127
|
+
permission = base.optional_child_text(element, "Permission")
|
|
128
|
+
if grantee_uri == _ALL_USERS and permission == "READ":
|
|
129
|
+
public = True
|
|
130
|
+
elif grantee_id == base.OWNER_ID and permission == "FULL_CONTROL":
|
|
131
|
+
owner = True
|
|
132
|
+
else:
|
|
133
|
+
raise S3Error(errors.NOT_IMPLEMENTED, "That grant is not implemented.")
|
|
134
|
+
if not owner:
|
|
135
|
+
raise S3Error(errors.NOT_IMPLEMENTED, "The owner keeps full control.")
|
|
136
|
+
return public
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def render_cors(rules: Iterable[CorsRule]) -> bytes:
|
|
140
|
+
return base.document("CORSConfiguration", *map(_render_cors_rule, rules))
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _render_cors_rule(rule: CorsRule) -> str:
|
|
144
|
+
inner = (
|
|
145
|
+
base.optional("ID", rule.rule_id)
|
|
146
|
+
+ "".join(base.element("AllowedOrigin", item) for item in rule.origins)
|
|
147
|
+
+ "".join(base.element("AllowedMethod", item) for item in rule.methods)
|
|
148
|
+
+ "".join(base.element("AllowedHeader", item) for item in rule.allowed_headers)
|
|
149
|
+
+ "".join(base.element("ExposeHeader", item) for item in rule.expose)
|
|
150
|
+
+ base.optional("MaxAgeSeconds", rule.max_age)
|
|
151
|
+
)
|
|
152
|
+
return f"<CORSRule>{inner}</CORSRule>"
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def parse_cors(body: bytes) -> tuple[CorsRule, ...]:
|
|
156
|
+
elements = list(base.parse_document(body))
|
|
157
|
+
if not elements or any(
|
|
158
|
+
base.local_name(item.tag) != "CORSRule" for item in elements
|
|
159
|
+
):
|
|
160
|
+
raise S3Error(errors.MALFORMED_XML, "A configuration is a list of rules.")
|
|
161
|
+
if len(elements) > MAX_CORS_RULES:
|
|
162
|
+
raise S3Error(errors.INVALID_REQUEST, "Too many CORS rules.")
|
|
163
|
+
return tuple(map(_parse_cors_rule, elements))
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _parse_cors_rule(element: Element) -> CorsRule:
|
|
167
|
+
values: dict[str, list[str]] = {name: [] for name in _CORS_PARTS}
|
|
168
|
+
for child in element:
|
|
169
|
+
name = base.local_name(child.tag)
|
|
170
|
+
if name not in values:
|
|
171
|
+
raise S3Error(errors.MALFORMED_XML, f"Unexpected element: {name}.")
|
|
172
|
+
values[name].append(child.text or "")
|
|
173
|
+
if not values["AllowedOrigin"] or not values["AllowedMethod"]:
|
|
174
|
+
raise S3Error(errors.MALFORMED_XML, "A rule needs an origin and a method.")
|
|
175
|
+
_check_cors_values(values)
|
|
176
|
+
return CorsRule(
|
|
177
|
+
tuple(values["AllowedOrigin"]),
|
|
178
|
+
tuple(values["AllowedMethod"]),
|
|
179
|
+
tuple(values["AllowedHeader"]),
|
|
180
|
+
tuple(values["ExposeHeader"]),
|
|
181
|
+
values["ID"][0] if values["ID"] else None,
|
|
182
|
+
int(values["MaxAgeSeconds"][0]) if values["MaxAgeSeconds"] else None,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _check_cors_values(values: Mapping[str, list[str]]) -> None:
|
|
187
|
+
if len(values["ID"]) > 1 or len(values["MaxAgeSeconds"]) > 1:
|
|
188
|
+
raise S3Error(errors.MALFORMED_XML, "Only one ID and MaxAgeSeconds in a rule.")
|
|
189
|
+
if any(len(rule_id) > MAX_RULE_ID for rule_id in values["ID"]):
|
|
190
|
+
raise S3Error(errors.INVALID_ARGUMENT, "That CORS rule ID is too long.")
|
|
191
|
+
if not all(text.isascii() and text.isdecimal() for text in values["MaxAgeSeconds"]):
|
|
192
|
+
raise S3Error(errors.MALFORMED_XML, "MaxAgeSeconds must be a whole number.")
|
|
193
|
+
if not cors.ALLOWED_METHODS.issuperset(values["AllowedMethod"]):
|
|
194
|
+
raise S3Error(errors.INVALID_REQUEST, "A rule names a method CORS can't allow.")
|
|
195
|
+
for pattern in (*values["AllowedOrigin"], *values["AllowedHeader"]):
|
|
196
|
+
if pattern.count("*") > 1:
|
|
197
|
+
raise S3Error(errors.INVALID_REQUEST, f"Two wildcards in {pattern!r}.")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def render_tagging(tags: Mapping[str, str]) -> bytes:
|
|
201
|
+
inner = "".join(starmap(_render_tag, tags.items()))
|
|
202
|
+
return base.document("Tagging", f"<TagSet>{inner}</TagSet>")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _render_tag(key: str, value: str) -> str:
|
|
206
|
+
return f"<Tag>{base.element('Key', key)}{base.element('Value', value)}</Tag>"
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def parse_tagging(body: bytes) -> dict[str, str]:
|
|
210
|
+
parts = _known_children(base.parse_document(body), frozenset({"TagSet"}))
|
|
211
|
+
if "TagSet" not in parts:
|
|
212
|
+
raise S3Error(errors.MALFORMED_XML, "Missing element: TagSet.")
|
|
213
|
+
if any(base.local_name(child.tag) != "Tag" for child in parts["TagSet"]):
|
|
214
|
+
raise S3Error(errors.MALFORMED_XML, "A TagSet is a list of tags.")
|
|
215
|
+
return tagging.build_tags(map(_parse_tag, parts["TagSet"]))
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _parse_tag(element: Element) -> tuple[str, str]:
|
|
219
|
+
parts = _known_children(element, _TAG_PARTS)
|
|
220
|
+
if parts.keys() != _TAG_PARTS:
|
|
221
|
+
raise S3Error(errors.MALFORMED_XML, "A tag needs a Key and a Value.")
|
|
222
|
+
return parts["Key"].text or "", parts["Value"].text or ""
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def render_lifecycle(rules: Iterable[LifecycleRule]) -> bytes:
|
|
226
|
+
return base.document("LifecycleConfiguration", *map(_render_rule, rules))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _render_rule(rule: LifecycleRule) -> str:
|
|
230
|
+
return (
|
|
231
|
+
f"<Rule>{base.element('ID', rule.rule_id)}{_render_scope(rule.scope)}"
|
|
232
|
+
f"{base.element('Status', 'Enabled' if rule.enabled else 'Disabled')}"
|
|
233
|
+
f"{_render_actions(rule.actions)}</Rule>"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _render_scope(scope: RuleScope) -> str:
|
|
238
|
+
if scope.legacy_prefix:
|
|
239
|
+
return base.element("Prefix", scope.prefix)
|
|
240
|
+
conditions = [
|
|
241
|
+
base.optional("Prefix", scope.prefix),
|
|
242
|
+
base.optional("ObjectSizeGreaterThan", scope.larger_than),
|
|
243
|
+
base.optional("ObjectSizeLessThan", scope.smaller_than),
|
|
244
|
+
*starmap(_render_tag, scope.tags),
|
|
245
|
+
]
|
|
246
|
+
inner = "".join(conditions)
|
|
247
|
+
several = sum(bool(condition) for condition in conditions) > 1
|
|
248
|
+
return (
|
|
249
|
+
f"<Filter><And>{inner}</And></Filter>"
|
|
250
|
+
if several
|
|
251
|
+
else f"<Filter>{inner}</Filter>"
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _render_actions(actions: RuleActions) -> str:
|
|
256
|
+
date = None if actions.expire_at is None else iso_date(actions.expire_at)
|
|
257
|
+
markers = "true" if actions.expire_lone_markers else None
|
|
258
|
+
expiration = (
|
|
259
|
+
base.optional("Days", actions.expire_days)
|
|
260
|
+
+ base.optional("Date", date)
|
|
261
|
+
+ base.optional("ExpiredObjectDeleteMarker", markers)
|
|
262
|
+
)
|
|
263
|
+
noncurrent = base.optional(
|
|
264
|
+
"NoncurrentDays", actions.noncurrent_days
|
|
265
|
+
) + base.optional("NewerNoncurrentVersions", actions.keep_noncurrent or None)
|
|
266
|
+
abort = base.optional("DaysAfterInitiation", actions.abort_upload_days)
|
|
267
|
+
return (
|
|
268
|
+
_wrap_if_any("Expiration", expiration)
|
|
269
|
+
+ _wrap_if_any("NoncurrentVersionExpiration", noncurrent)
|
|
270
|
+
+ _wrap_if_any("AbortIncompleteMultipartUpload", abort)
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _wrap_if_any(tag: str, inner: str) -> str:
|
|
275
|
+
return f"<{tag}>{inner}</{tag}>" if inner else ""
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def parse_lifecycle(body: bytes) -> tuple[LifecycleRule, ...]:
|
|
279
|
+
"""Return the rules a configuration asks for.
|
|
280
|
+
|
|
281
|
+
A rule deletes data, so anything not understood is refused. Skipping an element
|
|
282
|
+
could widen what a rule applies to, as ignoring a <Tag> filter would.
|
|
283
|
+
"""
|
|
284
|
+
elements = list(base.parse_document(body))
|
|
285
|
+
if not elements or any(base.local_name(item.tag) != "Rule" for item in elements):
|
|
286
|
+
raise S3Error(errors.MALFORMED_XML, "A configuration is a list of rules.")
|
|
287
|
+
if len(elements) > MAX_LIFECYCLE_RULES:
|
|
288
|
+
raise S3Error(errors.INVALID_ARGUMENT, "Too many lifecycle rules.")
|
|
289
|
+
rules = tuple(
|
|
290
|
+
_parse_rule(element, f"rule-{position}")
|
|
291
|
+
for position, element in enumerate(elements, 1)
|
|
292
|
+
)
|
|
293
|
+
if len({rule.rule_id for rule in rules}) < len(rules):
|
|
294
|
+
raise S3Error(errors.INVALID_ARGUMENT, "Lifecycle rule IDs must be unique.")
|
|
295
|
+
return rules
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _parse_rule(element: Element, default_id: str) -> LifecycleRule:
|
|
299
|
+
parts = _known_children(element, _RULE_PARTS)
|
|
300
|
+
rule_id = _text_of(parts, "ID") or default_id
|
|
301
|
+
if len(rule_id) > MAX_RULE_ID:
|
|
302
|
+
raise S3Error(errors.INVALID_ARGUMENT, "That lifecycle rule ID is too long.")
|
|
303
|
+
status = _text_of(parts, "Status")
|
|
304
|
+
if status not in {"Enabled", "Disabled"}:
|
|
305
|
+
raise S3Error(errors.MALFORMED_XML, f"Bad status: {status!r}.")
|
|
306
|
+
scope = _parse_scope(parts)
|
|
307
|
+
actions = _parse_actions(parts)
|
|
308
|
+
if actions == RuleActions():
|
|
309
|
+
raise S3Error(errors.MALFORMED_XML, "A lifecycle rule needs an action.")
|
|
310
|
+
if actions.abort_upload_days is not None and scope.looks_inside_objects():
|
|
311
|
+
# An upload has no size or tags yet: the rule would apply to more than it says.
|
|
312
|
+
raise S3Error(
|
|
313
|
+
errors.INVALID_REQUEST,
|
|
314
|
+
"A rule that aborts uploads can only filter on a prefix.",
|
|
315
|
+
)
|
|
316
|
+
if actions.expire_lone_markers and scope.tags:
|
|
317
|
+
raise S3Error(
|
|
318
|
+
errors.INVALID_REQUEST, "A delete marker has no tags to filter on."
|
|
319
|
+
)
|
|
320
|
+
return LifecycleRule(rule_id, status == "Enabled", scope, actions)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _parse_scope(parts: Mapping[str, Element]) -> RuleScope:
|
|
324
|
+
"""Require a scope: a rule over a whole bucket says so, with an empty filter."""
|
|
325
|
+
if ("Prefix" in parts) == ("Filter" in parts):
|
|
326
|
+
raise S3Error(errors.MALFORMED_XML, "A rule needs a Prefix or a Filter.")
|
|
327
|
+
if "Prefix" in parts:
|
|
328
|
+
return RuleScope(_text_of(parts, "Prefix") or "", legacy_prefix=True)
|
|
329
|
+
holder = parts["Filter"]
|
|
330
|
+
conditions = _known_children(holder, _FILTER_PARTS | {"And"}, _REPEATABLE)
|
|
331
|
+
if "And" in conditions:
|
|
332
|
+
if len(conditions) > 1:
|
|
333
|
+
raise S3Error(errors.MALFORMED_XML, "An And stands alone in a Filter.")
|
|
334
|
+
holder = conditions["And"]
|
|
335
|
+
conditions = _known_children(holder, _FILTER_PARTS, _REPEATABLE)
|
|
336
|
+
tags = tagging.build_tags(
|
|
337
|
+
_parse_tag(child) for child in holder if base.local_name(child.tag) == "Tag"
|
|
338
|
+
)
|
|
339
|
+
prefix = conditions.get("Prefix")
|
|
340
|
+
return RuleScope(
|
|
341
|
+
None if prefix is None else prefix.text or "",
|
|
342
|
+
larger_than=_whole_number(conditions, "ObjectSizeGreaterThan", minimum=0),
|
|
343
|
+
smaller_than=_whole_number(conditions, "ObjectSizeLessThan", minimum=1),
|
|
344
|
+
tags=tuple(tags.items()),
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _parse_actions(parts: Mapping[str, Element]) -> RuleActions:
|
|
349
|
+
expiration = _known_children_of(parts, "Expiration", _EXPIRATION_PARTS)
|
|
350
|
+
if len(expiration) > 1:
|
|
351
|
+
raise S3Error(errors.MALFORMED_XML, "An expiration takes one condition.")
|
|
352
|
+
noncurrent = _known_children_of(
|
|
353
|
+
parts, "NoncurrentVersionExpiration", _NONCURRENT_PARTS
|
|
354
|
+
)
|
|
355
|
+
if noncurrent and "NoncurrentDays" not in noncurrent:
|
|
356
|
+
raise S3Error(errors.MALFORMED_XML, "Missing element: NoncurrentDays.")
|
|
357
|
+
abort = _known_children_of(parts, "AbortIncompleteMultipartUpload", _ABORT_PARTS)
|
|
358
|
+
return RuleActions(
|
|
359
|
+
expire_days=_whole_number(expiration, "Days", minimum=1),
|
|
360
|
+
expire_at=_parse_rule_date(expiration),
|
|
361
|
+
expire_lone_markers=_parse_marker_flag(expiration),
|
|
362
|
+
noncurrent_days=_whole_number(noncurrent, "NoncurrentDays", minimum=1),
|
|
363
|
+
keep_noncurrent=_whole_number(noncurrent, "NewerNoncurrentVersions", minimum=1)
|
|
364
|
+
or 0,
|
|
365
|
+
abort_upload_days=_whole_number(abort, "DaysAfterInitiation", minimum=1),
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _parse_rule_date(expiration: Mapping[str, Element]) -> int | None:
|
|
370
|
+
if "Date" not in expiration:
|
|
371
|
+
return None
|
|
372
|
+
text = expiration["Date"].text or ""
|
|
373
|
+
try:
|
|
374
|
+
moment = datetime.fromisoformat(text)
|
|
375
|
+
except ValueError:
|
|
376
|
+
raise S3Error(errors.INVALID_ARGUMENT, f"Bad date: {text!r}.") from None
|
|
377
|
+
if moment.tzinfo is None:
|
|
378
|
+
moment = moment.replace(tzinfo=UTC)
|
|
379
|
+
if moment.astimezone(UTC).time() != time():
|
|
380
|
+
raise S3Error(errors.INVALID_ARGUMENT, "A date must be midnight UTC.")
|
|
381
|
+
return int(moment.timestamp()) * 1000
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _parse_marker_flag(expiration: Mapping[str, Element]) -> bool:
|
|
385
|
+
text = _text_of(expiration, "ExpiredObjectDeleteMarker")
|
|
386
|
+
if text not in {None, "true", "false"}:
|
|
387
|
+
raise S3Error(errors.MALFORMED_XML, f"Bad boolean: {text!r}.")
|
|
388
|
+
return text == "true"
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _whole_number(
|
|
392
|
+
parts: Mapping[str, Element], name: str, *, minimum: int
|
|
393
|
+
) -> int | None:
|
|
394
|
+
if name not in parts:
|
|
395
|
+
return None
|
|
396
|
+
# Present but empty is refused: dropping a size condition would widen the rule.
|
|
397
|
+
text = parts[name].text or ""
|
|
398
|
+
if not (text.isascii() and text.isdecimal()):
|
|
399
|
+
raise S3Error(errors.MALFORMED_XML, f"{name} must be a whole number.")
|
|
400
|
+
value = int(text)
|
|
401
|
+
if value < minimum:
|
|
402
|
+
raise S3Error(errors.INVALID_ARGUMENT, f"{name} must be at least {minimum}.")
|
|
403
|
+
return value
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _known_children_of(
|
|
407
|
+
parts: Mapping[str, Element], name: str, allowed: frozenset[str]
|
|
408
|
+
) -> dict[str, Element]:
|
|
409
|
+
return _known_children(parts[name], allowed) if name in parts else {}
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _known_children(
|
|
413
|
+
element: Element,
|
|
414
|
+
allowed: frozenset[str],
|
|
415
|
+
repeatable: frozenset[str] = frozenset(),
|
|
416
|
+
) -> dict[str, Element]:
|
|
417
|
+
"""An element's children by name, refusing surprises and repeats."""
|
|
418
|
+
found: dict[str, Element] = {}
|
|
419
|
+
for child in element:
|
|
420
|
+
name = base.local_name(child.tag)
|
|
421
|
+
if name in _LIFECYCLE_NOT_BUILT:
|
|
422
|
+
raise S3Error(errors.NOT_IMPLEMENTED, f"{name} is not implemented.")
|
|
423
|
+
repeated = name in found and name not in repeatable
|
|
424
|
+
if name not in allowed or repeated:
|
|
425
|
+
raise S3Error(errors.MALFORMED_XML, f"Unexpected element: {name}.")
|
|
426
|
+
found[name] = child
|
|
427
|
+
return found
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _text_of(parts: Mapping[str, Element], name: str) -> str | None:
|
|
431
|
+
return parts[name].text if name in parts else None
|
leblob/streams.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""An object's bytes on their way in and out: decoding, checking, encrypting, storing.
|
|
2
|
+
|
|
3
|
+
Handlers deal in requests and answers, and leave the bytes to this.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
from contextlib import aclosing
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
from leblob import blobstore
|
|
13
|
+
from leblob.model import BlobSlice
|
|
14
|
+
from leblob.s3 import conditions, errors, payload, sse
|
|
15
|
+
from leblob.s3.errors import S3Error
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from collections.abc import AsyncGenerator, Callable, Mapping
|
|
19
|
+
|
|
20
|
+
from leblob.handlers import Context, Request
|
|
21
|
+
from leblob.model import Blob, ByteSpan, Checksum, ChecksumPlan, ObjectRecord
|
|
22
|
+
|
|
23
|
+
MAX_OBJECT_BYTES = 5 * 1024**3
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ClientDisconnectedError(Exception):
|
|
27
|
+
"""The client went away during an upload; nothing was committed."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def decoded_body(
|
|
31
|
+
request: Request, decoder: payload.PlainDecoder | payload.ChunkedDecoder
|
|
32
|
+
) -> AsyncGenerator[bytes]:
|
|
33
|
+
while True:
|
|
34
|
+
message = await request.receive()
|
|
35
|
+
if message["type"] == "http.disconnect":
|
|
36
|
+
raise ClientDisconnectedError
|
|
37
|
+
data = message.get("body", b"")
|
|
38
|
+
if data:
|
|
39
|
+
yield decoder.feed(data)
|
|
40
|
+
if not message.get("more_body", False):
|
|
41
|
+
return
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def receive_blob(
|
|
45
|
+
ctx: Context,
|
|
46
|
+
request: Request,
|
|
47
|
+
plan: ChecksumPlan | None = None,
|
|
48
|
+
key: sse.CustomerKey | None = None,
|
|
49
|
+
) -> tuple[Blob, Checksum | None]:
|
|
50
|
+
"""Store a request's body as a durable blob, if it keeps its promises."""
|
|
51
|
+
spec = payload.parse_spec(request.headers)
|
|
52
|
+
if plan is not None:
|
|
53
|
+
spec = payload.requiring(spec, plan.algorithm)
|
|
54
|
+
decoder = payload.new_decoder(spec, request.stream_seed)
|
|
55
|
+
return await _write_blob(
|
|
56
|
+
ctx, spec, decoded_body(request, decoder), key, decoder.finish
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
async def copy_to_new_blob(
|
|
61
|
+
ctx: Context,
|
|
62
|
+
chunks: AsyncGenerator[bytes],
|
|
63
|
+
plan: ChecksumPlan | None = None,
|
|
64
|
+
key: sse.CustomerKey | None = None,
|
|
65
|
+
) -> tuple[Blob, Checksum | None]:
|
|
66
|
+
"""Store bytes already held again, as any new blob is written."""
|
|
67
|
+
spec = payload.parse_spec({})
|
|
68
|
+
if plan is not None:
|
|
69
|
+
spec = payload.requiring(spec, plan.algorithm)
|
|
70
|
+
async with aclosing(chunks) as plain:
|
|
71
|
+
return await _write_blob(ctx, spec, plain, key, dict)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
async def _write_blob(
|
|
75
|
+
ctx: Context,
|
|
76
|
+
spec: payload.PayloadSpec,
|
|
77
|
+
chunks: AsyncGenerator[bytes],
|
|
78
|
+
key: sse.CustomerKey | None,
|
|
79
|
+
trailers: Callable[[], Mapping[str, str]],
|
|
80
|
+
) -> tuple[Blob, Checksum | None]:
|
|
81
|
+
"""Make plain bytes a durable blob, once they have kept their promises.
|
|
82
|
+
|
|
83
|
+
Promises are about the plain bytes, and a blob's own MD5 about what is stored, which
|
|
84
|
+
with a key are different bytes, and so are hashed apart.
|
|
85
|
+
"""
|
|
86
|
+
if key is None:
|
|
87
|
+
with ctx.blobs.writer(payload.new_hashers(spec)) as writer:
|
|
88
|
+
await blobstore.write_stream(writer, chunks, MAX_OBJECT_BYTES)
|
|
89
|
+
checksum = payload.verify(spec, writer.digests(), writer.size, trailers())
|
|
90
|
+
return await asyncio.to_thread(writer.commit), checksum
|
|
91
|
+
|
|
92
|
+
hashers = payload.new_hashers(spec)
|
|
93
|
+
encryptor = sse.Encryptor(key)
|
|
94
|
+
size = 0
|
|
95
|
+
|
|
96
|
+
async def sealed() -> AsyncGenerator[bytes]:
|
|
97
|
+
nonlocal size
|
|
98
|
+
async for chunk in chunks:
|
|
99
|
+
size += len(chunk)
|
|
100
|
+
for hasher in hashers.values():
|
|
101
|
+
hasher.update(chunk)
|
|
102
|
+
yield encryptor.feed(chunk)
|
|
103
|
+
yield encryptor.finish()
|
|
104
|
+
|
|
105
|
+
with ctx.blobs.writer(payload.new_hashers(payload.parse_spec({}))) as writer:
|
|
106
|
+
await blobstore.write_stream(
|
|
107
|
+
writer, sealed(), sse.stored_size(MAX_OBJECT_BYTES)
|
|
108
|
+
)
|
|
109
|
+
digests = {name: hasher.digest() for name, hasher in hashers.items()}
|
|
110
|
+
checksum = payload.verify(spec, digests, size, trailers())
|
|
111
|
+
return await asyncio.to_thread(writer.commit), checksum
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def plain_bytes(
|
|
115
|
+
ctx: Context,
|
|
116
|
+
record: ObjectRecord,
|
|
117
|
+
span: ByteSpan,
|
|
118
|
+
key: sse.CustomerKey | None,
|
|
119
|
+
) -> AsyncGenerator[bytes]:
|
|
120
|
+
"""Stream a range of an object as its client knows it, decrypting if it needs to."""
|
|
121
|
+
if key is None:
|
|
122
|
+
slices = conditions.slice_blobs(record.blobs, span.start, span.length)
|
|
123
|
+
return blobstore.read_slices(ctx.blobs, slices)
|
|
124
|
+
return _decrypted_object(ctx, record, span, key)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
async def _decrypted_object(
|
|
128
|
+
ctx: Context,
|
|
129
|
+
record: ObjectRecord,
|
|
130
|
+
span: ByteSpan,
|
|
131
|
+
key: sse.CustomerKey,
|
|
132
|
+
) -> AsyncGenerator[bytes]:
|
|
133
|
+
start, length = span.start, span.length
|
|
134
|
+
position = 0
|
|
135
|
+
for blob in record.blobs:
|
|
136
|
+
size = sse.plain_size(blob.size)
|
|
137
|
+
first = max(start, position)
|
|
138
|
+
last = min(start + length, position + size)
|
|
139
|
+
if first < last:
|
|
140
|
+
salt = await _read_salt(ctx, blob)
|
|
141
|
+
plan = sse.plan_read(blob.size, first - position, last - first)
|
|
142
|
+
decryptor = sse.Decryptor(key, salt, blob.size, plan)
|
|
143
|
+
piece = BlobSlice(blob, plan.stored_offset, plan.stored_length)
|
|
144
|
+
stored = blobstore.read_slices(ctx.blobs, [piece])
|
|
145
|
+
try:
|
|
146
|
+
async for chunk in stored:
|
|
147
|
+
yield decryptor.feed(chunk)
|
|
148
|
+
finally:
|
|
149
|
+
await stored.aclose()
|
|
150
|
+
yield decryptor.finish()
|
|
151
|
+
position += size
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
async def _read_salt(ctx: Context, blob: Blob) -> bytes:
|
|
155
|
+
salt = bytearray()
|
|
156
|
+
head = [BlobSlice(blob, 0, sse.SALT_BYTES)]
|
|
157
|
+
async with aclosing(blobstore.read_slices(ctx.blobs, head)) as chunks:
|
|
158
|
+
async for chunk in chunks:
|
|
159
|
+
salt += chunk
|
|
160
|
+
return bytes(salt)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def customer_key(
|
|
164
|
+
ctx: Context, request: Request, prefix: str = sse.HEADER_PREFIX
|
|
165
|
+
) -> sse.CustomerKey | None:
|
|
166
|
+
"""Read a key sent with a request, which over plain HTTP anyone could read too."""
|
|
167
|
+
key = sse.parse_key(request.headers, prefix)
|
|
168
|
+
if key is not None and not request.secure and not ctx.sse_over_http:
|
|
169
|
+
raise S3Error(errors.INVALID_REQUEST, "An encryption key needs HTTPS.")
|
|
170
|
+
return key
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def client_size(blob: Blob, key: sse.CustomerKey | None) -> int:
|
|
174
|
+
"""How many bytes a client sent, which encrypting them takes more room to store."""
|
|
175
|
+
return blob.size if key is None else sse.plain_size(blob.size)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def key_headers(key: sse.CustomerKey | None) -> tuple[tuple[str, str], ...]:
|
|
179
|
+
return () if key is None else key.response_headers()
|