leblob 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- leblob/__init__.py +1 -0
- leblob/asgi.py +376 -0
- leblob/blobstore.py +594 -0
- leblob/cli.py +499 -0
- leblob/handlers.py +1119 -0
- leblob/maintenance.py +444 -0
- leblob/meta.py +2327 -0
- leblob/metrics.py +226 -0
- leblob/model.py +339 -0
- leblob/s3/__init__.py +1 -0
- leblob/s3/access.py +110 -0
- leblob/s3/chunked.py +175 -0
- leblob/s3/conditions.py +156 -0
- leblob/s3/cors.py +80 -0
- leblob/s3/errors.py +144 -0
- leblob/s3/lifecycle.py +119 -0
- leblob/s3/listing.py +196 -0
- leblob/s3/multipart.py +184 -0
- leblob/s3/payload.py +328 -0
- leblob/s3/post_form.py +344 -0
- leblob/s3/routing.py +443 -0
- leblob/s3/sigv4.py +333 -0
- leblob/s3/sse.py +229 -0
- leblob/s3/tagging.py +48 -0
- leblob/s3/xml.py +436 -0
- leblob/s3/xml_base.py +64 -0
- leblob/s3/xml_config.py +431 -0
- leblob/streams.py +179 -0
- leblob-0.4.0.dev1.dist-info/METADATA +109 -0
- leblob-0.4.0.dev1.dist-info/RECORD +33 -0
- leblob-0.4.0.dev1.dist-info/WHEEL +4 -0
- leblob-0.4.0.dev1.dist-info/entry_points.txt +3 -0
- leblob-0.4.0.dev1.dist-info/licenses/LICENSE +202 -0
leblob/s3/chunked.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Incremental decoder for the aws-chunked upload framing.
|
|
2
|
+
|
|
3
|
+
Bytes arrive in arbitrary pieces from the network, so the decoder is a state
|
|
4
|
+
machine that never assumes a chunk header or trailer line arrives whole.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import hmac
|
|
11
|
+
import re
|
|
12
|
+
from enum import Enum, auto
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
from leblob.s3 import errors, sigv4
|
|
16
|
+
from leblob.s3.errors import S3Error
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from collections.abc import Mapping
|
|
20
|
+
|
|
21
|
+
from leblob.s3.sigv4 import StreamSeed
|
|
22
|
+
|
|
23
|
+
_MAX_LINE_BYTES = 8192
|
|
24
|
+
_CRLF = b"\r\n"
|
|
25
|
+
_CHUNK_HEADER = re.compile(rb"([0-9a-fA-F]{1,16})(?:;chunk-signature=([0-9a-f]{64}))?")
|
|
26
|
+
_TRAILER_SIGNATURE = "x-amz-trailer-signature"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _State(Enum):
|
|
30
|
+
CHUNK_HEADER = auto()
|
|
31
|
+
CHUNK_DATA = auto()
|
|
32
|
+
CHUNK_END = auto()
|
|
33
|
+
TRAILER = auto()
|
|
34
|
+
DONE = auto()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class PlainDecoder:
|
|
38
|
+
"""A body with no framing: its bytes are the payload."""
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def feed(data: bytes) -> bytes:
|
|
42
|
+
return data
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def finish() -> Mapping[str, str]:
|
|
46
|
+
return {}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class ChunkedDecoder:
|
|
50
|
+
"""Strip aws-chunked framing, verifying chunk signatures when there is a seed."""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self, seed: StreamSeed | None, *, signed_trailer: bool = False
|
|
54
|
+
) -> None:
|
|
55
|
+
self._seed = seed
|
|
56
|
+
self._signed_trailer = signed_trailer
|
|
57
|
+
self._previous_signature = seed.signature if seed else ""
|
|
58
|
+
self._state = _State.CHUNK_HEADER
|
|
59
|
+
self._pending = b""
|
|
60
|
+
self._remaining = 0
|
|
61
|
+
self._chunk_signature = ""
|
|
62
|
+
self._chunk_hash = hashlib.sha256()
|
|
63
|
+
self._trailers: dict[str, str] = {}
|
|
64
|
+
|
|
65
|
+
def feed(self, data: bytes) -> bytes:
|
|
66
|
+
"""Consume network bytes and return the payload bytes they contain."""
|
|
67
|
+
buffer = self._pending + data if self._pending else data
|
|
68
|
+
self._pending = b""
|
|
69
|
+
output: list[bytes] = []
|
|
70
|
+
position = 0
|
|
71
|
+
while position < len(buffer) and self._state is not _State.DONE:
|
|
72
|
+
advanced = self._step(buffer, position, output)
|
|
73
|
+
if advanced is None:
|
|
74
|
+
self._pending = buffer[position:]
|
|
75
|
+
self._check_pending_size()
|
|
76
|
+
break
|
|
77
|
+
position = advanced
|
|
78
|
+
return b"".join(output)
|
|
79
|
+
|
|
80
|
+
def finish(self) -> Mapping[str, str]:
|
|
81
|
+
"""Return the trailers, or raise if the body stopped early."""
|
|
82
|
+
if self._state is not _State.DONE:
|
|
83
|
+
raise S3Error(errors.INCOMPLETE_BODY, "The aws-chunked body is truncated.")
|
|
84
|
+
return self._trailers
|
|
85
|
+
|
|
86
|
+
def _step(self, buffer: bytes, position: int, output: list[bytes]) -> int | None:
|
|
87
|
+
"""Run one state; return the new position, or None if more bytes are needed."""
|
|
88
|
+
if self._state is _State.CHUNK_DATA:
|
|
89
|
+
return self._read_data(buffer, position, output)
|
|
90
|
+
if self._state is _State.CHUNK_END:
|
|
91
|
+
return self._read_chunk_end(buffer, position)
|
|
92
|
+
|
|
93
|
+
end = buffer.find(_CRLF, position)
|
|
94
|
+
if end < 0:
|
|
95
|
+
return None
|
|
96
|
+
line = buffer[position:end]
|
|
97
|
+
if self._state is _State.CHUNK_HEADER:
|
|
98
|
+
self._read_chunk_header(line)
|
|
99
|
+
else:
|
|
100
|
+
self._read_trailer_line(line)
|
|
101
|
+
return end + len(_CRLF)
|
|
102
|
+
|
|
103
|
+
def _read_chunk_header(self, line: bytes) -> None:
|
|
104
|
+
match = _CHUNK_HEADER.fullmatch(line)
|
|
105
|
+
if match is None:
|
|
106
|
+
raise S3Error(errors.INCOMPLETE_BODY, "Malformed aws-chunked chunk header.")
|
|
107
|
+
self._remaining = int(match[1], 16)
|
|
108
|
+
if self._seed is not None and match[2] is None:
|
|
109
|
+
raise S3Error(errors.SIGNATURE_MISMATCH, "A chunk signature is missing.")
|
|
110
|
+
self._chunk_signature = match[2].decode() if match[2] else ""
|
|
111
|
+
self._chunk_hash = hashlib.sha256()
|
|
112
|
+
|
|
113
|
+
if self._remaining == 0:
|
|
114
|
+
self._verify_chunk()
|
|
115
|
+
self._state = _State.TRAILER
|
|
116
|
+
else:
|
|
117
|
+
self._state = _State.CHUNK_DATA
|
|
118
|
+
|
|
119
|
+
def _read_data(self, buffer: bytes, position: int, output: list[bytes]) -> int:
|
|
120
|
+
taken = buffer[position : position + self._remaining]
|
|
121
|
+
output.append(taken)
|
|
122
|
+
if self._seed is not None:
|
|
123
|
+
self._chunk_hash.update(taken)
|
|
124
|
+
self._remaining -= len(taken)
|
|
125
|
+
if self._remaining == 0:
|
|
126
|
+
self._verify_chunk()
|
|
127
|
+
self._state = _State.CHUNK_END
|
|
128
|
+
return position + len(taken)
|
|
129
|
+
|
|
130
|
+
def _read_chunk_end(self, buffer: bytes, position: int) -> int | None:
|
|
131
|
+
if len(buffer) - position < len(_CRLF):
|
|
132
|
+
return None
|
|
133
|
+
if buffer[position : position + len(_CRLF)] != _CRLF:
|
|
134
|
+
raise S3Error(errors.INCOMPLETE_BODY, "A chunk is longer than declared.")
|
|
135
|
+
self._state = _State.CHUNK_HEADER
|
|
136
|
+
return position + len(_CRLF)
|
|
137
|
+
|
|
138
|
+
def _read_trailer_line(self, line: bytes) -> None:
|
|
139
|
+
if not line:
|
|
140
|
+
self._verify_trailer_signature()
|
|
141
|
+
self._state = _State.DONE
|
|
142
|
+
return
|
|
143
|
+
name, separator, value = line.decode("latin-1").partition(":")
|
|
144
|
+
if not separator:
|
|
145
|
+
raise S3Error(errors.MALFORMED_TRAILER)
|
|
146
|
+
self._trailers[name.strip().lower()] = value.strip()
|
|
147
|
+
|
|
148
|
+
def _verify_chunk(self) -> None:
|
|
149
|
+
if self._seed is None:
|
|
150
|
+
return
|
|
151
|
+
expected = sigv4.chunk_signature(
|
|
152
|
+
self._seed, self._previous_signature, self._chunk_hash.hexdigest()
|
|
153
|
+
)
|
|
154
|
+
if not hmac.compare_digest(expected, self._chunk_signature):
|
|
155
|
+
raise S3Error(errors.SIGNATURE_MISMATCH, "A chunk signature is wrong.")
|
|
156
|
+
self._previous_signature = expected
|
|
157
|
+
|
|
158
|
+
def _verify_trailer_signature(self) -> None:
|
|
159
|
+
claimed = self._trailers.pop(_TRAILER_SIGNATURE, None)
|
|
160
|
+
if self._seed is None or not self._signed_trailer:
|
|
161
|
+
return
|
|
162
|
+
if claimed is None or not sigv4.is_signature(claimed):
|
|
163
|
+
raise S3Error(
|
|
164
|
+
errors.SIGNATURE_MISMATCH, "The trailer signature is missing."
|
|
165
|
+
)
|
|
166
|
+
trailer = "".join(f"{name}:{value}\n" for name, value in self._trailers.items())
|
|
167
|
+
expected = sigv4.trailer_signature(
|
|
168
|
+
self._seed, self._previous_signature, trailer.encode("latin-1")
|
|
169
|
+
)
|
|
170
|
+
if not hmac.compare_digest(expected, claimed):
|
|
171
|
+
raise S3Error(errors.SIGNATURE_MISMATCH, "The trailer signature is wrong.")
|
|
172
|
+
|
|
173
|
+
def _check_pending_size(self) -> None:
|
|
174
|
+
if len(self._pending) > _MAX_LINE_BYTES:
|
|
175
|
+
raise S3Error(errors.INCOMPLETE_BODY, "An aws-chunked line is too long.")
|
leblob/s3/conditions.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Conditional requests and byte ranges."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from email.utils import parsedate_to_datetime
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
from leblob.model import BlobSlice
|
|
11
|
+
from leblob.s3 import errors
|
|
12
|
+
from leblob.s3.errors import S3Error
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from collections.abc import Iterable, Mapping
|
|
16
|
+
|
|
17
|
+
from leblob.model import Blob
|
|
18
|
+
|
|
19
|
+
_RANGE = re.compile(r"bytes=(\d*)-(\d*)")
|
|
20
|
+
_COPY_RANGE = re.compile(r"bytes=([0-9]+)-([0-9]+)")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class WriteCondition:
|
|
25
|
+
"""A precondition on the current state of the key being written."""
|
|
26
|
+
|
|
27
|
+
if_match: str | None = None
|
|
28
|
+
if_none_match: str | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def parse_write_condition(headers: Mapping[str, str]) -> WriteCondition:
|
|
32
|
+
return WriteCondition(
|
|
33
|
+
_strip_etag(headers.get("if-match")), _strip_etag(headers.get("if-none-match"))
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def check_write_condition(condition: WriteCondition, current_etag: str | None) -> None:
|
|
38
|
+
"""Raise unless a write is allowed given the key's current ETag."""
|
|
39
|
+
if current_etag is not None and condition.if_none_match in {"*", current_etag}:
|
|
40
|
+
raise S3Error(errors.PRECONDITION_FAILED, "If-None-Match did not hold.")
|
|
41
|
+
if condition.if_match is None:
|
|
42
|
+
return
|
|
43
|
+
if current_etag is None:
|
|
44
|
+
raise S3Error(errors.NO_SUCH_KEY)
|
|
45
|
+
if condition.if_match not in {"*", current_etag}:
|
|
46
|
+
raise S3Error(errors.PRECONDITION_FAILED, "If-Match did not hold.")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def read_is_not_modified(
|
|
50
|
+
headers: Mapping[str, str],
|
|
51
|
+
etag: str,
|
|
52
|
+
modified_at_ms: int,
|
|
53
|
+
prefix: str = "",
|
|
54
|
+
) -> bool:
|
|
55
|
+
"""Evaluate read preconditions as RFC 9110 orders them.
|
|
56
|
+
|
|
57
|
+
Raise PreconditionFailed, or return True if the answer should be 304.
|
|
58
|
+
"""
|
|
59
|
+
modified_seconds = modified_at_ms // 1000
|
|
60
|
+
if_match = headers.get(f"{prefix}if-match")
|
|
61
|
+
if if_match is not None:
|
|
62
|
+
if not _etag_matches(if_match, etag):
|
|
63
|
+
raise S3Error(errors.PRECONDITION_FAILED, "If-Match did not hold.")
|
|
64
|
+
else:
|
|
65
|
+
limit = _parse_http_date(headers.get(f"{prefix}if-unmodified-since"))
|
|
66
|
+
if limit is not None and modified_seconds > limit:
|
|
67
|
+
raise S3Error(
|
|
68
|
+
errors.PRECONDITION_FAILED, "If-Unmodified-Since did not hold."
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
if_none_match = headers.get(f"{prefix}if-none-match")
|
|
72
|
+
if if_none_match is not None:
|
|
73
|
+
return _etag_matches(if_none_match, etag)
|
|
74
|
+
since = _parse_http_date(headers.get(f"{prefix}if-modified-since"))
|
|
75
|
+
return since is not None and modified_seconds <= since
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def resolve_range(header: str | None, size: int) -> tuple[int, int] | None:
|
|
79
|
+
"""Return (start, length), or None to serve the whole object.
|
|
80
|
+
|
|
81
|
+
Syntax that isn't a single byte range is ignored, as RFC 9110 requires.
|
|
82
|
+
"""
|
|
83
|
+
match = _RANGE.fullmatch(header.strip()) if header else None
|
|
84
|
+
if match is None:
|
|
85
|
+
return None
|
|
86
|
+
first, last = match.groups()
|
|
87
|
+
if not first and not last:
|
|
88
|
+
return None
|
|
89
|
+
if first:
|
|
90
|
+
start = int(first)
|
|
91
|
+
end = min(int(last), size - 1) if last else size - 1
|
|
92
|
+
else:
|
|
93
|
+
start = max(size - int(last), 0)
|
|
94
|
+
end = size - 1
|
|
95
|
+
if start > end or start >= size or (first and last and int(last) < int(first)):
|
|
96
|
+
raise S3Error(
|
|
97
|
+
errors.INVALID_RANGE, headers=(("content-range", f"bytes */{size}"),)
|
|
98
|
+
)
|
|
99
|
+
return start, end - start + 1
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def resolve_copy_range(header: str | None, size: int) -> tuple[int, int]:
|
|
103
|
+
"""Return (start, length) of what a part copies from its source.
|
|
104
|
+
|
|
105
|
+
Stricter than a read: the source of a part has to be exact, since a shorter part
|
|
106
|
+
would silently make a wrong object. No range means all of it.
|
|
107
|
+
"""
|
|
108
|
+
if header is None:
|
|
109
|
+
return 0, size
|
|
110
|
+
match = _COPY_RANGE.fullmatch(header)
|
|
111
|
+
if match is None or int(match[2]) < int(match[1]):
|
|
112
|
+
raise S3Error(
|
|
113
|
+
errors.INVALID_ARGUMENT,
|
|
114
|
+
"x-amz-copy-source-range must be of the form bytes=first-last.",
|
|
115
|
+
)
|
|
116
|
+
first, last = int(match[1]), int(match[2])
|
|
117
|
+
if last >= size:
|
|
118
|
+
raise S3Error(errors.INVALID_RANGE, f"The source has only {size} bytes.")
|
|
119
|
+
return first, last - first + 1
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def slice_blobs(blobs: Iterable[Blob], start: int, length: int) -> list[BlobSlice]:
|
|
123
|
+
"""Translate a byte range of an object into ranges of its blobs."""
|
|
124
|
+
slices = []
|
|
125
|
+
position = 0
|
|
126
|
+
remaining = length
|
|
127
|
+
for blob in blobs:
|
|
128
|
+
if remaining == 0:
|
|
129
|
+
break
|
|
130
|
+
if start < position + blob.size:
|
|
131
|
+
offset = max(start - position, 0)
|
|
132
|
+
taken = min(blob.size - offset, remaining)
|
|
133
|
+
slices.append(BlobSlice(blob, offset, taken))
|
|
134
|
+
remaining -= taken
|
|
135
|
+
position += blob.size
|
|
136
|
+
return slices
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _strip_etag(value: str | None) -> str | None:
|
|
140
|
+
return value.strip().strip('"') if value else None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _etag_matches(header: str, etag: str) -> bool:
|
|
144
|
+
candidates = {
|
|
145
|
+
item.strip().removeprefix("W/").strip('"') for item in header.split(",")
|
|
146
|
+
}
|
|
147
|
+
return "*" in candidates or etag in candidates
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _parse_http_date(value: str | None) -> int | None:
|
|
151
|
+
if value is None:
|
|
152
|
+
return None
|
|
153
|
+
try:
|
|
154
|
+
return int(parsedate_to_datetime(value).timestamp())
|
|
155
|
+
except ValueError:
|
|
156
|
+
return None
|
leblob/s3/cors.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Which browser requests a bucket's CORS rules allow, and what to tell the browser."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
from leblob.model import CorsRule
|
|
11
|
+
|
|
12
|
+
ALLOWED_METHODS = frozenset({"DELETE", "GET", "HEAD", "POST", "PUT"})
|
|
13
|
+
_WILDCARD = "*"
|
|
14
|
+
VARY = (("vary", "Origin"),)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def find_rule(
|
|
18
|
+
rules: Sequence[CorsRule],
|
|
19
|
+
origin: str,
|
|
20
|
+
method: str,
|
|
21
|
+
headers: Sequence[str] = (),
|
|
22
|
+
) -> CorsRule | None:
|
|
23
|
+
"""The first rule that allows all of a request, if any."""
|
|
24
|
+
for rule in rules:
|
|
25
|
+
if (
|
|
26
|
+
method in rule.methods
|
|
27
|
+
and any(_matches(allowed, origin) for allowed in rule.origins)
|
|
28
|
+
and all(_header_is_allowed(rule, header) for header in headers)
|
|
29
|
+
):
|
|
30
|
+
return rule
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def split_header_names(value: str) -> list[str]:
|
|
35
|
+
return [name.strip() for name in value.split(",") if name.strip()]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def response_headers(rule: CorsRule, origin: str) -> tuple[tuple[str, str], ...]:
|
|
39
|
+
"""What a browser needs to see in any answer the rule allows."""
|
|
40
|
+
allowed_origin = _WILDCARD if _WILDCARD in rule.origins else origin
|
|
41
|
+
expose = (
|
|
42
|
+
(("access-control-expose-headers", ", ".join(rule.expose)),)
|
|
43
|
+
if rule.expose
|
|
44
|
+
else ()
|
|
45
|
+
)
|
|
46
|
+
return (
|
|
47
|
+
("access-control-allow-origin", allowed_origin),
|
|
48
|
+
("access-control-allow-methods", ", ".join(rule.methods)),
|
|
49
|
+
*VARY,
|
|
50
|
+
*expose,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def preflight_headers(
|
|
55
|
+
rule: CorsRule, origin: str, headers: Sequence[str]
|
|
56
|
+
) -> tuple[tuple[str, str], ...]:
|
|
57
|
+
"""What a browser needs to go on to send the request it asked about."""
|
|
58
|
+
allowed = (("access-control-allow-headers", ", ".join(headers)),) if headers else ()
|
|
59
|
+
max_age = (
|
|
60
|
+
() if rule.max_age is None else (("access-control-max-age", str(rule.max_age)),)
|
|
61
|
+
)
|
|
62
|
+
return (*response_headers(rule, origin), *allowed, *max_age)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _header_is_allowed(rule: CorsRule, header: str) -> bool:
|
|
66
|
+
return any(
|
|
67
|
+
_matches(allowed.lower(), header.lower()) for allowed in rule.allowed_headers
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _matches(pattern: str, value: str) -> bool:
|
|
72
|
+
"""Compare with a pattern holding one wildcard at most, as parsing has ensured."""
|
|
73
|
+
before, wildcard, after = pattern.partition(_WILDCARD)
|
|
74
|
+
if not wildcard:
|
|
75
|
+
return pattern == value
|
|
76
|
+
return (
|
|
77
|
+
len(value) >= len(before) + len(after)
|
|
78
|
+
and value.startswith(before)
|
|
79
|
+
and value.endswith(after)
|
|
80
|
+
)
|
leblob/s3/errors.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""S3 error codes, and the exception that carries one to the HTTP boundary."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True, slots=True)
|
|
9
|
+
class ErrorCode:
|
|
10
|
+
code: str
|
|
11
|
+
status: int
|
|
12
|
+
message: str
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
ACCESS_DENIED = ErrorCode("AccessDenied", 403, "Access Denied.")
|
|
16
|
+
AUTH_HEADER_MALFORMED = ErrorCode(
|
|
17
|
+
"AuthorizationHeaderMalformed", 400, "The authorization header is malformed."
|
|
18
|
+
)
|
|
19
|
+
AUTH_QUERY_ERROR = ErrorCode(
|
|
20
|
+
"AuthorizationQueryParametersError",
|
|
21
|
+
400,
|
|
22
|
+
"The presigned URL parameters are malformed.",
|
|
23
|
+
)
|
|
24
|
+
BAD_DIGEST = ErrorCode(
|
|
25
|
+
"BadDigest", 400, "The checksum you specified did not match what we received."
|
|
26
|
+
)
|
|
27
|
+
BUCKET_ALREADY_OWNED = ErrorCode(
|
|
28
|
+
"BucketAlreadyOwnedByYou", 409, "You already own this bucket."
|
|
29
|
+
)
|
|
30
|
+
BUCKET_NOT_EMPTY = ErrorCode(
|
|
31
|
+
"BucketNotEmpty", 409, "The bucket you tried to delete is not empty."
|
|
32
|
+
)
|
|
33
|
+
CONTENT_SHA256_MISMATCH = ErrorCode(
|
|
34
|
+
"XAmzContentSHA256Mismatch",
|
|
35
|
+
400,
|
|
36
|
+
"The body does not match the declared x-amz-content-sha256.",
|
|
37
|
+
)
|
|
38
|
+
ENTITY_TOO_LARGE = ErrorCode(
|
|
39
|
+
"EntityTooLarge", 400, "Your upload exceeds the maximum allowed size."
|
|
40
|
+
)
|
|
41
|
+
ENTITY_TOO_SMALL = ErrorCode(
|
|
42
|
+
"EntityTooSmall", 400, "A part other than the last is below the minimum size."
|
|
43
|
+
)
|
|
44
|
+
INCOMPLETE_BODY = ErrorCode(
|
|
45
|
+
"IncompleteBody", 400, "The body does not have the declared length."
|
|
46
|
+
)
|
|
47
|
+
INTERNAL_ERROR = ErrorCode(
|
|
48
|
+
"InternalError", 500, "We encountered an internal error. Please try again."
|
|
49
|
+
)
|
|
50
|
+
ACCESS_FORBIDDEN = ErrorCode(
|
|
51
|
+
"AccessForbidden", 403, "CORSResponse: This CORS request is not allowed."
|
|
52
|
+
)
|
|
53
|
+
SERVICE_UNAVAILABLE = ErrorCode(
|
|
54
|
+
"ServiceUnavailable", 503, "Please try again: the data is not reachable just now."
|
|
55
|
+
)
|
|
56
|
+
INVALID_ACCESS_KEY = ErrorCode(
|
|
57
|
+
"InvalidAccessKeyId", 403, "The access key ID does not exist."
|
|
58
|
+
)
|
|
59
|
+
INVALID_ARGUMENT = ErrorCode("InvalidArgument", 400, "Invalid argument.")
|
|
60
|
+
INVALID_BUCKET_NAME = ErrorCode(
|
|
61
|
+
"InvalidBucketName", 400, "The specified bucket name is not valid."
|
|
62
|
+
)
|
|
63
|
+
INVALID_DIGEST = ErrorCode(
|
|
64
|
+
"InvalidDigest", 400, "The checksum you specified is not valid."
|
|
65
|
+
)
|
|
66
|
+
INVALID_PART = ErrorCode(
|
|
67
|
+
"InvalidPart", 400, "A listed part is missing or its ETag does not match."
|
|
68
|
+
)
|
|
69
|
+
INVALID_PART_ORDER = ErrorCode(
|
|
70
|
+
"InvalidPartOrder", 400, "The list of parts is not in ascending order."
|
|
71
|
+
)
|
|
72
|
+
INVALID_RANGE = ErrorCode(
|
|
73
|
+
"InvalidRange", 416, "The requested range is not satisfiable."
|
|
74
|
+
)
|
|
75
|
+
INVALID_POLICY = ErrorCode(
|
|
76
|
+
"InvalidPolicyDocument", 400, "The content of the form's policy is not valid."
|
|
77
|
+
)
|
|
78
|
+
INVALID_TAG = ErrorCode("InvalidTag", 400, "The tag provided was not valid.")
|
|
79
|
+
INVALID_REQUEST = ErrorCode("InvalidRequest", 400, "The request is not valid.")
|
|
80
|
+
KEY_TOO_LONG = ErrorCode(
|
|
81
|
+
"KeyTooLongError", 400, "Your key is longer than the 1024 bytes allowed."
|
|
82
|
+
)
|
|
83
|
+
MALFORMED_POST = ErrorCode(
|
|
84
|
+
"MalformedPOSTRequest", 400, "The body of your POST request is not valid."
|
|
85
|
+
)
|
|
86
|
+
MAX_POST_PRE_DATA = ErrorCode(
|
|
87
|
+
"MaxPostPreDataLengthExceeded",
|
|
88
|
+
400,
|
|
89
|
+
"Your POST request fields preceding the upload file were too large.",
|
|
90
|
+
)
|
|
91
|
+
MALFORMED_TRAILER = ErrorCode(
|
|
92
|
+
"MalformedTrailerError", 400, "The trailing checksum is missing or malformed."
|
|
93
|
+
)
|
|
94
|
+
MALFORMED_XML = ErrorCode("MalformedXML", 400, "The XML you provided is not valid.")
|
|
95
|
+
METADATA_TOO_LARGE = ErrorCode(
|
|
96
|
+
"MetadataTooLarge", 400, "Your metadata exceeds the 2 KB limit."
|
|
97
|
+
)
|
|
98
|
+
METHOD_NOT_ALLOWED = ErrorCode(
|
|
99
|
+
"MethodNotAllowed", 405, "The method is not allowed for this resource."
|
|
100
|
+
)
|
|
101
|
+
NO_SUCH_BUCKET = ErrorCode("NoSuchBucket", 404, "The specified bucket does not exist.")
|
|
102
|
+
NO_SUCH_KEY = ErrorCode("NoSuchKey", 404, "The specified key does not exist.")
|
|
103
|
+
NO_SUCH_VERSION = ErrorCode(
|
|
104
|
+
"NoSuchVersion", 404, "The specified version does not exist."
|
|
105
|
+
)
|
|
106
|
+
NO_SUCH_CORS = ErrorCode(
|
|
107
|
+
"NoSuchCORSConfiguration", 404, "The CORS configuration does not exist."
|
|
108
|
+
)
|
|
109
|
+
NO_SUCH_LIFECYCLE = ErrorCode(
|
|
110
|
+
"NoSuchLifecycleConfiguration", 404, "The lifecycle configuration does not exist."
|
|
111
|
+
)
|
|
112
|
+
NO_SUCH_UPLOAD = ErrorCode(
|
|
113
|
+
"NoSuchUpload", 404, "The specified multipart upload does not exist."
|
|
114
|
+
)
|
|
115
|
+
NOT_IMPLEMENTED = ErrorCode("NotImplemented", 501, "This feature is not implemented.")
|
|
116
|
+
PRECONDITION_FAILED = ErrorCode(
|
|
117
|
+
"PreconditionFailed", 412, "A condition you specified did not hold."
|
|
118
|
+
)
|
|
119
|
+
TIME_TOO_SKEWED = ErrorCode(
|
|
120
|
+
"RequestTimeTooSkewed",
|
|
121
|
+
403,
|
|
122
|
+
"The difference between the request time and the server time is too large.",
|
|
123
|
+
)
|
|
124
|
+
SIGNATURE_MISMATCH = ErrorCode(
|
|
125
|
+
"SignatureDoesNotMatch",
|
|
126
|
+
403,
|
|
127
|
+
"The request signature we calculated does not match the signature you provided.",
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class S3Error(Exception):
|
|
132
|
+
"""A failure that S3 clients understand; rendered once, at the HTTP boundary."""
|
|
133
|
+
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
error: ErrorCode,
|
|
137
|
+
message: str | None = None,
|
|
138
|
+
*,
|
|
139
|
+
headers: tuple[tuple[str, str], ...] = (),
|
|
140
|
+
) -> None:
|
|
141
|
+
self.error = error
|
|
142
|
+
self.message = message or error.message
|
|
143
|
+
self.headers = headers
|
|
144
|
+
super().__init__(f"{error.code}: {self.message}")
|
leblob/s3/lifecycle.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""What lifecycle rules say to do. Pure: `meta.expire_batch` carries it out."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from leblob.model import KeyVerdict
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from collections.abc import Sequence
|
|
12
|
+
|
|
13
|
+
from leblob.model import LifecycleRule, VersionSummary
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True, slots=True)
|
|
17
|
+
class Clock:
|
|
18
|
+
"""The time now, and how long a day lasts, which tests make short."""
|
|
19
|
+
|
|
20
|
+
now: int
|
|
21
|
+
day_ms: int
|
|
22
|
+
|
|
23
|
+
def has_passed(self, since: int, days: int) -> bool:
|
|
24
|
+
return since + days * self.day_ms <= self.now
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def judge_key(
|
|
28
|
+
rules: Sequence[LifecycleRule], versions: Sequence[VersionSummary], clock: Clock
|
|
29
|
+
) -> KeyVerdict:
|
|
30
|
+
"""Decide what happens to one key, given all its versions, newest first."""
|
|
31
|
+
latest, *older = versions
|
|
32
|
+
applying = [rule for rule in rules if _applies_to_key(rule, latest.key)]
|
|
33
|
+
expire_current = not latest.is_delete_marker and any(
|
|
34
|
+
_current_has_expired(rule, latest, clock) for rule in applying
|
|
35
|
+
)
|
|
36
|
+
erase = [
|
|
37
|
+
version.version_id
|
|
38
|
+
for position, (version, successor) in enumerate(
|
|
39
|
+
zip(older, versions, strict=False)
|
|
40
|
+
)
|
|
41
|
+
if any(
|
|
42
|
+
_noncurrent_has_expired(rule, version, successor, position, clock)
|
|
43
|
+
for rule in applying
|
|
44
|
+
)
|
|
45
|
+
]
|
|
46
|
+
if (
|
|
47
|
+
latest.is_delete_marker
|
|
48
|
+
and not older
|
|
49
|
+
and any(_lone_marker_has_expired(rule, latest, clock) for rule in applying)
|
|
50
|
+
):
|
|
51
|
+
erase.append(latest.version_id)
|
|
52
|
+
return KeyVerdict(expire_current, tuple(erase))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def upload_is_stale(
|
|
56
|
+
rules: Sequence[LifecycleRule], key: str, created_at: int, clock: Clock
|
|
57
|
+
) -> bool:
|
|
58
|
+
return any(
|
|
59
|
+
rule.actions.abort_upload_days is not None
|
|
60
|
+
and clock.has_passed(created_at, rule.actions.abort_upload_days)
|
|
61
|
+
for rule in rules
|
|
62
|
+
if _applies_to_key(rule, key)
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _applies_to_key(rule: LifecycleRule, key: str) -> bool:
|
|
67
|
+
return rule.enabled and key.startswith(rule.scope.prefix or "")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _applies_to_version(rule: LifecycleRule, version: VersionSummary) -> bool:
|
|
71
|
+
scope = rule.scope
|
|
72
|
+
return (
|
|
73
|
+
(scope.larger_than is None or version.size > scope.larger_than)
|
|
74
|
+
and (scope.smaller_than is None or version.size < scope.smaller_than)
|
|
75
|
+
and all(version.tags.get(key) == value for key, value in scope.tags)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _current_has_expired(
|
|
80
|
+
rule: LifecycleRule, current: VersionSummary, clock: Clock
|
|
81
|
+
) -> bool:
|
|
82
|
+
actions = rule.actions
|
|
83
|
+
if not _applies_to_version(rule, current):
|
|
84
|
+
return False
|
|
85
|
+
if actions.expire_at is not None:
|
|
86
|
+
return actions.expire_at <= clock.now
|
|
87
|
+
return actions.expire_days is not None and clock.has_passed(
|
|
88
|
+
current.modified_at, actions.expire_days
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _lone_marker_has_expired(
|
|
93
|
+
rule: LifecycleRule, marker: VersionSummary, clock: Clock
|
|
94
|
+
) -> bool:
|
|
95
|
+
"""A rule on current objects clears up after them too, as AWS documents."""
|
|
96
|
+
actions = rule.actions
|
|
97
|
+
if not _applies_to_version(rule, marker):
|
|
98
|
+
return False
|
|
99
|
+
return actions.expire_lone_markers or (
|
|
100
|
+
actions.expire_days is not None
|
|
101
|
+
and clock.has_passed(marker.modified_at, actions.expire_days)
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _noncurrent_has_expired(
|
|
106
|
+
rule: LifecycleRule,
|
|
107
|
+
version: VersionSummary,
|
|
108
|
+
successor: VersionSummary,
|
|
109
|
+
position: int,
|
|
110
|
+
clock: Clock,
|
|
111
|
+
) -> bool:
|
|
112
|
+
"""A version's age counts from when a newer one replaced it, not its creation."""
|
|
113
|
+
actions = rule.actions
|
|
114
|
+
return (
|
|
115
|
+
actions.noncurrent_days is not None
|
|
116
|
+
and position >= actions.keep_noncurrent
|
|
117
|
+
and _applies_to_version(rule, version)
|
|
118
|
+
and clock.has_passed(successor.modified_at, actions.noncurrent_days)
|
|
119
|
+
)
|