requence 0.8.0__tar.gz → 0.8.1.dev58__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {requence-0.8.0/requence.egg-info → requence-0.8.1.dev58}/PKG-INFO +2 -1
- {requence-0.8.0 → requence-0.8.1.dev58}/pyproject.toml +5 -1
- requence-0.8.1.dev58/requence/client_properties.py +211 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/exceptions.py +18 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/handlers/claim_check_handler.py +38 -11
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/handlers/context.py +14 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/internal_types.py +5 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/service/__init__.py +128 -18
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/service/boundary.py +13 -0
- requence-0.8.1.dev58/requence/wire_codec.py +169 -0
- {requence-0.8.0 → requence-0.8.1.dev58/requence.egg-info}/PKG-INFO +2 -1
- {requence-0.8.0 → requence-0.8.1.dev58}/requence.egg-info/SOURCES.txt +5 -1
- {requence-0.8.0 → requence-0.8.1.dev58}/requence.egg-info/requires.txt +1 -0
- requence-0.8.1.dev58/tests/test_client_properties.py +276 -0
- requence-0.8.1.dev58/tests/test_wire_codec.py +233 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/MANIFEST.in +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/README.md +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/__init__.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/callbacks/__init__.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/callbacks/context.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/callbacks/handler.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/context.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/__init__.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/errors.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/handlers/__init__.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/handlers/file_handler.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/handlers/stream_handler.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/pending_uploads.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/requence_file.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/requence_stream.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/sanitize_log_data.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/stream_framing.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/files/transport.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/obfuscase.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/py.typed +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/reachable.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/requence_callback.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/requence_file.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/requence_stream.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/service/cli.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/service/transformer.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/service/types.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/__init__.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/abort_task.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/ack_task.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/cli.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/get_task.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/ingress.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/protect_task.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/task.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/task_watcher.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/types.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/updates.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/task/utils.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/__init__.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/context.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/errors.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/guards.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/message_handler.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/transform.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/value_handler.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/transformer/walk.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/utils.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence/wire.py +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence.egg-info/dependency_links.txt +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence.egg-info/entry_points.txt +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/requence.egg-info/top_level.txt +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/setup.cfg +0 -0
- {requence-0.8.0 → requence-0.8.1.dev58}/tests/test_wire.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: requence
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.1.dev58
|
|
4
4
|
Summary: Requence Service
|
|
5
5
|
Author: Torsten Blindert
|
|
6
6
|
Requires-Python: >=3.12
|
|
@@ -9,6 +9,7 @@ Requires-Dist: pika>=1.3.2
|
|
|
9
9
|
Requires-Dist: requests>=2.32.2
|
|
10
10
|
Requires-Dist: colorama>=0.4.6
|
|
11
11
|
Requires-Dist: sseclient-py>=1.8.0
|
|
12
|
+
Requires-Dist: msgpack>=1.0.5
|
|
12
13
|
|
|
13
14
|
# requence
|
|
14
15
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "requence"
|
|
7
|
-
version = "0.8.
|
|
7
|
+
version = "0.8.1.dev58"
|
|
8
8
|
description = "Requence Service"
|
|
9
9
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
10
|
authors = [{ name = "Torsten Blindert" }]
|
|
@@ -14,6 +14,10 @@ dependencies = [
|
|
|
14
14
|
"requests>=2.32.2",
|
|
15
15
|
"colorama>=0.4.6",
|
|
16
16
|
"sseclient-py>=1.8.0",
|
|
17
|
+
# The portable msgpack wire (requence/wire_codec.py). NOT optional: the SDK
|
|
18
|
+
# declares `wire:msgpack1` in its AMQP client properties, which is a promise
|
|
19
|
+
# that it can decode a dispatch under that content type.
|
|
20
|
+
"msgpack>=1.0.5",
|
|
17
21
|
]
|
|
18
22
|
|
|
19
23
|
[project.scripts]
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""AMQP connection client properties — the Python mirror of
|
|
2
|
+
``packages/helpers/src/protocol/clientProperties.ts``.
|
|
3
|
+
|
|
4
|
+
That TS file is the contract and carries the full rationale. This module exists
|
|
5
|
+
so the Python SDK announces itself with the IDENTICAL key set; the parity is
|
|
6
|
+
locked by ``tests/test_client_properties.py``, which reads the TS source and
|
|
7
|
+
asserts the keys match. Changing a key is a wire change: both SDKs ship it
|
|
8
|
+
together.
|
|
9
|
+
|
|
10
|
+
Write side only for the announcements — reading connections back is
|
|
11
|
+
``ui/backend``'s job. The capability section at the bottom is the other way
|
|
12
|
+
round: the operator declares what a dispatch requires, and this SDK only reads.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"CLIENT_PROPERTY_KEYS",
|
|
19
|
+
"IDENTITY_HEADER_KEYS",
|
|
20
|
+
"CAPABILITY_HEADER_KEYS",
|
|
21
|
+
"MAX_SERVICE_NAME_LENGTH",
|
|
22
|
+
"MAX_CAPABILITIES_LENGTH",
|
|
23
|
+
"WIRE_JSON_1",
|
|
24
|
+
"WIRE_MSGPACK_1",
|
|
25
|
+
"SERVICE_WIRE_CAPABILITIES",
|
|
26
|
+
"build_service_client_properties",
|
|
27
|
+
"build_service_identity_headers",
|
|
28
|
+
"read_required_capabilities",
|
|
29
|
+
"unsupported_capabilities",
|
|
30
|
+
"unsupported_capability_message",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
# The pinned, namespaced property set. Flat strings on purpose.
|
|
34
|
+
CLIENT_PROPERTY_KEYS = {
|
|
35
|
+
"libVersion": "requence_lib_version",
|
|
36
|
+
"serviceVersion": "requence_service_version",
|
|
37
|
+
"capabilities": "requence_capabilities",
|
|
38
|
+
# Author-chosen debugging label. Optional, non-unique, unvalidated —
|
|
39
|
+
# NOTHING may key off or branch on it.
|
|
40
|
+
"name": "requence_name",
|
|
41
|
+
# Marks a developer's local dev-overlay connection. Only the marker travels,
|
|
42
|
+
# never the devKey (which is an access-token value).
|
|
43
|
+
"devOverlay": "requence_dev_overlay",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# The name is author-controlled text and the whole client-properties table
|
|
47
|
+
# travels inside the connection frame, so an unbounded string is a
|
|
48
|
+
# connection-level failure. Capped here and truncated AGAIN on read.
|
|
49
|
+
MAX_SERVICE_NAME_LENGTH = 64
|
|
50
|
+
MAX_CAPABILITIES_LENGTH = 512
|
|
51
|
+
|
|
52
|
+
WIRE_JSON_1 = "wire:json1"
|
|
53
|
+
# The portable msgpack subset in ``requence/wire_codec.py``. Claimed only by an
|
|
54
|
+
# SDK that can DECODE it on the dispatch wire — a service echoes the codec it was
|
|
55
|
+
# addressed in, so claiming this is a promise about READING, and the reply
|
|
56
|
+
# direction needs no capability of its own.
|
|
57
|
+
WIRE_MSGPACK_1 = "wire:msgpack1"
|
|
58
|
+
SERVICE_WIRE_CAPABILITIES = [WIRE_JSON_1, WIRE_MSGPACK_1]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _sanitize_label(value: str, max_length: int) -> str:
|
|
62
|
+
"""Drop control characters, trim, cap. Mirrors sanitizeLabel in the TS."""
|
|
63
|
+
return "".join(c for c in value if ord(c) >= 0x20 and ord(c) != 0x7F).strip()[
|
|
64
|
+
:max_length
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def build_service_client_properties(
|
|
69
|
+
*,
|
|
70
|
+
lib_version: str,
|
|
71
|
+
service_version: str,
|
|
72
|
+
name: Optional[str] = None,
|
|
73
|
+
dev_overlay: bool = False,
|
|
74
|
+
capabilities: Optional[Iterable[str]] = None,
|
|
75
|
+
) -> Dict[str, str]:
|
|
76
|
+
"""Build the properties handed to pika on connect (and therefore on every
|
|
77
|
+
reconnect, since the connection parameters are reused). Absent optional
|
|
78
|
+
fields are OMITTED, never sent blank."""
|
|
79
|
+
if capabilities is None:
|
|
80
|
+
capabilities = SERVICE_WIRE_CAPABILITIES
|
|
81
|
+
|
|
82
|
+
properties: Dict[str, str] = {
|
|
83
|
+
CLIENT_PROPERTY_KEYS["libVersion"]: lib_version,
|
|
84
|
+
CLIENT_PROPERTY_KEYS["serviceVersion"]: service_version,
|
|
85
|
+
CLIENT_PROPERTY_KEYS["capabilities"]: ",".join(capabilities)[
|
|
86
|
+
:MAX_CAPABILITIES_LENGTH
|
|
87
|
+
],
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
label = _sanitize_label(name, MAX_SERVICE_NAME_LENGTH) if name else ""
|
|
91
|
+
if label:
|
|
92
|
+
properties[CLIENT_PROPERTY_KEYS["name"]] = label
|
|
93
|
+
|
|
94
|
+
if dev_overlay:
|
|
95
|
+
properties[CLIENT_PROPERTY_KEYS["devOverlay"]] = "1"
|
|
96
|
+
|
|
97
|
+
return properties
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ─── The second announcement: per-run identity on the begin message ──────────
|
|
101
|
+
#
|
|
102
|
+
# The same three values as above, sent again on the ``begin`` publish, because
|
|
103
|
+
# they answer a DIFFERENT question: the client properties say "can this fleet
|
|
104
|
+
# decode msgpack?" (live, folded to the minimum, available before dispatch),
|
|
105
|
+
# these say "which build ran THIS node?" (exact, frozen into the NODE_START
|
|
106
|
+
# update). See the TS contract for the full rationale.
|
|
107
|
+
#
|
|
108
|
+
# Bare, un-namespaced names, matching the headers that already ride alongside
|
|
109
|
+
# them (``taskId``, ``index``, ``connectionId``, ``version``).
|
|
110
|
+
IDENTITY_HEADER_KEYS = {
|
|
111
|
+
"libVersion": "libVersion",
|
|
112
|
+
"name": "serviceName",
|
|
113
|
+
"capabilities": "capabilities",
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def build_service_identity_headers(
|
|
118
|
+
*,
|
|
119
|
+
lib_version: str,
|
|
120
|
+
name: Optional[str] = None,
|
|
121
|
+
capabilities: Optional[Iterable[str]] = None,
|
|
122
|
+
) -> Dict[str, str]:
|
|
123
|
+
"""Headers the service stamps on its own ``begin`` publish, built from this
|
|
124
|
+
process's own constants — nothing is looked up, so nothing can be stale."""
|
|
125
|
+
if capabilities is None:
|
|
126
|
+
capabilities = SERVICE_WIRE_CAPABILITIES
|
|
127
|
+
|
|
128
|
+
headers: Dict[str, str] = {
|
|
129
|
+
IDENTITY_HEADER_KEYS["libVersion"]: lib_version,
|
|
130
|
+
IDENTITY_HEADER_KEYS["capabilities"]: ",".join(capabilities)[
|
|
131
|
+
:MAX_CAPABILITIES_LENGTH
|
|
132
|
+
],
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
label = _sanitize_label(name, MAX_SERVICE_NAME_LENGTH) if name else ""
|
|
136
|
+
if label:
|
|
137
|
+
headers[IDENTITY_HEADER_KEYS["name"]] = label
|
|
138
|
+
|
|
139
|
+
return headers
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ─── The dispatch direction: what a message REQUIRES of its consumer ─────────
|
|
143
|
+
#
|
|
144
|
+
# The mirror of the TS contract's `requires` section. READ SIDE ONLY here: the
|
|
145
|
+
# tokens are declared by the operator, which is TypeScript, so Python never
|
|
146
|
+
# builds this header — it only ever answers "can this build satisfy it?".
|
|
147
|
+
#
|
|
148
|
+
# It is a HEADER, never a body field: the body is precisely the thing this build
|
|
149
|
+
# may be unable to decode, so a requirement stated inside it is unreadable
|
|
150
|
+
# exactly when it matters.
|
|
151
|
+
CAPABILITY_HEADER_KEYS = {
|
|
152
|
+
# dispatch → service: the tokens this message needs. Absent when it needs
|
|
153
|
+
# nothing beyond the floor every SDK speaks.
|
|
154
|
+
"requires": "requires",
|
|
155
|
+
# service → operator: this replica cannot satisfy `requires`. Sent by
|
|
156
|
+
# REPUBLISHING to `<exchange>-retry`, never by nacking — a nack cannot modify
|
|
157
|
+
# the message, these are classic queues (so there is no `x-delivery-count`
|
|
158
|
+
# either), and with no counter anywhere the message would requeue forever.
|
|
159
|
+
"mismatch": "capabilityMismatch",
|
|
160
|
+
# operator → service, on a reroute: how many reroutes are left. Named for
|
|
161
|
+
# reroutes and NOT for retries on purpose — it is its own allowance, never
|
|
162
|
+
# the node's `retry` budget. The operator owns the count; this SDK only ever
|
|
163
|
+
# echoes it back.
|
|
164
|
+
"reroutes": "capabilityReroutes",
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _parse_capability_list(raw: Optional[str]) -> List[str]:
|
|
169
|
+
"""Caps the whole string BEFORE splitting, so an unbounded writer cannot push
|
|
170
|
+
an unbounded list through. A token cut in half by that cap simply matches no
|
|
171
|
+
known capability, which can only ever UNDER-claim — the safe direction."""
|
|
172
|
+
if not raw:
|
|
173
|
+
return []
|
|
174
|
+
|
|
175
|
+
tokens: List[str] = []
|
|
176
|
+
for token in raw[:MAX_CAPABILITIES_LENGTH].split(","):
|
|
177
|
+
stripped = token.strip()
|
|
178
|
+
if stripped and stripped not in tokens:
|
|
179
|
+
tokens.append(stripped)
|
|
180
|
+
return tokens
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def read_required_capabilities(headers: Any) -> List[str]:
|
|
184
|
+
"""The tokens a dispatch declared in its `requires` header. Never raises and
|
|
185
|
+
never trusts the writer: an absent, blank or non-string header means
|
|
186
|
+
"requires nothing", which is how every message published before this existed
|
|
187
|
+
reads."""
|
|
188
|
+
if not isinstance(headers, dict):
|
|
189
|
+
return []
|
|
190
|
+
|
|
191
|
+
raw = headers.get(CAPABILITY_HEADER_KEYS["requires"])
|
|
192
|
+
return _parse_capability_list(raw if isinstance(raw, str) else None)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def unsupported_capabilities(
|
|
196
|
+
required: Iterable[str],
|
|
197
|
+
supported: Optional[Iterable[str]] = None,
|
|
198
|
+
) -> List[str]:
|
|
199
|
+
"""The tokens this build cannot satisfy. Anything unrecognised counts as
|
|
200
|
+
unsupported — an unknown token is a capability from a newer operator, and
|
|
201
|
+
guessing it is harmless would defeat the point of declaring it."""
|
|
202
|
+
known = set(SERVICE_WIRE_CAPABILITIES if supported is None else supported)
|
|
203
|
+
return [token for token in required if token not in known]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def unsupported_capability_message(unsupported: Iterable[str]) -> str:
|
|
207
|
+
"""The one diagnostic string, identical in both SDKs, so the operator can
|
|
208
|
+
surface it verbatim. Named and deliberate: the point of the mechanism is that
|
|
209
|
+
the failure is this message and never whatever incidental exception the
|
|
210
|
+
undecodable payload would have produced."""
|
|
211
|
+
return "unsupported wire capability: " + ", ".join(unsupported)
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
+
from .client_properties import unsupported_capability_message
|
|
4
|
+
|
|
5
|
+
|
|
3
6
|
class TaskException(Exception):
|
|
4
7
|
def __init__(self, message:str, task_template: str):
|
|
5
8
|
super().__init__(message)
|
|
@@ -18,6 +21,21 @@ class RequenceUnreachableException(Exception):
|
|
|
18
21
|
self.status = status
|
|
19
22
|
|
|
20
23
|
|
|
24
|
+
class UnsupportedCapabilityException(Exception):
|
|
25
|
+
"""Raised when a dispatch declares a capability (``requires`` header) this
|
|
26
|
+
build cannot satisfy, so it is refused before its body is ever decoded.
|
|
27
|
+
|
|
28
|
+
A NAMED exception on purpose: the point of the header is that a mismatch
|
|
29
|
+
reports the missing capability, instead of whatever incidental exception the
|
|
30
|
+
undecodable payload would otherwise have produced — a bare ``ValueError``
|
|
31
|
+
from the JSON decoder says nothing about why this box was the wrong one. The
|
|
32
|
+
message travels to the operator, which surfaces it on the node.
|
|
33
|
+
"""
|
|
34
|
+
def __init__(self, unsupported: list[str]):
|
|
35
|
+
super().__init__(unsupported_capability_message(unsupported))
|
|
36
|
+
self.capabilities = list(unsupported)
|
|
37
|
+
|
|
38
|
+
|
|
21
39
|
class RetryException(Exception):
|
|
22
40
|
def __init__(self, delay: Optional[int]):
|
|
23
41
|
self.delay = delay
|
|
@@ -38,12 +38,19 @@ _MIN_OFFLOAD_BYTES = 256
|
|
|
38
38
|
_DEFAULT_MAX_BYTES = 4 * 1024 * 1024
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def
|
|
41
|
+
def _json_body(value: Any) -> str | None:
|
|
42
|
+
"""The bytes PUT in an offloaded object — always JSON, whatever the wire
|
|
43
|
+
codec, because the ingress half below reads them back with ``json.loads``."""
|
|
42
44
|
try:
|
|
43
|
-
|
|
45
|
+
return json.dumps(value, separators=_SEPARATORS)
|
|
44
46
|
except (TypeError, ValueError):
|
|
45
|
-
return None
|
|
46
|
-
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _json_byte_length(value: Any) -> int:
|
|
51
|
+
"""The historical sizer: correct for any caller still publishing JSON."""
|
|
52
|
+
encoded = _json_body(value)
|
|
53
|
+
return 0 if encoded is None else len(encoded.encode("utf-8"))
|
|
47
54
|
|
|
48
55
|
|
|
49
56
|
def _offload(ctx: Any, upload_url: str, encoded: str) -> dict:
|
|
@@ -75,8 +82,17 @@ def _create_message_handler() -> MessageHandler:
|
|
|
75
82
|
threshold = _DEFAULT_MAX_BYTES
|
|
76
83
|
if threshold <= 0 or body is None:
|
|
77
84
|
return body
|
|
78
|
-
|
|
79
|
-
|
|
85
|
+
|
|
86
|
+
# Two different questions about the same value, and they no longer have
|
|
87
|
+
# the same answer:
|
|
88
|
+
# - how big will this be ON THE WIRE → `measure`, the publisher's codec;
|
|
89
|
+
# - what bytes do we PUT in the offloaded object → JSON, always.
|
|
90
|
+
# The stored object is an HTTP/S3 payload read back by `json.loads` in the
|
|
91
|
+
# ingress half below, so it stays JSON regardless of the wire codec. Only
|
|
92
|
+
# the threshold arithmetic follows the codec.
|
|
93
|
+
measure = getattr(ctx, "measure", None) or _json_byte_length
|
|
94
|
+
|
|
95
|
+
if measure(body) <= threshold:
|
|
80
96
|
return body
|
|
81
97
|
upload_url = getattr(ctx, "upload_url", None)
|
|
82
98
|
if not upload_url:
|
|
@@ -86,6 +102,9 @@ def _create_message_handler() -> MessageHandler:
|
|
|
86
102
|
return body
|
|
87
103
|
|
|
88
104
|
if not isinstance(body, dict):
|
|
105
|
+
encoded = _json_body(body)
|
|
106
|
+
if encoded is None:
|
|
107
|
+
return body
|
|
89
108
|
return _offload(ctx, upload_url, encoded)
|
|
90
109
|
|
|
91
110
|
# Record body: offload oversized top-level values largest-first until
|
|
@@ -94,22 +113,30 @@ def _create_message_handler() -> MessageHandler:
|
|
|
94
113
|
# key-preserving wrap can do — return it and let the broker decide.
|
|
95
114
|
entries = sorted(
|
|
96
115
|
(
|
|
97
|
-
{"key": key, "encoded":
|
|
116
|
+
{"key": key, "encoded": _json_body(value), "size": measure(value)}
|
|
98
117
|
for key, value in body.items()
|
|
99
|
-
for value_encoded, value_size in [_byte_length(value)]
|
|
100
118
|
),
|
|
101
119
|
key=lambda entry: entry["size"],
|
|
102
120
|
reverse=True,
|
|
103
121
|
)
|
|
104
122
|
|
|
105
123
|
wrapped = dict(body)
|
|
124
|
+
total = measure(body)
|
|
106
125
|
for entry in entries:
|
|
107
|
-
if total <= threshold or entry["
|
|
126
|
+
if total <= threshold or entry["size"] <= _MIN_OFFLOAD_BYTES:
|
|
108
127
|
break
|
|
128
|
+
if entry["encoded"] is None:
|
|
129
|
+
# An offloaded object is stored as JSON and read back with
|
|
130
|
+
# `json.loads`, so a value `json.dumps` refuses cannot be
|
|
131
|
+
# offloaded at all. SKIP it rather than break: entries are
|
|
132
|
+
# largest-first, and the msgpack wire made this reachable (it
|
|
133
|
+
# carries `bytes`, which JSON cannot express), so stopping here
|
|
134
|
+
# would strand every smaller value that CAN be offloaded — and
|
|
135
|
+
# the whole oversized body would reach the broker.
|
|
136
|
+
continue
|
|
109
137
|
ref = _offload(ctx, upload_url, entry["encoded"])
|
|
110
138
|
wrapped[entry["key"]] = ref
|
|
111
|
-
|
|
112
|
-
total += ref_size - entry["size"]
|
|
139
|
+
total += measure(ref) - entry["size"]
|
|
113
140
|
return wrapped
|
|
114
141
|
|
|
115
142
|
return MessageHandler(wrap_outbound=wrap_outbound)
|
|
@@ -56,4 +56,18 @@ class ClaimCheckEgressContext(Protocol):
|
|
|
56
56
|
# Per-task offload threshold in bytes, transmitted via task options.
|
|
57
57
|
# Absent ⇒ 4 MiB default.
|
|
58
58
|
max_payload_size: int | None
|
|
59
|
+
# Encoded size of a value under the codec the caller is about to publish
|
|
60
|
+
# with, in bytes.
|
|
61
|
+
#
|
|
62
|
+
# The claim-check decides what to offload by measuring the serialized body,
|
|
63
|
+
# so the codec and the measurement are coupled: **measured size == published
|
|
64
|
+
# size**. Since the wire codec is chosen per service node, both JSON and
|
|
65
|
+
# msgpack are live at once and the handler cannot assume either — it has to
|
|
66
|
+
# be told. Absent ⇒ `json.dumps` byte length, which is both the historical
|
|
67
|
+
# behaviour and correct for every caller still publishing JSON.
|
|
68
|
+
#
|
|
69
|
+
# Getting this wrong does not fail loudly: measuring larger than published
|
|
70
|
+
# offloads too eagerly (a needless S3 round-trip on the hot path), measuring
|
|
71
|
+
# smaller lets an oversized body reach the broker and get rejected.
|
|
72
|
+
measure: Callable[[object], int] | None
|
|
59
73
|
on_offload_skip: Callable[[], None] | None
|
|
@@ -4,6 +4,11 @@ from ssl import SSLContext
|
|
|
4
4
|
class ServiceConfig(TypedDict, total=False):
|
|
5
5
|
access_token: Optional[str]
|
|
6
6
|
version: str
|
|
7
|
+
# Optional, author-chosen label announced in the AMQP client properties and
|
|
8
|
+
# shown in the Requence UI next to the online count (mirrors
|
|
9
|
+
# ServiceOptions.name). A DEBUGGING LABEL, not an identifier: not unique,
|
|
10
|
+
# nothing keys off it. Capped at 64 characters, never rejected.
|
|
11
|
+
name: Optional[str]
|
|
7
12
|
prefetch: Optional[int]
|
|
8
13
|
ssl_context: Optional[SSLContext]
|
|
9
14
|
# heartbeat interval in seconds (mirrors ServiceOptions.heartbeatInterval)
|