requence 0.8.0__tar.gz → 0.8.1.dev55__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. {requence-0.8.0/requence.egg-info → requence-0.8.1.dev55}/PKG-INFO +2 -1
  2. {requence-0.8.0 → requence-0.8.1.dev55}/pyproject.toml +5 -1
  3. requence-0.8.1.dev55/requence/client_properties.py +211 -0
  4. {requence-0.8.0 → requence-0.8.1.dev55}/requence/exceptions.py +18 -0
  5. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/handlers/claim_check_handler.py +38 -11
  6. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/handlers/context.py +14 -0
  7. {requence-0.8.0 → requence-0.8.1.dev55}/requence/internal_types.py +5 -0
  8. {requence-0.8.0 → requence-0.8.1.dev55}/requence/service/__init__.py +112 -16
  9. {requence-0.8.0 → requence-0.8.1.dev55}/requence/service/boundary.py +13 -0
  10. requence-0.8.1.dev55/requence/wire_codec.py +169 -0
  11. {requence-0.8.0 → requence-0.8.1.dev55/requence.egg-info}/PKG-INFO +2 -1
  12. {requence-0.8.0 → requence-0.8.1.dev55}/requence.egg-info/SOURCES.txt +5 -1
  13. {requence-0.8.0 → requence-0.8.1.dev55}/requence.egg-info/requires.txt +1 -0
  14. requence-0.8.1.dev55/tests/test_client_properties.py +276 -0
  15. requence-0.8.1.dev55/tests/test_wire_codec.py +233 -0
  16. {requence-0.8.0 → requence-0.8.1.dev55}/MANIFEST.in +0 -0
  17. {requence-0.8.0 → requence-0.8.1.dev55}/README.md +0 -0
  18. {requence-0.8.0 → requence-0.8.1.dev55}/requence/__init__.py +0 -0
  19. {requence-0.8.0 → requence-0.8.1.dev55}/requence/callbacks/__init__.py +0 -0
  20. {requence-0.8.0 → requence-0.8.1.dev55}/requence/callbacks/context.py +0 -0
  21. {requence-0.8.0 → requence-0.8.1.dev55}/requence/callbacks/handler.py +0 -0
  22. {requence-0.8.0 → requence-0.8.1.dev55}/requence/context.py +0 -0
  23. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/__init__.py +0 -0
  24. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/errors.py +0 -0
  25. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/handlers/__init__.py +0 -0
  26. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/handlers/file_handler.py +0 -0
  27. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/handlers/stream_handler.py +0 -0
  28. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/pending_uploads.py +0 -0
  29. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/requence_file.py +0 -0
  30. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/requence_stream.py +0 -0
  31. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/sanitize_log_data.py +0 -0
  32. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/stream_framing.py +0 -0
  33. {requence-0.8.0 → requence-0.8.1.dev55}/requence/files/transport.py +0 -0
  34. {requence-0.8.0 → requence-0.8.1.dev55}/requence/obfuscase.py +0 -0
  35. {requence-0.8.0 → requence-0.8.1.dev55}/requence/py.typed +0 -0
  36. {requence-0.8.0 → requence-0.8.1.dev55}/requence/reachable.py +0 -0
  37. {requence-0.8.0 → requence-0.8.1.dev55}/requence/requence_callback.py +0 -0
  38. {requence-0.8.0 → requence-0.8.1.dev55}/requence/requence_file.py +0 -0
  39. {requence-0.8.0 → requence-0.8.1.dev55}/requence/requence_stream.py +0 -0
  40. {requence-0.8.0 → requence-0.8.1.dev55}/requence/service/cli.py +0 -0
  41. {requence-0.8.0 → requence-0.8.1.dev55}/requence/service/transformer.py +0 -0
  42. {requence-0.8.0 → requence-0.8.1.dev55}/requence/service/types.py +0 -0
  43. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/__init__.py +0 -0
  44. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/abort_task.py +0 -0
  45. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/ack_task.py +0 -0
  46. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/cli.py +0 -0
  47. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/get_task.py +0 -0
  48. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/ingress.py +0 -0
  49. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/protect_task.py +0 -0
  50. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/task.py +0 -0
  51. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/task_watcher.py +0 -0
  52. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/types.py +0 -0
  53. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/updates.py +0 -0
  54. {requence-0.8.0 → requence-0.8.1.dev55}/requence/task/utils.py +0 -0
  55. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/__init__.py +0 -0
  56. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/context.py +0 -0
  57. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/errors.py +0 -0
  58. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/guards.py +0 -0
  59. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/message_handler.py +0 -0
  60. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/transform.py +0 -0
  61. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/value_handler.py +0 -0
  62. {requence-0.8.0 → requence-0.8.1.dev55}/requence/transformer/walk.py +0 -0
  63. {requence-0.8.0 → requence-0.8.1.dev55}/requence/utils.py +0 -0
  64. {requence-0.8.0 → requence-0.8.1.dev55}/requence/wire.py +0 -0
  65. {requence-0.8.0 → requence-0.8.1.dev55}/requence.egg-info/dependency_links.txt +0 -0
  66. {requence-0.8.0 → requence-0.8.1.dev55}/requence.egg-info/entry_points.txt +0 -0
  67. {requence-0.8.0 → requence-0.8.1.dev55}/requence.egg-info/top_level.txt +0 -0
  68. {requence-0.8.0 → requence-0.8.1.dev55}/setup.cfg +0 -0
  69. {requence-0.8.0 → requence-0.8.1.dev55}/tests/test_wire.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: requence
3
- Version: 0.8.0
3
+ Version: 0.8.1.dev55
4
4
  Summary: Requence Service
5
5
  Author: Torsten Blindert
6
6
  Requires-Python: >=3.12
@@ -9,6 +9,7 @@ Requires-Dist: pika>=1.3.2
9
9
  Requires-Dist: requests>=2.32.2
10
10
  Requires-Dist: colorama>=0.4.6
11
11
  Requires-Dist: sseclient-py>=1.8.0
12
+ Requires-Dist: msgpack>=1.0.5
12
13
 
13
14
  # requence
14
15
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "requence"
7
- version = "0.8.0"
7
+ version = "0.8.1.dev55"
8
8
  description = "Requence Service"
9
9
  readme = { file = "README.md", content-type = "text/markdown" }
10
10
  authors = [{ name = "Torsten Blindert" }]
@@ -14,6 +14,10 @@ dependencies = [
14
14
  "requests>=2.32.2",
15
15
  "colorama>=0.4.6",
16
16
  "sseclient-py>=1.8.0",
17
+ # The portable msgpack wire (requence/wire_codec.py). NOT optional: the SDK
18
+ # declares `wire:msgpack1` in its AMQP client properties, which is a promise
19
+ # that it can decode a dispatch under that content type.
20
+ "msgpack>=1.0.5",
17
21
  ]
18
22
 
19
23
  [project.scripts]
@@ -0,0 +1,211 @@
1
+ """AMQP connection client properties — the Python mirror of
2
+ ``packages/helpers/src/protocol/clientProperties.ts``.
3
+
4
+ That TS file is the contract and carries the full rationale. This module exists
5
+ so the Python SDK announces itself with the IDENTICAL key set; the parity is
6
+ locked by ``tests/test_client_properties.py``, which reads the TS source and
7
+ asserts the keys match. Changing a key is a wire change: both SDKs ship it
8
+ together.
9
+
10
+ Write side only for the announcements — reading connections back is
11
+ ``ui/backend``'s job. The capability section at the bottom is the other way
12
+ round: the operator declares what a dispatch requires, and this SDK only reads.
13
+ """
14
+
15
+ from typing import Any, Dict, Iterable, List, Optional
16
+
17
+ __all__ = [
18
+ "CLIENT_PROPERTY_KEYS",
19
+ "IDENTITY_HEADER_KEYS",
20
+ "CAPABILITY_HEADER_KEYS",
21
+ "MAX_SERVICE_NAME_LENGTH",
22
+ "MAX_CAPABILITIES_LENGTH",
23
+ "WIRE_JSON_1",
24
+ "WIRE_MSGPACK_1",
25
+ "SERVICE_WIRE_CAPABILITIES",
26
+ "build_service_client_properties",
27
+ "build_service_identity_headers",
28
+ "read_required_capabilities",
29
+ "unsupported_capabilities",
30
+ "unsupported_capability_message",
31
+ ]
32
+
33
+ # The pinned, namespaced property set. Flat strings on purpose.
34
+ CLIENT_PROPERTY_KEYS = {
35
+ "libVersion": "requence_lib_version",
36
+ "serviceVersion": "requence_service_version",
37
+ "capabilities": "requence_capabilities",
38
+ # Author-chosen debugging label. Optional, non-unique, unvalidated —
39
+ # NOTHING may key off or branch on it.
40
+ "name": "requence_name",
41
+ # Marks a developer's local dev-overlay connection. Only the marker travels,
42
+ # never the devKey (which is an access-token value).
43
+ "devOverlay": "requence_dev_overlay",
44
+ }
45
+
46
+ # The name is author-controlled text and the whole client-properties table
47
+ # travels inside the connection frame, so an unbounded string is a
48
+ # connection-level failure. Capped here and truncated AGAIN on read.
49
+ MAX_SERVICE_NAME_LENGTH = 64
50
+ MAX_CAPABILITIES_LENGTH = 512
51
+
52
+ WIRE_JSON_1 = "wire:json1"
53
+ # The portable msgpack subset in ``requence/wire_codec.py``. Claimed only by an
54
+ # SDK that can DECODE it on the dispatch wire — a service echoes the codec it was
55
+ # addressed in, so claiming this is a promise about READING, and the reply
56
+ # direction needs no capability of its own.
57
+ WIRE_MSGPACK_1 = "wire:msgpack1"
58
+ SERVICE_WIRE_CAPABILITIES = [WIRE_JSON_1, WIRE_MSGPACK_1]
59
+
60
+
61
+ def _sanitize_label(value: str, max_length: int) -> str:
62
+ """Drop control characters, trim, cap. Mirrors sanitizeLabel in the TS."""
63
+ return "".join(c for c in value if ord(c) >= 0x20 and ord(c) != 0x7F).strip()[
64
+ :max_length
65
+ ]
66
+
67
+
68
+ def build_service_client_properties(
69
+ *,
70
+ lib_version: str,
71
+ service_version: str,
72
+ name: Optional[str] = None,
73
+ dev_overlay: bool = False,
74
+ capabilities: Optional[Iterable[str]] = None,
75
+ ) -> Dict[str, str]:
76
+ """Build the properties handed to pika on connect (and therefore on every
77
+ reconnect, since the connection parameters are reused). Absent optional
78
+ fields are OMITTED, never sent blank."""
79
+ if capabilities is None:
80
+ capabilities = SERVICE_WIRE_CAPABILITIES
81
+
82
+ properties: Dict[str, str] = {
83
+ CLIENT_PROPERTY_KEYS["libVersion"]: lib_version,
84
+ CLIENT_PROPERTY_KEYS["serviceVersion"]: service_version,
85
+ CLIENT_PROPERTY_KEYS["capabilities"]: ",".join(capabilities)[
86
+ :MAX_CAPABILITIES_LENGTH
87
+ ],
88
+ }
89
+
90
+ label = _sanitize_label(name, MAX_SERVICE_NAME_LENGTH) if name else ""
91
+ if label:
92
+ properties[CLIENT_PROPERTY_KEYS["name"]] = label
93
+
94
+ if dev_overlay:
95
+ properties[CLIENT_PROPERTY_KEYS["devOverlay"]] = "1"
96
+
97
+ return properties
98
+
99
+
100
+ # ─── The second announcement: per-run identity on the begin message ──────────
101
+ #
102
+ # The same three values as above, sent again on the ``begin`` publish, because
103
+ # they answer a DIFFERENT question: the client properties say "can this fleet
104
+ # decode msgpack?" (live, folded to the minimum, available before dispatch),
105
+ # these say "which build ran THIS node?" (exact, frozen into the NODE_START
106
+ # update). See the TS contract for the full rationale.
107
+ #
108
+ # Bare, un-namespaced names, matching the headers that already ride alongside
109
+ # them (``taskId``, ``index``, ``connectionId``, ``version``).
110
+ IDENTITY_HEADER_KEYS = {
111
+ "libVersion": "libVersion",
112
+ "name": "serviceName",
113
+ "capabilities": "capabilities",
114
+ }
115
+
116
+
117
+ def build_service_identity_headers(
118
+ *,
119
+ lib_version: str,
120
+ name: Optional[str] = None,
121
+ capabilities: Optional[Iterable[str]] = None,
122
+ ) -> Dict[str, str]:
123
+ """Headers the service stamps on its own ``begin`` publish, built from this
124
+ process's own constants — nothing is looked up, so nothing can be stale."""
125
+ if capabilities is None:
126
+ capabilities = SERVICE_WIRE_CAPABILITIES
127
+
128
+ headers: Dict[str, str] = {
129
+ IDENTITY_HEADER_KEYS["libVersion"]: lib_version,
130
+ IDENTITY_HEADER_KEYS["capabilities"]: ",".join(capabilities)[
131
+ :MAX_CAPABILITIES_LENGTH
132
+ ],
133
+ }
134
+
135
+ label = _sanitize_label(name, MAX_SERVICE_NAME_LENGTH) if name else ""
136
+ if label:
137
+ headers[IDENTITY_HEADER_KEYS["name"]] = label
138
+
139
+ return headers
140
+
141
+
142
+ # ─── The dispatch direction: what a message REQUIRES of its consumer ─────────
143
+ #
144
+ # The mirror of the TS contract's `requires` section. READ SIDE ONLY here: the
145
+ # tokens are declared by the operator, which is TypeScript, so Python never
146
+ # builds this header — it only ever answers "can this build satisfy it?".
147
+ #
148
+ # It is a HEADER, never a body field: the body is precisely the thing this build
149
+ # may be unable to decode, so a requirement stated inside it is unreadable
150
+ # exactly when it matters.
151
+ CAPABILITY_HEADER_KEYS = {
152
+ # dispatch → service: the tokens this message needs. Absent when it needs
153
+ # nothing beyond the floor every SDK speaks.
154
+ "requires": "requires",
155
+ # service → operator: this replica cannot satisfy `requires`. Sent by
156
+ # REPUBLISHING to `<exchange>-retry`, never by nacking — a nack cannot modify
157
+ # the message, these are classic queues (so there is no `x-delivery-count`
158
+ # either), and with no counter anywhere the message would requeue forever.
159
+ "mismatch": "capabilityMismatch",
160
+ # operator → service, on a reroute: how many reroutes are left. Named for
161
+ # reroutes and NOT for retries on purpose — it is its own allowance, never
162
+ # the node's `retry` budget. The operator owns the count; this SDK only ever
163
+ # echoes it back.
164
+ "reroutes": "capabilityReroutes",
165
+ }
166
+
167
+
168
+ def _parse_capability_list(raw: Optional[str]) -> List[str]:
169
+ """Caps the whole string BEFORE splitting, so an unbounded writer cannot push
170
+ an unbounded list through. A token cut in half by that cap simply matches no
171
+ known capability, which can only ever UNDER-claim — the safe direction."""
172
+ if not raw:
173
+ return []
174
+
175
+ tokens: List[str] = []
176
+ for token in raw[:MAX_CAPABILITIES_LENGTH].split(","):
177
+ stripped = token.strip()
178
+ if stripped and stripped not in tokens:
179
+ tokens.append(stripped)
180
+ return tokens
181
+
182
+
183
+ def read_required_capabilities(headers: Any) -> List[str]:
184
+ """The tokens a dispatch declared in its `requires` header. Never raises and
185
+ never trusts the writer: an absent, blank or non-string header means
186
+ "requires nothing", which is how every message published before this existed
187
+ reads."""
188
+ if not isinstance(headers, dict):
189
+ return []
190
+
191
+ raw = headers.get(CAPABILITY_HEADER_KEYS["requires"])
192
+ return _parse_capability_list(raw if isinstance(raw, str) else None)
193
+
194
+
195
+ def unsupported_capabilities(
196
+ required: Iterable[str],
197
+ supported: Optional[Iterable[str]] = None,
198
+ ) -> List[str]:
199
+ """The tokens this build cannot satisfy. Anything unrecognised counts as
200
+ unsupported — an unknown token is a capability from a newer operator, and
201
+ guessing it is harmless would defeat the point of declaring it."""
202
+ known = set(SERVICE_WIRE_CAPABILITIES if supported is None else supported)
203
+ return [token for token in required if token not in known]
204
+
205
+
206
+ def unsupported_capability_message(unsupported: Iterable[str]) -> str:
207
+ """The one diagnostic string, identical in both SDKs, so the operator can
208
+ surface it verbatim. Named and deliberate: the point of the mechanism is that
209
+ the failure is this message and never whatever incidental exception the
210
+ undecodable payload would have produced."""
211
+ return "unsupported wire capability: " + ", ".join(unsupported)
@@ -1,5 +1,8 @@
1
1
  from typing import Optional
2
2
 
3
+ from .client_properties import unsupported_capability_message
4
+
5
+
3
6
  class TaskException(Exception):
4
7
  def __init__(self, message:str, task_template: str):
5
8
  super().__init__(message)
@@ -18,6 +21,21 @@ class RequenceUnreachableException(Exception):
18
21
  self.status = status
19
22
 
20
23
 
24
+ class UnsupportedCapabilityException(Exception):
25
+ """Raised when a dispatch declares a capability (``requires`` header) this
26
+ build cannot satisfy, so it is refused before its body is ever decoded.
27
+
28
+ A NAMED exception on purpose: the point of the header is that a mismatch
29
+ reports the missing capability, instead of whatever incidental exception the
30
+ undecodable payload would otherwise have produced — a bare ``ValueError``
31
+ from the JSON decoder says nothing about why this box was the wrong one. The
32
+ message travels to the operator, which surfaces it on the node.
33
+ """
34
+ def __init__(self, unsupported: list[str]):
35
+ super().__init__(unsupported_capability_message(unsupported))
36
+ self.capabilities = list(unsupported)
37
+
38
+
21
39
  class RetryException(Exception):
22
40
  def __init__(self, delay: Optional[int]):
23
41
  self.delay = delay
@@ -38,12 +38,19 @@ _MIN_OFFLOAD_BYTES = 256
38
38
  _DEFAULT_MAX_BYTES = 4 * 1024 * 1024
39
39
 
40
40
 
41
- def _byte_length(value: Any) -> tuple[str | None, int]:
41
+ def _json_body(value: Any) -> str | None:
42
+ """The bytes PUT in an offloaded object — always JSON, whatever the wire
43
+ codec, because the ingress half below reads them back with ``json.loads``."""
42
44
  try:
43
- encoded = json.dumps(value, separators=_SEPARATORS)
45
+ return json.dumps(value, separators=_SEPARATORS)
44
46
  except (TypeError, ValueError):
45
- return None, 0
46
- return encoded, len(encoded.encode("utf-8"))
47
+ return None
48
+
49
+
50
+ def _json_byte_length(value: Any) -> int:
51
+ """The historical sizer: correct for any caller still publishing JSON."""
52
+ encoded = _json_body(value)
53
+ return 0 if encoded is None else len(encoded.encode("utf-8"))
47
54
 
48
55
 
49
56
  def _offload(ctx: Any, upload_url: str, encoded: str) -> dict:
@@ -75,8 +82,17 @@ def _create_message_handler() -> MessageHandler:
75
82
  threshold = _DEFAULT_MAX_BYTES
76
83
  if threshold <= 0 or body is None:
77
84
  return body
78
- encoded, total = _byte_length(body)
79
- if encoded is None or total <= threshold:
85
+
86
+ # Two different questions about the same value, and they no longer have
87
+ # the same answer:
88
+ # - how big will this be ON THE WIRE → `measure`, the publisher's codec;
89
+ # - what bytes do we PUT in the offloaded object → JSON, always.
90
+ # The stored object is an HTTP/S3 payload read back by `json.loads` in the
91
+ # ingress half below, so it stays JSON regardless of the wire codec. Only
92
+ # the threshold arithmetic follows the codec.
93
+ measure = getattr(ctx, "measure", None) or _json_byte_length
94
+
95
+ if measure(body) <= threshold:
80
96
  return body
81
97
  upload_url = getattr(ctx, "upload_url", None)
82
98
  if not upload_url:
@@ -86,6 +102,9 @@ def _create_message_handler() -> MessageHandler:
86
102
  return body
87
103
 
88
104
  if not isinstance(body, dict):
105
+ encoded = _json_body(body)
106
+ if encoded is None:
107
+ return body
89
108
  return _offload(ctx, upload_url, encoded)
90
109
 
91
110
  # Record body: offload oversized top-level values largest-first until
@@ -94,22 +113,30 @@ def _create_message_handler() -> MessageHandler:
94
113
  # key-preserving wrap can do — return it and let the broker decide.
95
114
  entries = sorted(
96
115
  (
97
- {"key": key, "encoded": value_encoded, "size": value_size}
116
+ {"key": key, "encoded": _json_body(value), "size": measure(value)}
98
117
  for key, value in body.items()
99
- for value_encoded, value_size in [_byte_length(value)]
100
118
  ),
101
119
  key=lambda entry: entry["size"],
102
120
  reverse=True,
103
121
  )
104
122
 
105
123
  wrapped = dict(body)
124
+ total = measure(body)
106
125
  for entry in entries:
107
- if total <= threshold or entry["encoded"] is None or entry["size"] <= _MIN_OFFLOAD_BYTES:
126
+ if total <= threshold or entry["size"] <= _MIN_OFFLOAD_BYTES:
108
127
  break
128
+ if entry["encoded"] is None:
129
+ # An offloaded object is stored as JSON and read back with
130
+ # `json.loads`, so a value `json.dumps` refuses cannot be
131
+ # offloaded at all. SKIP it rather than break: entries are
132
+ # largest-first, and the msgpack wire made this reachable (it
133
+ # carries `bytes`, which JSON cannot express), so stopping here
134
+ # would strand every smaller value that CAN be offloaded — and
135
+ # the whole oversized body would reach the broker.
136
+ continue
109
137
  ref = _offload(ctx, upload_url, entry["encoded"])
110
138
  wrapped[entry["key"]] = ref
111
- _ref_encoded, ref_size = _byte_length(ref)
112
- total += ref_size - entry["size"]
139
+ total += measure(ref) - entry["size"]
113
140
  return wrapped
114
141
 
115
142
  return MessageHandler(wrap_outbound=wrap_outbound)
@@ -56,4 +56,18 @@ class ClaimCheckEgressContext(Protocol):
56
56
  # Per-task offload threshold in bytes, transmitted via task options.
57
57
  # Absent ⇒ 4 MiB default.
58
58
  max_payload_size: int | None
59
+ # Encoded size of a value under the codec the caller is about to publish
60
+ # with, in bytes.
61
+ #
62
+ # The claim-check decides what to offload by measuring the serialized body,
63
+ # so the codec and the measurement are coupled: **measured size == published
64
+ # size**. Since the wire codec is chosen per service node, both JSON and
65
+ # msgpack are live at once and the handler cannot assume either — it has to
66
+ # be told. Absent ⇒ `json.dumps` byte length, which is both the historical
67
+ # behaviour and correct for every caller still publishing JSON.
68
+ #
69
+ # Getting this wrong does not fail loudly: measuring larger than published
70
+ # offloads too eagerly (a needless S3 round-trip on the hot path), measuring
71
+ # smaller lets an oversized body reach the broker and get rejected.
72
+ measure: Callable[[object], int] | None
59
73
  on_offload_skip: Callable[[], None] | None
@@ -4,6 +4,11 @@ from ssl import SSLContext
4
4
  class ServiceConfig(TypedDict, total=False):
5
5
  access_token: Optional[str]
6
6
  version: str
7
+ # Optional, author-chosen label announced in the AMQP client properties and
8
+ # shown in the Requence UI next to the online count (mirrors
9
+ # ServiceOptions.name). A DEBUGGING LABEL, not an identifier: not unique,
10
+ # nothing keys off it. Capped at 64 characters, never rejected.
11
+ name: Optional[str]
7
12
  prefetch: Optional[int]
8
13
  ssl_context: Optional[SSLContext]
9
14
  # heartbeat interval in seconds (mirrors ServiceOptions.heartbeatInterval)
@@ -1,18 +1,36 @@
1
1
  from pika.adapters.blocking_connection import BlockingChannel
2
2
  from pika.spec import Basic, BasicProperties
3
3
  from typing import Any, Callable, Generator, Optional, cast, Union, List, Tuple
4
+ from ..client_properties import (
5
+ CAPABILITY_HEADER_KEYS,
6
+ build_service_client_properties,
7
+ build_service_identity_headers,
8
+ read_required_capabilities,
9
+ unsupported_capabilities,
10
+ )
4
11
  from ..context import Context
5
12
  from ..utils import is_valid_uuid, libVersion, OutputValue
6
- from ..exceptions import RetryException, AbortException, SkipException, StopException
13
+ from ..exceptions import (
14
+ RetryException,
15
+ AbortException,
16
+ SkipException,
17
+ StopException,
18
+ UnsupportedCapabilityException,
19
+ )
7
20
  from uuid import uuid4
8
21
  from ..obfuscase import deobfuscate, obfuscate
22
+ from ..wire_codec import (
23
+ WireCodec,
24
+ decode_wire_body,
25
+ encode_wire_body,
26
+ wire_codec_from_content_type,
27
+ )
9
28
  import inspect
10
29
  import traceback
11
30
  import functools
12
31
  import os
13
32
  import pika
14
33
  import pika.exceptions
15
- import json
16
34
  import random
17
35
  import ssl
18
36
  import threading
@@ -136,11 +154,26 @@ class Service:
136
154
  if heartbeat_interval is not None:
137
155
  parameters.heartbeat = heartbeat_interval
138
156
 
157
+ # Who we are, what we run, what we can decode — declared in the AMQP
158
+ # connection frame so ui/backend knows it BEFORE the first dispatch. pika
159
+ # merges these over its own defaults, and the parameters are reused on
160
+ # every reconnect, so the announcement re-lands automatically.
161
+ parameters.client_properties = build_service_client_properties(
162
+ lib_version=libVersion,
163
+ service_version=version,
164
+ name=service_config.get("name"),
165
+ dev_overlay=dev_overlay is not None,
166
+ )
167
+
139
168
  if not parameters.credentials:
140
169
  raise ValueError("Url connection string needs a username")
141
170
 
142
171
  self.__parameters = parameters
143
172
  self.__version = version
173
+ # Author-chosen debugging label. Announced twice — in the client
174
+ # properties above (fleet view) and on every begin publish (per-run
175
+ # identity) — from this one value, so the two can never disagree.
176
+ self.__name = service_config.get("name")
144
177
  self.__prefetch = prefetch
145
178
  self.__dev_overlay = dev_overlay
146
179
  self.__exchange = cast(str, parameters.credentials.username)
@@ -290,17 +323,29 @@ class Service:
290
323
  except self.__CLOSED_ERRORS:
291
324
  print("skipping nack: channel/connection closed (will be redelivered)")
292
325
 
293
- def __publish(self, message_id: str, task_id: str, index: int, headers: dict[str,Any] = {}, data: Any = None, output_name: Union[str,None] = None, priority: Optional[int] = None, epoch: Optional[str] = None):
326
+ def __publish(self, message_id: str, task_id: str, index: int, headers: dict[str,Any] = {}, data: Any = None, output_name: Union[str,None] = None, priority: Optional[int] = None, epoch: Optional[str] = None, codec: WireCodec = "json"):
327
+ # A REPLY ECHOES THE CODEC OF ITS DISPATCH. That is what keeps this
328
+ # direction of the wire free of any capability logic: the service never
329
+ # chooses, never configures and never negotiates a codec — it answers in
330
+ # the language it was addressed in, and the operator reads the content
331
+ # type back off the reply.
332
+ #
333
+ # The codec is resolved by the caller, on the handler thread, and passed
334
+ # in: this method runs later on the connection's I/O thread, so looking it
335
+ # up here would race the handler's own teardown. `json` is the default —
336
+ # both the pre-migration status quo and the right answer for a publish
337
+ # that never went through a dispatch (`act()`).
338
+ body, content_type = encode_wire_body(data, codec)
339
+
294
340
  properties = pika.BasicProperties(
295
341
  correlation_id=message_id,
296
- content_type="application/json",
342
+ content_type=content_type,
297
343
  priority=priority,
298
344
  delivery_mode=pika.DeliveryMode.Persistent,
299
345
  headers={
300
346
  "taskId": task_id,
301
347
  "index": index,
302
348
  "connectionId": self.__connection_id,
303
- "libVersion": libVersion,
304
349
  "version": self.__version,
305
350
  **headers
306
351
  }
@@ -318,9 +363,7 @@ class Service:
318
363
  self.__channel.basic_publish(
319
364
  exchange=self.__exchange,
320
365
  routing_key='',
321
- # Compact separators: the claim-check measures the body with the
322
- # same separators, so measured size == published size.
323
- body=json.dumps(data, separators=(",", ":")).encode('utf-8'),
366
+ body=body,
324
367
  properties=properties
325
368
  )
326
369
 
@@ -328,7 +371,6 @@ class Service:
328
371
  if generation != self.__generation:
329
372
  return # settle from a superseded connection; broker redelivers.
330
373
  properties.headers["connectionId"] = self.__connection_id
331
- properties.headers["libVersion"] = libVersion
332
374
 
333
375
  try:
334
376
  self.__channel.basic_publish(
@@ -345,7 +387,6 @@ class Service:
345
387
  if generation != self.__generation:
346
388
  return # settle from a superseded connection; broker redelivers.
347
389
  properties.headers["connectionId"] = self.__connection_id
348
- properties.headers["libVersion"] = libVersion
349
390
 
350
391
  try:
351
392
  self.__channel.basic_publish(
@@ -362,7 +403,6 @@ class Service:
362
403
  if generation != self.__generation:
363
404
  return # settle from a superseded connection; broker redelivers.
364
405
  properties.headers["connectionId"] = self.__connection_id
365
- properties.headers["libVersion"] = libVersion
366
406
 
367
407
  try:
368
408
  self.__channel.basic_publish(
@@ -399,6 +439,14 @@ class Service:
399
439
  message_id = str(properties.correlation_id)
400
440
  registered_callback_ids: List[str] = []
401
441
 
442
+ # Never assume a codec: `ui/backend` bakes the choice per service node, so
443
+ # two nodes of one task can legitimately arrive under different codecs,
444
+ # and a replica just rolled forward still sees JSON dispatches for tasks
445
+ # created before it attached. Resolved before anything can publish, so
446
+ # every reply on this message — including one from an error path — echoes
447
+ # what it arrived under.
448
+ codec = wire_codec_from_content_type(properties.content_type)
449
+
402
450
  try:
403
451
  if not is_valid_uuid(message_id):
404
452
  raise Exception("Invalid or missing correlation ID.")
@@ -409,6 +457,19 @@ class Service:
409
457
  if not is_valid_uuid(task_id):
410
458
  raise Exception("Invalid or missing taskId header.")
411
459
 
460
+ # The capability gate, ahead of everything the dispatch could ask of
461
+ # this build — including its own decoding. A requirement stated in the
462
+ # body would be unreadable exactly when it matters, so it rides in the
463
+ # headers and is checked here, before the body is touched.
464
+ #
465
+ # An empty requirement (every message published before this existed)
466
+ # passes untouched.
467
+ unsupported = unsupported_capabilities(
468
+ read_required_capabilities(properties.headers)
469
+ )
470
+ if unsupported:
471
+ raise UnsupportedCapabilityException(unsupported)
472
+
412
473
  mode = str(properties.headers.get("mode", "LINEAR"))
413
474
 
414
475
  # One epoch per handler invocation. On redelivery a second invocation
@@ -416,7 +477,7 @@ class Service:
416
477
  # fence the superseded one.
417
478
  epoch = str(uuid4())
418
479
 
419
- payload_data = json.loads(body.decode('utf-8'))
480
+ payload_data = decode_wire_body(body, properties.content_type)
420
481
  if not isinstance(payload_data, dict):
421
482
  raise ValueError("Context must be a dictionary.")
422
483
 
@@ -440,7 +501,8 @@ class Service:
440
501
  data=data,
441
502
  headers=headers,
442
503
  priority=properties.priority,
443
- epoch=epoch
504
+ epoch=epoch,
505
+ codec=codec
444
506
  )
445
507
  )
446
508
 
@@ -473,6 +535,9 @@ class Service:
473
535
  stream_url=options.get("streamUrl"),
474
536
  max_payload_size=options.get("maxPayloadSize"),
475
537
  max_file_size=options.get("maxFileSize"),
538
+ # Replies echo the dispatch codec, so the claim-check must
539
+ # measure with it too — measured size == published size.
540
+ wire_codec=codec,
476
541
  terminated=terminated,
477
542
  warn=warn,
478
543
  )
@@ -506,6 +571,7 @@ class Service:
506
571
  "propertyPath": invocation.get("propertyPath"),
507
572
  },
508
573
  epoch=epoch,
574
+ codec=codec,
509
575
  )
510
576
  )
511
577
  outbound.track(send_and_publish)
@@ -540,8 +606,15 @@ class Service:
540
606
  terminated=terminated
541
607
  )
542
608
 
609
+ # The one stamp per node execution: this build, this box. It becomes
610
+ # the NODE_START update's `service`, written once and never
611
+ # re-derived — anything absent here is lost for that run.
543
612
  publish(None, {
544
- "begin": True
613
+ "begin": True,
614
+ **build_service_identity_headers(
615
+ lib_version=libVersion,
616
+ name=self.__name,
617
+ ),
545
618
  })
546
619
 
547
620
  response = self.__on_message_callback(
@@ -611,6 +684,27 @@ class Service:
611
684
  if not acked:
612
685
  self.__schedule(functools.partial(self.__handle_retry, body, properties, delivery_tag, generation))
613
686
 
687
+ except UnsupportedCapabilityException as e:
688
+ # A refusal, not a failure: this box is the wrong one, the message is
689
+ # fine. It goes back by REPUBLISHING to `<exchange>-retry` (the same
690
+ # publish the exception path uses), never by nacking, because a nack
691
+ # can carry no rejection count and a classic queue offers none either
692
+ # — see `CAPABILITY_HEADER_KEYS["mismatch"]` in
693
+ # `requence/client_properties.py` for the full argument, and
694
+ # `tests/service/test_capability_gate.py`, which pins it. The body
695
+ # travels verbatim: a reroute re-dispatches these bytes, and this build
696
+ # never decoded them.
697
+ print(str(e))
698
+
699
+ publish_index += 1
700
+ properties.headers[CAPABILITY_HEADER_KEYS["mismatch"]] = True
701
+ properties.headers["version"] = self.__version
702
+ properties.headers["error-message"] = str(e)
703
+ properties.headers["index"] = publish_index
704
+
705
+ if not acked:
706
+ self.__schedule(functools.partial(self.__handle_exception, body, properties, delivery_tag, generation))
707
+
614
708
  except AbortException as e:
615
709
  message = str(e)
616
710
  if (len(message) == 0):
@@ -651,7 +745,10 @@ class Service:
651
745
 
652
746
 
653
747
  def __on_backchannel_message(self, channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes):
654
- payload_data = json.loads(body.decode('utf-8'))
748
+ # The backchannel is its own wire: the operator publishes `stop` and
749
+ # `callback` here under whichever codec that task was baked with, so this
750
+ # decoder switches on the header exactly like the dispatch one.
751
+ payload_data = decode_wire_body(body, properties.content_type)
655
752
  message_id = properties.correlation_id
656
753
 
657
754
  if (payload_data["type"] == "stop"):
@@ -798,7 +895,6 @@ class Service:
798
895
  headers={
799
896
  "index": publish_index,
800
897
  "taskId": task_id,
801
- "libVersion": libVersion,
802
898
  "abort": True
803
899
  }
804
900
  )