dex-python-sdk 0.6.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/PKG-INFO +16 -2
  2. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/README.md +15 -1
  3. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/__init__.py +8 -3
  4. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_async_value_hydrator.py +54 -17
  5. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_grpc_errors.py +38 -11
  6. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_value_hydrator.py +41 -17
  7. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_value_mapper.py +5 -6
  8. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/async_client.py +102 -68
  9. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/client.py +103 -69
  10. dex_python_sdk-0.8.0/dex/dexpb/dex_pb2.py +503 -0
  11. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/dexpb/dex_pb2.pyi +14 -4
  12. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/dexpb/dex_pb2_grpc.py +18 -18
  13. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/runtime_errors.py +8 -0
  14. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/wait.py +6 -1
  15. dex_python_sdk-0.8.0/dex/wait_options.py +86 -0
  16. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/pyproject.toml +1 -1
  17. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/Cargo.lock +6 -6
  18. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/Cargo.toml +3 -3
  19. dex_python_sdk-0.6.0/dex/dexpb/dex_pb2.py +0 -501
  20. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/LEGACY_NOTICES.md +0 -0
  21. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/LICENSE +0 -0
  22. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_async_worker_dispatcher.py +0 -0
  23. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_async_worker_service.py +0 -0
  24. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_invocation_context.py +0 -0
  25. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_native.pyi +0 -0
  26. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_utils.py +0 -0
  27. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_worker_dispatcher.py +0 -0
  28. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_worker_service.py +0 -0
  29. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/async_worker.py +0 -0
  30. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/attribute.py +0 -0
  31. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/attribute_match.py +0 -0
  32. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/blob_cache.py +0 -0
  33. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/channel.py +0 -0
  34. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/client_options.py +0 -0
  35. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/codec.py +0 -0
  36. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/condition.py +0 -0
  37. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/context.py +0 -0
  38. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/dexpb/__init__.py +0 -0
  39. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow.py +0 -0
  40. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_config.py +0 -0
  41. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_info.py +0 -0
  42. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_options.py +0 -0
  43. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_result.py +0 -0
  44. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/py.typed +0 -0
  45. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/step.py +0 -0
  46. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/step_execution.py +0 -0
  47. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/stream.py +0 -0
  48. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/subflow.py +0 -0
  49. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/timer.py +0 -0
  50. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/worker.py +0 -0
  51. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/worker_options.py +0 -0
  52. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/Cargo.toml +0 -0
  53. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/LICENSE +0 -0
  54. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/config.rs +0 -0
  55. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/entry.rs +0 -0
  56. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/error.rs +0 -0
  57. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/format.rs +0 -0
  58. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/lib.rs +0 -0
  59. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/policy.rs +0 -0
  60. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/store.rs +0 -0
  61. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/tests/blob_cache_integration.rs +0 -0
  62. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache-python/Cargo.toml +0 -0
  63. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache-python/LICENSE +0 -0
  64. {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache-python/src/lib.rs +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dex-python-sdk
3
- Version: 0.6.0
3
+ Version: 0.8.0
4
4
  Requires-Dist: grpcio>=1.83.0
5
5
  Requires-Dist: grpcio-status>=1.83.0
6
6
  Requires-Dist: protobuf>=7.35.1
@@ -361,6 +361,17 @@ buffered sets and deletes. The matching `ChannelMap` methods are RPC-only,
361
361
  include buffered publishes, and omit empty instances. Keys are decoded and
362
362
  sorted. Use `force_complete_if_channels_empty(...)` for conditional completion.
363
363
 
364
+ Request IDs are optional for both durable waits. When omitted, the server
365
+ derives a namespaced stable ID from the Step execution or Attribute condition,
366
+ such as `wait-for-attribute:myInt>10`. Reuse an override only for the same
367
+ logical wait. The Client automatically reattaches transport long polls. If an
368
+ earlier Update with that ID exhausted its handler budget, the server appends an
369
+ increasing `-N` suffix and starts a new Update. `maximum_wait_time` is optional
370
+ and is the total handler budget across reattachments; zero waits indefinitely.
371
+ A positive budget expiry raises `WaitHandlerTimeoutError`. An abandoned infinite
372
+ wait remains accepted and counts against Temporal's in-flight Update limit until
373
+ it completes or the Flow closes.
374
+
364
375
  `Client.wait_for_flow` and `AsyncClient.wait_for_flow` return a
365
376
  `FlowResult` after hydrating every output-bearing completion. Use
366
377
  `single_output` only when the Flow contract produces exactly one output:
@@ -444,7 +455,10 @@ client operations, and assertions as the Java suite against an isolated
444
455
  The strongly typed contracts, registry, synchronous Client/Worker, optional
445
456
  `AsyncClient`/`AsyncWorker` (`grpc.aio`), and Rust-backed BlobCache are
446
457
  implemented. Python owns its gRPC transport; the native bridge is limited to
447
- the shared BlobCache. Design notes:
458
+ the shared BlobCache. Serialized object values use wire encoding `json` for JSON
459
+ and `raw` for raw bytes. Internal Blob references are opaque. Hydration sends the
460
+ owning Flow ID with each reference, and the cache is isolated by
461
+ `(flow_id, blob_ref)`. Design notes:
448
462
  [`python-sdk-async-apis.md`](../docs/design/plan/python-sdk-async-apis.md) and
449
463
  [`python-sdk-step-streaming.md`](../docs/design/plan/python-sdk-step-streaming.md).
450
464
 
@@ -345,6 +345,17 @@ buffered sets and deletes. The matching `ChannelMap` methods are RPC-only,
345
345
  include buffered publishes, and omit empty instances. Keys are decoded and
346
346
  sorted. Use `force_complete_if_channels_empty(...)` for conditional completion.
347
347
 
348
+ Request IDs are optional for both durable waits. When omitted, the server
349
+ derives a namespaced stable ID from the Step execution or Attribute condition,
350
+ such as `wait-for-attribute:myInt>10`. Reuse an override only for the same
351
+ logical wait. The Client automatically reattaches transport long polls. If an
352
+ earlier Update with that ID exhausted its handler budget, the server appends an
353
+ increasing `-N` suffix and starts a new Update. `maximum_wait_time` is optional
354
+ and is the total handler budget across reattachments; zero waits indefinitely.
355
+ A positive budget expiry raises `WaitHandlerTimeoutError`. An abandoned infinite
356
+ wait remains accepted and counts against Temporal's in-flight Update limit until
357
+ it completes or the Flow closes.
358
+
348
359
  `Client.wait_for_flow` and `AsyncClient.wait_for_flow` return a
349
360
  `FlowResult` after hydrating every output-bearing completion. Use
350
361
  `single_output` only when the Flow contract produces exactly one output:
@@ -428,7 +439,10 @@ client operations, and assertions as the Java suite against an isolated
428
439
  The strongly typed contracts, registry, synchronous Client/Worker, optional
429
440
  `AsyncClient`/`AsyncWorker` (`grpc.aio`), and Rust-backed BlobCache are
430
441
  implemented. Python owns its gRPC transport; the native bridge is limited to
431
- the shared BlobCache. Design notes:
442
+ the shared BlobCache. Serialized object values use wire encoding `json` for JSON
443
+ and `raw` for raw bytes. Internal Blob references are opaque. Hydration sends the
444
+ owning Flow ID with each reference, and the cache is isolated by
445
+ `(flow_id, blob_ref)`. Design notes:
432
446
  [`python-sdk-async-apis.md`](../docs/design/plan/python-sdk-async-apis.md) and
433
447
  [`python-sdk-step-streaming.md`](../docs/design/plan/python-sdk-step-streaming.md).
434
448
 
@@ -62,10 +62,11 @@ from dex.flow_options import (
62
62
  )
63
63
  from dex.flow_result import FlowResult, StepCompletion
64
64
  from dex.runtime_errors import (
65
- DexServiceError,
66
- ErrorSubStatus,
67
65
  AttributeMapNotLoadedError,
66
+ ChannelMessageNotFoundError,
68
67
  ChannelMessagesNotLoadedError,
68
+ DexServiceError,
69
+ ErrorSubStatus,
69
70
  FlowAlreadyStartedError,
70
71
  FlowDefinitionError,
71
72
  FlowErrorType,
@@ -73,9 +74,9 @@ from dex.runtime_errors import (
73
74
  FlowNotFoundError,
74
75
  InvalidStepResultError,
75
76
  LongPollTimeoutError,
76
- ChannelMessageNotFoundError,
77
77
  RpcLockConflictError,
78
78
  ValueMappingError,
79
+ WaitHandlerTimeoutError,
79
80
  WorkerInvocationError,
80
81
  )
81
82
  from dex.step import (
@@ -108,6 +109,7 @@ from dex.stream import (
108
109
  from dex.subflow import SubFlow
109
110
  from dex.timer import Timer
110
111
  from dex.wait import Wait
112
+ from dex.wait_options import WaitForAttributeOptions, WaitForStepCompletionOptions
111
113
  from dex.worker import Worker
112
114
  from dex.worker_options import WorkerOptions, WorkerTarget
113
115
 
@@ -164,6 +166,7 @@ __all__ = [
164
166
  "IndexType",
165
167
  "JsonCodec",
166
168
  "LongPollTimeoutError",
169
+ "WaitHandlerTimeoutError",
167
170
  "PersistenceSchema",
168
171
  "RPCResult",
169
172
  "RecoveryErrorInfo",
@@ -205,6 +208,8 @@ __all__ = [
205
208
  "WorkerInvocationError",
206
209
  "WorkerOptions",
207
210
  "WorkerTarget",
211
+ "WaitForAttributeOptions",
212
+ "WaitForStepCompletionOptions",
208
213
  "dead_end",
209
214
  "force_complete",
210
215
  "force_complete_if_channels_empty",
@@ -20,6 +20,7 @@ _LOGGER = logging.getLogger(__name__)
20
20
 
21
21
  @dataclass
22
22
  class _PendingBlob:
23
+ flow_id: str
23
24
  blob_id: str
24
25
  is_object: bool
25
26
  request: pb.Value
@@ -36,21 +37,37 @@ class AsyncValueHydrator:
36
37
  self._service = service
37
38
  self._cache = cache
38
39
 
39
- async def hydrate(self, value: pb.Value) -> pb.Value:
40
- return (await self.hydrate_all([value]))[0]
40
+ async def hydrate(self, flow_id: str, value: pb.Value) -> pb.Value:
41
+ return (await self.hydrate_all(flow_id, [value]))[0]
41
42
 
42
- async def hydrate_all(self, values: list[pb.Value]) -> list[pb.Value]:
43
+ async def hydrate_all(
44
+ self,
45
+ flow_id: str,
46
+ values: list[pb.Value],
47
+ ) -> list[pb.Value]:
48
+ return await self.hydrate_all_for_flows(
49
+ [(flow_id, value) for value in values]
50
+ )
51
+
52
+ async def hydrate_all_for_flows(
53
+ self,
54
+ flow_values: list[tuple[str, pb.Value]],
55
+ ) -> list[pb.Value]:
56
+ values = [value for _, value in flow_values]
43
57
  hydrated = list(values)
44
- pending: dict[tuple[str, bool], _PendingBlob] = {}
45
- for index, value in enumerate(values):
58
+ pending: dict[tuple[str, str, bool], _PendingBlob] = {}
59
+ for index, (flow_id, value) in enumerate(flow_values):
60
+ if not flow_id:
61
+ raise ValueError(f"Flow ID at index {index} is required")
46
62
  key = self._blob_key(value)
47
63
  if key is None:
48
64
  self._validate_concrete(value)
49
65
  continue
50
- blob = pending.get(key)
66
+ flow_blob_key = (flow_id, key[0], key[1])
67
+ blob = pending.get(flow_blob_key)
51
68
  if blob is None:
52
- blob = _PendingBlob(key[0], key[1], value)
53
- pending[key] = blob
69
+ blob = _PendingBlob(flow_id, key[0], key[1], value)
70
+ pending[flow_blob_key] = blob
54
71
  blob.indexes.append(index)
55
72
 
56
73
  misses: list[_PendingBlob] = []
@@ -86,7 +103,7 @@ class AsyncValueHydrator:
86
103
  for message in channel_values.messages
87
104
  ]
88
105
  values.extend(message.value for message in channel_messages)
89
- hydrated = iter(await self.hydrate_all(values))
106
+ hydrated = iter(await self.hydrate_all(request.context.flow_id, values))
90
107
  result.step_input.CopyFrom(next(hydrated))
91
108
  if has_heartbeat:
92
109
  result.context.last_heartbeat_value.CopyFrom(next(hydrated))
@@ -122,7 +139,11 @@ class AsyncValueHydrator:
122
139
  values.extend(message.value for message in channel_messages)
123
140
  for channel_result in request.condition_results.channel_results:
124
141
  values.extend(channel_result.values)
125
- hydrated = iter(await self.hydrate_all(values))
142
+ for flow_result in request.condition_results.sub_flow_results:
143
+ values.extend(
144
+ completion.completed_step_output for completion in flow_result.results
145
+ )
146
+ hydrated = iter(await self.hydrate_all(request.context.flow_id, values))
126
147
  if has_step_input:
127
148
  result.step_input.CopyFrom(next(hydrated))
128
149
  if has_heartbeat:
@@ -141,6 +162,9 @@ class AsyncValueHydrator:
141
162
  for channel_result in result.condition_results.channel_results:
142
163
  for value in channel_result.values:
143
164
  value.CopyFrom(next(hydrated))
165
+ for flow_result in result.condition_results.sub_flow_results:
166
+ for completion in flow_result.results:
167
+ completion.completed_step_output.CopyFrom(next(hydrated))
144
168
  return result
145
169
 
146
170
  async def rpc_request(
@@ -156,7 +180,7 @@ class AsyncValueHydrator:
156
180
  for message in channel_values.messages
157
181
  ]
158
182
  values.extend(message.value for message in channel_messages)
159
- hydrated = await self.hydrate_all(values)
183
+ hydrated = await self.hydrate_all(request.context.flow_id, values)
160
184
  result.input.CopyFrom(hydrated[0])
161
185
  hydrated_entries = iter(hydrated[1:])
162
186
  for entry in result.attributes:
@@ -172,10 +196,11 @@ class AsyncValueHydrator:
172
196
 
173
197
  async def step_outputs(
174
198
  self,
199
+ flow_id: str,
175
200
  outputs: list[pb.StepCompletionOutput],
176
201
  ) -> list[pb.StepCompletionOutput]:
177
202
  values = [output.completed_step_output for output in outputs]
178
- hydrated = await self.hydrate_all(values)
203
+ hydrated = await self.hydrate_all(flow_id, values)
179
204
  results: list[pb.StepCompletionOutput] = []
180
205
  for output, value in zip(outputs, hydrated):
181
206
  result = pb.StepCompletionOutput()
@@ -188,7 +213,15 @@ class AsyncValueHydrator:
188
213
  if not misses:
189
214
  return
190
215
  response = await self._service.LoadBlobs(
191
- pb.LoadBlobsRequest(values=[miss.request for miss in misses])
216
+ pb.LoadBlobsRequest(
217
+ entries=[
218
+ pb.LoadBlobRequestEntry(
219
+ flow_id=miss.flow_id,
220
+ blob_value=miss.request,
221
+ )
222
+ for miss in misses
223
+ ]
224
+ )
192
225
  )
193
226
  for miss in misses:
194
227
  concrete = response.values.get(miss.blob_id)
@@ -200,7 +233,7 @@ class AsyncValueHydrator:
200
233
 
201
234
  def _read_cache(self, blob: _PendingBlob) -> pb.Value | None:
202
235
  try:
203
- payload = self._cache.get(blob.blob_id)
236
+ payload = self._cache.get(self._cache_key(blob))
204
237
  if payload is None:
205
238
  return None
206
239
  if blob.is_object:
@@ -212,7 +245,7 @@ class AsyncValueHydrator:
212
245
  except Exception:
213
246
  _LOGGER.warning("cannot read cached blob %s", blob.blob_id, exc_info=True)
214
247
  try:
215
- self._cache.delete(blob.blob_id)
248
+ self._cache.delete(self._cache_key(blob))
216
249
  except Exception:
217
250
  _LOGGER.warning(
218
251
  "cannot delete cached blob %s",
@@ -228,7 +261,7 @@ class AsyncValueHydrator:
228
261
  if blob.is_object
229
262
  else concrete.string_value.encode("utf-8")
230
263
  )
231
- self._cache.put(blob.blob_id, payload)
264
+ self._cache.put(self._cache_key(blob), payload)
232
265
  except Exception:
233
266
  _LOGGER.warning("cannot cache blob %s", blob.blob_id, exc_info=True)
234
267
 
@@ -247,6 +280,10 @@ class AsyncValueHydrator:
247
280
  return blob_id, True
248
281
  return None
249
282
 
283
+ @staticmethod
284
+ def _cache_key(blob: _PendingBlob) -> str:
285
+ return f"{len(blob.flow_id)}:{blob.flow_id}{blob.blob_id}"
286
+
250
287
  @staticmethod
251
288
  def _validate_hydrated(blob: _PendingBlob, value: pb.Value) -> None:
252
289
  expected = "obj_value" if blob.is_object else "string_value"
@@ -268,7 +305,7 @@ class AsyncValueHydrator:
268
305
  raise ValueError("non-finite numbers are unsupported")
269
306
  return
270
307
  if kind == "obj_value":
271
- if value.obj_value.encoding not in ("json", "rawbytes"):
308
+ if value.obj_value.encoding not in ("json", "raw"):
272
309
  raise ValueError(
273
310
  f"unsupported object encoding {value.obj_value.encoding}"
274
311
  )
@@ -27,12 +27,17 @@ from dex.runtime_errors import (
27
27
  FlowNotFoundError,
28
28
  LongPollTimeoutError,
29
29
  RpcLockConflictError,
30
+ WaitHandlerTimeoutError,
30
31
  WorkerInvocationError,
31
32
  )
32
33
 
33
34
  FlowTargetRequirement = Literal["none", "existing", "active"]
34
35
 
35
- MAX_WORKER_STACK_TRACE_BYTES = 16 * 1024
36
+ MAX_WORKER_ERROR_DETAIL_BYTES = 1024
37
+ MAX_WORKER_ERROR_TYPE_BYTES = 256
38
+ MAX_WORKER_STACK_TRACE_BYTES = 4 * 1024
39
+ _ERROR_DETAIL_TRUNCATION_MARKER = b"\n... error detail truncated by Dex Python SDK ..."
40
+ _ERROR_TYPE_TRUNCATION_MARKER = b"\n... error type truncated by Dex Python SDK ..."
36
41
  _STACK_TRACE_TRUNCATION_MARKER = b"\n... stack trace truncated by Dex Python SDK ..."
37
42
 
38
43
 
@@ -139,6 +144,8 @@ def translate_rpc_error(
139
144
  return LongPollTimeoutError(*parameters)
140
145
  if sub_status is ErrorSubStatus.CHANNEL_MESSAGE_NOT_FOUND:
141
146
  return ChannelMessageNotFoundError(*parameters)
147
+ if sub_status is ErrorSubStatus.WAIT_HANDLER_TIMEOUT:
148
+ return WaitHandlerTimeoutError(*parameters)
142
149
  return DexServiceError(*parameters)
143
150
 
144
151
 
@@ -179,11 +186,19 @@ def _worker_error_status(error: BaseException) -> status_pb2.Status:
179
186
  retry_after_error = error
180
187
  reported = error.cause
181
188
 
182
- message = str(reported) or type(reported).__name__
189
+ message = _truncate_worker_failure_field(
190
+ str(reported) or type(reported).__name__,
191
+ MAX_WORKER_ERROR_DETAIL_BYTES,
192
+ _ERROR_DETAIL_TRUNCATION_MARKER,
193
+ )
183
194
  stack_trace_source = error if retry_after_error is not None else reported
184
195
  worker_error = pb.WorkerErrorResponse(
185
196
  detail=message,
186
- error_type=f"{type(reported).__module__}.{type(reported).__qualname__}",
197
+ error_type=_truncate_worker_failure_field(
198
+ f"{type(reported).__module__}.{type(reported).__qualname__}",
199
+ MAX_WORKER_ERROR_TYPE_BYTES,
200
+ _ERROR_TYPE_TRUNCATION_MARKER,
201
+ ),
187
202
  stack_trace=_worker_stack_trace(stack_trace_source),
188
203
  )
189
204
  if retry_after_error is not None:
@@ -201,16 +216,25 @@ def _worker_stack_trace(error: BaseException) -> str:
201
216
  if error.__traceback__ is None:
202
217
  return ""
203
218
  lines = traceback.format_exception(type(error), error, error.__traceback__)
204
- encoded = "".join(lines).encode()
205
- if len(encoded) <= MAX_WORKER_STACK_TRACE_BYTES:
206
- return encoded.decode()
207
- prefix_length = MAX_WORKER_STACK_TRACE_BYTES - len(_STACK_TRACE_TRUNCATION_MARKER)
219
+ return _truncate_worker_failure_field(
220
+ "".join(lines),
221
+ MAX_WORKER_STACK_TRACE_BYTES,
222
+ _STACK_TRACE_TRUNCATION_MARKER,
223
+ )
224
+
225
+
226
+ def _truncate_worker_failure_field(
227
+ value: str,
228
+ maximum_bytes: int,
229
+ truncation_marker: bytes,
230
+ ) -> str:
231
+ encoded = value.encode()
232
+ if len(encoded) <= maximum_bytes:
233
+ return value
234
+ prefix_length = maximum_bytes - len(truncation_marker)
208
235
  while prefix_length > 0 and (encoded[prefix_length] & 0xC0) == 0x80:
209
236
  prefix_length -= 1
210
- return (
211
- encoded[:prefix_length].decode(errors="replace")
212
- + _STACK_TRACE_TRUNCATION_MARKER.decode()
213
- )
237
+ return encoded[:prefix_length].decode() + truncation_marker.decode()
214
238
 
215
239
 
216
240
  def _map_sub_status(value: int) -> ErrorSubStatus:
@@ -225,5 +249,8 @@ def _map_sub_status(value: int) -> ErrorSubStatus:
225
249
  int(
226
250
  pb.ERROR_SUB_STATUS_CHANNEL_MESSAGE_NOT_FOUND
227
251
  ): ErrorSubStatus.CHANNEL_MESSAGE_NOT_FOUND,
252
+ int(
253
+ pb.ERROR_SUB_STATUS_WAIT_HANDLER_TIME_OUT
254
+ ): ErrorSubStatus.WAIT_HANDLER_TIMEOUT,
228
255
  }
229
256
  return statuses.get(value, ErrorSubStatus.UNCATEGORIZED)
@@ -20,6 +20,7 @@ _LOGGER = logging.getLogger(__name__)
20
20
 
21
21
  @dataclass
22
22
  class _PendingBlob:
23
+ flow_id: str
23
24
  blob_id: str
24
25
  is_object: bool
25
26
  request: pb.Value
@@ -36,21 +37,31 @@ class ValueHydrator:
36
37
  self._service = service
37
38
  self._cache = cache
38
39
 
39
- def hydrate(self, value: pb.Value) -> pb.Value:
40
- return self.hydrate_all([value])[0]
40
+ def hydrate(self, flow_id: str, value: pb.Value) -> pb.Value:
41
+ return self.hydrate_all(flow_id, [value])[0]
41
42
 
42
- def hydrate_all(self, values: list[pb.Value]) -> list[pb.Value]:
43
+ def hydrate_all(self, flow_id: str, values: list[pb.Value]) -> list[pb.Value]:
44
+ return self.hydrate_all_for_flows([(flow_id, value) for value in values])
45
+
46
+ def hydrate_all_for_flows(
47
+ self,
48
+ flow_values: list[tuple[str, pb.Value]],
49
+ ) -> list[pb.Value]:
50
+ values = [value for _, value in flow_values]
43
51
  hydrated = list(values)
44
- pending: dict[tuple[str, bool], _PendingBlob] = {}
45
- for index, value in enumerate(values):
52
+ pending: dict[tuple[str, str, bool], _PendingBlob] = {}
53
+ for index, (flow_id, value) in enumerate(flow_values):
54
+ if not flow_id:
55
+ raise ValueError(f"Flow ID at index {index} is required")
46
56
  key = self._blob_key(value)
47
57
  if key is None:
48
58
  self._validate_concrete(value)
49
59
  continue
50
- blob = pending.get(key)
60
+ flow_blob_key = (flow_id, key[0], key[1])
61
+ blob = pending.get(flow_blob_key)
51
62
  if blob is None:
52
- blob = _PendingBlob(key[0], key[1], value)
53
- pending[key] = blob
63
+ blob = _PendingBlob(flow_id, key[0], key[1], value)
64
+ pending[flow_blob_key] = blob
54
65
  blob.indexes.append(index)
55
66
 
56
67
  misses: list[_PendingBlob] = []
@@ -86,7 +97,7 @@ class ValueHydrator:
86
97
  for message in channel_values.messages
87
98
  ]
88
99
  values.extend(message.value for message in channel_messages)
89
- hydrated = iter(self.hydrate_all(values))
100
+ hydrated = iter(self.hydrate_all(request.context.flow_id, values))
90
101
  result.step_input.CopyFrom(next(hydrated))
91
102
  if has_heartbeat:
92
103
  result.context.last_heartbeat_value.CopyFrom(next(hydrated))
@@ -126,7 +137,7 @@ class ValueHydrator:
126
137
  values.extend(
127
138
  completion.completed_step_output for completion in flow_result.results
128
139
  )
129
- hydrated = iter(self.hydrate_all(values))
140
+ hydrated = iter(self.hydrate_all(request.context.flow_id, values))
130
141
  if has_step_input:
131
142
  result.step_input.CopyFrom(next(hydrated))
132
143
  if has_heartbeat:
@@ -163,7 +174,7 @@ class ValueHydrator:
163
174
  for message in channel_values.messages
164
175
  ]
165
176
  values.extend(message.value for message in channel_messages)
166
- hydrated = self.hydrate_all(values)
177
+ hydrated = self.hydrate_all(request.context.flow_id, values)
167
178
  result.input.CopyFrom(hydrated[0])
168
179
  hydrated_entries = iter(hydrated[1:])
169
180
  for entry in result.attributes:
@@ -179,10 +190,11 @@ class ValueHydrator:
179
190
 
180
191
  def step_outputs(
181
192
  self,
193
+ flow_id: str,
182
194
  outputs: list[pb.StepCompletionOutput],
183
195
  ) -> list[pb.StepCompletionOutput]:
184
196
  values = [output.completed_step_output for output in outputs]
185
- hydrated = self.hydrate_all(values)
197
+ hydrated = self.hydrate_all(flow_id, values)
186
198
  results: list[pb.StepCompletionOutput] = []
187
199
  for output, value in zip(outputs, hydrated):
188
200
  result = pb.StepCompletionOutput()
@@ -195,7 +207,15 @@ class ValueHydrator:
195
207
  if not misses:
196
208
  return
197
209
  response = self._service.LoadBlobs(
198
- pb.LoadBlobsRequest(values=[miss.request for miss in misses])
210
+ pb.LoadBlobsRequest(
211
+ entries=[
212
+ pb.LoadBlobRequestEntry(
213
+ flow_id=miss.flow_id,
214
+ blob_value=miss.request,
215
+ )
216
+ for miss in misses
217
+ ]
218
+ )
199
219
  )
200
220
  for miss in misses:
201
221
  concrete = response.values.get(miss.blob_id)
@@ -207,7 +227,7 @@ class ValueHydrator:
207
227
 
208
228
  def _read_cache(self, blob: _PendingBlob) -> pb.Value | None:
209
229
  try:
210
- payload = self._cache.get(blob.blob_id)
230
+ payload = self._cache.get(self._cache_key(blob))
211
231
  if payload is None:
212
232
  return None
213
233
  if blob.is_object:
@@ -219,7 +239,7 @@ class ValueHydrator:
219
239
  except Exception:
220
240
  _LOGGER.warning("cannot read cached blob %s", blob.blob_id, exc_info=True)
221
241
  try:
222
- self._cache.delete(blob.blob_id)
242
+ self._cache.delete(self._cache_key(blob))
223
243
  except Exception:
224
244
  _LOGGER.warning(
225
245
  "cannot delete cached blob %s",
@@ -235,7 +255,7 @@ class ValueHydrator:
235
255
  if blob.is_object
236
256
  else concrete.string_value.encode("utf-8")
237
257
  )
238
- self._cache.put(blob.blob_id, payload)
258
+ self._cache.put(self._cache_key(blob), payload)
239
259
  except Exception:
240
260
  _LOGGER.warning("cannot cache blob %s", blob.blob_id, exc_info=True)
241
261
 
@@ -254,6 +274,10 @@ class ValueHydrator:
254
274
  return blob_id, True
255
275
  return None
256
276
 
277
+ @staticmethod
278
+ def _cache_key(blob: _PendingBlob) -> str:
279
+ return f"{len(blob.flow_id)}:{blob.flow_id}{blob.blob_id}"
280
+
257
281
  @staticmethod
258
282
  def _validate_hydrated(blob: _PendingBlob, value: pb.Value) -> None:
259
283
  expected = "obj_value" if blob.is_object else "string_value"
@@ -275,7 +299,7 @@ class ValueHydrator:
275
299
  raise ValueError("non-finite numbers are unsupported")
276
300
  return
277
301
  if kind == "obj_value":
278
- if value.obj_value.encoding not in ("json", "rawbytes"):
302
+ if value.obj_value.encoding not in ("json", "raw"):
279
303
  raise ValueError(
280
304
  f"unsupported object encoding {value.obj_value.encoding}"
281
305
  )
@@ -22,7 +22,7 @@ from dex.dexpb import dex_pb2 as pb
22
22
  from dex.runtime_errors import ValueMappingError
23
23
 
24
24
  _JSON_ENCODING = "json"
25
- _RAW_BYTES_ENCODING = "rawbytes"
25
+ _RAW_BYTES_ENCODING = "raw"
26
26
 
27
27
 
28
28
  class ValueMapper:
@@ -43,10 +43,7 @@ class ValueMapper:
43
43
  def _encode(self, value: Any, codec: Codec[Any]) -> pb.Value:
44
44
  if value is None:
45
45
  return pb.Value(
46
- obj_value=pb.EncodedObject(
47
- encoding=_JSON_ENCODING,
48
- payload=b"null",
49
- )
46
+ null_value=cast(struct_pb2.NullValue, struct_pb2.NULL_VALUE)
50
47
  )
51
48
  logical = codec.encode(value)
52
49
  if logical.kind is WireKind.STRING:
@@ -134,7 +131,7 @@ class ValueMapper:
134
131
  ):
135
132
  raise ValueError("blob-backed Value was not hydrated")
136
133
  if kind == "null_value":
137
- raise ValueError("attribute deletion marker cannot be decoded")
134
+ return None
138
135
  raise TypeError(f"cannot decode {kind or 'empty Value'} as {codec.type_name}")
139
136
 
140
137
  def to_value(self, value: pb.Value) -> Value:
@@ -166,6 +163,8 @@ class ValueMapper:
166
163
  "internal_blob_id_for_obj_value",
167
164
  ):
168
165
  raise ValueError("blob-backed Value was not hydrated")
166
+ if kind == "null_value":
167
+ return Value(WireKind.JSON, "null")
169
168
  raise TypeError(f"cannot wrap {kind or 'empty Value'} as Value")
170
169
 
171
170
  @staticmethod