dex-python-sdk 0.6.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/PKG-INFO +16 -2
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/README.md +15 -1
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/__init__.py +8 -3
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_async_value_hydrator.py +54 -17
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_grpc_errors.py +38 -11
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_value_hydrator.py +41 -17
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_value_mapper.py +5 -6
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/async_client.py +102 -68
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/client.py +103 -69
- dex_python_sdk-0.8.0/dex/dexpb/dex_pb2.py +503 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/dexpb/dex_pb2.pyi +14 -4
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/dexpb/dex_pb2_grpc.py +18 -18
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/runtime_errors.py +8 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/wait.py +6 -1
- dex_python_sdk-0.8.0/dex/wait_options.py +86 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/pyproject.toml +1 -1
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/Cargo.lock +6 -6
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/Cargo.toml +3 -3
- dex_python_sdk-0.6.0/dex/dexpb/dex_pb2.py +0 -501
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/LEGACY_NOTICES.md +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/LICENSE +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_async_worker_dispatcher.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_async_worker_service.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_invocation_context.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_native.pyi +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_utils.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_worker_dispatcher.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/_worker_service.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/async_worker.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/attribute.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/attribute_match.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/blob_cache.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/channel.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/client_options.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/codec.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/condition.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/context.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/dexpb/__init__.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_config.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_info.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_options.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/flow_result.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/py.typed +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/step.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/step_execution.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/stream.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/subflow.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/timer.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/worker.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/dex/worker_options.py +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/Cargo.toml +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/LICENSE +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/config.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/entry.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/error.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/format.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/lib.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/policy.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/src/store.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache/tests/blob_cache_integration.rs +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache-python/Cargo.toml +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache-python/LICENSE +0 -0
- {dex_python_sdk-0.6.0 → dex_python_sdk-0.8.0}/sdk-rust/crates/dex-blob-cache-python/src/lib.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dex-python-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Requires-Dist: grpcio>=1.83.0
|
|
5
5
|
Requires-Dist: grpcio-status>=1.83.0
|
|
6
6
|
Requires-Dist: protobuf>=7.35.1
|
|
@@ -361,6 +361,17 @@ buffered sets and deletes. The matching `ChannelMap` methods are RPC-only,
|
|
|
361
361
|
include buffered publishes, and omit empty instances. Keys are decoded and
|
|
362
362
|
sorted. Use `force_complete_if_channels_empty(...)` for conditional completion.
|
|
363
363
|
|
|
364
|
+
Request IDs are optional for both durable waits. When omitted, the server
|
|
365
|
+
derives a namespaced stable ID from the Step execution or Attribute condition,
|
|
366
|
+
such as `wait-for-attribute:myInt>10`. Reuse an override only for the same
|
|
367
|
+
logical wait. The Client automatically reattaches transport long polls. If an
|
|
368
|
+
earlier Update with that ID exhausted its handler budget, the server appends an
|
|
369
|
+
increasing `-N` suffix and starts a new Update. `maximum_wait_time` is optional
|
|
370
|
+
and is the total handler budget across reattachments; zero waits indefinitely.
|
|
371
|
+
A positive budget expiry raises `WaitHandlerTimeoutError`. An abandoned infinite
|
|
372
|
+
wait remains accepted and counts against Temporal's in-flight Update limit until
|
|
373
|
+
it completes or the Flow closes.
|
|
374
|
+
|
|
364
375
|
`Client.wait_for_flow` and `AsyncClient.wait_for_flow` return a
|
|
365
376
|
`FlowResult` after hydrating every output-bearing completion. Use
|
|
366
377
|
`single_output` only when the Flow contract produces exactly one output:
|
|
@@ -444,7 +455,10 @@ client operations, and assertions as the Java suite against an isolated
|
|
|
444
455
|
The strongly typed contracts, registry, synchronous Client/Worker, optional
|
|
445
456
|
`AsyncClient`/`AsyncWorker` (`grpc.aio`), and Rust-backed BlobCache are
|
|
446
457
|
implemented. Python owns its gRPC transport; the native bridge is limited to
|
|
447
|
-
the shared BlobCache.
|
|
458
|
+
the shared BlobCache. Serialized object values use wire encoding `json` for JSON
|
|
459
|
+
and `raw` for raw bytes. Internal Blob references are opaque. Hydration sends the
|
|
460
|
+
owning Flow ID with each reference, and the cache is isolated by
|
|
461
|
+
`(flow_id, blob_ref)`. Design notes:
|
|
448
462
|
[`python-sdk-async-apis.md`](../docs/design/plan/python-sdk-async-apis.md) and
|
|
449
463
|
[`python-sdk-step-streaming.md`](../docs/design/plan/python-sdk-step-streaming.md).
|
|
450
464
|
|
|
@@ -345,6 +345,17 @@ buffered sets and deletes. The matching `ChannelMap` methods are RPC-only,
|
|
|
345
345
|
include buffered publishes, and omit empty instances. Keys are decoded and
|
|
346
346
|
sorted. Use `force_complete_if_channels_empty(...)` for conditional completion.
|
|
347
347
|
|
|
348
|
+
Request IDs are optional for both durable waits. When omitted, the server
|
|
349
|
+
derives a namespaced stable ID from the Step execution or Attribute condition,
|
|
350
|
+
such as `wait-for-attribute:myInt>10`. Reuse an override only for the same
|
|
351
|
+
logical wait. The Client automatically reattaches transport long polls. If an
|
|
352
|
+
earlier Update with that ID exhausted its handler budget, the server appends an
|
|
353
|
+
increasing `-N` suffix and starts a new Update. `maximum_wait_time` is optional
|
|
354
|
+
and is the total handler budget across reattachments; zero waits indefinitely.
|
|
355
|
+
A positive budget expiry raises `WaitHandlerTimeoutError`. An abandoned infinite
|
|
356
|
+
wait remains accepted and counts against Temporal's in-flight Update limit until
|
|
357
|
+
it completes or the Flow closes.
|
|
358
|
+
|
|
348
359
|
`Client.wait_for_flow` and `AsyncClient.wait_for_flow` return a
|
|
349
360
|
`FlowResult` after hydrating every output-bearing completion. Use
|
|
350
361
|
`single_output` only when the Flow contract produces exactly one output:
|
|
@@ -428,7 +439,10 @@ client operations, and assertions as the Java suite against an isolated
|
|
|
428
439
|
The strongly typed contracts, registry, synchronous Client/Worker, optional
|
|
429
440
|
`AsyncClient`/`AsyncWorker` (`grpc.aio`), and Rust-backed BlobCache are
|
|
430
441
|
implemented. Python owns its gRPC transport; the native bridge is limited to
|
|
431
|
-
the shared BlobCache.
|
|
442
|
+
the shared BlobCache. Serialized object values use wire encoding `json` for JSON
|
|
443
|
+
and `raw` for raw bytes. Internal Blob references are opaque. Hydration sends the
|
|
444
|
+
owning Flow ID with each reference, and the cache is isolated by
|
|
445
|
+
`(flow_id, blob_ref)`. Design notes:
|
|
432
446
|
[`python-sdk-async-apis.md`](../docs/design/plan/python-sdk-async-apis.md) and
|
|
433
447
|
[`python-sdk-step-streaming.md`](../docs/design/plan/python-sdk-step-streaming.md).
|
|
434
448
|
|
|
@@ -62,10 +62,11 @@ from dex.flow_options import (
|
|
|
62
62
|
)
|
|
63
63
|
from dex.flow_result import FlowResult, StepCompletion
|
|
64
64
|
from dex.runtime_errors import (
|
|
65
|
-
DexServiceError,
|
|
66
|
-
ErrorSubStatus,
|
|
67
65
|
AttributeMapNotLoadedError,
|
|
66
|
+
ChannelMessageNotFoundError,
|
|
68
67
|
ChannelMessagesNotLoadedError,
|
|
68
|
+
DexServiceError,
|
|
69
|
+
ErrorSubStatus,
|
|
69
70
|
FlowAlreadyStartedError,
|
|
70
71
|
FlowDefinitionError,
|
|
71
72
|
FlowErrorType,
|
|
@@ -73,9 +74,9 @@ from dex.runtime_errors import (
|
|
|
73
74
|
FlowNotFoundError,
|
|
74
75
|
InvalidStepResultError,
|
|
75
76
|
LongPollTimeoutError,
|
|
76
|
-
ChannelMessageNotFoundError,
|
|
77
77
|
RpcLockConflictError,
|
|
78
78
|
ValueMappingError,
|
|
79
|
+
WaitHandlerTimeoutError,
|
|
79
80
|
WorkerInvocationError,
|
|
80
81
|
)
|
|
81
82
|
from dex.step import (
|
|
@@ -108,6 +109,7 @@ from dex.stream import (
|
|
|
108
109
|
from dex.subflow import SubFlow
|
|
109
110
|
from dex.timer import Timer
|
|
110
111
|
from dex.wait import Wait
|
|
112
|
+
from dex.wait_options import WaitForAttributeOptions, WaitForStepCompletionOptions
|
|
111
113
|
from dex.worker import Worker
|
|
112
114
|
from dex.worker_options import WorkerOptions, WorkerTarget
|
|
113
115
|
|
|
@@ -164,6 +166,7 @@ __all__ = [
|
|
|
164
166
|
"IndexType",
|
|
165
167
|
"JsonCodec",
|
|
166
168
|
"LongPollTimeoutError",
|
|
169
|
+
"WaitHandlerTimeoutError",
|
|
167
170
|
"PersistenceSchema",
|
|
168
171
|
"RPCResult",
|
|
169
172
|
"RecoveryErrorInfo",
|
|
@@ -205,6 +208,8 @@ __all__ = [
|
|
|
205
208
|
"WorkerInvocationError",
|
|
206
209
|
"WorkerOptions",
|
|
207
210
|
"WorkerTarget",
|
|
211
|
+
"WaitForAttributeOptions",
|
|
212
|
+
"WaitForStepCompletionOptions",
|
|
208
213
|
"dead_end",
|
|
209
214
|
"force_complete",
|
|
210
215
|
"force_complete_if_channels_empty",
|
|
@@ -20,6 +20,7 @@ _LOGGER = logging.getLogger(__name__)
|
|
|
20
20
|
|
|
21
21
|
@dataclass
|
|
22
22
|
class _PendingBlob:
|
|
23
|
+
flow_id: str
|
|
23
24
|
blob_id: str
|
|
24
25
|
is_object: bool
|
|
25
26
|
request: pb.Value
|
|
@@ -36,21 +37,37 @@ class AsyncValueHydrator:
|
|
|
36
37
|
self._service = service
|
|
37
38
|
self._cache = cache
|
|
38
39
|
|
|
39
|
-
async def hydrate(self, value: pb.Value) -> pb.Value:
|
|
40
|
-
return (await self.hydrate_all([value]))[0]
|
|
40
|
+
async def hydrate(self, flow_id: str, value: pb.Value) -> pb.Value:
|
|
41
|
+
return (await self.hydrate_all(flow_id, [value]))[0]
|
|
41
42
|
|
|
42
|
-
async def hydrate_all(
|
|
43
|
+
async def hydrate_all(
|
|
44
|
+
self,
|
|
45
|
+
flow_id: str,
|
|
46
|
+
values: list[pb.Value],
|
|
47
|
+
) -> list[pb.Value]:
|
|
48
|
+
return await self.hydrate_all_for_flows(
|
|
49
|
+
[(flow_id, value) for value in values]
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
async def hydrate_all_for_flows(
|
|
53
|
+
self,
|
|
54
|
+
flow_values: list[tuple[str, pb.Value]],
|
|
55
|
+
) -> list[pb.Value]:
|
|
56
|
+
values = [value for _, value in flow_values]
|
|
43
57
|
hydrated = list(values)
|
|
44
|
-
pending: dict[tuple[str, bool], _PendingBlob] = {}
|
|
45
|
-
for index, value in enumerate(
|
|
58
|
+
pending: dict[tuple[str, str, bool], _PendingBlob] = {}
|
|
59
|
+
for index, (flow_id, value) in enumerate(flow_values):
|
|
60
|
+
if not flow_id:
|
|
61
|
+
raise ValueError(f"Flow ID at index {index} is required")
|
|
46
62
|
key = self._blob_key(value)
|
|
47
63
|
if key is None:
|
|
48
64
|
self._validate_concrete(value)
|
|
49
65
|
continue
|
|
50
|
-
|
|
66
|
+
flow_blob_key = (flow_id, key[0], key[1])
|
|
67
|
+
blob = pending.get(flow_blob_key)
|
|
51
68
|
if blob is None:
|
|
52
|
-
blob = _PendingBlob(key[0], key[1], value)
|
|
53
|
-
pending[
|
|
69
|
+
blob = _PendingBlob(flow_id, key[0], key[1], value)
|
|
70
|
+
pending[flow_blob_key] = blob
|
|
54
71
|
blob.indexes.append(index)
|
|
55
72
|
|
|
56
73
|
misses: list[_PendingBlob] = []
|
|
@@ -86,7 +103,7 @@ class AsyncValueHydrator:
|
|
|
86
103
|
for message in channel_values.messages
|
|
87
104
|
]
|
|
88
105
|
values.extend(message.value for message in channel_messages)
|
|
89
|
-
hydrated = iter(await self.hydrate_all(values))
|
|
106
|
+
hydrated = iter(await self.hydrate_all(request.context.flow_id, values))
|
|
90
107
|
result.step_input.CopyFrom(next(hydrated))
|
|
91
108
|
if has_heartbeat:
|
|
92
109
|
result.context.last_heartbeat_value.CopyFrom(next(hydrated))
|
|
@@ -122,7 +139,11 @@ class AsyncValueHydrator:
|
|
|
122
139
|
values.extend(message.value for message in channel_messages)
|
|
123
140
|
for channel_result in request.condition_results.channel_results:
|
|
124
141
|
values.extend(channel_result.values)
|
|
125
|
-
|
|
142
|
+
for flow_result in request.condition_results.sub_flow_results:
|
|
143
|
+
values.extend(
|
|
144
|
+
completion.completed_step_output for completion in flow_result.results
|
|
145
|
+
)
|
|
146
|
+
hydrated = iter(await self.hydrate_all(request.context.flow_id, values))
|
|
126
147
|
if has_step_input:
|
|
127
148
|
result.step_input.CopyFrom(next(hydrated))
|
|
128
149
|
if has_heartbeat:
|
|
@@ -141,6 +162,9 @@ class AsyncValueHydrator:
|
|
|
141
162
|
for channel_result in result.condition_results.channel_results:
|
|
142
163
|
for value in channel_result.values:
|
|
143
164
|
value.CopyFrom(next(hydrated))
|
|
165
|
+
for flow_result in result.condition_results.sub_flow_results:
|
|
166
|
+
for completion in flow_result.results:
|
|
167
|
+
completion.completed_step_output.CopyFrom(next(hydrated))
|
|
144
168
|
return result
|
|
145
169
|
|
|
146
170
|
async def rpc_request(
|
|
@@ -156,7 +180,7 @@ class AsyncValueHydrator:
|
|
|
156
180
|
for message in channel_values.messages
|
|
157
181
|
]
|
|
158
182
|
values.extend(message.value for message in channel_messages)
|
|
159
|
-
hydrated = await self.hydrate_all(values)
|
|
183
|
+
hydrated = await self.hydrate_all(request.context.flow_id, values)
|
|
160
184
|
result.input.CopyFrom(hydrated[0])
|
|
161
185
|
hydrated_entries = iter(hydrated[1:])
|
|
162
186
|
for entry in result.attributes:
|
|
@@ -172,10 +196,11 @@ class AsyncValueHydrator:
|
|
|
172
196
|
|
|
173
197
|
async def step_outputs(
|
|
174
198
|
self,
|
|
199
|
+
flow_id: str,
|
|
175
200
|
outputs: list[pb.StepCompletionOutput],
|
|
176
201
|
) -> list[pb.StepCompletionOutput]:
|
|
177
202
|
values = [output.completed_step_output for output in outputs]
|
|
178
|
-
hydrated = await self.hydrate_all(values)
|
|
203
|
+
hydrated = await self.hydrate_all(flow_id, values)
|
|
179
204
|
results: list[pb.StepCompletionOutput] = []
|
|
180
205
|
for output, value in zip(outputs, hydrated):
|
|
181
206
|
result = pb.StepCompletionOutput()
|
|
@@ -188,7 +213,15 @@ class AsyncValueHydrator:
|
|
|
188
213
|
if not misses:
|
|
189
214
|
return
|
|
190
215
|
response = await self._service.LoadBlobs(
|
|
191
|
-
pb.LoadBlobsRequest(
|
|
216
|
+
pb.LoadBlobsRequest(
|
|
217
|
+
entries=[
|
|
218
|
+
pb.LoadBlobRequestEntry(
|
|
219
|
+
flow_id=miss.flow_id,
|
|
220
|
+
blob_value=miss.request,
|
|
221
|
+
)
|
|
222
|
+
for miss in misses
|
|
223
|
+
]
|
|
224
|
+
)
|
|
192
225
|
)
|
|
193
226
|
for miss in misses:
|
|
194
227
|
concrete = response.values.get(miss.blob_id)
|
|
@@ -200,7 +233,7 @@ class AsyncValueHydrator:
|
|
|
200
233
|
|
|
201
234
|
def _read_cache(self, blob: _PendingBlob) -> pb.Value | None:
|
|
202
235
|
try:
|
|
203
|
-
payload = self._cache.get(blob
|
|
236
|
+
payload = self._cache.get(self._cache_key(blob))
|
|
204
237
|
if payload is None:
|
|
205
238
|
return None
|
|
206
239
|
if blob.is_object:
|
|
@@ -212,7 +245,7 @@ class AsyncValueHydrator:
|
|
|
212
245
|
except Exception:
|
|
213
246
|
_LOGGER.warning("cannot read cached blob %s", blob.blob_id, exc_info=True)
|
|
214
247
|
try:
|
|
215
|
-
self._cache.delete(blob
|
|
248
|
+
self._cache.delete(self._cache_key(blob))
|
|
216
249
|
except Exception:
|
|
217
250
|
_LOGGER.warning(
|
|
218
251
|
"cannot delete cached blob %s",
|
|
@@ -228,7 +261,7 @@ class AsyncValueHydrator:
|
|
|
228
261
|
if blob.is_object
|
|
229
262
|
else concrete.string_value.encode("utf-8")
|
|
230
263
|
)
|
|
231
|
-
self._cache.put(blob
|
|
264
|
+
self._cache.put(self._cache_key(blob), payload)
|
|
232
265
|
except Exception:
|
|
233
266
|
_LOGGER.warning("cannot cache blob %s", blob.blob_id, exc_info=True)
|
|
234
267
|
|
|
@@ -247,6 +280,10 @@ class AsyncValueHydrator:
|
|
|
247
280
|
return blob_id, True
|
|
248
281
|
return None
|
|
249
282
|
|
|
283
|
+
@staticmethod
|
|
284
|
+
def _cache_key(blob: _PendingBlob) -> str:
|
|
285
|
+
return f"{len(blob.flow_id)}:{blob.flow_id}{blob.blob_id}"
|
|
286
|
+
|
|
250
287
|
@staticmethod
|
|
251
288
|
def _validate_hydrated(blob: _PendingBlob, value: pb.Value) -> None:
|
|
252
289
|
expected = "obj_value" if blob.is_object else "string_value"
|
|
@@ -268,7 +305,7 @@ class AsyncValueHydrator:
|
|
|
268
305
|
raise ValueError("non-finite numbers are unsupported")
|
|
269
306
|
return
|
|
270
307
|
if kind == "obj_value":
|
|
271
|
-
if value.obj_value.encoding not in ("json", "
|
|
308
|
+
if value.obj_value.encoding not in ("json", "raw"):
|
|
272
309
|
raise ValueError(
|
|
273
310
|
f"unsupported object encoding {value.obj_value.encoding}"
|
|
274
311
|
)
|
|
@@ -27,12 +27,17 @@ from dex.runtime_errors import (
|
|
|
27
27
|
FlowNotFoundError,
|
|
28
28
|
LongPollTimeoutError,
|
|
29
29
|
RpcLockConflictError,
|
|
30
|
+
WaitHandlerTimeoutError,
|
|
30
31
|
WorkerInvocationError,
|
|
31
32
|
)
|
|
32
33
|
|
|
33
34
|
FlowTargetRequirement = Literal["none", "existing", "active"]
|
|
34
35
|
|
|
35
|
-
|
|
36
|
+
MAX_WORKER_ERROR_DETAIL_BYTES = 1024
|
|
37
|
+
MAX_WORKER_ERROR_TYPE_BYTES = 256
|
|
38
|
+
MAX_WORKER_STACK_TRACE_BYTES = 4 * 1024
|
|
39
|
+
_ERROR_DETAIL_TRUNCATION_MARKER = b"\n... error detail truncated by Dex Python SDK ..."
|
|
40
|
+
_ERROR_TYPE_TRUNCATION_MARKER = b"\n... error type truncated by Dex Python SDK ..."
|
|
36
41
|
_STACK_TRACE_TRUNCATION_MARKER = b"\n... stack trace truncated by Dex Python SDK ..."
|
|
37
42
|
|
|
38
43
|
|
|
@@ -139,6 +144,8 @@ def translate_rpc_error(
|
|
|
139
144
|
return LongPollTimeoutError(*parameters)
|
|
140
145
|
if sub_status is ErrorSubStatus.CHANNEL_MESSAGE_NOT_FOUND:
|
|
141
146
|
return ChannelMessageNotFoundError(*parameters)
|
|
147
|
+
if sub_status is ErrorSubStatus.WAIT_HANDLER_TIMEOUT:
|
|
148
|
+
return WaitHandlerTimeoutError(*parameters)
|
|
142
149
|
return DexServiceError(*parameters)
|
|
143
150
|
|
|
144
151
|
|
|
@@ -179,11 +186,19 @@ def _worker_error_status(error: BaseException) -> status_pb2.Status:
|
|
|
179
186
|
retry_after_error = error
|
|
180
187
|
reported = error.cause
|
|
181
188
|
|
|
182
|
-
message =
|
|
189
|
+
message = _truncate_worker_failure_field(
|
|
190
|
+
str(reported) or type(reported).__name__,
|
|
191
|
+
MAX_WORKER_ERROR_DETAIL_BYTES,
|
|
192
|
+
_ERROR_DETAIL_TRUNCATION_MARKER,
|
|
193
|
+
)
|
|
183
194
|
stack_trace_source = error if retry_after_error is not None else reported
|
|
184
195
|
worker_error = pb.WorkerErrorResponse(
|
|
185
196
|
detail=message,
|
|
186
|
-
error_type=
|
|
197
|
+
error_type=_truncate_worker_failure_field(
|
|
198
|
+
f"{type(reported).__module__}.{type(reported).__qualname__}",
|
|
199
|
+
MAX_WORKER_ERROR_TYPE_BYTES,
|
|
200
|
+
_ERROR_TYPE_TRUNCATION_MARKER,
|
|
201
|
+
),
|
|
187
202
|
stack_trace=_worker_stack_trace(stack_trace_source),
|
|
188
203
|
)
|
|
189
204
|
if retry_after_error is not None:
|
|
@@ -201,16 +216,25 @@ def _worker_stack_trace(error: BaseException) -> str:
|
|
|
201
216
|
if error.__traceback__ is None:
|
|
202
217
|
return ""
|
|
203
218
|
lines = traceback.format_exception(type(error), error, error.__traceback__)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
219
|
+
return _truncate_worker_failure_field(
|
|
220
|
+
"".join(lines),
|
|
221
|
+
MAX_WORKER_STACK_TRACE_BYTES,
|
|
222
|
+
_STACK_TRACE_TRUNCATION_MARKER,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _truncate_worker_failure_field(
|
|
227
|
+
value: str,
|
|
228
|
+
maximum_bytes: int,
|
|
229
|
+
truncation_marker: bytes,
|
|
230
|
+
) -> str:
|
|
231
|
+
encoded = value.encode()
|
|
232
|
+
if len(encoded) <= maximum_bytes:
|
|
233
|
+
return value
|
|
234
|
+
prefix_length = maximum_bytes - len(truncation_marker)
|
|
208
235
|
while prefix_length > 0 and (encoded[prefix_length] & 0xC0) == 0x80:
|
|
209
236
|
prefix_length -= 1
|
|
210
|
-
return (
|
|
211
|
-
encoded[:prefix_length].decode(errors="replace")
|
|
212
|
-
+ _STACK_TRACE_TRUNCATION_MARKER.decode()
|
|
213
|
-
)
|
|
237
|
+
return encoded[:prefix_length].decode() + truncation_marker.decode()
|
|
214
238
|
|
|
215
239
|
|
|
216
240
|
def _map_sub_status(value: int) -> ErrorSubStatus:
|
|
@@ -225,5 +249,8 @@ def _map_sub_status(value: int) -> ErrorSubStatus:
|
|
|
225
249
|
int(
|
|
226
250
|
pb.ERROR_SUB_STATUS_CHANNEL_MESSAGE_NOT_FOUND
|
|
227
251
|
): ErrorSubStatus.CHANNEL_MESSAGE_NOT_FOUND,
|
|
252
|
+
int(
|
|
253
|
+
pb.ERROR_SUB_STATUS_WAIT_HANDLER_TIME_OUT
|
|
254
|
+
): ErrorSubStatus.WAIT_HANDLER_TIMEOUT,
|
|
228
255
|
}
|
|
229
256
|
return statuses.get(value, ErrorSubStatus.UNCATEGORIZED)
|
|
@@ -20,6 +20,7 @@ _LOGGER = logging.getLogger(__name__)
|
|
|
20
20
|
|
|
21
21
|
@dataclass
|
|
22
22
|
class _PendingBlob:
|
|
23
|
+
flow_id: str
|
|
23
24
|
blob_id: str
|
|
24
25
|
is_object: bool
|
|
25
26
|
request: pb.Value
|
|
@@ -36,21 +37,31 @@ class ValueHydrator:
|
|
|
36
37
|
self._service = service
|
|
37
38
|
self._cache = cache
|
|
38
39
|
|
|
39
|
-
def hydrate(self, value: pb.Value) -> pb.Value:
|
|
40
|
-
return self.hydrate_all([value])[0]
|
|
40
|
+
def hydrate(self, flow_id: str, value: pb.Value) -> pb.Value:
|
|
41
|
+
return self.hydrate_all(flow_id, [value])[0]
|
|
41
42
|
|
|
42
|
-
def hydrate_all(self, values: list[pb.Value]) -> list[pb.Value]:
|
|
43
|
+
def hydrate_all(self, flow_id: str, values: list[pb.Value]) -> list[pb.Value]:
|
|
44
|
+
return self.hydrate_all_for_flows([(flow_id, value) for value in values])
|
|
45
|
+
|
|
46
|
+
def hydrate_all_for_flows(
|
|
47
|
+
self,
|
|
48
|
+
flow_values: list[tuple[str, pb.Value]],
|
|
49
|
+
) -> list[pb.Value]:
|
|
50
|
+
values = [value for _, value in flow_values]
|
|
43
51
|
hydrated = list(values)
|
|
44
|
-
pending: dict[tuple[str, bool], _PendingBlob] = {}
|
|
45
|
-
for index, value in enumerate(
|
|
52
|
+
pending: dict[tuple[str, str, bool], _PendingBlob] = {}
|
|
53
|
+
for index, (flow_id, value) in enumerate(flow_values):
|
|
54
|
+
if not flow_id:
|
|
55
|
+
raise ValueError(f"Flow ID at index {index} is required")
|
|
46
56
|
key = self._blob_key(value)
|
|
47
57
|
if key is None:
|
|
48
58
|
self._validate_concrete(value)
|
|
49
59
|
continue
|
|
50
|
-
|
|
60
|
+
flow_blob_key = (flow_id, key[0], key[1])
|
|
61
|
+
blob = pending.get(flow_blob_key)
|
|
51
62
|
if blob is None:
|
|
52
|
-
blob = _PendingBlob(key[0], key[1], value)
|
|
53
|
-
pending[
|
|
63
|
+
blob = _PendingBlob(flow_id, key[0], key[1], value)
|
|
64
|
+
pending[flow_blob_key] = blob
|
|
54
65
|
blob.indexes.append(index)
|
|
55
66
|
|
|
56
67
|
misses: list[_PendingBlob] = []
|
|
@@ -86,7 +97,7 @@ class ValueHydrator:
|
|
|
86
97
|
for message in channel_values.messages
|
|
87
98
|
]
|
|
88
99
|
values.extend(message.value for message in channel_messages)
|
|
89
|
-
hydrated = iter(self.hydrate_all(values))
|
|
100
|
+
hydrated = iter(self.hydrate_all(request.context.flow_id, values))
|
|
90
101
|
result.step_input.CopyFrom(next(hydrated))
|
|
91
102
|
if has_heartbeat:
|
|
92
103
|
result.context.last_heartbeat_value.CopyFrom(next(hydrated))
|
|
@@ -126,7 +137,7 @@ class ValueHydrator:
|
|
|
126
137
|
values.extend(
|
|
127
138
|
completion.completed_step_output for completion in flow_result.results
|
|
128
139
|
)
|
|
129
|
-
hydrated = iter(self.hydrate_all(values))
|
|
140
|
+
hydrated = iter(self.hydrate_all(request.context.flow_id, values))
|
|
130
141
|
if has_step_input:
|
|
131
142
|
result.step_input.CopyFrom(next(hydrated))
|
|
132
143
|
if has_heartbeat:
|
|
@@ -163,7 +174,7 @@ class ValueHydrator:
|
|
|
163
174
|
for message in channel_values.messages
|
|
164
175
|
]
|
|
165
176
|
values.extend(message.value for message in channel_messages)
|
|
166
|
-
hydrated = self.hydrate_all(values)
|
|
177
|
+
hydrated = self.hydrate_all(request.context.flow_id, values)
|
|
167
178
|
result.input.CopyFrom(hydrated[0])
|
|
168
179
|
hydrated_entries = iter(hydrated[1:])
|
|
169
180
|
for entry in result.attributes:
|
|
@@ -179,10 +190,11 @@ class ValueHydrator:
|
|
|
179
190
|
|
|
180
191
|
def step_outputs(
|
|
181
192
|
self,
|
|
193
|
+
flow_id: str,
|
|
182
194
|
outputs: list[pb.StepCompletionOutput],
|
|
183
195
|
) -> list[pb.StepCompletionOutput]:
|
|
184
196
|
values = [output.completed_step_output for output in outputs]
|
|
185
|
-
hydrated = self.hydrate_all(values)
|
|
197
|
+
hydrated = self.hydrate_all(flow_id, values)
|
|
186
198
|
results: list[pb.StepCompletionOutput] = []
|
|
187
199
|
for output, value in zip(outputs, hydrated):
|
|
188
200
|
result = pb.StepCompletionOutput()
|
|
@@ -195,7 +207,15 @@ class ValueHydrator:
|
|
|
195
207
|
if not misses:
|
|
196
208
|
return
|
|
197
209
|
response = self._service.LoadBlobs(
|
|
198
|
-
pb.LoadBlobsRequest(
|
|
210
|
+
pb.LoadBlobsRequest(
|
|
211
|
+
entries=[
|
|
212
|
+
pb.LoadBlobRequestEntry(
|
|
213
|
+
flow_id=miss.flow_id,
|
|
214
|
+
blob_value=miss.request,
|
|
215
|
+
)
|
|
216
|
+
for miss in misses
|
|
217
|
+
]
|
|
218
|
+
)
|
|
199
219
|
)
|
|
200
220
|
for miss in misses:
|
|
201
221
|
concrete = response.values.get(miss.blob_id)
|
|
@@ -207,7 +227,7 @@ class ValueHydrator:
|
|
|
207
227
|
|
|
208
228
|
def _read_cache(self, blob: _PendingBlob) -> pb.Value | None:
|
|
209
229
|
try:
|
|
210
|
-
payload = self._cache.get(blob
|
|
230
|
+
payload = self._cache.get(self._cache_key(blob))
|
|
211
231
|
if payload is None:
|
|
212
232
|
return None
|
|
213
233
|
if blob.is_object:
|
|
@@ -219,7 +239,7 @@ class ValueHydrator:
|
|
|
219
239
|
except Exception:
|
|
220
240
|
_LOGGER.warning("cannot read cached blob %s", blob.blob_id, exc_info=True)
|
|
221
241
|
try:
|
|
222
|
-
self._cache.delete(blob
|
|
242
|
+
self._cache.delete(self._cache_key(blob))
|
|
223
243
|
except Exception:
|
|
224
244
|
_LOGGER.warning(
|
|
225
245
|
"cannot delete cached blob %s",
|
|
@@ -235,7 +255,7 @@ class ValueHydrator:
|
|
|
235
255
|
if blob.is_object
|
|
236
256
|
else concrete.string_value.encode("utf-8")
|
|
237
257
|
)
|
|
238
|
-
self._cache.put(blob
|
|
258
|
+
self._cache.put(self._cache_key(blob), payload)
|
|
239
259
|
except Exception:
|
|
240
260
|
_LOGGER.warning("cannot cache blob %s", blob.blob_id, exc_info=True)
|
|
241
261
|
|
|
@@ -254,6 +274,10 @@ class ValueHydrator:
|
|
|
254
274
|
return blob_id, True
|
|
255
275
|
return None
|
|
256
276
|
|
|
277
|
+
@staticmethod
|
|
278
|
+
def _cache_key(blob: _PendingBlob) -> str:
|
|
279
|
+
return f"{len(blob.flow_id)}:{blob.flow_id}{blob.blob_id}"
|
|
280
|
+
|
|
257
281
|
@staticmethod
|
|
258
282
|
def _validate_hydrated(blob: _PendingBlob, value: pb.Value) -> None:
|
|
259
283
|
expected = "obj_value" if blob.is_object else "string_value"
|
|
@@ -275,7 +299,7 @@ class ValueHydrator:
|
|
|
275
299
|
raise ValueError("non-finite numbers are unsupported")
|
|
276
300
|
return
|
|
277
301
|
if kind == "obj_value":
|
|
278
|
-
if value.obj_value.encoding not in ("json", "
|
|
302
|
+
if value.obj_value.encoding not in ("json", "raw"):
|
|
279
303
|
raise ValueError(
|
|
280
304
|
f"unsupported object encoding {value.obj_value.encoding}"
|
|
281
305
|
)
|
|
@@ -22,7 +22,7 @@ from dex.dexpb import dex_pb2 as pb
|
|
|
22
22
|
from dex.runtime_errors import ValueMappingError
|
|
23
23
|
|
|
24
24
|
_JSON_ENCODING = "json"
|
|
25
|
-
_RAW_BYTES_ENCODING = "
|
|
25
|
+
_RAW_BYTES_ENCODING = "raw"
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class ValueMapper:
|
|
@@ -43,10 +43,7 @@ class ValueMapper:
|
|
|
43
43
|
def _encode(self, value: Any, codec: Codec[Any]) -> pb.Value:
|
|
44
44
|
if value is None:
|
|
45
45
|
return pb.Value(
|
|
46
|
-
|
|
47
|
-
encoding=_JSON_ENCODING,
|
|
48
|
-
payload=b"null",
|
|
49
|
-
)
|
|
46
|
+
null_value=cast(struct_pb2.NullValue, struct_pb2.NULL_VALUE)
|
|
50
47
|
)
|
|
51
48
|
logical = codec.encode(value)
|
|
52
49
|
if logical.kind is WireKind.STRING:
|
|
@@ -134,7 +131,7 @@ class ValueMapper:
|
|
|
134
131
|
):
|
|
135
132
|
raise ValueError("blob-backed Value was not hydrated")
|
|
136
133
|
if kind == "null_value":
|
|
137
|
-
|
|
134
|
+
return None
|
|
138
135
|
raise TypeError(f"cannot decode {kind or 'empty Value'} as {codec.type_name}")
|
|
139
136
|
|
|
140
137
|
def to_value(self, value: pb.Value) -> Value:
|
|
@@ -166,6 +163,8 @@ class ValueMapper:
|
|
|
166
163
|
"internal_blob_id_for_obj_value",
|
|
167
164
|
):
|
|
168
165
|
raise ValueError("blob-backed Value was not hydrated")
|
|
166
|
+
if kind == "null_value":
|
|
167
|
+
return Value(WireKind.JSON, "null")
|
|
169
168
|
raise TypeError(f"cannot wrap {kind or 'empty Value'} as Value")
|
|
170
169
|
|
|
171
170
|
@staticmethod
|