dex-python-sdk 1.10.0__tar.gz → 1.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/PKG-INFO +55 -5
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/README.md +54 -4
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_async_worker_dispatcher.py +11 -4
- dex_python_sdk-1.11.0/dex/_credential_transport.py +242 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_invocation_context.py +8 -1
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_pull_worker.py +17 -13
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_server_protocol.py +33 -1
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_worker_dispatcher.py +24 -5
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/async_client.py +80 -8
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/async_worker.py +37 -16
- dex_python_sdk-1.11.0/dex/attribute.py +546 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/client.py +66 -5
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/credential.py +10 -2
- dex_python_sdk-1.11.0/dex/dexpb/dex_pb2.py +545 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/dexpb/dex_pb2.pyi +14 -6
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/flow.py +17 -1
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/worker.py +21 -9
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/pyproject.toml +1 -1
- dex_python_sdk-1.10.0/dex/_credential_transport.py +0 -142
- dex_python_sdk-1.10.0/dex/attribute.py +0 -313
- dex_python_sdk-1.10.0/dex/dexpb/dex_pb2.py +0 -545
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/LEGACY_NOTICES.md +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/LICENSE +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/__init__.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_async_value_hydrator.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_async_worker_service.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_grpc_errors.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_native.pyi +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_utils.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_value_hydrator.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_value_mapper.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/_worker_service.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/attribute_match.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/blob_cache.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/channel.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/client_options.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/codec.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/condition.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/context.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/dexpb/__init__.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/dexpb/dex_pb2_grpc.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/flow_config.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/flow_info.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/flow_options.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/flow_result.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/py.typed +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/rpc_invoke_options.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/runtime_errors.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/step.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/step_execution.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/stream.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/subflow.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/timer.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/wait.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/wait_options.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/dex/worker_options.py +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/Cargo.lock +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/Cargo.toml +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/Cargo.toml +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/LICENSE +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/config.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/entry.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/error.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/format.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/lib.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/policy.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/store.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/tests/blob_cache_integration.rs +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache-python/Cargo.toml +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache-python/LICENSE +0 -0
- {dex_python_sdk-1.10.0 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache-python/src/lib.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dex-python-sdk
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.11.0
|
|
4
4
|
Requires-Dist: grpcio>=1.83.0
|
|
5
5
|
Requires-Dist: grpcio-status>=1.83.0
|
|
6
6
|
Requires-Dist: protobuf>=7.35.1
|
|
@@ -38,10 +38,19 @@ startup synchronization, blob hydration, and pull WorkerTaskService streams. A
|
|
|
38
38
|
pull stream reads the token when it opens and again on every reconnect. For a
|
|
39
39
|
custom credential, implement Credential.token() to return the current token;
|
|
40
40
|
the method may be called concurrently, and an exception fails the call before
|
|
41
|
-
it contacts Dex.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
it contacts Dex. Client and AsyncClient calls raise the credential's exception
|
|
42
|
+
unchanged, including TimeoutError when the RPC deadline has not expired.
|
|
43
|
+
AsyncClient and AsyncWorker also accept an awaitable token. Synchronous tokens
|
|
44
|
+
run in a dedicated four-thread executor per async Client or Worker, created
|
|
45
|
+
lazily and shut down without waiting for a blocked token call on close. Sync
|
|
46
|
+
calls run token lookup on the calling thread. Async calls bound token lookup
|
|
47
|
+
by the RPC deadline when set; pull stream opens use a 30-second lookup budget.
|
|
48
|
+
Declare token with `async def` to invoke it on the event loop. A plain token
|
|
49
|
+
function runs once in the SDK executor; if it returns a coroutine, that
|
|
50
|
+
coroutine runs on the event loop. A loop-bound Future from another loop raises
|
|
51
|
+
TypeError; use `async def` to create it on the right loop. A blocked token keeps
|
|
52
|
+
one executor thread busy until it returns, and an unclosed Client or Worker may
|
|
53
|
+
retain up to four idle credential threads. Sync token lookup must return
|
|
45
54
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
46
55
|
gRPC, so protect the network path that carries the token.
|
|
47
56
|
|
|
@@ -297,6 +306,39 @@ failures do not roll back Flow Attributes. `None` preserves current targets;
|
|
|
297
306
|
an explicit empty list disables future synchronization while retaining protocol
|
|
298
307
|
presence.
|
|
299
308
|
|
|
309
|
+
Declare short-lived secrets, such as API keys, with `hard_delete_on_null=True`:
|
|
310
|
+
|
|
311
|
+
```python
|
|
312
|
+
api_key = dex.Attribute("api-key", str, hard_delete_on_null=True)
|
|
313
|
+
provider_tokens = dex.AttributeMap("provider-token", str, hard_delete_on_null=True)
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
Dex stores these values only in its blob store, never in Flow history. Deleting
|
|
317
|
+
the Attribute or setting it to `None` permanently deletes the stored value, and a
|
|
318
|
+
deleted value reads as absent. Only string and object values, including
|
|
319
|
+
`bytes`, are supported; `int`, `bool`, and `float` types are rejected. The
|
|
320
|
+
option cannot be combined with an index or `sync_to_attribute_store`, and
|
|
321
|
+
`wait_for_attribute_match` rejects it. Handlers receive the value inline, and
|
|
322
|
+
the SDK never stores it in the local blob cache.
|
|
323
|
+
|
|
324
|
+
Hard-delete writes require a Server with an enabled blob store and Server
|
|
325
|
+
protocol version 3 or later. Workers still start against older Servers.
|
|
326
|
+
There, a Step, RPC, or timeout handler result that writes a hard-delete
|
|
327
|
+
Attribute fails that handler attempt with `RuntimeError`, as does a `wait_for`
|
|
328
|
+
result whose SubFlow start initializes one. A Client checks the Server protocol
|
|
329
|
+
once, before its first `start_flow` that initializes one.
|
|
330
|
+
|
|
331
|
+
Each Flow keeps one stored copy per Attribute key, overwritten in place. A Step
|
|
332
|
+
or RPC attempt writes or deletes the stored value before its result commits, so
|
|
333
|
+
retries, other Steps, and RPCs can observe a write or deletion from an attempt
|
|
334
|
+
that later failed. Concurrent writers of one key may leave a value other than
|
|
335
|
+
the last committed write. A failed `start_flow` or SubFlow start for a running
|
|
336
|
+
Flow ID still overwrites, or for `None` deletes, the running Flow's value.
|
|
337
|
+
Executions that reuse a Flow ID share the value, and time travel does not
|
|
338
|
+
restore a deleted value. Values remain until deleted or until retention and blob
|
|
339
|
+
cleanup remove the Flow. Blob store backups or bucket versioning can retain
|
|
340
|
+
copies.
|
|
341
|
+
|
|
300
342
|
```
|
|
301
343
|
pip install dex-python-sdk==0.1.0
|
|
302
344
|
```
|
|
@@ -628,6 +670,9 @@ environment:
|
|
|
628
670
|
./run-integration-tests.sh
|
|
629
671
|
```
|
|
630
672
|
|
|
673
|
+
Set `DEX_INTEG_WORKER_MODE=pull` to run the suite with pull Workers instead of
|
|
674
|
+
listening Workers. The default is `push`.
|
|
675
|
+
|
|
631
676
|
### Measure integration coverage
|
|
632
677
|
|
|
633
678
|
Run the same integration suite with Python source coverage:
|
|
@@ -702,6 +747,11 @@ Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
|
702
747
|
common protocol, synchronize Attribute indexes, and only then bind
|
|
703
748
|
WorkerService. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
704
749
|
invalid or disjoint intervals, and RPC failures stop startup before binding.
|
|
750
|
+
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
751
|
+
Attributes. Workers record the negotiated version at startup and reject each
|
|
752
|
+
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
753
|
+
Clients negotiate lazily, once, before the first `start_flow` that initializes
|
|
754
|
+
a hard-delete Attribute.
|
|
705
755
|
|
|
706
756
|
The diagnostic SDK version comes from Python distribution metadata for
|
|
707
757
|
`dex-python-sdk` and falls back to `dev` when the distribution is unavailable.
|
|
@@ -22,10 +22,19 @@ startup synchronization, blob hydration, and pull WorkerTaskService streams. A
|
|
|
22
22
|
pull stream reads the token when it opens and again on every reconnect. For a
|
|
23
23
|
custom credential, implement Credential.token() to return the current token;
|
|
24
24
|
the method may be called concurrently, and an exception fails the call before
|
|
25
|
-
it contacts Dex.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
25
|
+
it contacts Dex. Client and AsyncClient calls raise the credential's exception
|
|
26
|
+
unchanged, including TimeoutError when the RPC deadline has not expired.
|
|
27
|
+
AsyncClient and AsyncWorker also accept an awaitable token. Synchronous tokens
|
|
28
|
+
run in a dedicated four-thread executor per async Client or Worker, created
|
|
29
|
+
lazily and shut down without waiting for a blocked token call on close. Sync
|
|
30
|
+
calls run token lookup on the calling thread. Async calls bound token lookup
|
|
31
|
+
by the RPC deadline when set; pull stream opens use a 30-second lookup budget.
|
|
32
|
+
Declare token with `async def` to invoke it on the event loop. A plain token
|
|
33
|
+
function runs once in the SDK executor; if it returns a coroutine, that
|
|
34
|
+
coroutine runs on the event loop. A loop-bound Future from another loop raises
|
|
35
|
+
TypeError; use `async def` to create it on the right loop. A blocked token keeps
|
|
36
|
+
one executor thread busy until it returns, and an unclosed Client or Worker may
|
|
37
|
+
retain up to four idle credential threads. Sync token lookup must return
|
|
29
38
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
30
39
|
gRPC, so protect the network path that carries the token.
|
|
31
40
|
|
|
@@ -281,6 +290,39 @@ failures do not roll back Flow Attributes. `None` preserves current targets;
|
|
|
281
290
|
an explicit empty list disables future synchronization while retaining protocol
|
|
282
291
|
presence.
|
|
283
292
|
|
|
293
|
+
Declare short-lived secrets, such as API keys, with `hard_delete_on_null=True`:
|
|
294
|
+
|
|
295
|
+
```python
|
|
296
|
+
api_key = dex.Attribute("api-key", str, hard_delete_on_null=True)
|
|
297
|
+
provider_tokens = dex.AttributeMap("provider-token", str, hard_delete_on_null=True)
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
Dex stores these values only in its blob store, never in Flow history. Deleting
|
|
301
|
+
the Attribute or setting it to `None` permanently deletes the stored value, and a
|
|
302
|
+
deleted value reads as absent. Only string and object values, including
|
|
303
|
+
`bytes`, are supported; `int`, `bool`, and `float` types are rejected. The
|
|
304
|
+
option cannot be combined with an index or `sync_to_attribute_store`, and
|
|
305
|
+
`wait_for_attribute_match` rejects it. Handlers receive the value inline, and
|
|
306
|
+
the SDK never stores it in the local blob cache.
|
|
307
|
+
|
|
308
|
+
Hard-delete writes require a Server with an enabled blob store and Server
|
|
309
|
+
protocol version 3 or later. Workers still start against older Servers.
|
|
310
|
+
There, a Step, RPC, or timeout handler result that writes a hard-delete
|
|
311
|
+
Attribute fails that handler attempt with `RuntimeError`, as does a `wait_for`
|
|
312
|
+
result whose SubFlow start initializes one. A Client checks the Server protocol
|
|
313
|
+
once, before its first `start_flow` that initializes one.
|
|
314
|
+
|
|
315
|
+
Each Flow keeps one stored copy per Attribute key, overwritten in place. A Step
|
|
316
|
+
or RPC attempt writes or deletes the stored value before its result commits, so
|
|
317
|
+
retries, other Steps, and RPCs can observe a write or deletion from an attempt
|
|
318
|
+
that later failed. Concurrent writers of one key may leave a value other than
|
|
319
|
+
the last committed write. A failed `start_flow` or SubFlow start for a running
|
|
320
|
+
Flow ID still overwrites, or for `None` deletes, the running Flow's value.
|
|
321
|
+
Executions that reuse a Flow ID share the value, and time travel does not
|
|
322
|
+
restore a deleted value. Values remain until deleted or until retention and blob
|
|
323
|
+
cleanup remove the Flow. Blob store backups or bucket versioning can retain
|
|
324
|
+
copies.
|
|
325
|
+
|
|
284
326
|
```
|
|
285
327
|
pip install dex-python-sdk==0.1.0
|
|
286
328
|
```
|
|
@@ -612,6 +654,9 @@ environment:
|
|
|
612
654
|
./run-integration-tests.sh
|
|
613
655
|
```
|
|
614
656
|
|
|
657
|
+
Set `DEX_INTEG_WORKER_MODE=pull` to run the suite with pull Workers instead of
|
|
658
|
+
listening Workers. The default is `push`.
|
|
659
|
+
|
|
615
660
|
### Measure integration coverage
|
|
616
661
|
|
|
617
662
|
Run the same integration suite with Python source coverage:
|
|
@@ -686,6 +731,11 @@ Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
|
686
731
|
common protocol, synchronize Attribute indexes, and only then bind
|
|
687
732
|
WorkerService. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
688
733
|
invalid or disjoint intervals, and RPC failures stop startup before binding.
|
|
734
|
+
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
735
|
+
Attributes. Workers record the negotiated version at startup and reject each
|
|
736
|
+
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
737
|
+
Clients negotiate lazily, once, before the first `start_flow` that initializes
|
|
738
|
+
a hard-delete Attribute.
|
|
689
739
|
|
|
690
740
|
The diagnostic SDK version comes from Python distribution metadata for
|
|
691
741
|
`dex-python-sdk` and falls back to `dev` when the distribution is unavailable.
|
|
@@ -16,6 +16,7 @@ from typing import Any, Callable
|
|
|
16
16
|
|
|
17
17
|
from dex._async_value_hydrator import AsyncValueHydrator
|
|
18
18
|
from dex._invocation_context import InvocationContext, InvocationMethod
|
|
19
|
+
from dex._server_protocol import NegotiatedServerProtocol
|
|
19
20
|
from dex._value_mapper import ValueMapper
|
|
20
21
|
from dex._worker_dispatcher import _TIMEOUT_HANDLER_STEP_TYPE, WorkerDispatcher
|
|
21
22
|
from dex.dexpb import dex_pb2 as pb
|
|
@@ -64,10 +65,16 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
64
65
|
registry: Registry,
|
|
65
66
|
values: ValueMapper,
|
|
66
67
|
hydrator: AsyncValueHydrator,
|
|
68
|
+
server_protocol: NegotiatedServerProtocol,
|
|
67
69
|
) -> None:
|
|
70
|
+
if server_protocol is None:
|
|
71
|
+
raise ValueError(
|
|
72
|
+
"AsyncWorkerDispatcher requires a NegotiatedServerProtocol"
|
|
73
|
+
)
|
|
68
74
|
self._registry = registry
|
|
69
75
|
self._values = values
|
|
70
76
|
self._async_hydrator = hydrator
|
|
77
|
+
self._server_protocol = server_protocol
|
|
71
78
|
|
|
72
79
|
async def invoke_wait_for( # type: ignore[override]
|
|
73
80
|
self,
|
|
@@ -127,7 +134,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
127
134
|
if not isinstance(wait, Wait):
|
|
128
135
|
raise TypeError("wait_for must return Wait")
|
|
129
136
|
response = pb.InvokeWaitForMethodResponse(
|
|
130
|
-
upsert_attributes=
|
|
137
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
131
138
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
132
139
|
record_events=context.events,
|
|
133
140
|
publish_to_channel=context.publications,
|
|
@@ -229,7 +236,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
229
236
|
raise TypeError("execute must return StepDecision")
|
|
230
237
|
response = pb.InvokeExecuteMethodResponse(
|
|
231
238
|
step_decision=self._map_decision(flow, decision),
|
|
232
|
-
upsert_attributes=
|
|
239
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
233
240
|
record_events=context.events,
|
|
234
241
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
235
242
|
publish_to_channel=context.publications,
|
|
@@ -309,7 +316,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
309
316
|
raise TypeError("handle_timeout must return StepDecision")
|
|
310
317
|
response = pb.InvokeExecuteMethodResponse(
|
|
311
318
|
step_decision=self._map_decision(flow, decision),
|
|
312
|
-
upsert_attributes=
|
|
319
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
313
320
|
record_events=context.events,
|
|
314
321
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
315
322
|
publish_to_channel=context.publications,
|
|
@@ -361,7 +368,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
361
368
|
returned = await returned
|
|
362
369
|
try:
|
|
363
370
|
response = pb.InvokeWorkerRPCResponse(
|
|
364
|
-
upsert_attributes=
|
|
371
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
365
372
|
record_events=context.events,
|
|
366
373
|
delete_from_channel=context.channel_deletions,
|
|
367
374
|
publish_to_channel=context.publications,
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Sustainable Use License 1.0.
|
|
4
|
+
# You may not use this file except in compliance with the License.
|
|
5
|
+
# See the LICENSE file in the repository root.
|
|
6
|
+
#
|
|
7
|
+
# SPDX-License-Identifier: LicenseRef-Sustainable-Use-1.0
|
|
8
|
+
|
|
9
|
+
"""Attach bearer metadata to outbound gRPC calls."""
|
|
10
|
+
|
|
11
|
+
# mypy: disable-error-code=type-arg
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import inspect
|
|
17
|
+
import queue
|
|
18
|
+
import threading
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from concurrent.futures import Future
|
|
21
|
+
from typing import Any, NamedTuple, cast
|
|
22
|
+
|
|
23
|
+
import grpc
|
|
24
|
+
|
|
25
|
+
from dex.credential import Credential
|
|
26
|
+
|
|
27
|
+
_TOKEN_WORKERS = 4
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AsyncCredentialRunner:
|
|
31
|
+
def __init__(self, credential: Credential) -> None:
|
|
32
|
+
self._credential = credential
|
|
33
|
+
self._lock = threading.Lock()
|
|
34
|
+
self._jobs: queue.SimpleQueue[Future[Any] | None] = queue.SimpleQueue()
|
|
35
|
+
self._threads: list[threading.Thread] = []
|
|
36
|
+
self._pending: set[Future[Any]] = set()
|
|
37
|
+
self._closed = False
|
|
38
|
+
|
|
39
|
+
async def token(self) -> str:
|
|
40
|
+
token_method = self._credential.token
|
|
41
|
+
if inspect.iscoroutinefunction(token_method) or inspect.iscoroutinefunction(
|
|
42
|
+
getattr(token_method, "__call__", None)
|
|
43
|
+
):
|
|
44
|
+
token = token_method()
|
|
45
|
+
if inspect.isawaitable(token):
|
|
46
|
+
return await token
|
|
47
|
+
return token
|
|
48
|
+
token = await asyncio.wrap_future(self._submit())
|
|
49
|
+
if inspect.isawaitable(token):
|
|
50
|
+
if isinstance(token, asyncio.Future):
|
|
51
|
+
try:
|
|
52
|
+
token_loop = token.get_loop()
|
|
53
|
+
except RuntimeError as error:
|
|
54
|
+
raise TypeError(
|
|
55
|
+
"loop-bound credentials must use async def token()"
|
|
56
|
+
) from error
|
|
57
|
+
if token_loop is not asyncio.get_running_loop():
|
|
58
|
+
raise TypeError("loop-bound credentials must use async def token()")
|
|
59
|
+
return await token
|
|
60
|
+
return token
|
|
61
|
+
|
|
62
|
+
def _submit(self) -> Future[Any]:
|
|
63
|
+
with self._lock:
|
|
64
|
+
if self._closed:
|
|
65
|
+
raise RuntimeError("credential runner is closed")
|
|
66
|
+
future: Future[Any] = Future()
|
|
67
|
+
self._pending.add(future)
|
|
68
|
+
if len(self._threads) < _TOKEN_WORKERS:
|
|
69
|
+
thread = threading.Thread(
|
|
70
|
+
target=self._run,
|
|
71
|
+
name="dex-python-credential",
|
|
72
|
+
daemon=True,
|
|
73
|
+
)
|
|
74
|
+
self._threads.append(thread)
|
|
75
|
+
thread.start()
|
|
76
|
+
self._jobs.put(future)
|
|
77
|
+
return future
|
|
78
|
+
|
|
79
|
+
def _run(self) -> None:
|
|
80
|
+
while True:
|
|
81
|
+
future = self._jobs.get()
|
|
82
|
+
if future is None:
|
|
83
|
+
return
|
|
84
|
+
with self._lock:
|
|
85
|
+
if future.done() or not future.set_running_or_notify_cancel():
|
|
86
|
+
self._pending.discard(future)
|
|
87
|
+
continue
|
|
88
|
+
try:
|
|
89
|
+
token = self._credential.token()
|
|
90
|
+
failure = None
|
|
91
|
+
except BaseException as error:
|
|
92
|
+
token = None
|
|
93
|
+
failure = error
|
|
94
|
+
with self._lock:
|
|
95
|
+
if not future.done():
|
|
96
|
+
if failure is None:
|
|
97
|
+
future.set_result(token)
|
|
98
|
+
else:
|
|
99
|
+
future.set_exception(failure)
|
|
100
|
+
self._pending.discard(future)
|
|
101
|
+
|
|
102
|
+
def close(self) -> None:
|
|
103
|
+
with self._lock:
|
|
104
|
+
if self._closed:
|
|
105
|
+
return
|
|
106
|
+
self._closed = True
|
|
107
|
+
for future in self._pending:
|
|
108
|
+
if not future.done():
|
|
109
|
+
future.set_exception(RuntimeError("credential runner is closed"))
|
|
110
|
+
self._pending.clear()
|
|
111
|
+
while True:
|
|
112
|
+
try:
|
|
113
|
+
self._jobs.get_nowait()
|
|
114
|
+
except queue.Empty:
|
|
115
|
+
break
|
|
116
|
+
for _ in self._threads:
|
|
117
|
+
self._jobs.put(None)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _SyncCallDetails(NamedTuple):
|
|
121
|
+
method: str
|
|
122
|
+
timeout: float | None
|
|
123
|
+
metadata: Any
|
|
124
|
+
credentials: grpc.CallCredentials | None
|
|
125
|
+
wait_for_ready: bool | None
|
|
126
|
+
compression: grpc.Compression | None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def credential_metadata(credential: Credential | None, metadata: Any = None) -> Any:
|
|
130
|
+
if credential is None:
|
|
131
|
+
return metadata
|
|
132
|
+
token = credential.token()
|
|
133
|
+
if inspect.isawaitable(token):
|
|
134
|
+
if inspect.iscoroutine(token):
|
|
135
|
+
token.close()
|
|
136
|
+
raise TypeError("synchronous calls require a synchronous credential")
|
|
137
|
+
return tuple(metadata or ()) + (("authorization", "Bearer " + token),)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
async def async_credential_metadata(
|
|
141
|
+
runner: AsyncCredentialRunner, metadata: Any = None, timeout: float | None = None
|
|
142
|
+
) -> Any:
|
|
143
|
+
if timeout is None:
|
|
144
|
+
token = await runner.token()
|
|
145
|
+
else:
|
|
146
|
+
lookup = asyncio.create_task(runner.token())
|
|
147
|
+
try:
|
|
148
|
+
completed, _ = await asyncio.wait({lookup}, timeout=timeout)
|
|
149
|
+
if not completed:
|
|
150
|
+
lookup.cancel()
|
|
151
|
+
await asyncio.gather(lookup, return_exceptions=True)
|
|
152
|
+
raise credential_deadline_error()
|
|
153
|
+
token = lookup.result()
|
|
154
|
+
except BaseException:
|
|
155
|
+
lookup.cancel()
|
|
156
|
+
await asyncio.gather(lookup, return_exceptions=True)
|
|
157
|
+
raise
|
|
158
|
+
return tuple(metadata or ()) + (("authorization", "Bearer " + token),)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def credential_deadline_error() -> grpc.aio.AioRpcError:
|
|
162
|
+
return grpc.aio.AioRpcError(
|
|
163
|
+
grpc.StatusCode.DEADLINE_EXCEEDED,
|
|
164
|
+
grpc.aio.Metadata(),
|
|
165
|
+
grpc.aio.Metadata(),
|
|
166
|
+
"credential lookup exceeded RPC deadline",
|
|
167
|
+
None,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class _SyncCredentialInterceptor(grpc.UnaryUnaryClientInterceptor):
|
|
172
|
+
def __init__(self, credential: Credential) -> None:
|
|
173
|
+
self._credential = credential
|
|
174
|
+
|
|
175
|
+
def intercept_unary_unary(
|
|
176
|
+
self,
|
|
177
|
+
continuation: Callable[..., Any],
|
|
178
|
+
client_call_details: grpc.ClientCallDetails,
|
|
179
|
+
request: Any,
|
|
180
|
+
) -> Any:
|
|
181
|
+
details = _SyncCallDetails(
|
|
182
|
+
client_call_details.method,
|
|
183
|
+
client_call_details.timeout,
|
|
184
|
+
credential_metadata(self._credential, client_call_details.metadata),
|
|
185
|
+
client_call_details.credentials,
|
|
186
|
+
client_call_details.wait_for_ready,
|
|
187
|
+
client_call_details.compression,
|
|
188
|
+
)
|
|
189
|
+
return continuation(details, request)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class _AsyncCredentialInterceptor(grpc.aio.UnaryUnaryClientInterceptor):
|
|
193
|
+
def __init__(self, runner: AsyncCredentialRunner) -> None:
|
|
194
|
+
self._runner = runner
|
|
195
|
+
|
|
196
|
+
async def intercept_unary_unary(
|
|
197
|
+
self,
|
|
198
|
+
continuation: Callable[..., Any],
|
|
199
|
+
client_call_details: grpc.aio.ClientCallDetails,
|
|
200
|
+
request: Any,
|
|
201
|
+
) -> Any:
|
|
202
|
+
loop = asyncio.get_running_loop()
|
|
203
|
+
started_at = loop.time()
|
|
204
|
+
metadata = await async_credential_metadata(
|
|
205
|
+
self._runner, client_call_details.metadata, client_call_details.timeout
|
|
206
|
+
)
|
|
207
|
+
remaining_timeout = client_call_details.timeout
|
|
208
|
+
if remaining_timeout is not None:
|
|
209
|
+
remaining_timeout -= loop.time() - started_at
|
|
210
|
+
if remaining_timeout <= 0:
|
|
211
|
+
raise credential_deadline_error()
|
|
212
|
+
details = grpc.aio.ClientCallDetails(
|
|
213
|
+
client_call_details.method,
|
|
214
|
+
remaining_timeout,
|
|
215
|
+
metadata,
|
|
216
|
+
client_call_details.credentials,
|
|
217
|
+
client_call_details.wait_for_ready,
|
|
218
|
+
)
|
|
219
|
+
return await continuation(details, request)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def sync_channel(server_address: str, credential: Credential | None) -> grpc.Channel:
|
|
223
|
+
channel = grpc.insecure_channel(server_address)
|
|
224
|
+
if credential is None:
|
|
225
|
+
return channel
|
|
226
|
+
return grpc.intercept_channel(channel, _SyncCredentialInterceptor(credential))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def async_channel(
|
|
230
|
+
server_address: str, credential_runner: AsyncCredentialRunner | None
|
|
231
|
+
) -> grpc.aio.Channel:
|
|
232
|
+
if credential_runner is None:
|
|
233
|
+
return grpc.aio.insecure_channel(server_address)
|
|
234
|
+
return grpc.aio.insecure_channel(
|
|
235
|
+
server_address,
|
|
236
|
+
interceptors=[
|
|
237
|
+
cast(
|
|
238
|
+
grpc.aio.ClientInterceptor,
|
|
239
|
+
_AsyncCredentialInterceptor(credential_runner),
|
|
240
|
+
)
|
|
241
|
+
],
|
|
242
|
+
)
|
|
@@ -15,7 +15,12 @@ from urllib.parse import unquote
|
|
|
15
15
|
|
|
16
16
|
from dex._utils import require_map_instance, require_name
|
|
17
17
|
from dex._value_mapper import ValueMapper
|
|
18
|
-
from dex.attribute import
|
|
18
|
+
from dex.attribute import (
|
|
19
|
+
Attribute,
|
|
20
|
+
AttributeMap,
|
|
21
|
+
_apply_attribute_store_sync,
|
|
22
|
+
_apply_hard_delete_on_null,
|
|
23
|
+
)
|
|
19
24
|
from dex.channel import Channel, ChannelMap, ChannelMessage
|
|
20
25
|
from dex.codec import Codec
|
|
21
26
|
from dex.context import RecoveryErrorInfo
|
|
@@ -349,6 +354,7 @@ class InvocationContext:
|
|
|
349
354
|
if index is not None:
|
|
350
355
|
write.index_config.CopyFrom(index)
|
|
351
356
|
_apply_attribute_store_sync(write, definition)
|
|
357
|
+
_apply_hard_delete_on_null(write, definition)
|
|
352
358
|
self.attribute_writes[key] = write
|
|
353
359
|
|
|
354
360
|
def _delete_attribute(
|
|
@@ -366,6 +372,7 @@ class InvocationContext:
|
|
|
366
372
|
if index is not None:
|
|
367
373
|
write.index_config.CopyFrom(index)
|
|
368
374
|
_apply_attribute_store_sync(write, definition)
|
|
375
|
+
_apply_hard_delete_on_null(write, definition)
|
|
369
376
|
self.attribute_writes[key] = write
|
|
370
377
|
|
|
371
378
|
def _publish_channel(
|
|
@@ -22,13 +22,12 @@ from typing import Any
|
|
|
22
22
|
|
|
23
23
|
import grpc
|
|
24
24
|
|
|
25
|
-
from dex._credential_transport import async_credential_metadata
|
|
25
|
+
from dex._credential_transport import AsyncCredentialRunner, async_credential_metadata
|
|
26
26
|
from dex._grpc_errors import _worker_error_status
|
|
27
27
|
from dex._server_protocol import sdk_version
|
|
28
28
|
from dex.dexpb import dex_pb2 as pb
|
|
29
29
|
from dex.dexpb import dex_pb2_grpc
|
|
30
30
|
from dex.flow import Registry
|
|
31
|
-
from dex.credential import Credential
|
|
32
31
|
from dex.worker_options import PullWorkerOptions
|
|
33
32
|
|
|
34
33
|
_LOGGER = logging.getLogger(__name__)
|
|
@@ -81,7 +80,7 @@ class PullWorkerRuntime:
|
|
|
81
80
|
server_address: str,
|
|
82
81
|
executor: ThreadPoolExecutor | None,
|
|
83
82
|
app_worker_id: str | None = None,
|
|
84
|
-
|
|
83
|
+
credential_runner: AsyncCredentialRunner | None = None,
|
|
85
84
|
) -> None:
|
|
86
85
|
self.app_worker_id = app_worker_id or resolve_app_worker_id(registry, options)
|
|
87
86
|
self.flow_types = sorted(registry._registered_flows)
|
|
@@ -92,7 +91,7 @@ class PullWorkerRuntime:
|
|
|
92
91
|
self._dispatcher = dispatcher
|
|
93
92
|
self._server_address = server_address
|
|
94
93
|
self._executor = executor
|
|
95
|
-
self.
|
|
94
|
+
self._credential_runner = credential_runner
|
|
96
95
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
97
96
|
self._loop_thread_id: int | None = None
|
|
98
97
|
self._started = threading.Event()
|
|
@@ -187,6 +186,8 @@ class PullWorkerRuntime:
|
|
|
187
186
|
try:
|
|
188
187
|
for stream in tuple(self._streams):
|
|
189
188
|
stream.drain()
|
|
189
|
+
if not stream.is_busy and not stream.is_half_closing:
|
|
190
|
+
stream.run_task.cancel()
|
|
190
191
|
self.signal()
|
|
191
192
|
active = [
|
|
192
193
|
stream.handler_task for stream in self._streams if stream.handler_task
|
|
@@ -246,7 +247,7 @@ class PullStream:
|
|
|
246
247
|
self._active = threading.Event()
|
|
247
248
|
self._call: Any = None
|
|
248
249
|
self._send_lock = asyncio.Lock()
|
|
249
|
-
self.
|
|
250
|
+
self._is_reconnecting = False
|
|
250
251
|
self._deadline_handle: asyncio.TimerHandle | None = None
|
|
251
252
|
|
|
252
253
|
def on_done(self, completed: asyncio.Task[None]) -> None:
|
|
@@ -268,10 +269,9 @@ class PullStream:
|
|
|
268
269
|
)
|
|
269
270
|
if self.runtime.is_stopping:
|
|
270
271
|
break
|
|
271
|
-
if self.
|
|
272
|
-
self.
|
|
272
|
+
if self._is_reconnecting:
|
|
273
|
+
self._is_reconnecting = False
|
|
273
274
|
backoff = 0.1
|
|
274
|
-
continue
|
|
275
275
|
if was_registered:
|
|
276
276
|
backoff = 0.1
|
|
277
277
|
code = error.code() if error is not None else None
|
|
@@ -299,12 +299,16 @@ class PullStream:
|
|
|
299
299
|
backoff = min(backoff * 2, 5)
|
|
300
300
|
|
|
301
301
|
async def _poll(self) -> bool:
|
|
302
|
-
if self.runtime.
|
|
302
|
+
if self.runtime.is_stopping:
|
|
303
|
+
return False
|
|
304
|
+
if self.runtime._credential_runner is None:
|
|
303
305
|
self._call = self.stub.PollWorkerTask()
|
|
304
306
|
else:
|
|
305
307
|
metadata = await async_credential_metadata(
|
|
306
|
-
self.runtime.
|
|
308
|
+
self.runtime._credential_runner, timeout=30.0
|
|
307
309
|
)
|
|
310
|
+
if self.runtime.is_stopping:
|
|
311
|
+
return False
|
|
308
312
|
self._call = self.stub.PollWorkerTask(metadata=metadata)
|
|
309
313
|
await self._send(
|
|
310
314
|
pb.PollWorkerTaskRequest(
|
|
@@ -345,7 +349,7 @@ class PullStream:
|
|
|
345
349
|
await self._call.done_writing()
|
|
346
350
|
await self._await_stream_eof(reader)
|
|
347
351
|
reader = None
|
|
348
|
-
self.
|
|
352
|
+
self._is_reconnecting = True
|
|
349
353
|
return was_registered
|
|
350
354
|
if reader not in completed:
|
|
351
355
|
continue
|
|
@@ -356,7 +360,7 @@ class PullStream:
|
|
|
356
360
|
kind = frame.WhichOneof("frame")
|
|
357
361
|
if not was_registered:
|
|
358
362
|
if kind == "reconnect":
|
|
359
|
-
self.
|
|
363
|
+
self._is_reconnecting = True
|
|
360
364
|
return False
|
|
361
365
|
if kind != "registered":
|
|
362
366
|
raise RuntimeError("Worker task stream did not register")
|
|
@@ -375,7 +379,7 @@ class PullStream:
|
|
|
375
379
|
elif kind == "reconnect":
|
|
376
380
|
self.drain()
|
|
377
381
|
if not self.is_busy:
|
|
378
|
-
self.
|
|
382
|
+
self._is_reconnecting = True
|
|
379
383
|
return was_registered
|
|
380
384
|
finally:
|
|
381
385
|
self.is_half_closing = False
|
|
@@ -9,13 +9,15 @@
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
from importlib.metadata import PackageNotFoundError, version
|
|
12
|
+
from typing import Iterable
|
|
12
13
|
|
|
13
14
|
import grpc
|
|
14
15
|
|
|
15
16
|
from dex.dexpb import dex_pb2 as pb
|
|
16
17
|
|
|
17
18
|
MINIMUM_SUPPORTED_SERVER_PROTOCOL_VERSION = 1
|
|
18
|
-
MAXIMUM_SUPPORTED_SERVER_PROTOCOL_VERSION =
|
|
19
|
+
MAXIMUM_SUPPORTED_SERVER_PROTOCOL_VERSION = 3
|
|
20
|
+
HARD_DELETE_ATTRIBUTE_PROTOCOL_VERSION = 3
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
def sdk_version() -> str:
|
|
@@ -64,6 +66,36 @@ def negotiate_server_protocol(server_info: pb.ServerInfo, artifact_version: str)
|
|
|
64
66
|
return negotiated
|
|
65
67
|
|
|
66
68
|
|
|
69
|
+
class NegotiatedServerProtocol:
|
|
70
|
+
def __init__(self) -> None:
|
|
71
|
+
self._version = 0
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def version(self) -> int:
|
|
75
|
+
return self._version
|
|
76
|
+
|
|
77
|
+
def record_version(self, version: int) -> None:
|
|
78
|
+
self._version = version
|
|
79
|
+
|
|
80
|
+
def require_hard_delete_attribute_support(
|
|
81
|
+
self,
|
|
82
|
+
writes: Iterable[pb.AttributeWrite],
|
|
83
|
+
) -> None:
|
|
84
|
+
attribute_names = [
|
|
85
|
+
write.key for write in writes if write.is_hard_delete_on_null
|
|
86
|
+
]
|
|
87
|
+
if (
|
|
88
|
+
not attribute_names
|
|
89
|
+
or self._version >= HARD_DELETE_ATTRIBUTE_PROTOCOL_VERSION
|
|
90
|
+
):
|
|
91
|
+
return
|
|
92
|
+
raise RuntimeError(
|
|
93
|
+
"hard-delete Attribute writes require Dex Server protocol version "
|
|
94
|
+
f"{HARD_DELETE_ATTRIBUTE_PROTOCOL_VERSION} or later, but the negotiated "
|
|
95
|
+
f"protocol version is {self._version}: {', '.join(attribute_names)}"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
67
99
|
def server_info_request_error(
|
|
68
100
|
artifact_version: str, failure: grpc.RpcError
|
|
69
101
|
) -> RuntimeError:
|