dex-python-sdk 1.10.1__tar.gz → 1.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/PKG-INFO +60 -5
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/README.md +59 -4
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_async_worker_dispatcher.py +11 -4
- dex_python_sdk-1.12.0/dex/_credential_transport.py +242 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_invocation_context.py +8 -1
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_pull_worker.py +49 -8
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_server_protocol.py +33 -1
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_worker_dispatcher.py +24 -5
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/async_client.py +80 -8
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/async_worker.py +37 -16
- dex_python_sdk-1.12.0/dex/attribute.py +546 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/client.py +66 -5
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/credential.py +10 -2
- dex_python_sdk-1.12.0/dex/dexpb/dex_pb2.py +545 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/dexpb/dex_pb2.pyi +32 -10
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/flow.py +17 -1
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/worker.py +21 -9
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/pyproject.toml +1 -1
- dex_python_sdk-1.10.1/dex/_credential_transport.py +0 -142
- dex_python_sdk-1.10.1/dex/attribute.py +0 -313
- dex_python_sdk-1.10.1/dex/dexpb/dex_pb2.py +0 -545
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/LEGACY_NOTICES.md +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/LICENSE +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/__init__.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_async_value_hydrator.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_async_worker_service.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_grpc_errors.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_native.pyi +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_utils.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_value_hydrator.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_value_mapper.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/_worker_service.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/attribute_match.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/blob_cache.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/channel.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/client_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/codec.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/condition.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/context.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/dexpb/__init__.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/dexpb/dex_pb2_grpc.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/flow_config.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/flow_info.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/flow_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/flow_result.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/py.typed +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/rpc_invoke_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/runtime_errors.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/step.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/step_execution.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/stream.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/subflow.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/timer.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/wait.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/wait_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/dex/worker_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/Cargo.lock +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/Cargo.toml +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/Cargo.toml +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/LICENSE +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/config.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/entry.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/error.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/format.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/lib.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/policy.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/src/store.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache/tests/blob_cache_integration.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache-python/Cargo.toml +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache-python/LICENSE +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.12.0}/sdk-rust/crates/dex-blob-cache-python/src/lib.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dex-python-sdk
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.12.0
|
|
4
4
|
Requires-Dist: grpcio>=1.83.0
|
|
5
5
|
Requires-Dist: grpcio-status>=1.83.0
|
|
6
6
|
Requires-Dist: protobuf>=7.35.1
|
|
@@ -38,10 +38,19 @@ startup synchronization, blob hydration, and pull WorkerTaskService streams. A
|
|
|
38
38
|
pull stream reads the token when it opens and again on every reconnect. For a
|
|
39
39
|
custom credential, implement Credential.token() to return the current token;
|
|
40
40
|
the method may be called concurrently, and an exception fails the call before
|
|
41
|
-
it contacts Dex.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
it contacts Dex. Client and AsyncClient calls raise the credential's exception
|
|
42
|
+
unchanged, including TimeoutError when the RPC deadline has not expired.
|
|
43
|
+
AsyncClient and AsyncWorker also accept an awaitable token. Synchronous tokens
|
|
44
|
+
run in a dedicated four-thread executor per async Client or Worker, created
|
|
45
|
+
lazily and shut down without waiting for a blocked token call on close. Sync
|
|
46
|
+
calls run token lookup on the calling thread. Async calls bound token lookup
|
|
47
|
+
by the RPC deadline when set; pull stream opens use a 30-second lookup budget.
|
|
48
|
+
Declare token with `async def` to invoke it on the event loop. A plain token
|
|
49
|
+
function runs once in the SDK executor; if it returns a coroutine, that
|
|
50
|
+
coroutine runs on the event loop. A loop-bound Future from another loop raises
|
|
51
|
+
TypeError; use `async def` to create it on the right loop. A blocked token keeps
|
|
52
|
+
one executor thread busy until it returns, and an unclosed Client or Worker may
|
|
53
|
+
retain up to four idle credential threads. Sync token lookup must return
|
|
45
54
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
46
55
|
gRPC, so protect the network path that carries the token.
|
|
47
56
|
|
|
@@ -91,6 +100,14 @@ max_concurrent_tasks)` streams because busy streams are replaced with idle
|
|
|
91
100
|
streams and retained after tasks finish. Size these options for the server's
|
|
92
101
|
per-Worker stream cap (256 by default).
|
|
93
102
|
|
|
103
|
+
The Worker rejects an unregistered Step or RPC by name. Dex retries that task
|
|
104
|
+
on another pull Worker without consuming a Step attempt, including when the
|
|
105
|
+
Flow was sticky to the rejecting Worker. If no capable Worker accepts it within
|
|
106
|
+
the server's dispatch wait (10 seconds by default, or the local Step attempt),
|
|
107
|
+
the Step follows its failure and retry policy; an RPC caller receives the named
|
|
108
|
+
error. This reassignment requires a Dex Server that supports it. Against older
|
|
109
|
+
Servers, the Worker returns the not-registered error as before.
|
|
110
|
+
|
|
94
111
|
The generated `app_worker_id` uses the first nonempty `DEX_WORKER_INSTANCE`,
|
|
95
112
|
`POD_NAME`, `HOSTNAME`, or system hostname, plus eight random base32 characters.
|
|
96
113
|
An override must uniquely identify one Worker process and change after restart.
|
|
@@ -297,6 +314,39 @@ failures do not roll back Flow Attributes. `None` preserves current targets;
|
|
|
297
314
|
an explicit empty list disables future synchronization while retaining protocol
|
|
298
315
|
presence.
|
|
299
316
|
|
|
317
|
+
Declare short-lived secrets, such as API keys, with `hard_delete_on_null=True`:
|
|
318
|
+
|
|
319
|
+
```python
|
|
320
|
+
api_key = dex.Attribute("api-key", str, hard_delete_on_null=True)
|
|
321
|
+
provider_tokens = dex.AttributeMap("provider-token", str, hard_delete_on_null=True)
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
Dex stores these values only in its blob store, never in Flow history. Deleting
|
|
325
|
+
the Attribute or setting it to `None` permanently deletes the stored value, and a
|
|
326
|
+
deleted value reads as absent. Only string and object values, including
|
|
327
|
+
`bytes`, are supported; `int`, `bool`, and `float` types are rejected. The
|
|
328
|
+
option cannot be combined with an index or `sync_to_attribute_store`, and
|
|
329
|
+
`wait_for_attribute_match` rejects it. Handlers receive the value inline, and
|
|
330
|
+
the SDK never stores it in the local blob cache.
|
|
331
|
+
|
|
332
|
+
Hard-delete writes require a Server with an enabled blob store and Server
|
|
333
|
+
protocol version 3 or later. Workers still start against older Servers.
|
|
334
|
+
There, a Step, RPC, or timeout handler result that writes a hard-delete
|
|
335
|
+
Attribute fails that handler attempt with `RuntimeError`, as does a `wait_for`
|
|
336
|
+
result whose SubFlow start initializes one. A Client checks the Server protocol
|
|
337
|
+
once, before its first `start_flow` that initializes one.
|
|
338
|
+
|
|
339
|
+
Each Flow keeps one stored copy per Attribute key, overwritten in place. A Step
|
|
340
|
+
or RPC attempt writes or deletes the stored value before its result commits, so
|
|
341
|
+
retries, other Steps, and RPCs can observe a write or deletion from an attempt
|
|
342
|
+
that later failed. Concurrent writers of one key may leave a value other than
|
|
343
|
+
the last committed write. A failed `start_flow` or SubFlow start for a running
|
|
344
|
+
Flow ID still overwrites, or for `None` deletes, the running Flow's value.
|
|
345
|
+
Executions that reuse a Flow ID share the value, and time travel does not
|
|
346
|
+
restore a deleted value. Values remain until deleted or until retention and blob
|
|
347
|
+
cleanup remove the Flow. Blob store backups or bucket versioning can retain
|
|
348
|
+
copies.
|
|
349
|
+
|
|
300
350
|
```
|
|
301
351
|
pip install dex-python-sdk==0.1.0
|
|
302
352
|
```
|
|
@@ -705,6 +755,11 @@ Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
|
705
755
|
common protocol, synchronize Attribute indexes, and only then bind
|
|
706
756
|
WorkerService. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
707
757
|
invalid or disjoint intervals, and RPC failures stop startup before binding.
|
|
758
|
+
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
759
|
+
Attributes. Workers record the negotiated version at startup and reject each
|
|
760
|
+
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
761
|
+
Clients negotiate lazily, once, before the first `start_flow` that initializes
|
|
762
|
+
a hard-delete Attribute.
|
|
708
763
|
|
|
709
764
|
The diagnostic SDK version comes from Python distribution metadata for
|
|
710
765
|
`dex-python-sdk` and falls back to `dev` when the distribution is unavailable.
|
|
@@ -22,10 +22,19 @@ startup synchronization, blob hydration, and pull WorkerTaskService streams. A
|
|
|
22
22
|
pull stream reads the token when it opens and again on every reconnect. For a
|
|
23
23
|
custom credential, implement Credential.token() to return the current token;
|
|
24
24
|
the method may be called concurrently, and an exception fails the call before
|
|
25
|
-
it contacts Dex.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
25
|
+
it contacts Dex. Client and AsyncClient calls raise the credential's exception
|
|
26
|
+
unchanged, including TimeoutError when the RPC deadline has not expired.
|
|
27
|
+
AsyncClient and AsyncWorker also accept an awaitable token. Synchronous tokens
|
|
28
|
+
run in a dedicated four-thread executor per async Client or Worker, created
|
|
29
|
+
lazily and shut down without waiting for a blocked token call on close. Sync
|
|
30
|
+
calls run token lookup on the calling thread. Async calls bound token lookup
|
|
31
|
+
by the RPC deadline when set; pull stream opens use a 30-second lookup budget.
|
|
32
|
+
Declare token with `async def` to invoke it on the event loop. A plain token
|
|
33
|
+
function runs once in the SDK executor; if it returns a coroutine, that
|
|
34
|
+
coroutine runs on the event loop. A loop-bound Future from another loop raises
|
|
35
|
+
TypeError; use `async def` to create it on the right loop. A blocked token keeps
|
|
36
|
+
one executor thread busy until it returns, and an unclosed Client or Worker may
|
|
37
|
+
retain up to four idle credential threads. Sync token lookup must return
|
|
29
38
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
30
39
|
gRPC, so protect the network path that carries the token.
|
|
31
40
|
|
|
@@ -75,6 +84,14 @@ max_concurrent_tasks)` streams because busy streams are replaced with idle
|
|
|
75
84
|
streams and retained after tasks finish. Size these options for the server's
|
|
76
85
|
per-Worker stream cap (256 by default).
|
|
77
86
|
|
|
87
|
+
The Worker rejects an unregistered Step or RPC by name. Dex retries that task
|
|
88
|
+
on another pull Worker without consuming a Step attempt, including when the
|
|
89
|
+
Flow was sticky to the rejecting Worker. If no capable Worker accepts it within
|
|
90
|
+
the server's dispatch wait (10 seconds by default, or the local Step attempt),
|
|
91
|
+
the Step follows its failure and retry policy; an RPC caller receives the named
|
|
92
|
+
error. This reassignment requires a Dex Server that supports it. Against older
|
|
93
|
+
Servers, the Worker returns the not-registered error as before.
|
|
94
|
+
|
|
78
95
|
The generated `app_worker_id` uses the first nonempty `DEX_WORKER_INSTANCE`,
|
|
79
96
|
`POD_NAME`, `HOSTNAME`, or system hostname, plus eight random base32 characters.
|
|
80
97
|
An override must uniquely identify one Worker process and change after restart.
|
|
@@ -281,6 +298,39 @@ failures do not roll back Flow Attributes. `None` preserves current targets;
|
|
|
281
298
|
an explicit empty list disables future synchronization while retaining protocol
|
|
282
299
|
presence.
|
|
283
300
|
|
|
301
|
+
Declare short-lived secrets, such as API keys, with `hard_delete_on_null=True`:
|
|
302
|
+
|
|
303
|
+
```python
|
|
304
|
+
api_key = dex.Attribute("api-key", str, hard_delete_on_null=True)
|
|
305
|
+
provider_tokens = dex.AttributeMap("provider-token", str, hard_delete_on_null=True)
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Dex stores these values only in its blob store, never in Flow history. Deleting
|
|
309
|
+
the Attribute or setting it to `None` permanently deletes the stored value, and a
|
|
310
|
+
deleted value reads as absent. Only string and object values, including
|
|
311
|
+
`bytes`, are supported; `int`, `bool`, and `float` types are rejected. The
|
|
312
|
+
option cannot be combined with an index or `sync_to_attribute_store`, and
|
|
313
|
+
`wait_for_attribute_match` rejects it. Handlers receive the value inline, and
|
|
314
|
+
the SDK never stores it in the local blob cache.
|
|
315
|
+
|
|
316
|
+
Hard-delete writes require a Server with an enabled blob store and Server
|
|
317
|
+
protocol version 3 or later. Workers still start against older Servers.
|
|
318
|
+
There, a Step, RPC, or timeout handler result that writes a hard-delete
|
|
319
|
+
Attribute fails that handler attempt with `RuntimeError`, as does a `wait_for`
|
|
320
|
+
result whose SubFlow start initializes one. A Client checks the Server protocol
|
|
321
|
+
once, before its first `start_flow` that initializes one.
|
|
322
|
+
|
|
323
|
+
Each Flow keeps one stored copy per Attribute key, overwritten in place. A Step
|
|
324
|
+
or RPC attempt writes or deletes the stored value before its result commits, so
|
|
325
|
+
retries, other Steps, and RPCs can observe a write or deletion from an attempt
|
|
326
|
+
that later failed. Concurrent writers of one key may leave a value other than
|
|
327
|
+
the last committed write. A failed `start_flow` or SubFlow start for a running
|
|
328
|
+
Flow ID still overwrites, or for `None` deletes, the running Flow's value.
|
|
329
|
+
Executions that reuse a Flow ID share the value, and time travel does not
|
|
330
|
+
restore a deleted value. Values remain until deleted or until retention and blob
|
|
331
|
+
cleanup remove the Flow. Blob store backups or bucket versioning can retain
|
|
332
|
+
copies.
|
|
333
|
+
|
|
284
334
|
```
|
|
285
335
|
pip install dex-python-sdk==0.1.0
|
|
286
336
|
```
|
|
@@ -689,6 +739,11 @@ Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
|
689
739
|
common protocol, synchronize Attribute indexes, and only then bind
|
|
690
740
|
WorkerService. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
691
741
|
invalid or disjoint intervals, and RPC failures stop startup before binding.
|
|
742
|
+
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
743
|
+
Attributes. Workers record the negotiated version at startup and reject each
|
|
744
|
+
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
745
|
+
Clients negotiate lazily, once, before the first `start_flow` that initializes
|
|
746
|
+
a hard-delete Attribute.
|
|
692
747
|
|
|
693
748
|
The diagnostic SDK version comes from Python distribution metadata for
|
|
694
749
|
`dex-python-sdk` and falls back to `dev` when the distribution is unavailable.
|
|
@@ -16,6 +16,7 @@ from typing import Any, Callable
|
|
|
16
16
|
|
|
17
17
|
from dex._async_value_hydrator import AsyncValueHydrator
|
|
18
18
|
from dex._invocation_context import InvocationContext, InvocationMethod
|
|
19
|
+
from dex._server_protocol import NegotiatedServerProtocol
|
|
19
20
|
from dex._value_mapper import ValueMapper
|
|
20
21
|
from dex._worker_dispatcher import _TIMEOUT_HANDLER_STEP_TYPE, WorkerDispatcher
|
|
21
22
|
from dex.dexpb import dex_pb2 as pb
|
|
@@ -64,10 +65,16 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
64
65
|
registry: Registry,
|
|
65
66
|
values: ValueMapper,
|
|
66
67
|
hydrator: AsyncValueHydrator,
|
|
68
|
+
server_protocol: NegotiatedServerProtocol,
|
|
67
69
|
) -> None:
|
|
70
|
+
if server_protocol is None:
|
|
71
|
+
raise ValueError(
|
|
72
|
+
"AsyncWorkerDispatcher requires a NegotiatedServerProtocol"
|
|
73
|
+
)
|
|
68
74
|
self._registry = registry
|
|
69
75
|
self._values = values
|
|
70
76
|
self._async_hydrator = hydrator
|
|
77
|
+
self._server_protocol = server_protocol
|
|
71
78
|
|
|
72
79
|
async def invoke_wait_for( # type: ignore[override]
|
|
73
80
|
self,
|
|
@@ -127,7 +134,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
127
134
|
if not isinstance(wait, Wait):
|
|
128
135
|
raise TypeError("wait_for must return Wait")
|
|
129
136
|
response = pb.InvokeWaitForMethodResponse(
|
|
130
|
-
upsert_attributes=
|
|
137
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
131
138
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
132
139
|
record_events=context.events,
|
|
133
140
|
publish_to_channel=context.publications,
|
|
@@ -229,7 +236,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
229
236
|
raise TypeError("execute must return StepDecision")
|
|
230
237
|
response = pb.InvokeExecuteMethodResponse(
|
|
231
238
|
step_decision=self._map_decision(flow, decision),
|
|
232
|
-
upsert_attributes=
|
|
239
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
233
240
|
record_events=context.events,
|
|
234
241
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
235
242
|
publish_to_channel=context.publications,
|
|
@@ -309,7 +316,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
309
316
|
raise TypeError("handle_timeout must return StepDecision")
|
|
310
317
|
response = pb.InvokeExecuteMethodResponse(
|
|
311
318
|
step_decision=self._map_decision(flow, decision),
|
|
312
|
-
upsert_attributes=
|
|
319
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
313
320
|
record_events=context.events,
|
|
314
321
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
315
322
|
publish_to_channel=context.publications,
|
|
@@ -361,7 +368,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
361
368
|
returned = await returned
|
|
362
369
|
try:
|
|
363
370
|
response = pb.InvokeWorkerRPCResponse(
|
|
364
|
-
upsert_attributes=
|
|
371
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
365
372
|
record_events=context.events,
|
|
366
373
|
delete_from_channel=context.channel_deletions,
|
|
367
374
|
publish_to_channel=context.publications,
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Sustainable Use License 1.0.
|
|
4
|
+
# You may not use this file except in compliance with the License.
|
|
5
|
+
# See the LICENSE file in the repository root.
|
|
6
|
+
#
|
|
7
|
+
# SPDX-License-Identifier: LicenseRef-Sustainable-Use-1.0
|
|
8
|
+
|
|
9
|
+
"""Attach bearer metadata to outbound gRPC calls."""
|
|
10
|
+
|
|
11
|
+
# mypy: disable-error-code=type-arg
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import inspect
|
|
17
|
+
import queue
|
|
18
|
+
import threading
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from concurrent.futures import Future
|
|
21
|
+
from typing import Any, NamedTuple, cast
|
|
22
|
+
|
|
23
|
+
import grpc
|
|
24
|
+
|
|
25
|
+
from dex.credential import Credential
|
|
26
|
+
|
|
27
|
+
_TOKEN_WORKERS = 4
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AsyncCredentialRunner:
|
|
31
|
+
def __init__(self, credential: Credential) -> None:
|
|
32
|
+
self._credential = credential
|
|
33
|
+
self._lock = threading.Lock()
|
|
34
|
+
self._jobs: queue.SimpleQueue[Future[Any] | None] = queue.SimpleQueue()
|
|
35
|
+
self._threads: list[threading.Thread] = []
|
|
36
|
+
self._pending: set[Future[Any]] = set()
|
|
37
|
+
self._closed = False
|
|
38
|
+
|
|
39
|
+
async def token(self) -> str:
|
|
40
|
+
token_method = self._credential.token
|
|
41
|
+
if inspect.iscoroutinefunction(token_method) or inspect.iscoroutinefunction(
|
|
42
|
+
getattr(token_method, "__call__", None)
|
|
43
|
+
):
|
|
44
|
+
token = token_method()
|
|
45
|
+
if inspect.isawaitable(token):
|
|
46
|
+
return await token
|
|
47
|
+
return token
|
|
48
|
+
token = await asyncio.wrap_future(self._submit())
|
|
49
|
+
if inspect.isawaitable(token):
|
|
50
|
+
if isinstance(token, asyncio.Future):
|
|
51
|
+
try:
|
|
52
|
+
token_loop = token.get_loop()
|
|
53
|
+
except RuntimeError as error:
|
|
54
|
+
raise TypeError(
|
|
55
|
+
"loop-bound credentials must use async def token()"
|
|
56
|
+
) from error
|
|
57
|
+
if token_loop is not asyncio.get_running_loop():
|
|
58
|
+
raise TypeError("loop-bound credentials must use async def token()")
|
|
59
|
+
return await token
|
|
60
|
+
return token
|
|
61
|
+
|
|
62
|
+
def _submit(self) -> Future[Any]:
|
|
63
|
+
with self._lock:
|
|
64
|
+
if self._closed:
|
|
65
|
+
raise RuntimeError("credential runner is closed")
|
|
66
|
+
future: Future[Any] = Future()
|
|
67
|
+
self._pending.add(future)
|
|
68
|
+
if len(self._threads) < _TOKEN_WORKERS:
|
|
69
|
+
thread = threading.Thread(
|
|
70
|
+
target=self._run,
|
|
71
|
+
name="dex-python-credential",
|
|
72
|
+
daemon=True,
|
|
73
|
+
)
|
|
74
|
+
self._threads.append(thread)
|
|
75
|
+
thread.start()
|
|
76
|
+
self._jobs.put(future)
|
|
77
|
+
return future
|
|
78
|
+
|
|
79
|
+
def _run(self) -> None:
|
|
80
|
+
while True:
|
|
81
|
+
future = self._jobs.get()
|
|
82
|
+
if future is None:
|
|
83
|
+
return
|
|
84
|
+
with self._lock:
|
|
85
|
+
if future.done() or not future.set_running_or_notify_cancel():
|
|
86
|
+
self._pending.discard(future)
|
|
87
|
+
continue
|
|
88
|
+
try:
|
|
89
|
+
token = self._credential.token()
|
|
90
|
+
failure = None
|
|
91
|
+
except BaseException as error:
|
|
92
|
+
token = None
|
|
93
|
+
failure = error
|
|
94
|
+
with self._lock:
|
|
95
|
+
if not future.done():
|
|
96
|
+
if failure is None:
|
|
97
|
+
future.set_result(token)
|
|
98
|
+
else:
|
|
99
|
+
future.set_exception(failure)
|
|
100
|
+
self._pending.discard(future)
|
|
101
|
+
|
|
102
|
+
def close(self) -> None:
|
|
103
|
+
with self._lock:
|
|
104
|
+
if self._closed:
|
|
105
|
+
return
|
|
106
|
+
self._closed = True
|
|
107
|
+
for future in self._pending:
|
|
108
|
+
if not future.done():
|
|
109
|
+
future.set_exception(RuntimeError("credential runner is closed"))
|
|
110
|
+
self._pending.clear()
|
|
111
|
+
while True:
|
|
112
|
+
try:
|
|
113
|
+
self._jobs.get_nowait()
|
|
114
|
+
except queue.Empty:
|
|
115
|
+
break
|
|
116
|
+
for _ in self._threads:
|
|
117
|
+
self._jobs.put(None)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _SyncCallDetails(NamedTuple):
|
|
121
|
+
method: str
|
|
122
|
+
timeout: float | None
|
|
123
|
+
metadata: Any
|
|
124
|
+
credentials: grpc.CallCredentials | None
|
|
125
|
+
wait_for_ready: bool | None
|
|
126
|
+
compression: grpc.Compression | None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def credential_metadata(credential: Credential | None, metadata: Any = None) -> Any:
|
|
130
|
+
if credential is None:
|
|
131
|
+
return metadata
|
|
132
|
+
token = credential.token()
|
|
133
|
+
if inspect.isawaitable(token):
|
|
134
|
+
if inspect.iscoroutine(token):
|
|
135
|
+
token.close()
|
|
136
|
+
raise TypeError("synchronous calls require a synchronous credential")
|
|
137
|
+
return tuple(metadata or ()) + (("authorization", "Bearer " + token),)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
async def async_credential_metadata(
|
|
141
|
+
runner: AsyncCredentialRunner, metadata: Any = None, timeout: float | None = None
|
|
142
|
+
) -> Any:
|
|
143
|
+
if timeout is None:
|
|
144
|
+
token = await runner.token()
|
|
145
|
+
else:
|
|
146
|
+
lookup = asyncio.create_task(runner.token())
|
|
147
|
+
try:
|
|
148
|
+
completed, _ = await asyncio.wait({lookup}, timeout=timeout)
|
|
149
|
+
if not completed:
|
|
150
|
+
lookup.cancel()
|
|
151
|
+
await asyncio.gather(lookup, return_exceptions=True)
|
|
152
|
+
raise credential_deadline_error()
|
|
153
|
+
token = lookup.result()
|
|
154
|
+
except BaseException:
|
|
155
|
+
lookup.cancel()
|
|
156
|
+
await asyncio.gather(lookup, return_exceptions=True)
|
|
157
|
+
raise
|
|
158
|
+
return tuple(metadata or ()) + (("authorization", "Bearer " + token),)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def credential_deadline_error() -> grpc.aio.AioRpcError:
|
|
162
|
+
return grpc.aio.AioRpcError(
|
|
163
|
+
grpc.StatusCode.DEADLINE_EXCEEDED,
|
|
164
|
+
grpc.aio.Metadata(),
|
|
165
|
+
grpc.aio.Metadata(),
|
|
166
|
+
"credential lookup exceeded RPC deadline",
|
|
167
|
+
None,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class _SyncCredentialInterceptor(grpc.UnaryUnaryClientInterceptor):
|
|
172
|
+
def __init__(self, credential: Credential) -> None:
|
|
173
|
+
self._credential = credential
|
|
174
|
+
|
|
175
|
+
def intercept_unary_unary(
|
|
176
|
+
self,
|
|
177
|
+
continuation: Callable[..., Any],
|
|
178
|
+
client_call_details: grpc.ClientCallDetails,
|
|
179
|
+
request: Any,
|
|
180
|
+
) -> Any:
|
|
181
|
+
details = _SyncCallDetails(
|
|
182
|
+
client_call_details.method,
|
|
183
|
+
client_call_details.timeout,
|
|
184
|
+
credential_metadata(self._credential, client_call_details.metadata),
|
|
185
|
+
client_call_details.credentials,
|
|
186
|
+
client_call_details.wait_for_ready,
|
|
187
|
+
client_call_details.compression,
|
|
188
|
+
)
|
|
189
|
+
return continuation(details, request)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class _AsyncCredentialInterceptor(grpc.aio.UnaryUnaryClientInterceptor):
|
|
193
|
+
def __init__(self, runner: AsyncCredentialRunner) -> None:
|
|
194
|
+
self._runner = runner
|
|
195
|
+
|
|
196
|
+
async def intercept_unary_unary(
|
|
197
|
+
self,
|
|
198
|
+
continuation: Callable[..., Any],
|
|
199
|
+
client_call_details: grpc.aio.ClientCallDetails,
|
|
200
|
+
request: Any,
|
|
201
|
+
) -> Any:
|
|
202
|
+
loop = asyncio.get_running_loop()
|
|
203
|
+
started_at = loop.time()
|
|
204
|
+
metadata = await async_credential_metadata(
|
|
205
|
+
self._runner, client_call_details.metadata, client_call_details.timeout
|
|
206
|
+
)
|
|
207
|
+
remaining_timeout = client_call_details.timeout
|
|
208
|
+
if remaining_timeout is not None:
|
|
209
|
+
remaining_timeout -= loop.time() - started_at
|
|
210
|
+
if remaining_timeout <= 0:
|
|
211
|
+
raise credential_deadline_error()
|
|
212
|
+
details = grpc.aio.ClientCallDetails(
|
|
213
|
+
client_call_details.method,
|
|
214
|
+
remaining_timeout,
|
|
215
|
+
metadata,
|
|
216
|
+
client_call_details.credentials,
|
|
217
|
+
client_call_details.wait_for_ready,
|
|
218
|
+
)
|
|
219
|
+
return await continuation(details, request)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def sync_channel(server_address: str, credential: Credential | None) -> grpc.Channel:
|
|
223
|
+
channel = grpc.insecure_channel(server_address)
|
|
224
|
+
if credential is None:
|
|
225
|
+
return channel
|
|
226
|
+
return grpc.intercept_channel(channel, _SyncCredentialInterceptor(credential))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def async_channel(
|
|
230
|
+
server_address: str, credential_runner: AsyncCredentialRunner | None
|
|
231
|
+
) -> grpc.aio.Channel:
|
|
232
|
+
if credential_runner is None:
|
|
233
|
+
return grpc.aio.insecure_channel(server_address)
|
|
234
|
+
return grpc.aio.insecure_channel(
|
|
235
|
+
server_address,
|
|
236
|
+
interceptors=[
|
|
237
|
+
cast(
|
|
238
|
+
grpc.aio.ClientInterceptor,
|
|
239
|
+
_AsyncCredentialInterceptor(credential_runner),
|
|
240
|
+
)
|
|
241
|
+
],
|
|
242
|
+
)
|
|
@@ -15,7 +15,12 @@ from urllib.parse import unquote
|
|
|
15
15
|
|
|
16
16
|
from dex._utils import require_map_instance, require_name
|
|
17
17
|
from dex._value_mapper import ValueMapper
|
|
18
|
-
from dex.attribute import
|
|
18
|
+
from dex.attribute import (
|
|
19
|
+
Attribute,
|
|
20
|
+
AttributeMap,
|
|
21
|
+
_apply_attribute_store_sync,
|
|
22
|
+
_apply_hard_delete_on_null,
|
|
23
|
+
)
|
|
19
24
|
from dex.channel import Channel, ChannelMap, ChannelMessage
|
|
20
25
|
from dex.codec import Codec
|
|
21
26
|
from dex.context import RecoveryErrorInfo
|
|
@@ -349,6 +354,7 @@ class InvocationContext:
|
|
|
349
354
|
if index is not None:
|
|
350
355
|
write.index_config.CopyFrom(index)
|
|
351
356
|
_apply_attribute_store_sync(write, definition)
|
|
357
|
+
_apply_hard_delete_on_null(write, definition)
|
|
352
358
|
self.attribute_writes[key] = write
|
|
353
359
|
|
|
354
360
|
def _delete_attribute(
|
|
@@ -366,6 +372,7 @@ class InvocationContext:
|
|
|
366
372
|
if index is not None:
|
|
367
373
|
write.index_config.CopyFrom(index)
|
|
368
374
|
_apply_attribute_store_sync(write, definition)
|
|
375
|
+
_apply_hard_delete_on_null(write, definition)
|
|
369
376
|
self.attribute_writes[key] = write
|
|
370
377
|
|
|
371
378
|
def _publish_channel(
|
|
@@ -22,13 +22,12 @@ from typing import Any
|
|
|
22
22
|
|
|
23
23
|
import grpc
|
|
24
24
|
|
|
25
|
-
from dex._credential_transport import async_credential_metadata
|
|
25
|
+
from dex._credential_transport import AsyncCredentialRunner, async_credential_metadata
|
|
26
26
|
from dex._grpc_errors import _worker_error_status
|
|
27
27
|
from dex._server_protocol import sdk_version
|
|
28
28
|
from dex.dexpb import dex_pb2 as pb
|
|
29
29
|
from dex.dexpb import dex_pb2_grpc
|
|
30
30
|
from dex.flow import Registry
|
|
31
|
-
from dex.credential import Credential
|
|
32
31
|
from dex.worker_options import PullWorkerOptions
|
|
33
32
|
|
|
34
33
|
_LOGGER = logging.getLogger(__name__)
|
|
@@ -81,8 +80,9 @@ class PullWorkerRuntime:
|
|
|
81
80
|
server_address: str,
|
|
82
81
|
executor: ThreadPoolExecutor | None,
|
|
83
82
|
app_worker_id: str | None = None,
|
|
84
|
-
|
|
83
|
+
credential_runner: AsyncCredentialRunner | None = None,
|
|
85
84
|
) -> None:
|
|
85
|
+
self._registry = registry
|
|
86
86
|
self.app_worker_id = app_worker_id or resolve_app_worker_id(registry, options)
|
|
87
87
|
self.flow_types = sorted(registry._registered_flows)
|
|
88
88
|
self.queues = (self.app_worker_id, *self.flow_types)
|
|
@@ -92,7 +92,7 @@ class PullWorkerRuntime:
|
|
|
92
92
|
self._dispatcher = dispatcher
|
|
93
93
|
self._server_address = server_address
|
|
94
94
|
self._executor = executor
|
|
95
|
-
self.
|
|
95
|
+
self._credential_runner = credential_runner
|
|
96
96
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
97
97
|
self._loop_thread_id: int | None = None
|
|
98
98
|
self._started = threading.Event()
|
|
@@ -104,6 +104,26 @@ class PullWorkerRuntime:
|
|
|
104
104
|
self._busy_tasks = 0
|
|
105
105
|
self._unsupported_logged = False
|
|
106
106
|
|
|
107
|
+
def unknown_task_reason(
|
|
108
|
+
self, task: pb.PollWorkerTask
|
|
109
|
+
) -> pb.PollWorkerTaskRejectReason | None:
|
|
110
|
+
kind = task.WhichOneof("request")
|
|
111
|
+
if kind == "rpc":
|
|
112
|
+
flow = self._registry._registered_flows.get(task.rpc.flow_type)
|
|
113
|
+
if flow is None or task.rpc.rpc_name not in flow.rpcs:
|
|
114
|
+
return pb.POLL_WORKER_TASK_REJECT_REASON_UNKNOWN_RPC
|
|
115
|
+
elif kind in ("wait_for", "execute"):
|
|
116
|
+
request = task.wait_for if kind == "wait_for" else task.execute
|
|
117
|
+
flow = self._registry._registered_flows.get(request.flow_type)
|
|
118
|
+
if flow is None:
|
|
119
|
+
return pb.POLL_WORKER_TASK_REJECT_REASON_UNKNOWN_STEP
|
|
120
|
+
if kind == "execute" and request.step_type == "sys:timeout_handler":
|
|
121
|
+
if not flow.has_timeout_handler:
|
|
122
|
+
return pb.POLL_WORKER_TASK_REJECT_REASON_UNKNOWN_STEP
|
|
123
|
+
elif request.step_type not in flow.steps:
|
|
124
|
+
return pb.POLL_WORKER_TASK_REJECT_REASON_UNKNOWN_STEP
|
|
125
|
+
return None
|
|
126
|
+
|
|
107
127
|
async def run(self) -> None:
|
|
108
128
|
self._loop = asyncio.get_running_loop()
|
|
109
129
|
self._loop_thread_id = threading.get_ident()
|
|
@@ -250,6 +270,7 @@ class PullStream:
|
|
|
250
270
|
self._send_lock = asyncio.Lock()
|
|
251
271
|
self._is_reconnecting = False
|
|
252
272
|
self._deadline_handle: asyncio.TimerHandle | None = None
|
|
273
|
+
self._supports_unknown_name_rejection = False
|
|
253
274
|
|
|
254
275
|
def on_done(self, completed: asyncio.Task[None]) -> None:
|
|
255
276
|
self.runtime._stream_done(self, completed)
|
|
@@ -300,12 +321,17 @@ class PullStream:
|
|
|
300
321
|
backoff = min(backoff * 2, 5)
|
|
301
322
|
|
|
302
323
|
async def _poll(self) -> bool:
|
|
303
|
-
if self.runtime.
|
|
324
|
+
if self.runtime.is_stopping:
|
|
325
|
+
return False
|
|
326
|
+
self._supports_unknown_name_rejection = False
|
|
327
|
+
if self.runtime._credential_runner is None:
|
|
304
328
|
self._call = self.stub.PollWorkerTask()
|
|
305
329
|
else:
|
|
306
330
|
metadata = await async_credential_metadata(
|
|
307
|
-
self.runtime.
|
|
331
|
+
self.runtime._credential_runner, timeout=30.0
|
|
308
332
|
)
|
|
333
|
+
if self.runtime.is_stopping:
|
|
334
|
+
return False
|
|
309
335
|
self._call = self.stub.PollWorkerTask(metadata=metadata)
|
|
310
336
|
await self._send(
|
|
311
337
|
pb.PollWorkerTaskRequest(
|
|
@@ -361,6 +387,9 @@ class PullStream:
|
|
|
361
387
|
return False
|
|
362
388
|
if kind != "registered":
|
|
363
389
|
raise RuntimeError("Worker task stream did not register")
|
|
390
|
+
self._supports_unknown_name_rejection = (
|
|
391
|
+
frame.registered.supports_unknown_name_rejection
|
|
392
|
+
)
|
|
364
393
|
was_registered = True
|
|
365
394
|
continue
|
|
366
395
|
if kind == "task":
|
|
@@ -440,6 +469,14 @@ class PullStream:
|
|
|
440
469
|
pass
|
|
441
470
|
|
|
442
471
|
async def _start_task(self, task: pb.PollWorkerTask) -> None:
|
|
472
|
+
unknown_reason = (
|
|
473
|
+
self.runtime.unknown_task_reason(task)
|
|
474
|
+
if self._supports_unknown_name_rejection
|
|
475
|
+
else None
|
|
476
|
+
)
|
|
477
|
+
if unknown_reason is not None:
|
|
478
|
+
await self._reject_task(task, unknown_reason)
|
|
479
|
+
return
|
|
443
480
|
if self.is_busy or self.is_draining or not self.runtime.accept_task():
|
|
444
481
|
await self._reject_task(task)
|
|
445
482
|
return
|
|
@@ -455,12 +492,16 @@ class PullStream:
|
|
|
455
492
|
)
|
|
456
493
|
self.runtime.signal()
|
|
457
494
|
|
|
458
|
-
async def _reject_task(
|
|
495
|
+
async def _reject_task(
|
|
496
|
+
self,
|
|
497
|
+
task: pb.PollWorkerTask,
|
|
498
|
+
reason: pb.PollWorkerTaskRejectReason = pb.POLL_WORKER_TASK_REJECT_REASON_NO_CAPACITY,
|
|
499
|
+
) -> None:
|
|
459
500
|
await self._send(
|
|
460
501
|
pb.PollWorkerTaskRequest(
|
|
461
502
|
reject=pb.PollWorkerTaskReject(
|
|
462
503
|
task_id=task.task_id,
|
|
463
|
-
reason=
|
|
504
|
+
reason=reason,
|
|
464
505
|
)
|
|
465
506
|
)
|
|
466
507
|
)
|