dex-python-sdk 1.10.1__tar.gz → 1.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/PKG-INFO +52 -5
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/README.md +51 -4
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_async_worker_dispatcher.py +11 -4
- dex_python_sdk-1.11.0/dex/_credential_transport.py +242 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_invocation_context.py +8 -1
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_pull_worker.py +9 -6
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_server_protocol.py +33 -1
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_worker_dispatcher.py +24 -5
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/async_client.py +80 -8
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/async_worker.py +37 -16
- dex_python_sdk-1.11.0/dex/attribute.py +546 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/client.py +66 -5
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/credential.py +10 -2
- dex_python_sdk-1.11.0/dex/dexpb/dex_pb2.py +545 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/dexpb/dex_pb2.pyi +4 -2
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/flow.py +17 -1
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/worker.py +21 -9
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/pyproject.toml +1 -1
- dex_python_sdk-1.10.1/dex/_credential_transport.py +0 -142
- dex_python_sdk-1.10.1/dex/attribute.py +0 -313
- dex_python_sdk-1.10.1/dex/dexpb/dex_pb2.py +0 -545
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/LEGACY_NOTICES.md +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/LICENSE +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/__init__.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_async_value_hydrator.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_async_worker_service.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_grpc_errors.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_native.pyi +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_utils.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_value_hydrator.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_value_mapper.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/_worker_service.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/attribute_match.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/blob_cache.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/channel.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/client_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/codec.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/condition.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/context.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/dexpb/__init__.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/dexpb/dex_pb2_grpc.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/flow_config.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/flow_info.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/flow_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/flow_result.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/py.typed +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/rpc_invoke_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/runtime_errors.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/step.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/step_execution.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/stream.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/subflow.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/timer.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/wait.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/wait_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/dex/worker_options.py +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/Cargo.lock +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/Cargo.toml +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/Cargo.toml +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/LICENSE +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/config.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/entry.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/error.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/format.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/lib.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/policy.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/src/store.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache/tests/blob_cache_integration.rs +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache-python/Cargo.toml +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache-python/LICENSE +0 -0
- {dex_python_sdk-1.10.1 → dex_python_sdk-1.11.0}/sdk-rust/crates/dex-blob-cache-python/src/lib.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dex-python-sdk
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.11.0
|
|
4
4
|
Requires-Dist: grpcio>=1.83.0
|
|
5
5
|
Requires-Dist: grpcio-status>=1.83.0
|
|
6
6
|
Requires-Dist: protobuf>=7.35.1
|
|
@@ -38,10 +38,19 @@ startup synchronization, blob hydration, and pull WorkerTaskService streams. A
|
|
|
38
38
|
pull stream reads the token when it opens and again on every reconnect. For a
|
|
39
39
|
custom credential, implement Credential.token() to return the current token;
|
|
40
40
|
the method may be called concurrently, and an exception fails the call before
|
|
41
|
-
it contacts Dex.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
it contacts Dex. Client and AsyncClient calls raise the credential's exception
|
|
42
|
+
unchanged, including TimeoutError when the RPC deadline has not expired.
|
|
43
|
+
AsyncClient and AsyncWorker also accept an awaitable token. Synchronous tokens
|
|
44
|
+
run in a dedicated four-thread executor per async Client or Worker, created
|
|
45
|
+
lazily and shut down without waiting for a blocked token call on close. Sync
|
|
46
|
+
calls run token lookup on the calling thread. Async calls bound token lookup
|
|
47
|
+
by the RPC deadline when set; pull stream opens use a 30-second lookup budget.
|
|
48
|
+
Declare token with `async def` to invoke it on the event loop. A plain token
|
|
49
|
+
function runs once in the SDK executor; if it returns a coroutine, that
|
|
50
|
+
coroutine runs on the event loop. A loop-bound Future from another loop raises
|
|
51
|
+
TypeError; use `async def` to create it on the right loop. A blocked token keeps
|
|
52
|
+
one executor thread busy until it returns, and an unclosed Client or Worker may
|
|
53
|
+
retain up to four idle credential threads. Sync token lookup must return
|
|
45
54
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
46
55
|
gRPC, so protect the network path that carries the token.
|
|
47
56
|
|
|
@@ -297,6 +306,39 @@ failures do not roll back Flow Attributes. `None` preserves current targets;
|
|
|
297
306
|
an explicit empty list disables future synchronization while retaining protocol
|
|
298
307
|
presence.
|
|
299
308
|
|
|
309
|
+
Declare short-lived secrets, such as API keys, with `hard_delete_on_null=True`:
|
|
310
|
+
|
|
311
|
+
```python
|
|
312
|
+
api_key = dex.Attribute("api-key", str, hard_delete_on_null=True)
|
|
313
|
+
provider_tokens = dex.AttributeMap("provider-token", str, hard_delete_on_null=True)
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
Dex stores these values only in its blob store, never in Flow history. Deleting
|
|
317
|
+
the Attribute or setting it to `None` permanently deletes the stored value, and a
|
|
318
|
+
deleted value reads as absent. Only string and object values, including
|
|
319
|
+
`bytes`, are supported; `int`, `bool`, and `float` types are rejected. The
|
|
320
|
+
option cannot be combined with an index or `sync_to_attribute_store`, and
|
|
321
|
+
`wait_for_attribute_match` rejects it. Handlers receive the value inline, and
|
|
322
|
+
the SDK never stores it in the local blob cache.
|
|
323
|
+
|
|
324
|
+
Hard-delete writes require a Server with an enabled blob store and Server
|
|
325
|
+
protocol version 3 or later. Workers still start against older Servers.
|
|
326
|
+
There, a Step, RPC, or timeout handler result that writes a hard-delete
|
|
327
|
+
Attribute fails that handler attempt with `RuntimeError`, as does a `wait_for`
|
|
328
|
+
result whose SubFlow start initializes one. A Client checks the Server protocol
|
|
329
|
+
once, before its first `start_flow` that initializes one.
|
|
330
|
+
|
|
331
|
+
Each Flow keeps one stored copy per Attribute key, overwritten in place. A Step
|
|
332
|
+
or RPC attempt writes or deletes the stored value before its result commits, so
|
|
333
|
+
retries, other Steps, and RPCs can observe a write or deletion from an attempt
|
|
334
|
+
that later failed. Concurrent writers of one key may leave a value other than
|
|
335
|
+
the last committed write. A failed `start_flow` or SubFlow start for a running
|
|
336
|
+
Flow ID still overwrites, or for `None` deletes, the running Flow's value.
|
|
337
|
+
Executions that reuse a Flow ID share the value, and time travel does not
|
|
338
|
+
restore a deleted value. Values remain until deleted or until retention and blob
|
|
339
|
+
cleanup remove the Flow. Blob store backups or bucket versioning can retain
|
|
340
|
+
copies.
|
|
341
|
+
|
|
300
342
|
```
|
|
301
343
|
pip install dex-python-sdk==0.1.0
|
|
302
344
|
```
|
|
@@ -705,6 +747,11 @@ Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
|
705
747
|
common protocol, synchronize Attribute indexes, and only then bind
|
|
706
748
|
WorkerService. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
707
749
|
invalid or disjoint intervals, and RPC failures stop startup before binding.
|
|
750
|
+
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
751
|
+
Attributes. Workers record the negotiated version at startup and reject each
|
|
752
|
+
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
753
|
+
Clients negotiate lazily, once, before the first `start_flow` that initializes
|
|
754
|
+
a hard-delete Attribute.
|
|
708
755
|
|
|
709
756
|
The diagnostic SDK version comes from Python distribution metadata for
|
|
710
757
|
`dex-python-sdk` and falls back to `dev` when the distribution is unavailable.
|
|
@@ -22,10 +22,19 @@ startup synchronization, blob hydration, and pull WorkerTaskService streams. A
|
|
|
22
22
|
pull stream reads the token when it opens and again on every reconnect. For a
|
|
23
23
|
custom credential, implement Credential.token() to return the current token;
|
|
24
24
|
the method may be called concurrently, and an exception fails the call before
|
|
25
|
-
it contacts Dex.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
25
|
+
it contacts Dex. Client and AsyncClient calls raise the credential's exception
|
|
26
|
+
unchanged, including TimeoutError when the RPC deadline has not expired.
|
|
27
|
+
AsyncClient and AsyncWorker also accept an awaitable token. Synchronous tokens
|
|
28
|
+
run in a dedicated four-thread executor per async Client or Worker, created
|
|
29
|
+
lazily and shut down without waiting for a blocked token call on close. Sync
|
|
30
|
+
calls run token lookup on the calling thread. Async calls bound token lookup
|
|
31
|
+
by the RPC deadline when set; pull stream opens use a 30-second lookup budget.
|
|
32
|
+
Declare token with `async def` to invoke it on the event loop. A plain token
|
|
33
|
+
function runs once in the SDK executor; if it returns a coroutine, that
|
|
34
|
+
coroutine runs on the event loop. A loop-bound Future from another loop raises
|
|
35
|
+
TypeError; use `async def` to create it on the right loop. A blocked token keeps
|
|
36
|
+
one executor thread busy until it returns, and an unclosed Client or Worker may
|
|
37
|
+
retain up to four idle credential threads. Sync token lookup must return
|
|
29
38
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
30
39
|
gRPC, so protect the network path that carries the token.
|
|
31
40
|
|
|
@@ -281,6 +290,39 @@ failures do not roll back Flow Attributes. `None` preserves current targets;
|
|
|
281
290
|
an explicit empty list disables future synchronization while retaining protocol
|
|
282
291
|
presence.
|
|
283
292
|
|
|
293
|
+
Declare short-lived secrets, such as API keys, with `hard_delete_on_null=True`:
|
|
294
|
+
|
|
295
|
+
```python
|
|
296
|
+
api_key = dex.Attribute("api-key", str, hard_delete_on_null=True)
|
|
297
|
+
provider_tokens = dex.AttributeMap("provider-token", str, hard_delete_on_null=True)
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
Dex stores these values only in its blob store, never in Flow history. Deleting
|
|
301
|
+
the Attribute or setting it to `None` permanently deletes the stored value, and a
|
|
302
|
+
deleted value reads as absent. Only string and object values, including
|
|
303
|
+
`bytes`, are supported; `int`, `bool`, and `float` types are rejected. The
|
|
304
|
+
option cannot be combined with an index or `sync_to_attribute_store`, and
|
|
305
|
+
`wait_for_attribute_match` rejects it. Handlers receive the value inline, and
|
|
306
|
+
the SDK never stores it in the local blob cache.
|
|
307
|
+
|
|
308
|
+
Hard-delete writes require a Server with an enabled blob store and Server
|
|
309
|
+
protocol version 3 or later. Workers still start against older Servers.
|
|
310
|
+
There, a Step, RPC, or timeout handler result that writes a hard-delete
|
|
311
|
+
Attribute fails that handler attempt with `RuntimeError`, as does a `wait_for`
|
|
312
|
+
result whose SubFlow start initializes one. A Client checks the Server protocol
|
|
313
|
+
once, before its first `start_flow` that initializes one.
|
|
314
|
+
|
|
315
|
+
Each Flow keeps one stored copy per Attribute key, overwritten in place. A Step
|
|
316
|
+
or RPC attempt writes or deletes the stored value before its result commits, so
|
|
317
|
+
retries, other Steps, and RPCs can observe a write or deletion from an attempt
|
|
318
|
+
that later failed. Concurrent writers of one key may leave a value other than
|
|
319
|
+
the last committed write. A failed `start_flow` or SubFlow start for a running
|
|
320
|
+
Flow ID still overwrites, or for `None` deletes, the running Flow's value.
|
|
321
|
+
Executions that reuse a Flow ID share the value, and time travel does not
|
|
322
|
+
restore a deleted value. Values remain until deleted or until retention and blob
|
|
323
|
+
cleanup remove the Flow. Blob store backups or bucket versioning can retain
|
|
324
|
+
copies.
|
|
325
|
+
|
|
284
326
|
```
|
|
285
327
|
pip install dex-python-sdk==0.1.0
|
|
286
328
|
```
|
|
@@ -689,6 +731,11 @@ Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
|
689
731
|
common protocol, synchronize Attribute indexes, and only then bind
|
|
690
732
|
WorkerService. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
691
733
|
invalid or disjoint intervals, and RPC failures stop startup before binding.
|
|
734
|
+
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
735
|
+
Attributes. Workers record the negotiated version at startup and reject each
|
|
736
|
+
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
737
|
+
Clients negotiate lazily, once, before the first `start_flow` that initializes
|
|
738
|
+
a hard-delete Attribute.
|
|
692
739
|
|
|
693
740
|
The diagnostic SDK version comes from Python distribution metadata for
|
|
694
741
|
`dex-python-sdk` and falls back to `dev` when the distribution is unavailable.
|
|
@@ -16,6 +16,7 @@ from typing import Any, Callable
|
|
|
16
16
|
|
|
17
17
|
from dex._async_value_hydrator import AsyncValueHydrator
|
|
18
18
|
from dex._invocation_context import InvocationContext, InvocationMethod
|
|
19
|
+
from dex._server_protocol import NegotiatedServerProtocol
|
|
19
20
|
from dex._value_mapper import ValueMapper
|
|
20
21
|
from dex._worker_dispatcher import _TIMEOUT_HANDLER_STEP_TYPE, WorkerDispatcher
|
|
21
22
|
from dex.dexpb import dex_pb2 as pb
|
|
@@ -64,10 +65,16 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
64
65
|
registry: Registry,
|
|
65
66
|
values: ValueMapper,
|
|
66
67
|
hydrator: AsyncValueHydrator,
|
|
68
|
+
server_protocol: NegotiatedServerProtocol,
|
|
67
69
|
) -> None:
|
|
70
|
+
if server_protocol is None:
|
|
71
|
+
raise ValueError(
|
|
72
|
+
"AsyncWorkerDispatcher requires a NegotiatedServerProtocol"
|
|
73
|
+
)
|
|
68
74
|
self._registry = registry
|
|
69
75
|
self._values = values
|
|
70
76
|
self._async_hydrator = hydrator
|
|
77
|
+
self._server_protocol = server_protocol
|
|
71
78
|
|
|
72
79
|
async def invoke_wait_for( # type: ignore[override]
|
|
73
80
|
self,
|
|
@@ -127,7 +134,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
127
134
|
if not isinstance(wait, Wait):
|
|
128
135
|
raise TypeError("wait_for must return Wait")
|
|
129
136
|
response = pb.InvokeWaitForMethodResponse(
|
|
130
|
-
upsert_attributes=
|
|
137
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
131
138
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
132
139
|
record_events=context.events,
|
|
133
140
|
publish_to_channel=context.publications,
|
|
@@ -229,7 +236,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
229
236
|
raise TypeError("execute must return StepDecision")
|
|
230
237
|
response = pb.InvokeExecuteMethodResponse(
|
|
231
238
|
step_decision=self._map_decision(flow, decision),
|
|
232
|
-
upsert_attributes=
|
|
239
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
233
240
|
record_events=context.events,
|
|
234
241
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
235
242
|
publish_to_channel=context.publications,
|
|
@@ -309,7 +316,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
309
316
|
raise TypeError("handle_timeout must return StepDecision")
|
|
310
317
|
response = pb.InvokeExecuteMethodResponse(
|
|
311
318
|
step_decision=self._map_decision(flow, decision),
|
|
312
|
-
upsert_attributes=
|
|
319
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
313
320
|
record_events=context.events,
|
|
314
321
|
upsert_step_exe_locals=list(context.local_writes.values()),
|
|
315
322
|
publish_to_channel=context.publications,
|
|
@@ -361,7 +368,7 @@ class AsyncWorkerDispatcher(WorkerDispatcher):
|
|
|
361
368
|
returned = await returned
|
|
362
369
|
try:
|
|
363
370
|
response = pb.InvokeWorkerRPCResponse(
|
|
364
|
-
upsert_attributes=
|
|
371
|
+
upsert_attributes=self._result_attribute_writes(context),
|
|
365
372
|
record_events=context.events,
|
|
366
373
|
delete_from_channel=context.channel_deletions,
|
|
367
374
|
publish_to_channel=context.publications,
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Sustainable Use License 1.0.
|
|
4
|
+
# You may not use this file except in compliance with the License.
|
|
5
|
+
# See the LICENSE file in the repository root.
|
|
6
|
+
#
|
|
7
|
+
# SPDX-License-Identifier: LicenseRef-Sustainable-Use-1.0
|
|
8
|
+
|
|
9
|
+
"""Attach bearer metadata to outbound gRPC calls."""
|
|
10
|
+
|
|
11
|
+
# mypy: disable-error-code=type-arg
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import inspect
|
|
17
|
+
import queue
|
|
18
|
+
import threading
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from concurrent.futures import Future
|
|
21
|
+
from typing import Any, NamedTuple, cast
|
|
22
|
+
|
|
23
|
+
import grpc
|
|
24
|
+
|
|
25
|
+
from dex.credential import Credential
|
|
26
|
+
|
|
27
|
+
_TOKEN_WORKERS = 4
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AsyncCredentialRunner:
|
|
31
|
+
def __init__(self, credential: Credential) -> None:
|
|
32
|
+
self._credential = credential
|
|
33
|
+
self._lock = threading.Lock()
|
|
34
|
+
self._jobs: queue.SimpleQueue[Future[Any] | None] = queue.SimpleQueue()
|
|
35
|
+
self._threads: list[threading.Thread] = []
|
|
36
|
+
self._pending: set[Future[Any]] = set()
|
|
37
|
+
self._closed = False
|
|
38
|
+
|
|
39
|
+
async def token(self) -> str:
|
|
40
|
+
token_method = self._credential.token
|
|
41
|
+
if inspect.iscoroutinefunction(token_method) or inspect.iscoroutinefunction(
|
|
42
|
+
getattr(token_method, "__call__", None)
|
|
43
|
+
):
|
|
44
|
+
token = token_method()
|
|
45
|
+
if inspect.isawaitable(token):
|
|
46
|
+
return await token
|
|
47
|
+
return token
|
|
48
|
+
token = await asyncio.wrap_future(self._submit())
|
|
49
|
+
if inspect.isawaitable(token):
|
|
50
|
+
if isinstance(token, asyncio.Future):
|
|
51
|
+
try:
|
|
52
|
+
token_loop = token.get_loop()
|
|
53
|
+
except RuntimeError as error:
|
|
54
|
+
raise TypeError(
|
|
55
|
+
"loop-bound credentials must use async def token()"
|
|
56
|
+
) from error
|
|
57
|
+
if token_loop is not asyncio.get_running_loop():
|
|
58
|
+
raise TypeError("loop-bound credentials must use async def token()")
|
|
59
|
+
return await token
|
|
60
|
+
return token
|
|
61
|
+
|
|
62
|
+
def _submit(self) -> Future[Any]:
|
|
63
|
+
with self._lock:
|
|
64
|
+
if self._closed:
|
|
65
|
+
raise RuntimeError("credential runner is closed")
|
|
66
|
+
future: Future[Any] = Future()
|
|
67
|
+
self._pending.add(future)
|
|
68
|
+
if len(self._threads) < _TOKEN_WORKERS:
|
|
69
|
+
thread = threading.Thread(
|
|
70
|
+
target=self._run,
|
|
71
|
+
name="dex-python-credential",
|
|
72
|
+
daemon=True,
|
|
73
|
+
)
|
|
74
|
+
self._threads.append(thread)
|
|
75
|
+
thread.start()
|
|
76
|
+
self._jobs.put(future)
|
|
77
|
+
return future
|
|
78
|
+
|
|
79
|
+
def _run(self) -> None:
|
|
80
|
+
while True:
|
|
81
|
+
future = self._jobs.get()
|
|
82
|
+
if future is None:
|
|
83
|
+
return
|
|
84
|
+
with self._lock:
|
|
85
|
+
if future.done() or not future.set_running_or_notify_cancel():
|
|
86
|
+
self._pending.discard(future)
|
|
87
|
+
continue
|
|
88
|
+
try:
|
|
89
|
+
token = self._credential.token()
|
|
90
|
+
failure = None
|
|
91
|
+
except BaseException as error:
|
|
92
|
+
token = None
|
|
93
|
+
failure = error
|
|
94
|
+
with self._lock:
|
|
95
|
+
if not future.done():
|
|
96
|
+
if failure is None:
|
|
97
|
+
future.set_result(token)
|
|
98
|
+
else:
|
|
99
|
+
future.set_exception(failure)
|
|
100
|
+
self._pending.discard(future)
|
|
101
|
+
|
|
102
|
+
def close(self) -> None:
|
|
103
|
+
with self._lock:
|
|
104
|
+
if self._closed:
|
|
105
|
+
return
|
|
106
|
+
self._closed = True
|
|
107
|
+
for future in self._pending:
|
|
108
|
+
if not future.done():
|
|
109
|
+
future.set_exception(RuntimeError("credential runner is closed"))
|
|
110
|
+
self._pending.clear()
|
|
111
|
+
while True:
|
|
112
|
+
try:
|
|
113
|
+
self._jobs.get_nowait()
|
|
114
|
+
except queue.Empty:
|
|
115
|
+
break
|
|
116
|
+
for _ in self._threads:
|
|
117
|
+
self._jobs.put(None)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _SyncCallDetails(NamedTuple):
|
|
121
|
+
method: str
|
|
122
|
+
timeout: float | None
|
|
123
|
+
metadata: Any
|
|
124
|
+
credentials: grpc.CallCredentials | None
|
|
125
|
+
wait_for_ready: bool | None
|
|
126
|
+
compression: grpc.Compression | None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def credential_metadata(credential: Credential | None, metadata: Any = None) -> Any:
|
|
130
|
+
if credential is None:
|
|
131
|
+
return metadata
|
|
132
|
+
token = credential.token()
|
|
133
|
+
if inspect.isawaitable(token):
|
|
134
|
+
if inspect.iscoroutine(token):
|
|
135
|
+
token.close()
|
|
136
|
+
raise TypeError("synchronous calls require a synchronous credential")
|
|
137
|
+
return tuple(metadata or ()) + (("authorization", "Bearer " + token),)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
async def async_credential_metadata(
|
|
141
|
+
runner: AsyncCredentialRunner, metadata: Any = None, timeout: float | None = None
|
|
142
|
+
) -> Any:
|
|
143
|
+
if timeout is None:
|
|
144
|
+
token = await runner.token()
|
|
145
|
+
else:
|
|
146
|
+
lookup = asyncio.create_task(runner.token())
|
|
147
|
+
try:
|
|
148
|
+
completed, _ = await asyncio.wait({lookup}, timeout=timeout)
|
|
149
|
+
if not completed:
|
|
150
|
+
lookup.cancel()
|
|
151
|
+
await asyncio.gather(lookup, return_exceptions=True)
|
|
152
|
+
raise credential_deadline_error()
|
|
153
|
+
token = lookup.result()
|
|
154
|
+
except BaseException:
|
|
155
|
+
lookup.cancel()
|
|
156
|
+
await asyncio.gather(lookup, return_exceptions=True)
|
|
157
|
+
raise
|
|
158
|
+
return tuple(metadata or ()) + (("authorization", "Bearer " + token),)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def credential_deadline_error() -> grpc.aio.AioRpcError:
|
|
162
|
+
return grpc.aio.AioRpcError(
|
|
163
|
+
grpc.StatusCode.DEADLINE_EXCEEDED,
|
|
164
|
+
grpc.aio.Metadata(),
|
|
165
|
+
grpc.aio.Metadata(),
|
|
166
|
+
"credential lookup exceeded RPC deadline",
|
|
167
|
+
None,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class _SyncCredentialInterceptor(grpc.UnaryUnaryClientInterceptor):
|
|
172
|
+
def __init__(self, credential: Credential) -> None:
|
|
173
|
+
self._credential = credential
|
|
174
|
+
|
|
175
|
+
def intercept_unary_unary(
|
|
176
|
+
self,
|
|
177
|
+
continuation: Callable[..., Any],
|
|
178
|
+
client_call_details: grpc.ClientCallDetails,
|
|
179
|
+
request: Any,
|
|
180
|
+
) -> Any:
|
|
181
|
+
details = _SyncCallDetails(
|
|
182
|
+
client_call_details.method,
|
|
183
|
+
client_call_details.timeout,
|
|
184
|
+
credential_metadata(self._credential, client_call_details.metadata),
|
|
185
|
+
client_call_details.credentials,
|
|
186
|
+
client_call_details.wait_for_ready,
|
|
187
|
+
client_call_details.compression,
|
|
188
|
+
)
|
|
189
|
+
return continuation(details, request)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class _AsyncCredentialInterceptor(grpc.aio.UnaryUnaryClientInterceptor):
|
|
193
|
+
def __init__(self, runner: AsyncCredentialRunner) -> None:
|
|
194
|
+
self._runner = runner
|
|
195
|
+
|
|
196
|
+
async def intercept_unary_unary(
|
|
197
|
+
self,
|
|
198
|
+
continuation: Callable[..., Any],
|
|
199
|
+
client_call_details: grpc.aio.ClientCallDetails,
|
|
200
|
+
request: Any,
|
|
201
|
+
) -> Any:
|
|
202
|
+
loop = asyncio.get_running_loop()
|
|
203
|
+
started_at = loop.time()
|
|
204
|
+
metadata = await async_credential_metadata(
|
|
205
|
+
self._runner, client_call_details.metadata, client_call_details.timeout
|
|
206
|
+
)
|
|
207
|
+
remaining_timeout = client_call_details.timeout
|
|
208
|
+
if remaining_timeout is not None:
|
|
209
|
+
remaining_timeout -= loop.time() - started_at
|
|
210
|
+
if remaining_timeout <= 0:
|
|
211
|
+
raise credential_deadline_error()
|
|
212
|
+
details = grpc.aio.ClientCallDetails(
|
|
213
|
+
client_call_details.method,
|
|
214
|
+
remaining_timeout,
|
|
215
|
+
metadata,
|
|
216
|
+
client_call_details.credentials,
|
|
217
|
+
client_call_details.wait_for_ready,
|
|
218
|
+
)
|
|
219
|
+
return await continuation(details, request)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def sync_channel(server_address: str, credential: Credential | None) -> grpc.Channel:
|
|
223
|
+
channel = grpc.insecure_channel(server_address)
|
|
224
|
+
if credential is None:
|
|
225
|
+
return channel
|
|
226
|
+
return grpc.intercept_channel(channel, _SyncCredentialInterceptor(credential))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def async_channel(
|
|
230
|
+
server_address: str, credential_runner: AsyncCredentialRunner | None
|
|
231
|
+
) -> grpc.aio.Channel:
|
|
232
|
+
if credential_runner is None:
|
|
233
|
+
return grpc.aio.insecure_channel(server_address)
|
|
234
|
+
return grpc.aio.insecure_channel(
|
|
235
|
+
server_address,
|
|
236
|
+
interceptors=[
|
|
237
|
+
cast(
|
|
238
|
+
grpc.aio.ClientInterceptor,
|
|
239
|
+
_AsyncCredentialInterceptor(credential_runner),
|
|
240
|
+
)
|
|
241
|
+
],
|
|
242
|
+
)
|
|
@@ -15,7 +15,12 @@ from urllib.parse import unquote
|
|
|
15
15
|
|
|
16
16
|
from dex._utils import require_map_instance, require_name
|
|
17
17
|
from dex._value_mapper import ValueMapper
|
|
18
|
-
from dex.attribute import
|
|
18
|
+
from dex.attribute import (
|
|
19
|
+
Attribute,
|
|
20
|
+
AttributeMap,
|
|
21
|
+
_apply_attribute_store_sync,
|
|
22
|
+
_apply_hard_delete_on_null,
|
|
23
|
+
)
|
|
19
24
|
from dex.channel import Channel, ChannelMap, ChannelMessage
|
|
20
25
|
from dex.codec import Codec
|
|
21
26
|
from dex.context import RecoveryErrorInfo
|
|
@@ -349,6 +354,7 @@ class InvocationContext:
|
|
|
349
354
|
if index is not None:
|
|
350
355
|
write.index_config.CopyFrom(index)
|
|
351
356
|
_apply_attribute_store_sync(write, definition)
|
|
357
|
+
_apply_hard_delete_on_null(write, definition)
|
|
352
358
|
self.attribute_writes[key] = write
|
|
353
359
|
|
|
354
360
|
def _delete_attribute(
|
|
@@ -366,6 +372,7 @@ class InvocationContext:
|
|
|
366
372
|
if index is not None:
|
|
367
373
|
write.index_config.CopyFrom(index)
|
|
368
374
|
_apply_attribute_store_sync(write, definition)
|
|
375
|
+
_apply_hard_delete_on_null(write, definition)
|
|
369
376
|
self.attribute_writes[key] = write
|
|
370
377
|
|
|
371
378
|
def _publish_channel(
|
|
@@ -22,13 +22,12 @@ from typing import Any
|
|
|
22
22
|
|
|
23
23
|
import grpc
|
|
24
24
|
|
|
25
|
-
from dex._credential_transport import async_credential_metadata
|
|
25
|
+
from dex._credential_transport import AsyncCredentialRunner, async_credential_metadata
|
|
26
26
|
from dex._grpc_errors import _worker_error_status
|
|
27
27
|
from dex._server_protocol import sdk_version
|
|
28
28
|
from dex.dexpb import dex_pb2 as pb
|
|
29
29
|
from dex.dexpb import dex_pb2_grpc
|
|
30
30
|
from dex.flow import Registry
|
|
31
|
-
from dex.credential import Credential
|
|
32
31
|
from dex.worker_options import PullWorkerOptions
|
|
33
32
|
|
|
34
33
|
_LOGGER = logging.getLogger(__name__)
|
|
@@ -81,7 +80,7 @@ class PullWorkerRuntime:
|
|
|
81
80
|
server_address: str,
|
|
82
81
|
executor: ThreadPoolExecutor | None,
|
|
83
82
|
app_worker_id: str | None = None,
|
|
84
|
-
|
|
83
|
+
credential_runner: AsyncCredentialRunner | None = None,
|
|
85
84
|
) -> None:
|
|
86
85
|
self.app_worker_id = app_worker_id or resolve_app_worker_id(registry, options)
|
|
87
86
|
self.flow_types = sorted(registry._registered_flows)
|
|
@@ -92,7 +91,7 @@ class PullWorkerRuntime:
|
|
|
92
91
|
self._dispatcher = dispatcher
|
|
93
92
|
self._server_address = server_address
|
|
94
93
|
self._executor = executor
|
|
95
|
-
self.
|
|
94
|
+
self._credential_runner = credential_runner
|
|
96
95
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
97
96
|
self._loop_thread_id: int | None = None
|
|
98
97
|
self._started = threading.Event()
|
|
@@ -300,12 +299,16 @@ class PullStream:
|
|
|
300
299
|
backoff = min(backoff * 2, 5)
|
|
301
300
|
|
|
302
301
|
async def _poll(self) -> bool:
|
|
303
|
-
if self.runtime.
|
|
302
|
+
if self.runtime.is_stopping:
|
|
303
|
+
return False
|
|
304
|
+
if self.runtime._credential_runner is None:
|
|
304
305
|
self._call = self.stub.PollWorkerTask()
|
|
305
306
|
else:
|
|
306
307
|
metadata = await async_credential_metadata(
|
|
307
|
-
self.runtime.
|
|
308
|
+
self.runtime._credential_runner, timeout=30.0
|
|
308
309
|
)
|
|
310
|
+
if self.runtime.is_stopping:
|
|
311
|
+
return False
|
|
309
312
|
self._call = self.stub.PollWorkerTask(metadata=metadata)
|
|
310
313
|
await self._send(
|
|
311
314
|
pb.PollWorkerTaskRequest(
|
|
@@ -9,13 +9,15 @@
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
from importlib.metadata import PackageNotFoundError, version
|
|
12
|
+
from typing import Iterable
|
|
12
13
|
|
|
13
14
|
import grpc
|
|
14
15
|
|
|
15
16
|
from dex.dexpb import dex_pb2 as pb
|
|
16
17
|
|
|
17
18
|
MINIMUM_SUPPORTED_SERVER_PROTOCOL_VERSION = 1
|
|
18
|
-
MAXIMUM_SUPPORTED_SERVER_PROTOCOL_VERSION =
|
|
19
|
+
MAXIMUM_SUPPORTED_SERVER_PROTOCOL_VERSION = 3
|
|
20
|
+
HARD_DELETE_ATTRIBUTE_PROTOCOL_VERSION = 3
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
def sdk_version() -> str:
|
|
@@ -64,6 +66,36 @@ def negotiate_server_protocol(server_info: pb.ServerInfo, artifact_version: str)
|
|
|
64
66
|
return negotiated
|
|
65
67
|
|
|
66
68
|
|
|
69
|
+
class NegotiatedServerProtocol:
|
|
70
|
+
def __init__(self) -> None:
|
|
71
|
+
self._version = 0
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def version(self) -> int:
|
|
75
|
+
return self._version
|
|
76
|
+
|
|
77
|
+
def record_version(self, version: int) -> None:
|
|
78
|
+
self._version = version
|
|
79
|
+
|
|
80
|
+
def require_hard_delete_attribute_support(
|
|
81
|
+
self,
|
|
82
|
+
writes: Iterable[pb.AttributeWrite],
|
|
83
|
+
) -> None:
|
|
84
|
+
attribute_names = [
|
|
85
|
+
write.key for write in writes if write.is_hard_delete_on_null
|
|
86
|
+
]
|
|
87
|
+
if (
|
|
88
|
+
not attribute_names
|
|
89
|
+
or self._version >= HARD_DELETE_ATTRIBUTE_PROTOCOL_VERSION
|
|
90
|
+
):
|
|
91
|
+
return
|
|
92
|
+
raise RuntimeError(
|
|
93
|
+
"hard-delete Attribute writes require Dex Server protocol version "
|
|
94
|
+
f"{HARD_DELETE_ATTRIBUTE_PROTOCOL_VERSION} or later, but the negotiated "
|
|
95
|
+
f"protocol version is {self._version}: {', '.join(attribute_names)}"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
67
99
|
def server_info_request_error(
|
|
68
100
|
artifact_version: str, failure: grpc.RpcError
|
|
69
101
|
) -> RuntimeError:
|