dex-python-sdk 1.23.0__tar.gz → 1.25.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/PKG-INFO +21 -21
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/README.md +20 -20
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/__init__.py +1 -2
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_grpc_errors.py +1 -18
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_pull_worker.py +15 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_worker_dispatcher.py +4 -8
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/async_client.py +12 -22
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/async_worker.py +33 -106
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/client.py +10 -23
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/client_options.py +1 -5
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/flow_config.py +2 -5
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/pull_registration.py +1 -1
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/runtime_errors.py +1 -1
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/worker.py +35 -124
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/worker_options.py +13 -70
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/pyproject.toml +1 -1
- dex_python_sdk-1.23.0/dex/_async_worker_service.py +0 -122
- dex_python_sdk-1.23.0/dex/_worker_service.py +0 -94
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/LEGACY_NOTICES.md +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/LICENSE +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_async_value_hydrator.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_async_worker_dispatcher.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_credential_transport.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_invocation_context.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_native.pyi +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_server_protocol.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_utils.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_value_hydrator.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/_value_mapper.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/attribute.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/attribute_match.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/blob_cache.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/channel.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/codec.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/condition.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/context.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/credential.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/dexpb/__init__.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/dexpb/dex_pb2.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/dexpb/dex_pb2.pyi +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/dexpb/dex_pb2_grpc.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/flow.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/flow_info.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/flow_options.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/flow_result.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/py.typed +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/rpc_invoke_options.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/step.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/step_execution.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/stream.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/subflow.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/timer.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/wait.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/dex/wait_options.py +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/Cargo.lock +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/Cargo.toml +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/Cargo.toml +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/LICENSE +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/README.md +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/config.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/entry.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/error.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/format.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/lib.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/policy.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/src/store.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache/tests/blob_cache_integration.rs +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache-python/Cargo.toml +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache-python/LICENSE +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache-python/README.md +0 -0
- {dex_python_sdk-1.23.0 → dex_python_sdk-1.25.0}/sdk-rust/crates/dex-blob-cache-python/src/lib.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dex-python-sdk
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.25.0
|
|
4
4
|
Requires-Dist: grpcio>=1.83.0
|
|
5
5
|
Requires-Dist: grpcio-status>=1.83.0
|
|
6
6
|
Requires-Dist: protobuf>=7.35.1
|
|
@@ -95,14 +95,17 @@ retain up to four idle credential threads. Sync token lookup must return
|
|
|
95
95
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
96
96
|
gRPC, so protect the network path that carries the token.
|
|
97
97
|
|
|
98
|
-
##
|
|
98
|
+
## Workers
|
|
99
99
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
`
|
|
104
|
-
|
|
105
|
-
SubFlow
|
|
100
|
+
Workers receive tasks through outbound pull streams. `Worker` and `AsyncWorker`
|
|
101
|
+
need only an outbound connection to the Dex Server at `server_address`. Configure
|
|
102
|
+
capacity and shutdown with `WorkerOptions.pull`; its default is
|
|
103
|
+
`PullWorkerOptions()`. Clients and Flows use pull routing automatically.
|
|
104
|
+
|
|
105
|
+
StartFlow and SubFlow always select pull Workers. Updating Flow configuration
|
|
106
|
+
preserves its existing Worker target; an empty `FlowConfig()` changes nothing.
|
|
107
|
+
Before upgrading an application with active push Flows, use an SDK version that
|
|
108
|
+
supports Worker targets to migrate those Flows to pull routing.
|
|
106
109
|
|
|
107
110
|
```python
|
|
108
111
|
import threading
|
|
@@ -118,10 +121,11 @@ worker = Worker(
|
|
|
118
121
|
)
|
|
119
122
|
worker_thread = threading.Thread(target=worker.start)
|
|
120
123
|
worker_thread.start()
|
|
124
|
+
worker.wait_pull_registered(timeout=30)
|
|
121
125
|
client = Client(
|
|
122
126
|
registry,
|
|
123
127
|
cache,
|
|
124
|
-
ClientOptions(server_address="dex.example:8801"
|
|
128
|
+
ClientOptions(server_address="dex.example:8801"),
|
|
125
129
|
)
|
|
126
130
|
try:
|
|
127
131
|
client.start_flow(flow, flow_id, input_value)
|
|
@@ -160,10 +164,7 @@ The generated `app_worker_id` uses the first nonempty `DEX_WORKER_INSTANCE`,
|
|
|
160
164
|
`POD_NAME`, `HOSTNAME`, or system hostname, plus eight random base32 characters.
|
|
161
165
|
An override must uniquely identify one Worker process and change after restart.
|
|
162
166
|
It must contain 1-63 ASCII letters, digits, dots, underscores, or hyphens and
|
|
163
|
-
must not equal a registered Flow type.
|
|
164
|
-
or an advertised address or headless target. Use `WorkerTarget(pull=True)` for
|
|
165
|
-
Client defaults or Flow overrides when the Worker target is not directly in
|
|
166
|
-
scope.
|
|
167
|
+
must not equal a registered Flow type.
|
|
167
168
|
|
|
168
169
|
Pull streams reconnect after failures. A server-directed reconnect waits for
|
|
169
170
|
an active task to finish, then waits up to `stop_drain_timeout` for the server
|
|
@@ -186,7 +187,7 @@ Worker ID queue to register at least once. Timeout raises `TimeoutError`;
|
|
|
186
187
|
async cancellation cancels only that wait. Startup failures reach waiters.
|
|
187
188
|
A permanent rejection before readiness raises `PullRegistrationError`, whose
|
|
188
189
|
`queue` and `error` identify the rejected queue and original transport error.
|
|
189
|
-
|
|
190
|
+
Stopping before readiness raises `RuntimeError`.
|
|
190
191
|
|
|
191
192
|
Use `worker.pull_registration_state()` for subsequent readiness probes. Its
|
|
192
193
|
immutable `queues` expose `queue`, `has_registered_stream`, and
|
|
@@ -240,7 +241,6 @@ to distinguish reserve pressure from full-capacity rejections. The count exclude
|
|
|
240
241
|
full-capacity rejections and unknown names. It counts decline events, including
|
|
241
242
|
repeated declines of the same Step dispatch while Dex retries it. Read its rate
|
|
242
243
|
to assess reserve pressure. It never resets and remains available after shutdown.
|
|
243
|
-
Listening Workers return zero.
|
|
244
244
|
|
|
245
245
|
## Flow state I/O
|
|
246
246
|
|
|
@@ -398,7 +398,7 @@ explicit codec only for a custom encoding or a type Registry cannot derive.
|
|
|
398
398
|
partitions them by definition type.
|
|
399
399
|
|
|
400
400
|
`Worker` and `AsyncWorker` synchronize all registered Indexed Attributes with
|
|
401
|
-
Dex Server before opening
|
|
401
|
+
Dex Server before opening outbound task streams. Existing indexes return immediately;
|
|
402
402
|
failure or the default two-minute deadline aborts startup. An indexed
|
|
403
403
|
`AttributeMap` must provide one fixed `index_key`.
|
|
404
404
|
|
|
@@ -794,8 +794,8 @@ environment:
|
|
|
794
794
|
./run-integration-tests.sh
|
|
795
795
|
```
|
|
796
796
|
|
|
797
|
-
|
|
798
|
-
|
|
797
|
+
The suite always uses pull Workers. Set `DEX_INTEG_ENGINE=zd` to test against
|
|
798
|
+
server-zd, or leave it unset to use the Temporal-backed server.
|
|
799
799
|
|
|
800
800
|
### Measure integration coverage
|
|
801
801
|
|
|
@@ -868,9 +868,9 @@ See [CONTRIBUTING.md](../CONTRIBUTING.md#releases-monorepo-tags) for monorepo ta
|
|
|
868
868
|
## Server protocol compatibility
|
|
869
869
|
|
|
870
870
|
Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
871
|
-
common protocol, synchronize Attribute indexes, and only then
|
|
872
|
-
|
|
873
|
-
invalid or disjoint intervals, and RPC failures stop startup before
|
|
871
|
+
common protocol, synchronize Attribute indexes, and only then open outbound
|
|
872
|
+
task streams. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
873
|
+
invalid or disjoint intervals, and RPC failures stop startup before polling.
|
|
874
874
|
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
875
875
|
Attributes. Workers record the negotiated version at startup and reject each
|
|
876
876
|
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
@@ -79,14 +79,17 @@ retain up to four idle credential threads. Sync token lookup must return
|
|
|
79
79
|
promptly. By default, no authorization metadata is sent. Dex uses plaintext
|
|
80
80
|
gRPC, so protect the network path that carries the token.
|
|
81
81
|
|
|
82
|
-
##
|
|
82
|
+
## Workers
|
|
83
83
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
`
|
|
88
|
-
|
|
89
|
-
SubFlow
|
|
84
|
+
Workers receive tasks through outbound pull streams. `Worker` and `AsyncWorker`
|
|
85
|
+
need only an outbound connection to the Dex Server at `server_address`. Configure
|
|
86
|
+
capacity and shutdown with `WorkerOptions.pull`; its default is
|
|
87
|
+
`PullWorkerOptions()`. Clients and Flows use pull routing automatically.
|
|
88
|
+
|
|
89
|
+
StartFlow and SubFlow always select pull Workers. Updating Flow configuration
|
|
90
|
+
preserves its existing Worker target; an empty `FlowConfig()` changes nothing.
|
|
91
|
+
Before upgrading an application with active push Flows, use an SDK version that
|
|
92
|
+
supports Worker targets to migrate those Flows to pull routing.
|
|
90
93
|
|
|
91
94
|
```python
|
|
92
95
|
import threading
|
|
@@ -102,10 +105,11 @@ worker = Worker(
|
|
|
102
105
|
)
|
|
103
106
|
worker_thread = threading.Thread(target=worker.start)
|
|
104
107
|
worker_thread.start()
|
|
108
|
+
worker.wait_pull_registered(timeout=30)
|
|
105
109
|
client = Client(
|
|
106
110
|
registry,
|
|
107
111
|
cache,
|
|
108
|
-
ClientOptions(server_address="dex.example:8801"
|
|
112
|
+
ClientOptions(server_address="dex.example:8801"),
|
|
109
113
|
)
|
|
110
114
|
try:
|
|
111
115
|
client.start_flow(flow, flow_id, input_value)
|
|
@@ -144,10 +148,7 @@ The generated `app_worker_id` uses the first nonempty `DEX_WORKER_INSTANCE`,
|
|
|
144
148
|
`POD_NAME`, `HOSTNAME`, or system hostname, plus eight random base32 characters.
|
|
145
149
|
An override must uniquely identify one Worker process and change after restart.
|
|
146
150
|
It must contain 1-63 ASCII letters, digits, dots, underscores, or hyphens and
|
|
147
|
-
must not equal a registered Flow type.
|
|
148
|
-
or an advertised address or headless target. Use `WorkerTarget(pull=True)` for
|
|
149
|
-
Client defaults or Flow overrides when the Worker target is not directly in
|
|
150
|
-
scope.
|
|
151
|
+
must not equal a registered Flow type.
|
|
151
152
|
|
|
152
153
|
Pull streams reconnect after failures. A server-directed reconnect waits for
|
|
153
154
|
an active task to finish, then waits up to `stop_drain_timeout` for the server
|
|
@@ -170,7 +171,7 @@ Worker ID queue to register at least once. Timeout raises `TimeoutError`;
|
|
|
170
171
|
async cancellation cancels only that wait. Startup failures reach waiters.
|
|
171
172
|
A permanent rejection before readiness raises `PullRegistrationError`, whose
|
|
172
173
|
`queue` and `error` identify the rejected queue and original transport error.
|
|
173
|
-
|
|
174
|
+
Stopping before readiness raises `RuntimeError`.
|
|
174
175
|
|
|
175
176
|
Use `worker.pull_registration_state()` for subsequent readiness probes. Its
|
|
176
177
|
immutable `queues` expose `queue`, `has_registered_stream`, and
|
|
@@ -224,7 +225,6 @@ to distinguish reserve pressure from full-capacity rejections. The count exclude
|
|
|
224
225
|
full-capacity rejections and unknown names. It counts decline events, including
|
|
225
226
|
repeated declines of the same Step dispatch while Dex retries it. Read its rate
|
|
226
227
|
to assess reserve pressure. It never resets and remains available after shutdown.
|
|
227
|
-
Listening Workers return zero.
|
|
228
228
|
|
|
229
229
|
## Flow state I/O
|
|
230
230
|
|
|
@@ -382,7 +382,7 @@ explicit codec only for a custom encoding or a type Registry cannot derive.
|
|
|
382
382
|
partitions them by definition type.
|
|
383
383
|
|
|
384
384
|
`Worker` and `AsyncWorker` synchronize all registered Indexed Attributes with
|
|
385
|
-
Dex Server before opening
|
|
385
|
+
Dex Server before opening outbound task streams. Existing indexes return immediately;
|
|
386
386
|
failure or the default two-minute deadline aborts startup. An indexed
|
|
387
387
|
`AttributeMap` must provide one fixed `index_key`.
|
|
388
388
|
|
|
@@ -778,8 +778,8 @@ environment:
|
|
|
778
778
|
./run-integration-tests.sh
|
|
779
779
|
```
|
|
780
780
|
|
|
781
|
-
|
|
782
|
-
|
|
781
|
+
The suite always uses pull Workers. Set `DEX_INTEG_ENGINE=zd` to test against
|
|
782
|
+
server-zd, or leave it unset to use the Temporal-backed server.
|
|
783
783
|
|
|
784
784
|
### Measure integration coverage
|
|
785
785
|
|
|
@@ -852,9 +852,9 @@ See [CONTRIBUTING.md](../CONTRIBUTING.md#releases-monorepo-tags) for monorepo ta
|
|
|
852
852
|
## Server protocol compatibility
|
|
853
853
|
|
|
854
854
|
Both `Worker` and `AsyncWorker` call `GetServerInfo`, negotiate the highest
|
|
855
|
-
common protocol, synchronize Attribute indexes, and only then
|
|
856
|
-
|
|
857
|
-
invalid or disjoint intervals, and RPC failures stop startup before
|
|
855
|
+
common protocol, synchronize Attribute indexes, and only then open outbound
|
|
856
|
+
task streams. The initial Python SDK interval is `[1,1]`. Missing information,
|
|
857
|
+
invalid or disjoint intervals, and RPC failures stop startup before polling.
|
|
858
858
|
The current Python SDK interval is `[1,3]`. Protocol 3 adds hard-delete
|
|
859
859
|
Attributes. Workers record the negotiated version at startup and reject each
|
|
860
860
|
handler result that contains a hard-delete Attribute write when it is below 3.
|
|
@@ -124,7 +124,7 @@ from dex.timer import Timer
|
|
|
124
124
|
from dex.wait import Wait
|
|
125
125
|
from dex.wait_options import WaitForAttributeOptions, WaitForStepCompletionOptions
|
|
126
126
|
from dex.worker import Worker
|
|
127
|
-
from dex.worker_options import PullWorkerOptions, WorkerOptions
|
|
127
|
+
from dex.worker_options import PullWorkerOptions, WorkerOptions
|
|
128
128
|
|
|
129
129
|
__all__ = [
|
|
130
130
|
"BOOL",
|
|
@@ -226,7 +226,6 @@ __all__ = [
|
|
|
226
226
|
"Worker",
|
|
227
227
|
"WorkerInvocationError",
|
|
228
228
|
"WorkerOptions",
|
|
229
|
-
"WorkerTarget",
|
|
230
229
|
"PullWorkerOptions",
|
|
231
230
|
"PullRegistrationError",
|
|
232
231
|
"PullRegistrationState",
|
|
@@ -10,7 +10,7 @@ from __future__ import annotations
|
|
|
10
10
|
|
|
11
11
|
import traceback
|
|
12
12
|
from dataclasses import dataclass
|
|
13
|
-
from typing import
|
|
13
|
+
from typing import Literal, cast
|
|
14
14
|
|
|
15
15
|
import grpc
|
|
16
16
|
from google.protobuf import any_pb2
|
|
@@ -165,23 +165,6 @@ def _worker_code(details: pb.ServiceErrorResponse | None) -> grpc.StatusCode | N
|
|
|
165
165
|
)
|
|
166
166
|
|
|
167
167
|
|
|
168
|
-
def abort_worker_error(
|
|
169
|
-
context: grpc.ServicerContext,
|
|
170
|
-
error: BaseException,
|
|
171
|
-
) -> None:
|
|
172
|
-
context.abort_with_status(rpc_status.to_status(_worker_error_status(error)))
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
async def async_abort_worker_error(
|
|
176
|
-
context: grpc.aio.ServicerContext[Any, Any],
|
|
177
|
-
error: BaseException,
|
|
178
|
-
) -> None:
|
|
179
|
-
# types-grpcio on 3.11 omits aio abort_with_status; runtime provides it.
|
|
180
|
-
await cast(Any, context).abort_with_status(
|
|
181
|
-
rpc_status.to_status(_worker_error_status(error))
|
|
182
|
-
)
|
|
183
|
-
|
|
184
|
-
|
|
185
168
|
def _worker_error_status(error: BaseException) -> status_pb2.Status:
|
|
186
169
|
retry_after_error: RetryAfterError | None = None
|
|
187
170
|
reported = error
|
|
@@ -805,6 +805,21 @@ class PullStream:
|
|
|
805
805
|
async with self._send_lock:
|
|
806
806
|
if request.HasField("task_frame"):
|
|
807
807
|
frame = request.task_frame
|
|
808
|
+
is_progress_frame = (
|
|
809
|
+
frame.HasField("wait_for")
|
|
810
|
+
and frame.wait_for.WhichOneof("output")
|
|
811
|
+
in ("heartbeat", "stream_write")
|
|
812
|
+
or frame.HasField("execute")
|
|
813
|
+
and frame.execute.WhichOneof("output")
|
|
814
|
+
in ("heartbeat", "stream_write")
|
|
815
|
+
)
|
|
816
|
+
if (
|
|
817
|
+
frame.task_id != self.task_id
|
|
818
|
+
or self._is_final_frame_sent
|
|
819
|
+
or not self._active.is_set()
|
|
820
|
+
and is_progress_frame
|
|
821
|
+
):
|
|
822
|
+
return
|
|
808
823
|
if (
|
|
809
824
|
frame.HasField("rpc")
|
|
810
825
|
or frame.HasField("worker_error")
|
|
@@ -711,6 +711,9 @@ class WorkerDispatcher:
|
|
|
711
711
|
mapped.flow_config_override.CopyFrom(
|
|
712
712
|
self._map_sub_flow_config(options.config_override)
|
|
713
713
|
)
|
|
714
|
+
mapped.flow_config_override.worker_target.CopyFrom(
|
|
715
|
+
pb.WorkerTarget(is_pull_mode=True)
|
|
716
|
+
)
|
|
714
717
|
return mapped
|
|
715
718
|
|
|
716
719
|
def _result_attribute_writes(
|
|
@@ -761,14 +764,7 @@ class WorkerDispatcher:
|
|
|
761
764
|
StepDurability.SYNC: pb.STEP_DURABILITY_SYNC,
|
|
762
765
|
StepDurability.ASYNC: pb.STEP_DURABILITY_ASYNC,
|
|
763
766
|
}[config.step_durability]
|
|
764
|
-
|
|
765
|
-
mapped.worker_target.CopyFrom(
|
|
766
|
-
pb.WorkerTarget(
|
|
767
|
-
address=config.worker_target.address,
|
|
768
|
-
is_headless_address=config.worker_target.headless,
|
|
769
|
-
is_pull_mode=config.worker_target.pull,
|
|
770
|
-
)
|
|
771
|
-
)
|
|
767
|
+
mapped.worker_target.CopyFrom(pb.WorkerTarget(is_pull_mode=True))
|
|
772
768
|
if config.attribute_store_names is not None:
|
|
773
769
|
mapped.attribute_store_names.CopyFrom(
|
|
774
770
|
pb.AttributeStoreNames(names=config.attribute_store_names)
|
|
@@ -107,7 +107,7 @@ class AsyncClient:
|
|
|
107
107
|
Args:
|
|
108
108
|
registry: The Flow Registry used for validation and codecs.
|
|
109
109
|
blob_cache: An open cache used for asynchronous hydration.
|
|
110
|
-
options: Service address
|
|
110
|
+
options: Service address and bearer credential;
|
|
111
111
|
``None`` uses defaults and sends no authorization metadata.
|
|
112
112
|
"""
|
|
113
113
|
self.registry = registry
|
|
@@ -1038,7 +1038,11 @@ class AsyncClient:
|
|
|
1038
1038
|
raise
|
|
1039
1039
|
|
|
1040
1040
|
async def update_flow_config(self, flow_id: str, config: FlowConfig) -> None:
|
|
1041
|
-
"""
|
|
1041
|
+
"""Update specified mutable configuration fields for an active Flow.
|
|
1042
|
+
|
|
1043
|
+
The update affects later decisions and does not recall work already
|
|
1044
|
+
dispatched. Unspecified fields and the existing Worker target remain
|
|
1045
|
+
unchanged. An empty ``FlowConfig()`` leaves all configuration unchanged.
|
|
1042
1046
|
|
|
1043
1047
|
Args:
|
|
1044
1048
|
flow_id: The non-empty active Flow ID.
|
|
@@ -1193,13 +1197,12 @@ class AsyncClient:
|
|
|
1193
1197
|
_apply_attribute_store_sync(write, definition)
|
|
1194
1198
|
_apply_hard_delete_on_null(write, definition)
|
|
1195
1199
|
mapped.attributes.append(write)
|
|
1196
|
-
|
|
1197
|
-
options.config_override
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
)
|
|
1200
|
+
mapped.flow_config_override.CopyFrom(
|
|
1201
|
+
self._map_flow_config(options.config_override)
|
|
1202
|
+
)
|
|
1203
|
+
mapped.flow_config_override.worker_target.CopyFrom(
|
|
1204
|
+
pb.WorkerTarget(is_pull_mode=True)
|
|
1205
|
+
)
|
|
1203
1206
|
return mapped
|
|
1204
1207
|
|
|
1205
1208
|
def _map_flow_config(self, config: FlowConfig | None) -> pb.FlowConfig:
|
|
@@ -1236,19 +1239,6 @@ class AsyncClient:
|
|
|
1236
1239
|
mapped.attribute_store_names.CopyFrom(
|
|
1237
1240
|
pb.AttributeStoreNames(names=config.attribute_store_names)
|
|
1238
1241
|
)
|
|
1239
|
-
target = (
|
|
1240
|
-
config.worker_target
|
|
1241
|
-
if config is not None and config.worker_target is not None
|
|
1242
|
-
else self.options.worker_target
|
|
1243
|
-
)
|
|
1244
|
-
if target is not None:
|
|
1245
|
-
mapped.worker_target.CopyFrom(
|
|
1246
|
-
pb.WorkerTarget(
|
|
1247
|
-
address=target.address,
|
|
1248
|
-
is_headless_address=target.headless,
|
|
1249
|
-
is_pull_mode=target.pull,
|
|
1250
|
-
)
|
|
1251
|
-
)
|
|
1252
1242
|
return mapped
|
|
1253
1243
|
|
|
1254
1244
|
@staticmethod
|
|
@@ -14,10 +14,9 @@ from types import TracebackType
|
|
|
14
14
|
import grpc
|
|
15
15
|
from google.protobuf import empty_pb2
|
|
16
16
|
|
|
17
|
-
from dex._credential_transport import AsyncCredentialRunner, async_channel
|
|
18
17
|
from dex._async_value_hydrator import AsyncValueHydrator
|
|
19
18
|
from dex._async_worker_dispatcher import AsyncWorkerDispatcher
|
|
20
|
-
from dex.
|
|
19
|
+
from dex._credential_transport import AsyncCredentialRunner, async_channel
|
|
21
20
|
from dex._pull_worker import PullWorkerRuntime, resolve_app_worker_id
|
|
22
21
|
from dex._server_protocol import (
|
|
23
22
|
NegotiatedServerProtocol,
|
|
@@ -31,15 +30,15 @@ from dex.dexpb import dex_pb2 as pb
|
|
|
31
30
|
from dex.dexpb import dex_pb2_grpc
|
|
32
31
|
from dex.flow import Registry
|
|
33
32
|
from dex.pull_registration import PullRegistrationState
|
|
34
|
-
from dex.worker_options import WorkerOptions
|
|
33
|
+
from dex.worker_options import WorkerOptions
|
|
35
34
|
|
|
36
35
|
|
|
37
36
|
class AsyncWorker:
|
|
38
|
-
"""Host asynchronous Step and RPC handlers through
|
|
37
|
+
"""Host asynchronous Step and RPC handlers through outbound pull streams.
|
|
39
38
|
|
|
40
39
|
A Worker is one-shot. Await ``start`` in a background task because it waits for
|
|
41
|
-
termination, and await ``stop`` or ``close`` during shutdown.
|
|
42
|
-
|
|
40
|
+
termination, and await ``stop`` or ``close`` during shutdown. Workers need
|
|
41
|
+
only an outbound connection to the Dex Server. Application handlers
|
|
43
42
|
may overlap on the event loop and must avoid blocking operations.
|
|
44
43
|
|
|
45
44
|
Attributes:
|
|
@@ -60,7 +59,7 @@ class AsyncWorker:
|
|
|
60
59
|
blob_cache: BlobCache,
|
|
61
60
|
options: WorkerOptions | None = None,
|
|
62
61
|
) -> None:
|
|
63
|
-
"""Construct an AsyncWorker without starting its
|
|
62
|
+
"""Construct an AsyncWorker without starting its outbound pull streams.
|
|
64
63
|
|
|
65
64
|
Args:
|
|
66
65
|
registry: A Registry created with ``allow_async_handlers=True``.
|
|
@@ -69,16 +68,12 @@ class AsyncWorker:
|
|
|
69
68
|
``None`` uses defaults and sends no authorization metadata.
|
|
70
69
|
|
|
71
70
|
Raises:
|
|
72
|
-
ValueError: If
|
|
71
|
+
ValueError: If the pull Worker ID is invalid.
|
|
73
72
|
"""
|
|
74
73
|
self.registry = registry
|
|
75
74
|
self.blob_cache = blob_cache
|
|
76
75
|
self.options = options or WorkerOptions()
|
|
77
|
-
app_worker_id = (
|
|
78
|
-
resolve_app_worker_id(registry, self.options.pull)
|
|
79
|
-
if self.options.pull is not None
|
|
80
|
-
else None
|
|
81
|
-
)
|
|
76
|
+
app_worker_id = resolve_app_worker_id(registry, self.options.pull)
|
|
82
77
|
self._state = "created"
|
|
83
78
|
self._credential_runner = (
|
|
84
79
|
AsyncCredentialRunner(self.options.credential)
|
|
@@ -100,49 +95,22 @@ class AsyncWorker:
|
|
|
100
95
|
AsyncValueHydrator(self._flow_service, blob_cache),
|
|
101
96
|
self._server_protocol,
|
|
102
97
|
)
|
|
103
|
-
self._server = None
|
|
104
98
|
self._has_start_called = False
|
|
105
|
-
self._pull =
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
app_worker_id,
|
|
114
|
-
self._credential_runner,
|
|
115
|
-
)
|
|
116
|
-
else:
|
|
117
|
-
self._server = grpc.aio.server()
|
|
118
|
-
dex_pb2_grpc.add_WorkerServiceServicer_to_server( # type: ignore[no-untyped-call]
|
|
119
|
-
AsyncWorkerService(dispatcher),
|
|
120
|
-
self._server,
|
|
121
|
-
)
|
|
122
|
-
self._bound_port = 0
|
|
123
|
-
self._worker_target = (
|
|
124
|
-
WorkerTarget(pull=True)
|
|
125
|
-
if self._pull is not None
|
|
126
|
-
else self.options.worker_target
|
|
127
|
-
or WorkerTarget(self._target_address(self.options.bind_address, 0))
|
|
99
|
+
self._pull = PullWorkerRuntime(
|
|
100
|
+
registry,
|
|
101
|
+
dispatcher,
|
|
102
|
+
self.options.pull,
|
|
103
|
+
self.options.server_address,
|
|
104
|
+
None,
|
|
105
|
+
app_worker_id,
|
|
106
|
+
self._credential_runner,
|
|
128
107
|
)
|
|
129
108
|
self._stopped = asyncio.Event()
|
|
130
109
|
|
|
131
|
-
@property
|
|
132
|
-
def worker_target(self) -> WorkerTarget:
|
|
133
|
-
"""Return the effective Worker target advertised to Dex.
|
|
134
|
-
|
|
135
|
-
Pull mode returns ``WorkerTarget(pull=True)`` without binding a port.
|
|
136
|
-
|
|
137
|
-
Returns:
|
|
138
|
-
The WorkerTarget, including the actual port after binding.
|
|
139
|
-
"""
|
|
140
|
-
return self._worker_target
|
|
141
|
-
|
|
142
110
|
async def __aenter__(self) -> AsyncWorker:
|
|
143
111
|
"""Return this Worker for asynchronous context-manager use.
|
|
144
112
|
|
|
145
|
-
Entering does not start
|
|
113
|
+
Entering does not start outbound task streams.
|
|
146
114
|
|
|
147
115
|
Returns:
|
|
148
116
|
This AsyncWorker instance.
|
|
@@ -167,14 +135,15 @@ class AsyncWorker:
|
|
|
167
135
|
async def start(self) -> None:
|
|
168
136
|
"""Negotiate protocol, synchronize Attribute indexes, and serve.
|
|
169
137
|
|
|
170
|
-
``start`` may be awaited exactly once. It contacts FlowService before
|
|
171
|
-
and completes only after ``stop``
|
|
138
|
+
``start`` may be awaited exactly once. It contacts FlowService before
|
|
139
|
+
opening outbound task streams and completes only after ``stop``
|
|
140
|
+
terminates those streams.
|
|
172
141
|
Permanent pull stream rejections raise PullRegistrationError when
|
|
173
142
|
stop_on_permanent_stream_rejection is enabled; otherwise streams retry.
|
|
174
143
|
|
|
175
144
|
Raises:
|
|
176
145
|
PullRegistrationError: A permanent stream rejection stops this Worker.
|
|
177
|
-
RuntimeError: If lifecycle state
|
|
146
|
+
RuntimeError: If lifecycle state is invalid or index synchronization fails.
|
|
178
147
|
"""
|
|
179
148
|
if self._has_start_called:
|
|
180
149
|
raise RuntimeError("Worker can only start once")
|
|
@@ -186,10 +155,9 @@ class AsyncWorker:
|
|
|
186
155
|
start_error = failure
|
|
187
156
|
raise
|
|
188
157
|
finally:
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
await self.stop()
|
|
158
|
+
self._pull.registration.finish_start(start_error)
|
|
159
|
+
if self._state == "running":
|
|
160
|
+
await self.stop()
|
|
193
161
|
|
|
194
162
|
async def wait_pull_registered(self, timeout: float | None = None) -> None:
|
|
195
163
|
"""Wait until every pull queue has registered at least once.
|
|
@@ -208,7 +176,7 @@ class AsyncWorker:
|
|
|
208
176
|
PullRegistrationError: A queue is permanently rejected before readiness.
|
|
209
177
|
TimeoutError: The timeout expires.
|
|
210
178
|
asyncio.CancelledError: The calling task is canceled; the Worker keeps running.
|
|
211
|
-
RuntimeError:
|
|
179
|
+
RuntimeError: The Worker stops before readiness.
|
|
212
180
|
BaseException: The original startup failure, if start fails.
|
|
213
181
|
|
|
214
182
|
Examples:
|
|
@@ -217,8 +185,6 @@ class AsyncWorker:
|
|
|
217
185
|
>>> await worker.wait_pull_registered(timeout=30)
|
|
218
186
|
>>> is_ready = worker.pull_registration_state().is_ready
|
|
219
187
|
"""
|
|
220
|
-
if self._pull is None:
|
|
221
|
-
raise RuntimeError("wait_pull_registered requires a pull Worker")
|
|
222
188
|
await self._pull.registration.wait_async(timeout)
|
|
223
189
|
|
|
224
190
|
@property
|
|
@@ -229,16 +195,16 @@ class AsyncWorker:
|
|
|
229
195
|
repeatedly while retrying it, so read the counter as a rate.
|
|
230
196
|
Full-capacity rejections and unknown names are excluded. The count never
|
|
231
197
|
resets during this Worker's lifetime and stays available after stop.
|
|
232
|
-
Reading it alongside handlers is safe.
|
|
198
|
+
Reading it alongside handlers is safe.
|
|
233
199
|
Export this value to a monitoring counter to identify reserve pressure.
|
|
234
200
|
|
|
235
201
|
Returns:
|
|
236
|
-
The cumulative reserve-decline count
|
|
202
|
+
The cumulative reserve-decline count.
|
|
237
203
|
|
|
238
204
|
Examples:
|
|
239
205
|
>>> declines = worker.pull_reserved_rpc_declines
|
|
240
206
|
"""
|
|
241
|
-
return
|
|
207
|
+
return self._pull.reserved_rpc_declines
|
|
242
208
|
|
|
243
209
|
def pull_registration_state(self) -> PullRegistrationState:
|
|
244
210
|
"""Return an immutable snapshot of pull queue registrations.
|
|
@@ -250,13 +216,9 @@ class AsyncWorker:
|
|
|
250
216
|
for an example.
|
|
251
217
|
|
|
252
218
|
Returns:
|
|
253
|
-
Current per-queue state
|
|
219
|
+
Current per-queue state.
|
|
254
220
|
"""
|
|
255
|
-
return (
|
|
256
|
-
PullRegistrationState()
|
|
257
|
-
if self._pull is None
|
|
258
|
-
else self._pull.registration.snapshot()
|
|
259
|
-
)
|
|
221
|
+
return self._pull.registration.snapshot()
|
|
260
222
|
|
|
261
223
|
async def _start(self) -> None:
|
|
262
224
|
if self._state != "created":
|
|
@@ -302,27 +264,8 @@ class AsyncWorker:
|
|
|
302
264
|
await self._close_failed_start_channel()
|
|
303
265
|
self._stopped.set()
|
|
304
266
|
raise RuntimeError("cannot synchronize Attribute indexes") from failure
|
|
305
|
-
if self._pull is None:
|
|
306
|
-
assert self._server is not None
|
|
307
|
-
self._bound_port = self._server.add_insecure_port(self.options.bind_address)
|
|
308
|
-
if self._bound_port == 0:
|
|
309
|
-
self._state = "stopped"
|
|
310
|
-
await self._close_failed_start_channel()
|
|
311
|
-
self._stopped.set()
|
|
312
|
-
raise RuntimeError(
|
|
313
|
-
f"cannot bind Python AsyncWorker to {self.options.bind_address}"
|
|
314
|
-
)
|
|
315
|
-
if self.options.worker_target is None:
|
|
316
|
-
self._worker_target = WorkerTarget(
|
|
317
|
-
self._target_address(self.options.bind_address, self._bound_port)
|
|
318
|
-
)
|
|
319
267
|
self._state = "running"
|
|
320
|
-
|
|
321
|
-
await self._pull.run()
|
|
322
|
-
else:
|
|
323
|
-
assert self._server is not None
|
|
324
|
-
await self._server.start()
|
|
325
|
-
await self._server.wait_for_termination()
|
|
268
|
+
await self._pull.run()
|
|
326
269
|
self._stopped.set()
|
|
327
270
|
|
|
328
271
|
async def stop(self) -> None:
|
|
@@ -334,16 +277,11 @@ class AsyncWorker:
|
|
|
334
277
|
Half-closed pull streams wait for server EOF until the remaining drain
|
|
335
278
|
deadline before the channel closes.
|
|
336
279
|
"""
|
|
337
|
-
|
|
338
|
-
self._pull.registration.record_stopping()
|
|
280
|
+
self._pull.registration.record_stopping()
|
|
339
281
|
if self._state in ("stopped", "closed"):
|
|
340
282
|
return
|
|
341
283
|
self._state = "stopping"
|
|
342
|
-
|
|
343
|
-
await self._pull.stop()
|
|
344
|
-
else:
|
|
345
|
-
assert self._server is not None
|
|
346
|
-
await self._server.stop(grace=5)
|
|
284
|
+
await self._pull.stop()
|
|
347
285
|
try:
|
|
348
286
|
await self._flow_channel.close(None)
|
|
349
287
|
finally:
|
|
@@ -372,14 +310,3 @@ class AsyncWorker:
|
|
|
372
310
|
finally:
|
|
373
311
|
if self._credential_runner is not None:
|
|
374
312
|
self._credential_runner.close()
|
|
375
|
-
|
|
376
|
-
@staticmethod
|
|
377
|
-
def _target_address(bind_address: str, bound_port: int) -> str:
|
|
378
|
-
host, separator, port = bind_address.rpartition(":")
|
|
379
|
-
if not separator or not port:
|
|
380
|
-
raise ValueError("Worker bind address requires a port")
|
|
381
|
-
if bound_port == 0:
|
|
382
|
-
bound_port = int(port)
|
|
383
|
-
if host in ("", "0.0.0.0", "::", "[::]"):
|
|
384
|
-
host = "localhost"
|
|
385
|
-
return f"{host}:{bound_port}"
|