async-task-api-worker 0.1.0rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- async_task_api_worker-0.1.0rc2/.gitignore +61 -0
- async_task_api_worker-0.1.0rc2/CHANGELOG.md +63 -0
- async_task_api_worker-0.1.0rc2/LICENSE +21 -0
- async_task_api_worker-0.1.0rc2/PKG-INFO +269 -0
- async_task_api_worker-0.1.0rc2/README.md +248 -0
- async_task_api_worker-0.1.0rc2/pyproject.toml +89 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/__init__.py +42 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/__main__.py +5 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/__init__.py +1 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/amqp_transport.py +251 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/declare.py +40 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/engine.py +543 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/inflight.py +38 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/lifecycle.py +463 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/log_context.py +20 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/observer.py +78 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/progress.py +143 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/retry.py +29 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/sync_bridge.py +68 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_runtime/transport.py +62 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/_version.py +11 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/cli.py +88 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/context.py +70 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/errors.py +116 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/observability/__init__.py +1 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/observability/health.py +299 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/observability/logging.py +89 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/observability/loop_lag.py +56 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/observability/metrics.py +188 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/protocol/__init__.py +41 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/protocol/envelope.py +103 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/protocol/json_types.py +7 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/protocol/messages.py +121 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/protocol/version.py +11 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/py.typed +0 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/settings.py +127 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/testing/__init__.py +27 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/testing/_context.py +93 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/testing/_harness.py +251 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/testing/_memory_broker.py +162 -0
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/worker.py +254 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
.venv
|
|
2
|
+
.env
|
|
3
|
+
.coverage
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
__pycache__
|
|
6
|
+
htmlcov/
|
|
7
|
+
nodemodules/
|
|
8
|
+
junit.xml
|
|
9
|
+
.vscode
|
|
10
|
+
.ruff_cache
|
|
11
|
+
node_modules/
|
|
12
|
+
package-lock.json
|
|
13
|
+
*.egg-info/
|
|
14
|
+
dist/
|
|
15
|
+
build/
|
|
16
|
+
*.log
|
|
17
|
+
*.pot
|
|
18
|
+
*.pyc
|
|
19
|
+
# charts/ intentionnellement pour ArgoCD (pas d'accès aux repos externes)
|
|
20
|
+
.DS_Store
|
|
21
|
+
demo.db
|
|
22
|
+
junit.xml
|
|
23
|
+
coverage.xml
|
|
24
|
+
|
|
25
|
+
# Sorties locales sensibles (audit, homologation, tests d'intrusion) — ne pas committer
|
|
26
|
+
private/
|
|
27
|
+
|
|
28
|
+
# Dépôt de fixtures de la boucle de révision, monté par docker-compose.revision.yml et
|
|
29
|
+
# reconstruit par `revision-ui/recette/bout_en_bout.py --racine`. Contient des vignettes
|
|
30
|
+
# générées et des jetons signés : produit, jamais source.
|
|
31
|
+
revision-ui/.fixtures/
|
|
32
|
+
|
|
33
|
+
# Secrets et matériel cryptographique
|
|
34
|
+
.env.*
|
|
35
|
+
!.env.example
|
|
36
|
+
*.pem
|
|
37
|
+
*.key
|
|
38
|
+
*.p12
|
|
39
|
+
*.pfx
|
|
40
|
+
id_rsa*
|
|
41
|
+
id_ed25519*
|
|
42
|
+
*.kubeconfig
|
|
43
|
+
kubeconfig*
|
|
44
|
+
*.tfstate
|
|
45
|
+
*.tfstate.*
|
|
46
|
+
secrets/
|
|
47
|
+
*.secret
|
|
48
|
+
|
|
49
|
+
# Dumps et sauvegardes
|
|
50
|
+
*.sql.gz
|
|
51
|
+
*.dump
|
|
52
|
+
*.bak-*
|
|
53
|
+
|
|
54
|
+
# Console d'exemple : secrets locaux et fragment Keycloak rempli. Le `.gitignore` couvre
|
|
55
|
+
# déjà `.env` et `.env.*` à toute profondeur — ces lignes sont EXPLICITES parce qu'un secret
|
|
56
|
+
# protégé par accident cesse de l'être à la première réécriture du fichier.
|
|
57
|
+
exemple-front/.env
|
|
58
|
+
exemple-front/keycloak/*.local.json
|
|
59
|
+
|
|
60
|
+
# Captures du smoke navigateur : régénérées à chaque exécution.
|
|
61
|
+
exemple-front/recette/captures/
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.1.0-rc.2](https://github.com/IA-Generative/async-api/compare/async-task-api-worker-v0.1.0-rc.1...async-task-api-worker-v0.1.0-rc.2) (2026-10-23)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* **sdk:** add a harness to run a worker's task in memory ([27a0ca7](https://github.com/IA-Generative/async-api/commit/27a0ca7558f15b5f7f589e84feb391ba152ddbde))
|
|
9
|
+
* **sdk:** add context factories to unit-test a handler directly ([0429cc7](https://github.com/IA-Generative/async-api/commit/0429cc78e70c2d9de9966c350d50b95d5f2e2af5))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
### Bug Fixes
|
|
13
|
+
|
|
14
|
+
* **sdk:** close the channels before the connection so the last ack is not lost ([1118516](https://github.com/IA-Generative/async-api/commit/1118516300c9d56e0ff023a8cecc5af6994fefbd))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
### Documentation
|
|
18
|
+
|
|
19
|
+
* **sdk:** add a Testing your worker section ([77284da](https://github.com/IA-Generative/async-api/commit/77284dafdae038357ca33cf1cb1dca8cef2f0f76))
|
|
20
|
+
|
|
21
|
+
## 0.1.0-rc.1 (2026-10-23)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
### Features
|
|
25
|
+
|
|
26
|
+
* **sdk:** add a transport on aio-pika ([89f90d2](https://github.com/IA-Generative/async-api/commit/89f90d293fcd12e9ec893adc17f7fe138c1f5bee))
|
|
27
|
+
* **sdk:** add JSON logging ([c02fbf9](https://github.com/IA-Generative/async-api/commit/c02fbf905e7eb6372bffb645da1119ff34d130d3))
|
|
28
|
+
* **sdk:** add the delivery engine and the Worker definition API ([99b9636](https://github.com/IA-Generative/async-api/commit/99b96365fedcace29b4e5cbbea6e90860138e73d))
|
|
29
|
+
* **sdk:** add the probe and metrics HTTP server ([4694871](https://github.com/IA-Generative/async-api/commit/4694871a673b1526ea260bf4fdb786996f013e91))
|
|
30
|
+
* **sdk:** add the process lifecycle, Worker.run and the run command ([38c93da](https://github.com/IA-Generative/async-api/commit/38c93da62f4446c8a001352a76e5d8f9dc39a70c))
|
|
31
|
+
* **sdk:** add the Prometheus metrics and the event loop lag probe ([ae1b82c](https://github.com/IA-Generative/async-api/commit/ae1b82c2b3ef8f8d71faf5e28d2f1e235e77b14a))
|
|
32
|
+
* **sdk:** add the RabbitMQ transport and the queue declaration ([709d1f3](https://github.com/IA-Generative/async-api/commit/709d1f3fbab892009ff7a9d4dec7c007dadaffda))
|
|
33
|
+
* **sdk:** add the runtime transport contract, retry backoff and in-flight registry ([03a65f8](https://github.com/IA-Generative/async-api/commit/03a65f858d7816223b93ec63a02a5bf7ecca733c))
|
|
34
|
+
* **sdk:** add the task contexts and the sync handler bridge ([3e50261](https://github.com/IA-Generative/async-api/commit/3e502617fd8ca50f4d0a5254b268f58d814b50a5))
|
|
35
|
+
* **sdk:** add the time-based progress throttle ([7d85fe1](https://github.com/IA-Generative/async-api/commit/7d85fe11496890d4dbd3580b130b82bcfd0905f1))
|
|
36
|
+
* **sdk:** add WORKER_MODE settings and force a prefetch of 1 in job mode ([94703d0](https://github.com/IA-Generative/async-api/commit/94703d05bd450717cd914784fcf75abadd4ce660))
|
|
37
|
+
* **sdk:** let a transport report whether the broker connection is up ([82338fa](https://github.com/IA-Generative/async-api/commit/82338fae6a0c988647fa2a47fd1aadb54fb0311e))
|
|
38
|
+
* **sdk:** log one decision line per settled delivery ([162467a](https://github.com/IA-Generative/async-api/commit/162467abeead6776973c1bd72137d067e0137923))
|
|
39
|
+
* **sdk:** run the lifecycle as a one-shot job when WORKER_MODE=job ([82f9034](https://github.com/IA-Generative/async-api/commit/82f90345341f40bf080aa3df44a371e5a2fab63b))
|
|
40
|
+
* **sdk:** serve the probes and the metrics from the lifecycle ([6df9b75](https://github.com/IA-Generative/async-api/commit/6df9b7584922b48c756cdc0f5c1fd64d2586e93f))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
### Bug Fixes
|
|
44
|
+
|
|
45
|
+
* **sdk:** abort the probe clients when the server stops ([0ad2dd4](https://github.com/IA-Generative/async-api/commit/0ad2dd4fa1862d2175076f2ff6f5ddb61c9a5e80))
|
|
46
|
+
* **sdk:** bound the shutdown tail and stop a startup on a signal ([0ce9e19](https://github.com/IA-Generative/async-api/commit/0ce9e1993ed830261146bd7b3b7a3785c97cc679))
|
|
47
|
+
* **sdk:** cancel the consumer before the subscriber stops accepting deliveries ([a5959a3](https://github.com/IA-Generative/async-api/commit/a5959a30356a5b960902aa4bc01510a2dc13c1cb))
|
|
48
|
+
* **sdk:** do not wait for default-executor threads at exit ([f5a2f1e](https://github.com/IA-Generative/async-api/commit/f5a2f1e643364b1d9f05d544998e55b394ec5a3c))
|
|
49
|
+
* **sdk:** harden the encoder, the decoder and the settings ([7f91e23](https://github.com/IA-Generative/async-api/commit/7f91e2393094da9a272cc19c400547313d9a5936))
|
|
50
|
+
* **sdk:** log the reconnect errors of aio-pika as warnings ([591526d](https://github.com/IA-Generative/async-api/commit/591526df2c17c99c7b12ea72f3941e4308ba98bd))
|
|
51
|
+
* **sdk:** retry a raw connect timeout in service mode ([d6d1fcd](https://github.com/IA-Generative/async-api/commit/d6d1fcd5876c2031800163ea9a15b4f789651875))
|
|
52
|
+
* **sdk:** ship only the sources in the sdist and polish messages ([e6e6ee1](https://github.com/IA-Generative/async-api/commit/e6e6ee128d231a8bb942215ab746122929719362))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
### Documentation
|
|
56
|
+
|
|
57
|
+
* **sdk:** describe the protocol itself in the packaged sources ([67cd9f0](https://github.com/IA-Generative/async-api/commit/67cd9f02ddc730c388128d53b214c7e13f841280))
|
|
58
|
+
* **sdk:** document the deployment or job modes ([5e94fb9](https://github.com/IA-Generative/async-api/commit/5e94fb9a36958fd29a449138e7286a9ece47cc5d))
|
|
59
|
+
* **sdk:** document the probes, the metrics and the logs ([29c22ca](https://github.com/IA-Generative/async-api/commit/29c22cae66498201cc02b0b0087f54513e58b51b))
|
|
60
|
+
* **sdk:** document the worker definition and progress rules ([9a1b710](https://github.com/IA-Generative/async-api/commit/9a1b71052fe47c18ea7c8d09e779e20d78bb4d91))
|
|
61
|
+
* **sdk:** leave the changelog to release-please ([f7cb380](https://github.com/IA-Generative/async-api/commit/f7cb380912601fb2bbb59636f5bbe1ce98b06fb8))
|
|
62
|
+
|
|
63
|
+
## Changelog
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 IA-Generative
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: async-task-api-worker
|
|
3
|
+
Version: 0.1.0-rc.2
|
|
4
|
+
Summary: SDK to write workers for the AsyncTaskAPI platform: protocol models, runtime and conformance kit.
|
|
5
|
+
Project-URL: Repository, https://github.com/IA-Generative/async-api
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: async-api,rabbitmq,sdk,worker
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Python: >=3.13
|
|
16
|
+
Requires-Dist: aio-pika>=9.5.5
|
|
17
|
+
Requires-Dist: prometheus-client>=0.22.1
|
|
18
|
+
Requires-Dist: pydantic-settings>=2.10.1
|
|
19
|
+
Requires-Dist: pydantic>=2.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# async-task-api-worker
|
|
23
|
+
|
|
24
|
+
SDK to write workers for the AsyncTaskAPI platform.
|
|
25
|
+
|
|
26
|
+
This is a work in progress: the package ships the protocol models (inbound envelope decoder and the byte-exact
|
|
27
|
+
outbound message encoder), the error types, the settings, the runtime engine with the `Worker` definition API, the
|
|
28
|
+
RabbitMQ transport, the process lifecycle (graceful drain), the probes, the metrics and the JSON logs. The conformance
|
|
29
|
+
kit and the extras land in later releases.
|
|
30
|
+
|
|
31
|
+
## Declaring a worker
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from pydantic import BaseModel, Field
|
|
35
|
+
|
|
36
|
+
from async_task_api_worker import TaskContext, Worker
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Input(BaseModel):
|
|
40
|
+
text: str = Field(min_length=1, max_length=100_000)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Output(BaseModel):
|
|
44
|
+
words: int
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
worker = Worker("count-words")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@worker.task(timeout=30.0)
|
|
51
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
52
|
+
await ctx.progress(0.5)
|
|
53
|
+
return Output(words=len(task_input.text.split()))
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
- `async def` handlers run on the event loop; `def` handlers (with a `SyncTaskContext`) run in a thread pool. The
|
|
57
|
+
context parameter must be annotated exactly `TaskContext` or `SyncTaskContext` (an identity check: no subclass or
|
|
58
|
+
`Annotated` wrapper). Hooks and probes (`on_startup`, `on_shutdown`, `readiness_probe`) must be `async def`.
|
|
59
|
+
- The input and output models are read from the annotations. Exactly one task per worker.
|
|
60
|
+
- Raise `FatalError(code, message)` to fail a task with a code the client can match, `RetryableError` for a
|
|
61
|
+
transient failure. Any other exception is reported as `INTERNAL_ERROR` without its text.
|
|
62
|
+
- `ctx.progress(fraction, payload)` reports the **latest state** of the task, not an event stream: updates are
|
|
63
|
+
throttled (at most one per `WORKER_PROGRESS_MIN_INTERVAL_S`, default 1 s), a newer update replaces a held one,
|
|
64
|
+
`fraction` is in `[0, 1)` and never goes back. The terminal success or failure message is the 100%.
|
|
65
|
+
|
|
66
|
+
## Running a worker
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
export BROKER_URL=amqp://user:password@broker:5672/ IN_QUEUE_NAME=count-words_queue_in OUT_QUEUE_NAME=count-words_queue_out
|
|
70
|
+
async-task-api-worker run my_module:worker # or: python -m async_task_api_worker run my_module:worker
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`my_module:worker` is a `Worker`, or a zero-argument factory function returning one; modules in the current directory can
|
|
74
|
+
be imported. A definition error in the module exits with code 2 and one line. From code, `worker.run()` blocks until the
|
|
75
|
+
process is stopped and `await worker.serve()` serves from an existing event loop.
|
|
76
|
+
|
|
77
|
+
- The settings are read and validated first: an invalid configuration, or a task timeout that does not fit under the
|
|
78
|
+
broker's consumer timeout (`task timeout + WORKER_RETRY_BACKOFF_MAX_S + 60 s < WORKER_CONSUMER_TIMEOUT_S`), exits with
|
|
79
|
+
code 2 before any connection. Exit codes: 0 clean stop, 1 failure, 2 configuration or usage error.
|
|
80
|
+
- The queues are declared durable and with no arguments (the broker decides their type and policies), and consumed with
|
|
81
|
+
manual acknowledgements: a task is acked only after its result is published.
|
|
82
|
+
- SIGTERM or SIGINT stops consuming, lets the in-flight tasks finish within `WORKER_SHUTDOWN_GRACE_S` (default 25 s),
|
|
83
|
+
requeues the ones that did not, closes the connection and exits 0. A second signal skips the rest of the grace period.
|
|
84
|
+
A `def` handler whose thread is still running after the drain, or an `async def` handler whose `asyncio.to_thread`
|
|
85
|
+
thread is (given 2 s after the drain), is logged and the process exits at once with code 1 (only for `run`, never for
|
|
86
|
+
`serve()`). The whole shutdown is bounded: the process exits at most
|
|
87
|
+
`max(WORKER_SHUTDOWN_GRACE_S, 5)` + 17 s after the signal (5 s for the nacks of the cancelled tasks, 5 s to close the
|
|
88
|
+
connection, and 5 s in total, `WORKER_SHUTDOWN_HOOKS_TIMEOUT_S`, for all the shutdown hooks together, each getting what is
|
|
89
|
+
left). Set a pod's `terminationGracePeriodSeconds` to at least that. A signal during startup does not wait for it: the
|
|
90
|
+
connection attempt and the startup hooks are cancelled and the process exits 0 (a hook that blocks the event loop
|
|
91
|
+
cannot be interrupted).
|
|
92
|
+
- **Retries and crash-loop protection need quorum queues** (the broker's default queue type, as in the platform's
|
|
93
|
+
chart). A classic queue gives no delivery counter: there, a `RetryableError` gets one retry, then concludes with its
|
|
94
|
+
own code, and a warning says so.
|
|
95
|
+
- Refused broker credentials exit with code 2 right away instead of being retried.
|
|
96
|
+
- The attempt number of a delivery (`ctx.attempt`) counts every time the broker handed the message out. A
|
|
97
|
+
`RetryableError` requeues until attempt `WORKER_DELIVERY_LIMIT`, then concludes with its own code. A delivery past
|
|
98
|
+
that limit (a crash loop) is concluded with `TASK_ABANDONED_AFTER_REDELIVERIES` without running the handler.
|
|
99
|
+
|
|
100
|
+
## Probes, metrics and logs
|
|
101
|
+
|
|
102
|
+
A small HTTP server runs on `HEALTH_CHECK_HOST:HEALTH_CHECK_PORT` (default `0.0.0.0:8083`) in both modes, from the very
|
|
103
|
+
start of the process until the end of the shutdown.
|
|
104
|
+
|
|
105
|
+
| Path | Answer |
|
|
106
|
+
|---|---|
|
|
107
|
+
| `GET /health` | Liveness. 200 `{"status": "ok"}`, or 503 `TASK_OVERRUN` when a `def` handler is still running `WORKER_TASK_OVERRUN_GRACE_S` after its deadline. It consults no dependency. |
|
|
108
|
+
| `GET /ready` | Readiness. 200 with a `components` map, or 503 `DEPENDENCIES_NOT_READY` (with the same `components` in `error`) when the broker is disconnected, the worker is draining, or one of the probes registered with `@worker.readiness_probe("name")` raises or exceeds `WORKER_PROBE_TIMEOUT_S` (3 s). Probes run in parallel. A `ProbeError` message is shown as the detail; any other exception only by its class name. |
|
|
109
|
+
| `GET /metrics`, `GET /internal/metrics` | Prometheus text exposition. |
|
|
110
|
+
| `GET /__version__` | `/app/version.json` of the image, or a `dev` document. |
|
|
111
|
+
|
|
112
|
+
Errors use the platform's format, `{"status": "error", "error": {"code", "message"}}` (404 `NOT_FOUND`, 405
|
|
113
|
+
`METHOD_NOT_ALLOWED`, 500 `PROBE_FAILED`). `broker` and `draining` are reserved component names. `/ready` is also 503 in
|
|
114
|
+
job mode once the quota is reached and the consumer cancelled.
|
|
115
|
+
|
|
116
|
+
The server is bounded: at most 32 connections are served at once (the others are closed at once), reading the request,
|
|
117
|
+
writing the answer and closing each take at most 5 s, and concurrent `/ready` requests share a single evaluation of the
|
|
118
|
+
probes.
|
|
119
|
+
|
|
120
|
+
**Startup.** `/health` answers while the startup hooks run, but only as long as the event loop is free: a hook that loads a
|
|
121
|
+
model or a large file synchronously blocks it, `/health` stops answering and the kubelet restarts the pod in a loop. Run
|
|
122
|
+
heavy loading in `asyncio.to_thread`, or give the container a `startupProbe`.
|
|
123
|
+
|
|
124
|
+
Metrics have no task, client or service label (the scrape adds the pod and service): `worker_tasks_total{status}`,
|
|
125
|
+
`worker_task_duration_seconds`, `worker_messages_published_total{message_type}`, `event_loop_lag_seconds{component}`,
|
|
126
|
+
`worker_task_failures_total{code}`, `worker_task_retries_total`, `worker_poison_messages_total{reason}`,
|
|
127
|
+
`worker_publish_failures_total{message_type}`, `worker_ack_failures_total`, `worker_tasks_in_flight`,
|
|
128
|
+
`worker_shutdown_requeued_total`, `worker_progress_dropped_total{reason}` and `worker_sdk_info`, plus the process, platform
|
|
129
|
+
and GC collectors.
|
|
130
|
+
|
|
131
|
+
Logs are one JSON object per line on stdout (`ts`, `level`, `logger`, `msg`, and `task_id`, `service`, `attempt` when
|
|
132
|
+
bound; `ctx.logger` carries them, and so do the records of any library logging inside a handler, on the SDK's handler).
|
|
133
|
+
`LOG_LEVEL` sets the level. If the application configured the root logger before running the worker, its handlers are
|
|
134
|
+
kept and the SDK adds none (no line is printed twice): the format is then the application's, and so is the level of
|
|
135
|
+
the `aio_pika.tools` logger, whose two ERROR lines on a dropped connection the SDK otherwise logs as warnings (the
|
|
136
|
+
connection then reconnects by itself). To still get the task fields on your own handler, chain a record factory:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import logging
|
|
140
|
+
from async_task_api_worker.observability.logging import TaskContextFilter
|
|
141
|
+
|
|
142
|
+
task_fields = TaskContextFilter()
|
|
143
|
+
previous_factory = logging.getLogRecordFactory()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def factory(*args, **kwargs):
|
|
147
|
+
record = previous_factory(*args, **kwargs)
|
|
148
|
+
task_fields.filter(record)
|
|
149
|
+
return record
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
logging.setLogRecordFactory(factory)
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Every delivery ends with exactly one INFO decision line, for instance `Delivery settled: outcome=failure action=conclude
|
|
156
|
+
code=NOT_ALLOWED attempt=1/5`, with the fields `task_id`, `attempt`, `delivery_limit`, `outcome` (`success`, `failure`,
|
|
157
|
+
`poison`, `retry`, `publish_failed`, `shutdown_requeued`, `cancelled`, `engine_error`), `action` (`ack`, `conclude`,
|
|
158
|
+
`nack-requeue`) and `code`: `grep <task_id>`
|
|
159
|
+
tells the whole story of a task. Payloads and secrets are never logged.
|
|
160
|
+
|
|
161
|
+
## Deployment or Job
|
|
162
|
+
|
|
163
|
+
The same worker code and image run as a long-lived consumer (a Kubernetes Deployment) or as a one-shot process (a
|
|
164
|
+
Job, for instance one started per message by KEDA's `ScaledJob`). The deployment picks the mode with `WORKER_MODE`; the
|
|
165
|
+
`Worker` definition does not change.
|
|
166
|
+
|
|
167
|
+
| Variable | Default | Meaning |
|
|
168
|
+
|---|---|---|
|
|
169
|
+
| `WORKER_MODE` | `service` | `service` consumes until SIGTERM; `job` is one-shot. Any other value exits with code 2. |
|
|
170
|
+
| `WORKER_JOB_MAX_TASKS` | `1` | Job mode: stop consuming at this many deliveries, finish them, and exit. |
|
|
171
|
+
| `WORKER_JOB_IDLE_TIMEOUT_S` | `10` | Job mode: exit when no delivery is in hand and none arrives for this long (another pod probably took the message). |
|
|
172
|
+
| `WORKER_JOB_CONNECT_TIMEOUT_S` | `30` | Job mode: give up and exit 1 when the broker stays unreachable for this long. Service mode retries forever. |
|
|
173
|
+
|
|
174
|
+
In job mode the prefetch is forced to 1, whatever `WORKER_CONCURRENCY` says (it then only sizes the handler semaphore
|
|
175
|
+
and the thread pool). The quota counts deliveries **on arrival**: any delivery counts, poison ones and redeliveries past
|
|
176
|
+
the limit included, and consuming stops as soon as the last one is in hand. The exit code reports the **pod**, not the
|
|
177
|
+
task: a task that succeeds, fails (`FatalError`, timeout, invalid input), or is requeued (`RetryableError`) all exit 0,
|
|
178
|
+
and so does an idle exit, because a failed task is a business outcome already published to the client. Exit 1 (an
|
|
179
|
+
infrastructure failure such as a broker unreachable for `WORKER_JOB_CONNECT_TIMEOUT_S`, or a `def` handler stuck after
|
|
180
|
+
the drain) and 2 (invalid configuration) stay non-zero, so the Job's `backoffLimit` applies to them. SIGTERM and
|
|
181
|
+
`activeDeadlineSeconds` drain exactly as in service mode.
|
|
182
|
+
|
|
183
|
+
## Testing your worker
|
|
184
|
+
|
|
185
|
+
`async_task_api_worker.testing` runs your task in memory through the real engine (decoding, time limit, retries, error
|
|
186
|
+
mapping, acknowledgements), with no broker and no Docker:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from pydantic import BaseModel
|
|
190
|
+
|
|
191
|
+
from async_task_api_worker import FatalError, TaskContext, Worker
|
|
192
|
+
from async_task_api_worker.testing import Outcome, ProgressRecorder, WorkerHarness, make_task_context
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class Input(BaseModel):
|
|
196
|
+
text: str
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class Output(BaseModel):
|
|
200
|
+
words: int
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
worker = Worker("count-words")
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@worker.task(timeout=30.0)
|
|
207
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
208
|
+
if not task_input.text.strip():
|
|
209
|
+
raise FatalError("EMPTY_TEXT", "There is nothing to count.")
|
|
210
|
+
await ctx.progress(0.5)
|
|
211
|
+
return Output(words=len(task_input.text.split()))
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
async def test_counts_the_words() -> None:
|
|
215
|
+
async with WorkerHarness(worker) as harness:
|
|
216
|
+
run = await harness.submit({"text": "hello big world"})
|
|
217
|
+
|
|
218
|
+
run.assert_succeeded()
|
|
219
|
+
assert run.response == {"words": 3}
|
|
220
|
+
assert run.message_types == ("started", "progress", "success")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
async def test_an_empty_text_fails_with_its_code() -> None:
|
|
224
|
+
async with WorkerHarness(worker) as harness:
|
|
225
|
+
run = await harness.submit({"text": " "})
|
|
226
|
+
|
|
227
|
+
run.assert_failed(code="EMPTY_TEXT")
|
|
228
|
+
assert run.outcome is Outcome.ACKED
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
async def test_the_handler_alone_with_a_context_you_build() -> None:
|
|
232
|
+
recorder = ProgressRecorder()
|
|
233
|
+
ctx = make_task_context(recorder=recorder)
|
|
234
|
+
|
|
235
|
+
assert await handle(Input(text="a b"), ctx) == Output(words=2)
|
|
236
|
+
assert [update.fraction for update in recorder.updates] == [0.5]
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
- `harness.submit(body, client_id=..., input_refs=..., delivery_count=...)` builds the envelope the API publishes;
|
|
240
|
+
`submit_raw(bytes)` sends bytes as they are (a poison message). The result is a `TaskRun`: `messages` (decoded) and
|
|
241
|
+
`raw` (the published bytes), `message_types` (all three are tuples), `outcome` (`ACKED`, `NACKED_REQUEUE`), `response`,
|
|
242
|
+
`error`, and the two assertions. `task_id` is the envelope's, or `None` for bytes that do not carry one.
|
|
243
|
+
- `delivery_count` is the broker's `x-delivery-count`: pass `0`, then `delivery_count=n` to see a `RetryableError`
|
|
244
|
+
requeue until the last attempt (`worker_delivery_limit`), which concludes with the error's own code. Retry backoffs are
|
|
245
|
+
a millisecond; every other setting is the production default (progress throttling included).
|
|
246
|
+
- `settings_overrides={"worker_task_timeout_s": 0.1, ...}` takes `WorkerSettings` field names; no environment variable is
|
|
247
|
+
read, and the time budget is checked as at startup. `harness.fail_next_publish("success")` makes the next publish (of
|
|
248
|
+
that message type, or of any) fail once, to test the requeue. Use `async with` (or `await harness.aclose()`).
|
|
249
|
+
- `make_task_context` and `make_sync_task_context` build the context of an `async def` and of a `def` handler, to call
|
|
250
|
+
the handler directly; `ProgressRecorder` collects what it reports.
|
|
251
|
+
|
|
252
|
+
## Development
|
|
253
|
+
|
|
254
|
+
From this directory:
|
|
255
|
+
|
|
256
|
+
```bash
|
|
257
|
+
uv sync --locked --all-extras
|
|
258
|
+
uv run ruff check . && uv run ruff format --check .
|
|
259
|
+
uv run ty check src
|
|
260
|
+
uv run pytest -m "not integration" -q --cov=async_task_api_worker --cov-branch --cov-fail-under=95
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
The integration tests need Docker (they start RabbitMQ with the platform's policy through testcontainers):
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
uv run pytest -m integration -q
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Add dependencies with `uv add` from this directory, never by editing `pyproject.toml`.
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# async-task-api-worker
|
|
2
|
+
|
|
3
|
+
SDK to write workers for the AsyncTaskAPI platform.
|
|
4
|
+
|
|
5
|
+
This is a work in progress: the package ships the protocol models (inbound envelope decoder and the byte-exact
|
|
6
|
+
outbound message encoder), the error types, the settings, the runtime engine with the `Worker` definition API, the
|
|
7
|
+
RabbitMQ transport, the process lifecycle (graceful drain), the probes, the metrics and the JSON logs. The conformance
|
|
8
|
+
kit and the extras land in later releases.
|
|
9
|
+
|
|
10
|
+
## Declaring a worker
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
from pydantic import BaseModel, Field
|
|
14
|
+
|
|
15
|
+
from async_task_api_worker import TaskContext, Worker
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Input(BaseModel):
|
|
19
|
+
text: str = Field(min_length=1, max_length=100_000)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Output(BaseModel):
|
|
23
|
+
words: int
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
worker = Worker("count-words")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@worker.task(timeout=30.0)
|
|
30
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
31
|
+
await ctx.progress(0.5)
|
|
32
|
+
return Output(words=len(task_input.text.split()))
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
- `async def` handlers run on the event loop; `def` handlers (with a `SyncTaskContext`) run in a thread pool. The
|
|
36
|
+
context parameter must be annotated exactly `TaskContext` or `SyncTaskContext` (an identity check: no subclass or
|
|
37
|
+
`Annotated` wrapper). Hooks and probes (`on_startup`, `on_shutdown`, `readiness_probe`) must be `async def`.
|
|
38
|
+
- The input and output models are read from the annotations. Exactly one task per worker.
|
|
39
|
+
- Raise `FatalError(code, message)` to fail a task with a code the client can match, `RetryableError` for a
|
|
40
|
+
transient failure. Any other exception is reported as `INTERNAL_ERROR` without its text.
|
|
41
|
+
- `ctx.progress(fraction, payload)` reports the **latest state** of the task, not an event stream: updates are
|
|
42
|
+
throttled (at most one per `WORKER_PROGRESS_MIN_INTERVAL_S`, default 1 s), a newer update replaces a held one,
|
|
43
|
+
`fraction` is in `[0, 1)` and never goes back. The terminal success or failure message is the 100%.
|
|
44
|
+
|
|
45
|
+
## Running a worker
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
export BROKER_URL=amqp://user:password@broker:5672/ IN_QUEUE_NAME=count-words_queue_in OUT_QUEUE_NAME=count-words_queue_out
|
|
49
|
+
async-task-api-worker run my_module:worker # or: python -m async_task_api_worker run my_module:worker
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`my_module:worker` is a `Worker`, or a zero-argument factory function returning one; modules in the current directory can
|
|
53
|
+
be imported. A definition error in the module exits with code 2 and one line. From code, `worker.run()` blocks until the
|
|
54
|
+
process is stopped and `await worker.serve()` serves from an existing event loop.
|
|
55
|
+
|
|
56
|
+
- The settings are read and validated first: an invalid configuration, or a task timeout that does not fit under the
|
|
57
|
+
broker's consumer timeout (`task timeout + WORKER_RETRY_BACKOFF_MAX_S + 60 s < WORKER_CONSUMER_TIMEOUT_S`), exits with
|
|
58
|
+
code 2 before any connection. Exit codes: 0 clean stop, 1 failure, 2 configuration or usage error.
|
|
59
|
+
- The queues are declared durable and with no arguments (the broker decides their type and policies), and consumed with
|
|
60
|
+
manual acknowledgements: a task is acked only after its result is published.
|
|
61
|
+
- SIGTERM or SIGINT stops consuming, lets the in-flight tasks finish within `WORKER_SHUTDOWN_GRACE_S` (default 25 s),
|
|
62
|
+
requeues the ones that did not, closes the connection and exits 0. A second signal skips the rest of the grace period.
|
|
63
|
+
A `def` handler whose thread is still running after the drain, or an `async def` handler whose `asyncio.to_thread`
|
|
64
|
+
thread is (given 2 s after the drain), is logged and the process exits at once with code 1 (only for `run`, never for
|
|
65
|
+
`serve()`). The whole shutdown is bounded: the process exits at most
|
|
66
|
+
`max(WORKER_SHUTDOWN_GRACE_S, 5)` + 17 s after the signal (5 s for the nacks of the cancelled tasks, 5 s to close the
|
|
67
|
+
connection, and 5 s in total, `WORKER_SHUTDOWN_HOOKS_TIMEOUT_S`, for all the shutdown hooks together, each getting what is
|
|
68
|
+
left). Set a pod's `terminationGracePeriodSeconds` to at least that. A signal during startup does not wait for it: the
|
|
69
|
+
connection attempt and the startup hooks are cancelled and the process exits 0 (a hook that blocks the event loop
|
|
70
|
+
cannot be interrupted).
|
|
71
|
+
- **Retries and crash-loop protection need quorum queues** (the broker's default queue type, as in the platform's
|
|
72
|
+
chart). A classic queue gives no delivery counter: there, a `RetryableError` gets one retry, then concludes with its
|
|
73
|
+
own code, and a warning says so.
|
|
74
|
+
- Refused broker credentials exit with code 2 right away instead of being retried.
|
|
75
|
+
- The attempt number of a delivery (`ctx.attempt`) counts every time the broker handed the message out. A
|
|
76
|
+
`RetryableError` requeues until attempt `WORKER_DELIVERY_LIMIT`, then concludes with its own code. A delivery past
|
|
77
|
+
that limit (a crash loop) is concluded with `TASK_ABANDONED_AFTER_REDELIVERIES` without running the handler.
|
|
78
|
+
|
|
79
|
+
## Probes, metrics and logs
|
|
80
|
+
|
|
81
|
+
A small HTTP server runs on `HEALTH_CHECK_HOST:HEALTH_CHECK_PORT` (default `0.0.0.0:8083`) in both modes, from the very
|
|
82
|
+
start of the process until the end of the shutdown.
|
|
83
|
+
|
|
84
|
+
| Path | Answer |
|
|
85
|
+
|---|---|
|
|
86
|
+
| `GET /health` | Liveness. 200 `{"status": "ok"}`, or 503 `TASK_OVERRUN` when a `def` handler is still running `WORKER_TASK_OVERRUN_GRACE_S` after its deadline. It consults no dependency. |
|
|
87
|
+
| `GET /ready` | Readiness. 200 with a `components` map, or 503 `DEPENDENCIES_NOT_READY` (with the same `components` in `error`) when the broker is disconnected, the worker is draining, or one of the probes registered with `@worker.readiness_probe("name")` raises or exceeds `WORKER_PROBE_TIMEOUT_S` (3 s). Probes run in parallel. A `ProbeError` message is shown as the detail; any other exception only by its class name. |
|
|
88
|
+
| `GET /metrics`, `GET /internal/metrics` | Prometheus text exposition. |
|
|
89
|
+
| `GET /__version__` | `/app/version.json` of the image, or a `dev` document. |
|
|
90
|
+
|
|
91
|
+
Errors use the platform's format, `{"status": "error", "error": {"code", "message"}}` (404 `NOT_FOUND`, 405
|
|
92
|
+
`METHOD_NOT_ALLOWED`, 500 `PROBE_FAILED`). `broker` and `draining` are reserved component names. `/ready` is also 503 in
|
|
93
|
+
job mode once the quota is reached and the consumer cancelled.
|
|
94
|
+
|
|
95
|
+
The server is bounded: at most 32 connections are served at once (the others are closed at once), reading the request,
|
|
96
|
+
writing the answer and closing each take at most 5 s, and concurrent `/ready` requests share a single evaluation of the
|
|
97
|
+
probes.
|
|
98
|
+
|
|
99
|
+
**Startup.** `/health` answers while the startup hooks run, but only as long as the event loop is free: a hook that loads a
|
|
100
|
+
model or a large file synchronously blocks it, `/health` stops answering and the kubelet restarts the pod in a loop. Run
|
|
101
|
+
heavy loading in `asyncio.to_thread`, or give the container a `startupProbe`.
|
|
102
|
+
|
|
103
|
+
Metrics have no task, client or service label (the scrape adds the pod and service): `worker_tasks_total{status}`,
|
|
104
|
+
`worker_task_duration_seconds`, `worker_messages_published_total{message_type}`, `event_loop_lag_seconds{component}`,
|
|
105
|
+
`worker_task_failures_total{code}`, `worker_task_retries_total`, `worker_poison_messages_total{reason}`,
|
|
106
|
+
`worker_publish_failures_total{message_type}`, `worker_ack_failures_total`, `worker_tasks_in_flight`,
|
|
107
|
+
`worker_shutdown_requeued_total`, `worker_progress_dropped_total{reason}` and `worker_sdk_info`, plus the process, platform
|
|
108
|
+
and GC collectors.
|
|
109
|
+
|
|
110
|
+
Logs are one JSON object per line on stdout (`ts`, `level`, `logger`, `msg`, and `task_id`, `service`, `attempt` when
|
|
111
|
+
bound; `ctx.logger` carries them, and so do the records of any library logging inside a handler, on the SDK's handler).
|
|
112
|
+
`LOG_LEVEL` sets the level. If the application configured the root logger before running the worker, its handlers are
|
|
113
|
+
kept and the SDK adds none (no line is printed twice): the format is then the application's, and so is the level of
|
|
114
|
+
the `aio_pika.tools` logger, whose two ERROR lines on a dropped connection the SDK otherwise logs as warnings (the
|
|
115
|
+
connection then reconnects by itself). To still get the task fields on your own handler, chain a record factory:
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
import logging
|
|
119
|
+
from async_task_api_worker.observability.logging import TaskContextFilter
|
|
120
|
+
|
|
121
|
+
task_fields = TaskContextFilter()
|
|
122
|
+
previous_factory = logging.getLogRecordFactory()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def factory(*args, **kwargs):
|
|
126
|
+
record = previous_factory(*args, **kwargs)
|
|
127
|
+
task_fields.filter(record)
|
|
128
|
+
return record
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
logging.setLogRecordFactory(factory)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Every delivery ends with exactly one INFO decision line, for instance `Delivery settled: outcome=failure action=conclude
|
|
135
|
+
code=NOT_ALLOWED attempt=1/5`, with the fields `task_id`, `attempt`, `delivery_limit`, `outcome` (`success`, `failure`,
|
|
136
|
+
`poison`, `retry`, `publish_failed`, `shutdown_requeued`, `cancelled`, `engine_error`), `action` (`ack`, `conclude`,
|
|
137
|
+
`nack-requeue`) and `code`: `grep <task_id>`
|
|
138
|
+
tells the whole story of a task. Payloads and secrets are never logged.
|
|
139
|
+
|
|
140
|
+
## Deployment or Job
|
|
141
|
+
|
|
142
|
+
The same worker code and image run as a long-lived consumer (a Kubernetes Deployment) or as a one-shot process (a
|
|
143
|
+
Job, for instance one started per message by KEDA's `ScaledJob`). The deployment picks the mode with `WORKER_MODE`; the
|
|
144
|
+
`Worker` definition does not change.
|
|
145
|
+
|
|
146
|
+
| Variable | Default | Meaning |
|
|
147
|
+
|---|---|---|
|
|
148
|
+
| `WORKER_MODE` | `service` | `service` consumes until SIGTERM; `job` is one-shot. Any other value exits with code 2. |
|
|
149
|
+
| `WORKER_JOB_MAX_TASKS` | `1` | Job mode: stop consuming at this many deliveries, finish them, and exit. |
|
|
150
|
+
| `WORKER_JOB_IDLE_TIMEOUT_S` | `10` | Job mode: exit when no delivery is in hand and none arrives for this long (another pod probably took the message). |
|
|
151
|
+
| `WORKER_JOB_CONNECT_TIMEOUT_S` | `30` | Job mode: give up and exit 1 when the broker stays unreachable for this long. Service mode retries forever. |
|
|
152
|
+
|
|
153
|
+
In job mode the prefetch is forced to 1, whatever `WORKER_CONCURRENCY` says (it then only sizes the handler semaphore
|
|
154
|
+
and the thread pool). The quota counts deliveries **on arrival**: any delivery counts, poison ones and redeliveries past
|
|
155
|
+
the limit included, and consuming stops as soon as the last one is in hand. The exit code reports the **pod**, not the
|
|
156
|
+
task: a task that succeeds, fails (`FatalError`, timeout, invalid input), or is requeued (`RetryableError`) all exit 0,
|
|
157
|
+
and so does an idle exit, because a failed task is a business outcome already published to the client. Exit 1 (an
|
|
158
|
+
infrastructure failure such as a broker unreachable for `WORKER_JOB_CONNECT_TIMEOUT_S`, or a `def` handler stuck after
|
|
159
|
+
the drain) and 2 (invalid configuration) stay non-zero, so the Job's `backoffLimit` applies to them. SIGTERM and
|
|
160
|
+
`activeDeadlineSeconds` drain exactly as in service mode.
|
|
161
|
+
|
|
162
|
+
## Testing your worker
|
|
163
|
+
|
|
164
|
+
`async_task_api_worker.testing` runs your task in memory through the real engine (decoding, time limit, retries, error
|
|
165
|
+
mapping, acknowledgements), with no broker and no Docker:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from pydantic import BaseModel
|
|
169
|
+
|
|
170
|
+
from async_task_api_worker import FatalError, TaskContext, Worker
|
|
171
|
+
from async_task_api_worker.testing import Outcome, ProgressRecorder, WorkerHarness, make_task_context
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class Input(BaseModel):
|
|
175
|
+
text: str
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class Output(BaseModel):
|
|
179
|
+
words: int
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
worker = Worker("count-words")
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@worker.task(timeout=30.0)
|
|
186
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
187
|
+
if not task_input.text.strip():
|
|
188
|
+
raise FatalError("EMPTY_TEXT", "There is nothing to count.")
|
|
189
|
+
await ctx.progress(0.5)
|
|
190
|
+
return Output(words=len(task_input.text.split()))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
async def test_counts_the_words() -> None:
|
|
194
|
+
async with WorkerHarness(worker) as harness:
|
|
195
|
+
run = await harness.submit({"text": "hello big world"})
|
|
196
|
+
|
|
197
|
+
run.assert_succeeded()
|
|
198
|
+
assert run.response == {"words": 3}
|
|
199
|
+
assert run.message_types == ("started", "progress", "success")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
async def test_an_empty_text_fails_with_its_code() -> None:
|
|
203
|
+
async with WorkerHarness(worker) as harness:
|
|
204
|
+
run = await harness.submit({"text": " "})
|
|
205
|
+
|
|
206
|
+
run.assert_failed(code="EMPTY_TEXT")
|
|
207
|
+
assert run.outcome is Outcome.ACKED
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
async def test_the_handler_alone_with_a_context_you_build() -> None:
|
|
211
|
+
recorder = ProgressRecorder()
|
|
212
|
+
ctx = make_task_context(recorder=recorder)
|
|
213
|
+
|
|
214
|
+
assert await handle(Input(text="a b"), ctx) == Output(words=2)
|
|
215
|
+
assert [update.fraction for update in recorder.updates] == [0.5]
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
- `harness.submit(body, client_id=..., input_refs=..., delivery_count=...)` builds the envelope the API publishes;
|
|
219
|
+
`submit_raw(bytes)` sends bytes as they are (a poison message). The result is a `TaskRun`: `messages` (decoded) and
|
|
220
|
+
`raw` (the published bytes), `message_types` (all three are tuples), `outcome` (`ACKED`, `NACKED_REQUEUE`), `response`,
|
|
221
|
+
`error`, and the two assertions. `task_id` is the envelope's, or `None` for bytes that do not carry one.
|
|
222
|
+
- `delivery_count` is the broker's `x-delivery-count`: pass `0`, then `delivery_count=n` to see a `RetryableError`
|
|
223
|
+
requeue until the last attempt (`worker_delivery_limit`), which concludes with the error's own code. Retry backoffs are
|
|
224
|
+
a millisecond; every other setting is the production default (progress throttling included).
|
|
225
|
+
- `settings_overrides={"worker_task_timeout_s": 0.1, ...}` takes `WorkerSettings` field names; no environment variable is
|
|
226
|
+
read, and the time budget is checked as at startup. `harness.fail_next_publish("success")` makes the next publish (of
|
|
227
|
+
that message type, or of any) fail once, to test the requeue. Use `async with` (or `await harness.aclose()`).
|
|
228
|
+
- `make_task_context` and `make_sync_task_context` build the context of an `async def` and of a `def` handler, to call
|
|
229
|
+
the handler directly; `ProgressRecorder` collects what it reports.
|
|
230
|
+
|
|
231
|
+
## Development
|
|
232
|
+
|
|
233
|
+
From this directory:
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
uv sync --locked --all-extras
|
|
237
|
+
uv run ruff check . && uv run ruff format --check .
|
|
238
|
+
uv run ty check src
|
|
239
|
+
uv run pytest -m "not integration" -q --cov=async_task_api_worker --cov-branch --cov-fail-under=95
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
The integration tests need Docker (they start RabbitMQ with the platform's policy through testcontainers):
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
uv run pytest -m integration -q
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Add dependencies with `uv add` from this directory, never by editing `pyproject.toml`.
|