async-task-api-worker 0.1.0rc2__tar.gz → 0.1.0rc5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- async_task_api_worker-0.1.0rc5/.gitignore +52 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/CHANGELOG.md +38 -0
- async_task_api_worker-0.1.0rc5/PKG-INFO +525 -0
- async_task_api_worker-0.1.0rc5/README.md +498 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/pyproject.toml +12 -1
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/_extras_settings.py +39 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/amqp_transport.py +8 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/lifecycle.py +26 -2
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/cli.py +255 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/errors.py +22 -1
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/llm.py +401 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/manifest.py +251 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/observability/logging.py +8 -1
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/protocol/asyncapi.py +258 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/protocol/envelope.py +3 -1
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/protocol/messages.py +20 -7
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/s3.py +456 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/settings.py +12 -3
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/__init__.py +79 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/testing/_harness.py +29 -30
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/testing/_memory_broker.py +12 -5
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/cases.py +100 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/conformance.py +738 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/fake_api.py +49 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/fake_listener.py +119 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/memory_broker.py +281 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/pytest_plugin.py +113 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/reference.py +46 -0
- async_task_api_worker-0.1.0rc5/src/async_task_api_worker/testing/reference_cases.json +27 -0
- async_task_api_worker-0.1.0rc2/.gitignore +0 -61
- async_task_api_worker-0.1.0rc2/PKG-INFO +0 -269
- async_task_api_worker-0.1.0rc2/README.md +0 -248
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/cli.py +0 -88
- async_task_api_worker-0.1.0rc2/src/async_task_api_worker/testing/__init__.py +0 -27
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/LICENSE +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/__init__.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/__main__.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/__init__.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/declare.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/engine.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/inflight.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/log_context.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/observer.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/progress.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/retry.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/sync_bridge.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_runtime/transport.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/_version.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/context.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/observability/__init__.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/observability/health.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/observability/loop_lag.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/observability/metrics.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/protocol/__init__.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/protocol/json_types.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/protocol/version.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/py.typed +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/testing/_context.py +0 -0
- {async_task_api_worker-0.1.0rc2 → async_task_api_worker-0.1.0rc5}/src/async_task_api_worker/worker.py +0 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
.venv
|
|
2
|
+
.env
|
|
3
|
+
.coverage
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
__pycache__
|
|
6
|
+
htmlcov/
|
|
7
|
+
junit.xml
|
|
8
|
+
.vscode/
|
|
9
|
+
.ruff_cache
|
|
10
|
+
node_modules/
|
|
11
|
+
package-lock.json
|
|
12
|
+
*.egg-info/
|
|
13
|
+
dist/
|
|
14
|
+
build/
|
|
15
|
+
*.log
|
|
16
|
+
*.pot
|
|
17
|
+
*.pyc
|
|
18
|
+
.scratch
|
|
19
|
+
# helm/charts/ is deliberately NOT ignored: ArgoCD cannot reach external repositories.
|
|
20
|
+
.DS_Store
|
|
21
|
+
demo.db
|
|
22
|
+
coverage.xml
|
|
23
|
+
|
|
24
|
+
# Sorties locales sensibles (audit, homologation, tests d'intrusion) — ne pas committer
|
|
25
|
+
private/
|
|
26
|
+
|
|
27
|
+
# Secrets et matériel cryptographique
|
|
28
|
+
.env.*
|
|
29
|
+
!.env.example
|
|
30
|
+
*.pem
|
|
31
|
+
*.key
|
|
32
|
+
*.p12
|
|
33
|
+
*.pfx
|
|
34
|
+
id_rsa*
|
|
35
|
+
id_ed25519*
|
|
36
|
+
*.kubeconfig
|
|
37
|
+
kubeconfig*
|
|
38
|
+
*.tfstate
|
|
39
|
+
*.tfstate.*
|
|
40
|
+
secrets/
|
|
41
|
+
*.secret
|
|
42
|
+
|
|
43
|
+
# Dumps et sauvegardes
|
|
44
|
+
*.sql.gz
|
|
45
|
+
*.dump
|
|
46
|
+
*.bak-*
|
|
47
|
+
|
|
48
|
+
# Assistant configuration is local to each machine: nothing under .claude/ is tracked.
|
|
49
|
+
.claude/
|
|
50
|
+
|
|
51
|
+
# Private working plans and notes: never committed.
|
|
52
|
+
docs/workers-v2/
|
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.1.0-rc.5](https://github.com/IA-Generative/async-api/compare/async-task-api-worker-v0.1.0-rc.4...async-task-api-worker-v0.1.0-rc.5) (2026-10-26)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* **sdk:** name the wait that timed out in conformance failures ([73cc271](https://github.com/IA-Generative/async-api/commit/73cc271d67950b5b272740af291b085d1b7e1546))
|
|
9
|
+
|
|
10
|
+
## [0.1.0-rc.4](https://github.com/IA-Generative/async-api/compare/async-task-api-worker-v0.1.0-rc.3...async-task-api-worker-v0.1.0-rc.4) (2026-10-26)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Features
|
|
14
|
+
|
|
15
|
+
* **sdk:** add the conformance command and the pytest plugin ([03a1c7b](https://github.com/IA-Generative/async-api/commit/03a1c7b06d45ecfe4410d5cfa3d9b254a4c7dad0))
|
|
16
|
+
* **sdk:** add the conformance suite and its reference worker ([d3e9fc7](https://github.com/IA-Generative/async-api/commit/d3e9fc791e5847e7c42f2b26ef153993545b0430))
|
|
17
|
+
* **sdk:** add the llm extra ([d650023](https://github.com/IA-Generative/async-api/commit/d650023d958890dede1b75d86e60039da55332be))
|
|
18
|
+
* **sdk:** add the manifest and spec commands ([700331c](https://github.com/IA-Generative/async-api/commit/700331c14dccf0625c6897decb4b525e3946c802))
|
|
19
|
+
* **sdk:** add the manifest of a service, derived from the input model ([8ab58d6](https://github.com/IA-Generative/async-api/commit/8ab58d6612a3bd718d7581cccccd595f9e6f6b47))
|
|
20
|
+
* **sdk:** add the quorum queue emulator, the fake API and the fake listener ([8706837](https://github.com/IA-Generative/async-api/commit/8706837238897aefeb5679b8f35ea4ce7931b2bf))
|
|
21
|
+
* **sdk:** add the s3 extra ([5ac8168](https://github.com/IA-Generative/async-api/commit/5ac81689d4c8974d51ddd849fe8cb8f5a1863238))
|
|
22
|
+
* **sdk:** generate the AsyncAPI 3.0 document of the protocol from its models ([749207b](https://github.com/IA-Generative/async-api/commit/749207bc797b475f09bac3e5c93480fa4ad58c20))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
### Documentation
|
|
26
|
+
|
|
27
|
+
* **sdk:** add the guides to write a worker and to register a service ([c5a6fe8](https://github.com/IA-Generative/async-api/commit/c5a6fe8bdc116a2a0a30dfbc3766cd19efd4d89e))
|
|
28
|
+
* **sdk:** document the conformance kit ([ae0bc9c](https://github.com/IA-Generative/async-api/commit/ae0bc9c2dc0bd55b732c4d1396cb55f33e0c296d))
|
|
29
|
+
* **sdk:** document the protocol, its versioning and the manifest and spec commands ([3e8d510](https://github.com/IA-Generative/async-api/commit/3e8d510a84387a46c81548c2eb3195c8b3b5ce97))
|
|
30
|
+
* **sdk:** document the s3 and llm extras ([e3bc270](https://github.com/IA-Generative/async-api/commit/e3bc2704a5acc520972dfdcc7ff1bb20180361e6))
|
|
31
|
+
|
|
32
|
+
## [0.1.0-rc.3](https://github.com/IA-Generative/async-api/compare/async-task-api-worker-v0.1.0-rc.2...async-task-api-worker-v0.1.0-rc.3) (2026-10-23)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
### Bug Fixes
|
|
36
|
+
|
|
37
|
+
* **sdk:** close an already closed channel quietly and tie the close budgets together ([51e0281](https://github.com/IA-Generative/async-api/commit/51e0281dfe7ea22686f0967d4c8ce55efa470d0e))
|
|
38
|
+
* **sdk:** name the invalid settings instead of quoting pydantic's text about them ([df6c747](https://github.com/IA-Generative/async-api/commit/df6c7475464600f31f767997269d3f86b1cf432e))
|
|
39
|
+
* **sdk:** treat a pydantic validation error in a startup hook as a configuration error ([5e98af1](https://github.com/IA-Generative/async-api/commit/5e98af109a027fc2903a3f98d5d0028f936efbba))
|
|
40
|
+
|
|
3
41
|
## [0.1.0-rc.2](https://github.com/IA-Generative/async-api/compare/async-task-api-worker-v0.1.0-rc.1...async-task-api-worker-v0.1.0-rc.2) (2026-10-23)
|
|
4
42
|
|
|
5
43
|
|
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: async-task-api-worker
|
|
3
|
+
Version: 0.1.0-rc.5
|
|
4
|
+
Summary: SDK to write workers for the AsyncTaskAPI platform: protocol models, runtime and conformance kit.
|
|
5
|
+
Project-URL: Repository, https://github.com/IA-Generative/async-api
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: async-api,rabbitmq,sdk,worker
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Python: >=3.13
|
|
16
|
+
Requires-Dist: aio-pika>=9.5.5
|
|
17
|
+
Requires-Dist: prometheus-client>=0.22.1
|
|
18
|
+
Requires-Dist: pydantic-settings>=2.10.1
|
|
19
|
+
Requires-Dist: pydantic>=2.10
|
|
20
|
+
Provides-Extra: llm
|
|
21
|
+
Requires-Dist: httpx>=0.28.1; extra == 'llm'
|
|
22
|
+
Provides-Extra: s3
|
|
23
|
+
Requires-Dist: boto3>=1.43.118; extra == 's3'
|
|
24
|
+
Provides-Extra: testing
|
|
25
|
+
Requires-Dist: pytest>=9.1.1; extra == 'testing'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# async-task-api-worker
|
|
29
|
+
|
|
30
|
+
SDK to write workers for the AsyncTaskAPI platform.
|
|
31
|
+
|
|
32
|
+
This is a work in progress: the package ships the protocol models (inbound envelope decoder and the byte-exact
|
|
33
|
+
outbound message encoder), the error types, the settings, the runtime engine with the `Worker` definition API, the
|
|
34
|
+
RabbitMQ transport, the process lifecycle (graceful drain), the probes, the metrics and the JSON logs. The conformance
|
|
35
|
+
suite is part of the testing kit below; the `[s3]` and `[llm]` extras are described below.
|
|
36
|
+
|
|
37
|
+
## Declaring a worker
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from pydantic import BaseModel, Field
|
|
41
|
+
|
|
42
|
+
from async_task_api_worker import TaskContext, Worker
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Input(BaseModel):
|
|
46
|
+
text: str = Field(min_length=1, max_length=100_000)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Output(BaseModel):
|
|
50
|
+
words: int
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
worker = Worker("count-words")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@worker.task(timeout=30.0)
|
|
57
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
58
|
+
await ctx.progress(0.5)
|
|
59
|
+
return Output(words=len(task_input.text.split()))
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
- `async def` handlers run on the event loop; `def` handlers (with a `SyncTaskContext`) run in a thread pool. The
|
|
63
|
+
context parameter must be annotated exactly `TaskContext` or `SyncTaskContext` (an identity check: no subclass or
|
|
64
|
+
`Annotated` wrapper). Hooks and probes (`on_startup`, `on_shutdown`, `readiness_probe`) must be `async def`.
|
|
65
|
+
- The input and output models are read from the annotations. Exactly one task per worker.
|
|
66
|
+
- Raise `FatalError(code, message)` to fail a task with a code the client can match, `RetryableError` for a
|
|
67
|
+
transient failure. Any other exception is reported as `INTERNAL_ERROR` without its text.
|
|
68
|
+
- `ctx.progress(fraction, payload)` reports the **latest state** of the task, not an event stream: updates are
|
|
69
|
+
throttled (at most one per `WORKER_PROGRESS_MIN_INTERVAL_S`, default 1 s), a newer update replaces a held one,
|
|
70
|
+
`fraction` is in `[0, 1)` and never goes back. The terminal success or failure message is the 100%.
|
|
71
|
+
|
|
72
|
+
## Running a worker
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
export BROKER_URL=amqp://user:password@broker:5672/ IN_QUEUE_NAME=count-words_queue_in OUT_QUEUE_NAME=count-words_queue_out
|
|
76
|
+
async-task-api-worker run my_module:worker # or: python -m async_task_api_worker run my_module:worker
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`my_module:worker` is a `Worker`, or a zero-argument factory function returning one; modules in the current directory can
|
|
80
|
+
be imported. A definition error in the module exits with code 2 and one line. From code, `worker.run()` blocks until the
|
|
81
|
+
process is stopped and `await worker.serve()` serves from an existing event loop.
|
|
82
|
+
|
|
83
|
+
- The settings are read and validated first: an invalid configuration, or a task timeout that does not fit under the
|
|
84
|
+
broker's consumer timeout (`task timeout + WORKER_RETRY_BACKOFF_MAX_S + 60 s < WORKER_CONSUMER_TIMEOUT_S`), exits with
|
|
85
|
+
code 2 before any connection. Exit codes: 0 clean stop, 1 failure, 2 configuration or usage error.
|
|
86
|
+
- A pydantic `ValidationError` raised by a startup hook (a settings class of your own, say) is a configuration error too:
|
|
87
|
+
one line naming the invalid fields, never their values (and `<Model> (a model-level check failed)` for a check that
|
|
88
|
+
belongs to no field), exit code 2. In job mode that fails the Job without a retry. So a startup hook must only validate
|
|
89
|
+
**configuration**: a hook that validates external data (a downloaded document, a response) must catch its own errors and
|
|
90
|
+
raise a `FatalError` or `RetryableError` from the task, where the task-level error model applies.
|
|
91
|
+
- The queues are declared durable and with no arguments (the broker decides their type and policies), and consumed with
|
|
92
|
+
manual acknowledgements: a task is acked only after its result is published.
|
|
93
|
+
- SIGTERM or SIGINT stops consuming, lets the in-flight tasks finish within `WORKER_SHUTDOWN_GRACE_S` (default 25 s),
|
|
94
|
+
requeues the ones that did not, closes the connection and exits 0. A second signal skips the rest of the grace period.
|
|
95
|
+
A `def` handler whose thread is still running after the drain, or an `async def` handler whose `asyncio.to_thread`
|
|
96
|
+
thread is (given 2 s after the drain), is logged and the process exits at once with code 1 (only for `run`, never for
|
|
97
|
+
`serve()`). The whole shutdown is bounded: the process exits at most
|
|
98
|
+
`max(WORKER_SHUTDOWN_GRACE_S, 5)` + 17 s after the signal (5 s for the nacks of the cancelled tasks, 5 s to close the
|
|
99
|
+
connection, and 5 s in total, `WORKER_SHUTDOWN_HOOKS_TIMEOUT_S`, for all the shutdown hooks together, each getting what is
|
|
100
|
+
left). Set a pod's `terminationGracePeriodSeconds` to at least that. A signal during startup does not wait for it: the
|
|
101
|
+
connection attempt and the startup hooks are cancelled and the process exits 0 (a hook that blocks the event loop
|
|
102
|
+
cannot be interrupted).
|
|
103
|
+
- **Retries and crash-loop protection need quorum queues** (the broker's default queue type, as in the platform's
|
|
104
|
+
chart). A classic queue gives no delivery counter: there, a `RetryableError` gets one retry, then concludes with its
|
|
105
|
+
own code, and a warning says so.
|
|
106
|
+
- Refused broker credentials exit with code 2 right away instead of being retried.
|
|
107
|
+
- The attempt number of a delivery (`ctx.attempt`) counts every time the broker handed the message out. A
|
|
108
|
+
`RetryableError` requeues until attempt `WORKER_DELIVERY_LIMIT`, then concludes with its own code. A delivery past
|
|
109
|
+
that limit (a crash loop) is concluded with `TASK_ABANDONED_AFTER_REDELIVERIES` without running the handler.
|
|
110
|
+
|
|
111
|
+
## Probes, metrics and logs
|
|
112
|
+
|
|
113
|
+
A small HTTP server runs on `HEALTH_CHECK_HOST:HEALTH_CHECK_PORT` (default `0.0.0.0:8083`) in both modes, from the very
|
|
114
|
+
start of the process until the end of the shutdown.
|
|
115
|
+
|
|
116
|
+
| Path | Answer |
|
|
117
|
+
|---|---|
|
|
118
|
+
| `GET /health` | Liveness. 200 `{"status": "ok"}`, or 503 `TASK_OVERRUN` when a `def` handler is still running `WORKER_TASK_OVERRUN_GRACE_S` after its deadline. It consults no dependency. |
|
|
119
|
+
| `GET /ready` | Readiness. 200 with a `components` map, or 503 `DEPENDENCIES_NOT_READY` (with the same `components` in `error`) when the broker is disconnected, the worker is draining, or one of the probes registered with `@worker.readiness_probe("name")` raises or exceeds `WORKER_PROBE_TIMEOUT_S` (3 s). Probes run in parallel. A `ProbeError` message is shown as the detail; any other exception only by its class name. |
|
|
120
|
+
| `GET /metrics`, `GET /internal/metrics` | Prometheus text exposition. |
|
|
121
|
+
| `GET /__version__` | `/app/version.json` of the image, or a `dev` document. |
|
|
122
|
+
|
|
123
|
+
Errors use the platform's format, `{"status": "error", "error": {"code", "message"}}` (404 `NOT_FOUND`, 405
|
|
124
|
+
`METHOD_NOT_ALLOWED`, 500 `PROBE_FAILED`). `broker` and `draining` are reserved component names. `/ready` is also 503 in
|
|
125
|
+
job mode once the quota is reached and the consumer cancelled.
|
|
126
|
+
|
|
127
|
+
The server is bounded: at most 32 connections are served at once (the others are closed at once), reading the request,
|
|
128
|
+
writing the answer and closing each take at most 5 s, and concurrent `/ready` requests share a single evaluation of the
|
|
129
|
+
probes.
|
|
130
|
+
|
|
131
|
+
**Startup.** `/health` answers while the startup hooks run, but only as long as the event loop is free: a hook that loads a
|
|
132
|
+
model or a large file synchronously blocks it, `/health` stops answering and the kubelet restarts the pod in a loop. Run
|
|
133
|
+
heavy loading in `asyncio.to_thread`, or give the container a `startupProbe`.
|
|
134
|
+
|
|
135
|
+
Metrics have no task, client or service label (the scrape adds the pod and service): `worker_tasks_total{status}`,
|
|
136
|
+
`worker_task_duration_seconds`, `worker_messages_published_total{message_type}`, `event_loop_lag_seconds{component}`,
|
|
137
|
+
`worker_task_failures_total{code}`, `worker_task_retries_total`, `worker_poison_messages_total{reason}`,
|
|
138
|
+
`worker_publish_failures_total{message_type}`, `worker_ack_failures_total`, `worker_tasks_in_flight`,
|
|
139
|
+
`worker_shutdown_requeued_total`, `worker_progress_dropped_total{reason}` and `worker_sdk_info`, plus the process, platform
|
|
140
|
+
and GC collectors.
|
|
141
|
+
|
|
142
|
+
Logs are one JSON object per line on stdout (`ts`, `level`, `logger`, `msg`, and `task_id`, `service`, `attempt` when
|
|
143
|
+
bound; `ctx.logger` carries them, and so do the records of any library logging inside a handler, on the SDK's handler).
|
|
144
|
+
`LOG_LEVEL` sets the level. If the application configured the root logger before running the worker, its handlers are
|
|
145
|
+
kept and the SDK adds none (no line is printed twice): the format is then the application's, and so is the level of
|
|
146
|
+
the `aio_pika.tools` logger, whose two ERROR lines on a dropped connection the SDK otherwise logs as warnings (the
|
|
147
|
+
connection then reconnects by itself). To still get the task fields on your own handler, chain a record factory:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
import logging
|
|
151
|
+
from async_task_api_worker.observability.logging import TaskContextFilter
|
|
152
|
+
|
|
153
|
+
task_fields = TaskContextFilter()
|
|
154
|
+
previous_factory = logging.getLogRecordFactory()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def factory(*args, **kwargs):
|
|
158
|
+
record = previous_factory(*args, **kwargs)
|
|
159
|
+
task_fields.filter(record)
|
|
160
|
+
return record
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
logging.setLogRecordFactory(factory)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Every delivery ends with exactly one INFO decision line, for instance `Delivery settled: outcome=failure action=conclude
|
|
167
|
+
code=NOT_ALLOWED attempt=1/5`, with the fields `task_id`, `attempt`, `delivery_limit`, `outcome` (`success`, `failure`,
|
|
168
|
+
`poison`, `retry`, `publish_failed`, `shutdown_requeued`, `cancelled`, `engine_error`), `action` (`ack`, `conclude`,
|
|
169
|
+
`nack-requeue`) and `code`: `grep <task_id>`
|
|
170
|
+
tells the whole story of a task. Payloads and secrets are never logged.
|
|
171
|
+
|
|
172
|
+
## Deployment or Job
|
|
173
|
+
|
|
174
|
+
The same worker code and image run as a long-lived consumer (a Kubernetes Deployment) or as a one-shot process (a
|
|
175
|
+
Job, for instance one started per message by KEDA's `ScaledJob`). The deployment picks the mode with `WORKER_MODE`; the
|
|
176
|
+
`Worker` definition does not change.
|
|
177
|
+
|
|
178
|
+
| Variable | Default | Meaning |
|
|
179
|
+
|---|---|---|
|
|
180
|
+
| `WORKER_MODE` | `service` | `service` consumes until SIGTERM; `job` is one-shot. Any other value exits with code 2. |
|
|
181
|
+
| `WORKER_JOB_MAX_TASKS` | `1` | Job mode: stop consuming at this many deliveries, finish them, and exit. |
|
|
182
|
+
| `WORKER_JOB_IDLE_TIMEOUT_S` | `10` | Job mode: exit when no delivery is in hand and none arrives for this long (another pod probably took the message). |
|
|
183
|
+
| `WORKER_JOB_CONNECT_TIMEOUT_S` | `30` | Job mode: give up and exit 1 when the broker stays unreachable for this long. Service mode retries forever. |
|
|
184
|
+
|
|
185
|
+
In job mode the prefetch is forced to 1, whatever `WORKER_CONCURRENCY` says (it then only sizes the handler semaphore
|
|
186
|
+
and the thread pool). The quota counts deliveries **on arrival**: any delivery counts, poison ones and redeliveries past
|
|
187
|
+
the limit included, and consuming stops as soon as the last one is in hand. The exit code reports the **pod**, not the
|
|
188
|
+
task: a task that succeeds, fails (`FatalError`, timeout, invalid input), or is requeued (`RetryableError`) all exit 0,
|
|
189
|
+
and so does an idle exit, because a failed task is a business outcome already published to the client. Exit 1 (an
|
|
190
|
+
infrastructure failure such as a broker unreachable for `WORKER_JOB_CONNECT_TIMEOUT_S`, or a `def` handler stuck after
|
|
191
|
+
the drain) and 2 (invalid configuration) stay non-zero, so the Job's `backoffLimit` applies to them. SIGTERM and
|
|
192
|
+
`activeDeadlineSeconds` drain exactly as in service mode.
|
|
193
|
+
|
|
194
|
+
## S3 extra
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
pip install 'async-task-api-worker[s3]'
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
`async_task_api_worker.s3` reads and writes objects on an S3-compatible store (AWS, Scaleway, MinIO). The core package
|
|
201
|
+
never imports it, and importing it without the extra raises an `ImportError` that names `[s3]`. Blocking boto3 calls run
|
|
202
|
+
in a thread, so the event loop is never blocked.
|
|
203
|
+
|
|
204
|
+
| Variable | Default | Meaning |
|
|
205
|
+
|---|---|---|
|
|
206
|
+
| `S3_BUCKET_NAME` | required | The bucket. |
|
|
207
|
+
| `S3_ENDPOINT_URL` | AWS | The endpoint of a non-AWS store. |
|
|
208
|
+
| `S3_REGION_NAME` | `fr-par` | The region the requests and the presigned URLs are signed for. Scaleway rejects the one boto falls back to (`us-east-1`), so the default is the deployments' region: set it for any other store. |
|
|
209
|
+
| `S3_ACCESS_KEY`, `S3_SECRET_KEY` | boto's credential chain | Both or neither. Held as secrets: never printed or logged. |
|
|
210
|
+
| `S3_VERIFY_SSL` | `true` | Verify the certificate of the endpoint. |
|
|
211
|
+
| `S3_CONNECT_TIMEOUT_S`, `S3_READ_TIMEOUT_S` | `10`, `60` | botocore timeouts of one attempt (capped by the time left before `deadline`, see below). |
|
|
212
|
+
| `S3_MAX_ATTEMPTS` | `3` | Attempts of botocore's standard retry mode (the initial one included): the only retry layer. Past it, a `RetryableError` hands the task back to the broker. |
|
|
213
|
+
|
|
214
|
+
```python
|
|
215
|
+
import time
|
|
216
|
+
|
|
217
|
+
from async_task_api_worker import TaskContext, Worker
|
|
218
|
+
from async_task_api_worker.s3 import ObjectStorage, S3Settings
|
|
219
|
+
|
|
220
|
+
worker = Worker("summarize")
|
|
221
|
+
storage = ObjectStorage(S3Settings()) # reads the S3_* variables
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@worker.readiness_probe("s3")
|
|
225
|
+
async def s3_ready() -> None:
|
|
226
|
+
await storage.check_access() # raises ProbeError: /ready answers 503 with this dependency down
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@worker.on_shutdown
|
|
230
|
+
async def close_storage() -> None:
|
|
231
|
+
await storage.aclose()
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@worker.task(timeout=60.0)
|
|
235
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
236
|
+
data = await storage.download(task_input.file_key, max_bytes=20_000_000, deadline=ctx.deadline)
|
|
237
|
+
...
|
|
238
|
+
await storage.upload("results/42.json", payload, content_type="application/json", deadline=ctx.deadline)
|
|
239
|
+
return Output(download_url=await storage.presign_get("results/42.json", expires_in=3600))
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
- `head_size(key, *, deadline)`, `download(key, *, max_bytes, deadline)`, `download_to(key, path, *, max_bytes, deadline)`
|
|
243
|
+
(streams to a file, which a failure removes), `upload(key, data, *, content_type=None, deadline)`,
|
|
244
|
+
`presign_get(key, *, expires_in)` (1 second to 7 days, no request), `check_access()`, `aclose()`; the storage is also an
|
|
245
|
+
async context manager. `content_type` is stored as given; without it the store applies its default.
|
|
246
|
+
- `deadline` is `ctx.deadline`. A call that starts past it, or still runs when it passes, raises
|
|
247
|
+
`RetryableError(UPSTREAM_UNAVAILABLE)`. The botocore timeouts of a call are capped by the time left (rounded up to 1,
|
|
248
|
+
2, 5, 10, 30 or 60 s: a client is built per value), so a call abandoned at its deadline ends soon after it.
|
|
249
|
+
- A blocking boto3 call cannot be interrupted. It runs in a **daemon thread the SDK owns**, not in the event loop's
|
|
250
|
+
default executor: an abandoned call (past its deadline, or when the worker stops) is left behind, its result
|
|
251
|
+
discarded, and never delays the exit nor turns a clean exit (code 0, a Job's included) into exit 1. Worst case, it
|
|
252
|
+
holds one thread and one connection for the botocore bound of its call: at most `S3_MAX_ATTEMPTS` times the capped
|
|
253
|
+
connect and read timeouts, and at most 3 times 70 s (about 3.5 minutes) for a call with more than a minute left.
|
|
254
|
+
An upload abandoned at its deadline can still complete in the background after the task was requeued: write outputs to
|
|
255
|
+
idempotent or per-attempt keys.
|
|
256
|
+
`check_access()` answers within 3 s whatever the settings (`ObjectStorage(probe_budget_s=...)`), and its thread
|
|
257
|
+
lingers the same way.
|
|
258
|
+
- The size is checked three times: on the HEAD, before any byte is read; on the GET, since the object may have been
|
|
259
|
+
replaced in between; and on the bytes actually read. Over `max_bytes` is `FatalError(FILE_TOO_LARGE)`, and memory
|
|
260
|
+
never holds more than `max_bytes` plus one byte.
|
|
261
|
+
- Errors: a missing key is `FatalError(DOCUMENT_NOT_FOUND)`. Connection errors, timeouts, 5xx, 408 and 429, and the
|
|
262
|
+
codes `RequestTimeout`, `SlowDown`, `InternalError`, `ServiceUnavailable`, `Throttling` whatever the status, are
|
|
263
|
+
`RetryableError(UPSTREAM_UNAVAILABLE)`. Anything else (`AccessDenied`, `NoSuchBucket`, no credentials) is an
|
|
264
|
+
`ObjectStorageError` carrying the operation and the S3 code, which the runtime reports as `INTERNAL_ERROR`. No error
|
|
265
|
+
text contains the endpoint, a key or a credential, and the botocore exception is not chained to it.
|
|
266
|
+
- **AWS answers 403, not 404, to a HEAD on a missing key when the credentials lack `s3:ListBucket`.** That is
|
|
267
|
+
indistinguishable from a real permission error and ends as an `ObjectStorageError`: grant `s3:ListBucket` to get
|
|
268
|
+
`DOCUMENT_NOT_FOUND`. `check_access()` sends a HEAD on the bucket, so it needs that permission too.
|
|
269
|
+
- A presigned URL embeds the endpoint and the key: give it to the client, do not log it.
|
|
270
|
+
|
|
271
|
+
## LLM extra
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
pip install 'async-task-api-worker[llm]'
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
`async_task_api_worker.llm` is a chat client for OpenAI-compatible servers (vLLM, OpenAI, Scaleway...) over `httpx`.
|
|
278
|
+
There is no `openai` package behind it, so no hidden retry layer. Importing it without the extra raises an
|
|
279
|
+
`ImportError` that names `[llm]`.
|
|
280
|
+
|
|
281
|
+
| Variable | Default | Meaning |
|
|
282
|
+
|---|---|---|
|
|
283
|
+
| `LLM_BASE_URL` | required | The API root, for instance `http://vllm:8000/v1`. |
|
|
284
|
+
| `LLM_API_KEY` | none | Sent as a bearer token. Held as a secret. A deployment maps its own secret onto this name at the environment level, for instance with a Kubernetes `secretKeyRef`. |
|
|
285
|
+
| `LLM_VERIFY_SSL` | `true` | `false` turns off the certificate check (a hub with a private CA). |
|
|
286
|
+
| `LLM_CONNECT_TIMEOUT_S` | `10` | Connection timeout, capped by the time left before the task deadline. |
|
|
287
|
+
| `LLM_TIMEOUT_S` | `120` | Timeout of each operation of a request (read, write), capped by the time left. The whole request is also cut at the task deadline. |
|
|
288
|
+
| `LLM_RETRY_ATTEMPTS` | `3` | Requests in total (the first one included). |
|
|
289
|
+
| `LLM_RETRY_AFTER_MAX_S` | `60` | Cap of the wait asked for by `Retry-After` or `retry-after-ms`. |
|
|
290
|
+
| `LLM_RETRY_BACKOFF_BASE_S`, `LLM_RETRY_BACKOFF_MAX_S` | `1`, `30` | Without `Retry-After`: full jitter below `base * 2^(attempt-1)`, at most `max`. |
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
from async_task_api_worker import TaskContext, Worker
|
|
294
|
+
from async_task_api_worker.llm import ChatMessage, ImageUrlPart, LLMClient, LLMSettings, Role, TextPart
|
|
295
|
+
|
|
296
|
+
worker = Worker("describe-page")
|
|
297
|
+
llm = LLMClient(LLMSettings()) # reads the LLM_* variables
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@worker.on_shutdown
|
|
301
|
+
async def close_llm() -> None:
|
|
302
|
+
await llm.aclose()
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
@worker.task(timeout=120.0)
|
|
306
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
307
|
+
page = ImageUrlPart.from_bytes(task_input.png, media_type="image/png")
|
|
308
|
+
result = await llm.chat(
|
|
309
|
+
[ChatMessage(Role.USER, [TextPart("Transcribe this page."), page])],
|
|
310
|
+
model="my-vision-model",
|
|
311
|
+
deadline=ctx.deadline,
|
|
312
|
+
max_tokens=4096,
|
|
313
|
+
temperature=0.0,
|
|
314
|
+
extra_body={"skip_special_tokens": False}, # vLLM-specific fields, merged at the top level
|
|
315
|
+
)
|
|
316
|
+
return Output(text=result.text)
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
- `chat(messages, *, model, deadline, max_tokens=None, temperature=None, response_format=None, extra_body=None)`
|
|
320
|
+
returns a `ChatResult(text, finish_reason, model, usage)`. A message is plain text or a list of `TextPart` and
|
|
321
|
+
`ImageUrlPart` (a `data:image/...` URL or an `https://` one: anything else is a `ValueError`).
|
|
322
|
+
`response_format` and `extra_body` are JSON objects; an `extra_body` key that would replace `model`, `messages`,
|
|
323
|
+
`stream` or a field set by an argument is a `ValueError`.
|
|
324
|
+
- Retries happen on 408, 429, 500, 502, 503, 504 and on any `httpx` error (connection errors, timeouts, an undecodable body). `Retry-After` (seconds or HTTP date)
|
|
325
|
+
and `retry-after-ms` set the wait. The client never sleeps past `deadline` and never sends a request after it: when
|
|
326
|
+
the wait does not fit in the time left it gives up at once.
|
|
327
|
+
- Giving up on a transient failure (attempts used up, or no time left) is `RetryableError(UPSTREAM_UNAVAILABLE)`, and so
|
|
328
|
+
is any other 5xx, without in-process retry, except 501 and 505 (a server that does not implement the route or the protocol: a deployment problem). Other statuses are `FatalError(LLM_REQUEST_REJECTED)`; an answer that is not
|
|
329
|
+
JSON, has no choice or no text is `LLM_OUTPUT_INVALID`; `finish_reason` `length` is `LLM_OUTPUT_TRUNCATED` and
|
|
330
|
+
`content_filter` is `LLM_OUTPUT_FILTERED`. No error text contains the URL, a header or a response body.
|
|
331
|
+
- Use `async with LLMClient(settings) as llm:` or call `aclose()`.
|
|
332
|
+
- `embed()` is not provided yet.
|
|
333
|
+
- `LLMClient` also takes `transport=` (an `httpx` transport) and `clock`, `sleep`, `jitter` and `now` (time and
|
|
334
|
+
randomness). They are **test seams, outside the stability guarantee**: they may change or go with a minor release.
|
|
335
|
+
`ObjectStorage`'s `client`, `clock` and `probe_budget_s` are test seams too.
|
|
336
|
+
- `httpx` logs every request URL at INFO. `configure_logging` keeps the `httpx` and `httpcore` loggers at WARNING when
|
|
337
|
+
the SDK installs its own log handler. An application that installs its own handler before running the worker still
|
|
338
|
+
gets those INFO lines (URLs included) unless it raises these two loggers itself.
|
|
339
|
+
|
|
340
|
+
## Testing your worker
|
|
341
|
+
|
|
342
|
+
`async_task_api_worker.testing` runs your task in memory through the real engine (decoding, time limit, retries, error
|
|
343
|
+
mapping, acknowledgements), with no broker and no Docker:
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
from pydantic import BaseModel
|
|
347
|
+
|
|
348
|
+
from async_task_api_worker import FatalError, TaskContext, Worker
|
|
349
|
+
from async_task_api_worker.testing import Outcome, ProgressRecorder, WorkerHarness, make_task_context
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
class Input(BaseModel):
|
|
353
|
+
text: str
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
class Output(BaseModel):
|
|
357
|
+
words: int
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
worker = Worker("count-words")
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@worker.task(timeout=30.0)
|
|
364
|
+
async def handle(task_input: Input, ctx: TaskContext) -> Output:
|
|
365
|
+
if not task_input.text.strip():
|
|
366
|
+
raise FatalError("EMPTY_TEXT", "There is nothing to count.")
|
|
367
|
+
await ctx.progress(0.5)
|
|
368
|
+
return Output(words=len(task_input.text.split()))
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
async def test_counts_the_words() -> None:
|
|
372
|
+
async with WorkerHarness(worker) as harness:
|
|
373
|
+
run = await harness.submit({"text": "hello big world"})
|
|
374
|
+
|
|
375
|
+
run.assert_succeeded()
|
|
376
|
+
assert run.response == {"words": 3}
|
|
377
|
+
assert run.message_types == ("started", "progress", "success")
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
async def test_an_empty_text_fails_with_its_code() -> None:
|
|
381
|
+
async with WorkerHarness(worker) as harness:
|
|
382
|
+
run = await harness.submit({"text": " "})
|
|
383
|
+
|
|
384
|
+
run.assert_failed(code="EMPTY_TEXT")
|
|
385
|
+
assert run.outcome is Outcome.ACKED
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
async def test_the_handler_alone_with_a_context_you_build() -> None:
|
|
389
|
+
recorder = ProgressRecorder()
|
|
390
|
+
ctx = make_task_context(recorder=recorder)
|
|
391
|
+
|
|
392
|
+
assert await handle(Input(text="a b"), ctx) == Output(words=2)
|
|
393
|
+
assert [update.fraction for update in recorder.updates] == [0.5]
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
- `harness.submit(body, client_id=..., input_refs=..., delivery_count=...)` builds the envelope the API publishes;
|
|
397
|
+
`submit_raw(bytes)` sends bytes as they are (a poison message). The result is a `TaskRun`: `messages` (decoded) and
|
|
398
|
+
`raw` (the published bytes), `message_types` (all three are tuples), `outcome` (`ACKED`, `NACKED_REQUEUE`), `response`,
|
|
399
|
+
`error`, and the two assertions. `task_id` is the envelope's, or `None` for bytes that do not carry one.
|
|
400
|
+
- `delivery_count` is the broker's `x-delivery-count`: pass `0`, then `delivery_count=n` to see a `RetryableError`
|
|
401
|
+
requeue until the last attempt (`worker_delivery_limit`), which concludes with the error's own code. Retry backoffs are
|
|
402
|
+
a millisecond; every other setting is the production default (progress throttling included).
|
|
403
|
+
- `settings_overrides={"worker_task_timeout_s": 0.1, ...}` takes `WorkerSettings` field names; no environment variable is
|
|
404
|
+
read, and the time budget is checked as at startup. `harness.fail_next_publish("success")` makes the next publish (of
|
|
405
|
+
that message type, or of any) fail once, to test the requeue. Use `async with` (or `await harness.aclose()`).
|
|
406
|
+
- `make_task_context` and `make_sync_task_context` build the context of an `async def` and of a `def` handler, to call
|
|
407
|
+
the handler directly; `ProgressRecorder` collects what it reports.
|
|
408
|
+
|
|
409
|
+
## Conformance
|
|
410
|
+
|
|
411
|
+
The conformance suite checks, in memory, that a worker honours the platform contract with the SDK runtime under it.
|
|
412
|
+
It needs no broker and no Docker, so it runs in your own CI.
|
|
413
|
+
|
|
414
|
+
```bash
|
|
415
|
+
pip install 'async-task-api-worker[testing]' # the [testing] extra only adds pytest, for the plugin
|
|
416
|
+
async-task-api-worker conformance my_package.worker:worker --cases conformance.json --junit conformance.xml
|
|
417
|
+
pytest -p async_task_api_worker.testing.pytest_plugin --async-task-api-worker my_package.worker:worker \
|
|
418
|
+
--async-task-api-worker-cases conformance.json
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
The command exits **0** when every check ran and none failed, **1** with a readable report when one failed, **2** when
|
|
422
|
+
the target or a file (cases, contract) cannot be read, and **3** when nothing failed but a check was skipped for lack
|
|
423
|
+
of input (see below): a green run must not prove less than it claims. The opt-in C11 is the only benign skip.
|
|
424
|
+
|
|
425
|
+
The pytest plugin adds one test per check (a check that cannot run is a skipped test with its reason). The
|
|
426
|
+
plugin is not loaded by default: name it with `-p`, or put `pytest_plugins = ["async_task_api_worker.testing.pytest_plugin"]`
|
|
427
|
+
in your `conftest.py`. `async_task_api_worker.testing.reference:worker` is the SDK's own reference worker: the suite
|
|
428
|
+
always passes on it, with the cases it ships: `--cases "$(python -c 'from async_task_api_worker.testing.reference import CASES_FILE; print(CASES_FILE)')"`
|
|
429
|
+
(body `{"sleep_s": 0, "fail_with": null, "retryable": false}`).
|
|
430
|
+
|
|
431
|
+
`conformance.json` (or `--async-task-api-worker-cases` for pytest) lists the cases of your service:
|
|
432
|
+
|
|
433
|
+
```json
|
|
434
|
+
[
|
|
435
|
+
{"name": "counts words", "body": {"text": "hello world"}, "expect": "success"},
|
|
436
|
+
{"name": "rejects empty text", "body": {"text": ""}, "expect": {"failure": "INPUT_INVALID"}}
|
|
437
|
+
]
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
| Check | Exercises | What it proves |
|
|
441
|
+
|---|---|---|
|
|
442
|
+
| C01 | worker | Needs cases: skipped without them (the handler's own behaviour is not checked). Each case: `started`, then exactly one terminal message, then the ack; the outcome is the expected one. A delivery that is not settled exactly once fails here. |
|
|
443
|
+
| C02 | worker | Every published message is a valid protocol message, byte-encoded canonically, with a zone-aware timestamp. |
|
|
444
|
+
| C03 | worker | An invalid input (built from your input model: a missing field, a wrong type, an unknown field) fails with `INPUT_INVALID` and never reaches the handler. Skipped when the model accepts everything tried. |
|
|
445
|
+
| C04 | sdk | Every kind of poison message is acked and produces no message. |
|
|
446
|
+
| C05 | sdk | An unexpected exception becomes `INTERNAL_ERROR`, and its text never reaches a published message. |
|
|
447
|
+
| C06 | sdk | On an emulated quorum queue, a `RetryableError` is requeued until `WORKER_DELIVERY_LIMIT`, then concluded with its own code: one ack, nothing dead-lettered. |
|
|
448
|
+
| C07 | sdk | A terminal (or `started`) message that fails to publish requeues the delivery, and never acks it. |
|
|
449
|
+
| C08 | sdk | A task past its time limit fails with `TASK_TIMEOUT` and is acked. |
|
|
450
|
+
| C09 | sdk | A drain lets an in-flight task finish and ack, requeues one that outlives the grace period, and consumes nothing after the stop. |
|
|
451
|
+
| C10 | sdk | A burst of progress updates is throttled, stays in `[0, 1)` and never goes back, and nothing follows the terminal message. |
|
|
452
|
+
| C11 | worker | **Opt-in** (`--contract contrat.json`): the file `{"nom": <service>, "entree": <JSON schema>}` matches the service name and the input model's schema. |
|
|
453
|
+
| C12 | sdk | `/health` and `/metrics` answer, and `/ready` is 200 while consuming and 503 once the broker is gone. |
|
|
454
|
+
|
|
455
|
+
**`worker` checks exercise your own code** (C01 to C03, and C11 on your model); **`sdk` checks verify the guarantees of the
|
|
456
|
+
SDK runtime** under your models and settings (retry, drain, probes...): C05 to C10 and C12 run them with a handler of
|
|
457
|
+
their own (a failure, a sleeper) in place of yours, so they cannot fail because of your handler. A worker whose handler
|
|
458
|
+
fails on every input passes them, and fails C01 as soon as you give it a `success` case. Every check but C03 and C11
|
|
459
|
+
needs an input your model accepts: the first `success` case, else one built from the schema. Without one
|
|
460
|
+
they are skipped, the report lists them above its summary ("Not checked, for lack of input") and the exit code is 3. Your
|
|
461
|
+
startup hooks, shutdown hooks and readiness probes are not run by default (they need the real dependencies), so a worker
|
|
462
|
+
whose own probe always fails still passes C12, which proves the SDK's `/ready` semantics: pass `--with-probes`
|
|
463
|
+
(`--async-task-api-worker-with-probes`, `with_probes=True`) to run them in C12 when their dependencies are reachable. The
|
|
464
|
+
settings are the production defaults, not your environment (`settings_overrides` of `ConformanceSuite` applies yours).
|
|
465
|
+
|
|
466
|
+
The contract file of C11 is `{"nom": <service>, "entree": <JSON schema>}`: the keys are those of the existing `contrat.json`
|
|
467
|
+
manifest, kept so that those files work as they are. A malformed one is an error of usage (exit 2), not a failed check.
|
|
468
|
+
|
|
469
|
+
The pieces are public, for your own tests:
|
|
470
|
+
|
|
471
|
+
- `InMemoryBroker(delivery_limit=5, prefetch=1)` emulates the quorum queue with the counters measured on RabbitMQ 4.3 and
|
|
472
|
+
later (ADR 0005): a voluntary `nack(requeue=True)` only increments `x-acquired-count` and never dead-letters, a delivery
|
|
473
|
+
returned after a failure (`disconnect()`) increments both counters and is dead-lettered after `delivery_limit + 1`
|
|
474
|
+
deliveries. It is a transport: `submit(bytes)`, `published`, `published_types()`, `dead_lettered`, `ready`, `unacked`,
|
|
475
|
+
`fail_next_publish()`, `disconnect()` / `reconnect()`, `wait_idle()`, `acks` and `nacks` (the `requeue` flags, in order). The suite runs the real process lifecycle on it.
|
|
476
|
+
- `FakeApi().submission(body, client_id=..., input_refs=...)` builds the envelope the API publishes, byte for byte;
|
|
477
|
+
`FakeListener` validates what a worker publishes against the protocol models and applies the listener's rules (a
|
|
478
|
+
terminal task is never overwritten; a progress of 1 or more is ignored and the task stays in progress, as the real listener does, although the models refuse it): `register(task_id)`, `ingest(raw)`,
|
|
479
|
+
`status(task_id)`.
|
|
480
|
+
- `ConformanceSuite(worker, cases, contract=None, settings_overrides=None).run()` returns a `ConformanceReport`
|
|
481
|
+
(`passed`, `complete`, `results`, `to_text()`, `to_junit_xml()`); `run_check("C06")` runs one check. The text and JUnit
|
|
482
|
+
reports give each check's kind. The keyword arguments of `ConformanceSuite` that start with an underscore are test seams
|
|
483
|
+
of the suite's own tests, outside the stability guarantee.
|
|
484
|
+
|
|
485
|
+
## Manifest and protocol document
|
|
486
|
+
|
|
487
|
+
```bash
|
|
488
|
+
async-task-api-worker manifest my_module:worker --title "Count words" --description "Counts the words of a text." \
|
|
489
|
+
--class court --output contrat.json # the manifest of the service (the input schema is derived)
|
|
490
|
+
async-task-api-worker manifest my_module:worker --check contrat.json # exit 1 when the file drifted from the model
|
|
491
|
+
async-task-api-worker spec # the AsyncAPI 3.0 document of the protocol, from the SDK models
|
|
492
|
+
async-task-api-worker spec --check asyncapi.json # exit 1 when a committed copy is out of date
|
|
493
|
+
```
|
|
494
|
+
|
|
495
|
+
- The manifest uses the keys of the platform's `contrat.json` format (`nom`, `titre`, `description`, `classe`, `entree`,
|
|
496
|
+
and the optional `description_courte`, `icone`, `taille_max_fichier_mio`, `gpu`). `nom` is the service name and `entree` is
|
|
497
|
+
`input_model.model_json_schema()`: it can never be written wrong by hand, and `--check` proves a committed file is
|
|
498
|
+
still right. `--class` is one of `immediat`, `court`, `long`; the catalogue reads it, so give it when the file feeds
|
|
499
|
+
the catalogue. `build_manifest`, `parse_manifest`, `load_manifest` and `check_manifest` are in
|
|
500
|
+
`async_task_api_worker.manifest`.
|
|
501
|
+
- The file is what the conformance check C11 reads (`conformance --contract contrat.json`): both sides use the same keys, so the manifest you generate is the contract file you check.
|
|
502
|
+
- Set `WORKER_MANIFEST_PATH` to a manifest in the image to make the worker refuse to start (exit 2, before any
|
|
503
|
+
connection) when it does not read, names another service or no longer matches the input model.
|
|
504
|
+
- Exit codes of the commands: 0 ok, 1 a `--check` found a drift, 2 usage or an unreadable file.
|
|
505
|
+
- The protocol document is also published in the repository, under `docs/protocol/`, with the versioning rules, the
|
|
506
|
+
error codes and the acknowledgement semantics.
|
|
507
|
+
|
|
508
|
+
## Development
|
|
509
|
+
|
|
510
|
+
From this directory:
|
|
511
|
+
|
|
512
|
+
```bash
|
|
513
|
+
uv sync --locked --all-extras
|
|
514
|
+
uv run ruff check . && uv run ruff format --check .
|
|
515
|
+
uv run ty check src
|
|
516
|
+
uv run pytest -m "not integration" -q --cov=async_task_api_worker --cov-branch --cov-fail-under=95
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
The integration tests need Docker (they start RabbitMQ with the platform's policy through testcontainers):
|
|
520
|
+
|
|
521
|
+
```bash
|
|
522
|
+
uv run pytest -m integration -q
|
|
523
|
+
```
|
|
524
|
+
|
|
525
|
+
Add dependencies with `uv add` from this directory, never by editing `pyproject.toml`.
|