milvaion-worker 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- milvaion_worker-1.0.0/.gitignore +6 -0
- milvaion_worker-1.0.0/PKG-INFO +166 -0
- milvaion_worker-1.0.0/README.md +147 -0
- milvaion_worker-1.0.0/examples/basic_worker.py +61 -0
- milvaion_worker-1.0.0/pyproject.toml +37 -0
- milvaion_worker-1.0.0/src/milvaion/__init__.py +22 -0
- milvaion_worker-1.0.0/src/milvaion/cancellation.py +76 -0
- milvaion_worker-1.0.0/src/milvaion/context.py +152 -0
- milvaion_worker-1.0.0/src/milvaion/errors.py +48 -0
- milvaion_worker-1.0.0/src/milvaion/log_buffer.py +104 -0
- milvaion_worker-1.0.0/src/milvaion/metrics.py +145 -0
- milvaion_worker-1.0.0/src/milvaion/options.py +187 -0
- milvaion_worker-1.0.0/src/milvaion/processor.py +545 -0
- milvaion_worker-1.0.0/src/milvaion/protocol.py +35 -0
- milvaion_worker-1.0.0/src/milvaion/py.typed +0 -0
- milvaion_worker-1.0.0/src/milvaion/worker.py +451 -0
- milvaion_worker-1.0.0/tests/test_processor.py +381 -0
- milvaion_worker-1.0.0/tests/test_units.py +160 -0
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: milvaion-worker
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Python worker SDK for the Milvaion job scheduler
|
|
5
|
+
Project-URL: Homepage, https://github.com/Milvasoft/milvaion
|
|
6
|
+
Project-URL: Source, https://github.com/Milvasoft/milvaion/tree/master/sdks/python/milvaion-worker
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Keywords: cron,job-scheduler,milvaion,rabbitmq,worker
|
|
9
|
+
Classifier: Framework :: AsyncIO
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Typing :: Typed
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Requires-Dist: aio-pika>=9.4
|
|
14
|
+
Requires-Dist: redis>=5.0
|
|
15
|
+
Provides-Extra: test
|
|
16
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
|
|
17
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# Milvaion Python Worker SDK
|
|
21
|
+
|
|
22
|
+
Run [Milvaion](https://github.com/Milvasoft/milvaion) jobs in Python. A Python worker shows up in the dashboard like a .NET worker. The scheduler dispatches jobs to it, and the dashboard shows its live logs, status, retries, cancellation and timeouts.
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install milvaion-worker
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
It requires Python 3.10 or later, and connects to the same RabbitMQ and Redis instances as your Milvaion API.
|
|
29
|
+
|
|
30
|
+
## Quick start
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from milvaion import JobContext, PermanentJobError, Worker
|
|
34
|
+
|
|
35
|
+
worker = Worker(
|
|
36
|
+
worker_id="report-worker",
|
|
37
|
+
rabbitmq_url="amqp://guest:guest@localhost:5672/",
|
|
38
|
+
redis_url="redis://localhost:6379",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@worker.job("BuildReportJob", max_retries=3, execution_timeout=600)
|
|
43
|
+
async def build_report(ctx: JobContext) -> dict:
|
|
44
|
+
data = ctx.data or {}
|
|
45
|
+
|
|
46
|
+
if "month" not in data:
|
|
47
|
+
raise PermanentJobError('Job data needs "month".') # no retries, straight to the DLQ
|
|
48
|
+
|
|
49
|
+
ctx.log(f"Building the report for {data['month']}")
|
|
50
|
+
rows = await fetch_rows(data["month"]) # cancelled if the job is cancelled or times out
|
|
51
|
+
|
|
52
|
+
return {"rows": len(rows)} # shown as the occurrence result
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@worker.job("CleanupJob")
|
|
56
|
+
def cleanup(ctx: JobContext) -> None: # plain functions run in a thread
|
|
57
|
+
for path in old_files():
|
|
58
|
+
ctx.raise_if_aborted() # threads cannot be stopped from outside; check in long loops
|
|
59
|
+
path.unlink()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
worker.run() # blocks; SIGINT/SIGTERM shut down gracefully
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Then create a job in the dashboard: pick the worker `report-worker` and the job `BuildReportJob`, and set a schedule.
|
|
67
|
+
|
|
68
|
+
To run the worker inside your own event loop, call `await worker.start()` and `await worker.stop()` instead of `run()`.
|
|
69
|
+
|
|
70
|
+
## Jobs
|
|
71
|
+
|
|
72
|
+
`@worker.job(name, **options)` or `worker.add_job(name, handler, JobOptions(...))` registers a handler under the name the dashboard dispatches it by. Names are unique within a worker and matched case-insensitively.
|
|
73
|
+
|
|
74
|
+
Handlers can be `async def` or plain functions. A plain function runs in a thread, so blocking code does not stall other jobs.
|
|
75
|
+
|
|
76
|
+
The handler receives a `JobContext`:
|
|
77
|
+
|
|
78
|
+
| Member | Description |
|
|
79
|
+
|---|---|
|
|
80
|
+
| `ctx.data` | Parsed job data, or `None`. Data that is not valid JSON fails the job permanently. |
|
|
81
|
+
| `ctx.raw_data` | Job data as the raw JSON string. |
|
|
82
|
+
| `ctx.occurrence_id` | Id of this execution. It stays the same across retries and redeliveries, so you can use it as an idempotency key. |
|
|
83
|
+
| `ctx.retry_count` | `0` on the first attempt, then `1`, `2` and so on. |
|
|
84
|
+
| `ctx.is_aborted`, `ctx.abort_reason`, `ctx.raise_if_aborted()` | Whether the job was cancelled, timed out or interrupted by shutdown. |
|
|
85
|
+
| `ctx.log` / `debug` / `warn` / `error(message, error=None, data=None)` | Write to the occurrence log in the dashboard. |
|
|
86
|
+
| `ctx.job` | The scheduled job as the API sent it. |
|
|
87
|
+
|
|
88
|
+
How the handler ends decides the outcome:
|
|
89
|
+
|
|
90
|
+
| The handler... | Outcome |
|
|
91
|
+
|---|---|
|
|
92
|
+
| returns a value | `Completed`. A string is stored as-is; anything else is stored as JSON. |
|
|
93
|
+
| raises | Retried with exponential backoff. After `max_retries`, the occurrence is `Failed` and the job goes to the dead letter queue. |
|
|
94
|
+
| raises `PermanentJobError` | `Failed` without retrying, and the job goes to the dead letter queue. |
|
|
95
|
+
| runs past its timeout | `TimedOut` |
|
|
96
|
+
| is cancelled from the dashboard | `Cancelled` |
|
|
97
|
+
| is still running when the worker stops | The job goes back to the queue and runs again. |
|
|
98
|
+
|
|
99
|
+
When a job is aborted, an `async` handler is cancelled, so it gets `asyncio.CancelledError` at its next `await`. A plain function cannot be stopped from outside. The occurrence still settles right away, but the thread keeps running until the function returns, so check `ctx.raise_if_aborted()` in long loops.
|
|
100
|
+
|
|
101
|
+
### Job options
|
|
102
|
+
|
|
103
|
+
| Option | Default | Description |
|
|
104
|
+
|---|---|---|
|
|
105
|
+
| `max_parallel_jobs` | min(10, worker `max_parallel_jobs`) | Jobs of this type that may run at once on this instance |
|
|
106
|
+
| `execution_timeout` | worker default (3600 s) | Seconds. `0` disables the timeout. A timeout set on the job in the dashboard takes precedence. |
|
|
107
|
+
| `max_retries` | 3 | Retries after the first failure |
|
|
108
|
+
| `base_retry_delay` | 5 | Retry *n* waits `base × 2^(n-1)` seconds |
|
|
109
|
+
| `data_schema` | none | JSON Schema of the job data, as a dict or string. The dashboard builds the job data form from it; with Pydantic, use `Model.model_json_schema()`. |
|
|
110
|
+
| `result_schema` | none | JSON Schema of the result |
|
|
111
|
+
| `routing_pattern` | `{worker_id}.{name without "Job"}.*` | RabbitMQ routing pattern |
|
|
112
|
+
| `consumer_id` | `{name without "Job"}-consumer` | Shown in the dashboard |
|
|
113
|
+
|
|
114
|
+
## Worker options
|
|
115
|
+
|
|
116
|
+
Pass these to `Worker(...)` or `WorkerOptions(...)`:
|
|
117
|
+
|
|
118
|
+
| Option | Default | Description |
|
|
119
|
+
|---|---|---|
|
|
120
|
+
| `worker_id` | required | Identifies the worker application. Use the same value on every replica. |
|
|
121
|
+
| `rabbitmq_url` | `amqp://guest:guest@localhost:5672/` | |
|
|
122
|
+
| `queue_type` | `classic` | Must match the API's `MilvaionConfig:RabbitMQ:QueueType` |
|
|
123
|
+
| `redis_url` | `redis://localhost:6379` | Redis only carries cancellation requests. `None` disables them. |
|
|
124
|
+
| `redis_key_prefix` | `Milvaion:JobScheduler:` | Must match the API's `MilvaionConfig:Redis:KeyPrefix` |
|
|
125
|
+
| `max_parallel_jobs` | CPU count × 2 | Capacity reported for this instance |
|
|
126
|
+
| `execution_timeout` | 3600 | Default job timeout in seconds |
|
|
127
|
+
| `heartbeat_interval` | 30 | |
|
|
128
|
+
| `job_heartbeat_interval` | 60 | Keeps long jobs from being flagged as zombies |
|
|
129
|
+
| `shutdown_timeout` | 30 | How long `stop()` lets running jobs finish |
|
|
130
|
+
| `startup_timeout` | 30 | How long `start()` waits for RabbitMQ and Redis |
|
|
131
|
+
| `version` | `1.0.0` | Shown in the dashboard |
|
|
132
|
+
| `logger` | `logging.getLogger("milvaion")` | |
|
|
133
|
+
|
|
134
|
+
For containers, `Worker(WorkerOptions.from_env())` reads the following variables:
|
|
135
|
+
|
|
136
|
+
- `MILVAION_WORKER_ID`
|
|
137
|
+
- `MILVAION_RABBITMQ_URL`
|
|
138
|
+
- `MILVAION_RABBITMQ_QUEUE_TYPE`
|
|
139
|
+
- `MILVAION_REDIS_URL` (`false` disables Redis)
|
|
140
|
+
- `MILVAION_REDIS_KEY_PREFIX`
|
|
141
|
+
- `MILVAION_MAX_PARALLEL_JOBS`
|
|
142
|
+
- `MILVAION_EXECUTION_TIMEOUT_SECONDS`
|
|
143
|
+
- `MILVAION_HEARTBEAT_INTERVAL_SECONDS`
|
|
144
|
+
|
|
145
|
+
## Reliability
|
|
146
|
+
|
|
147
|
+
- **Reconnects.** If RabbitMQ goes away, the connection is restored with its queues and consumers, and the worker registers again. The Redis listener reconnects and resubscribes.
|
|
148
|
+
- **At-least-once delivery.** A message is acknowledged only after its outcome has been reported, so a crash makes the job run again. Make handlers idempotent; `ctx.occurrence_id` helps with that.
|
|
149
|
+
- **Graceful shutdown.** `stop()` does the following in order:
|
|
150
|
+
1. Stops taking new jobs.
|
|
151
|
+
2. Waits up to `shutdown_timeout` for running jobs to finish.
|
|
152
|
+
3. Returns the jobs that are still running to the queue.
|
|
153
|
+
4. Flushes logs and tells the API the instance has left.
|
|
154
|
+
|
|
155
|
+
## How it works
|
|
156
|
+
|
|
157
|
+
The SDK implements the language-neutral [worker protocol](https://github.com/Milvasoft/milvaion/blob/master/docs/githubdocs/WORKER-PROTOCOL.md): the RabbitMQ queues and message formats the .NET SDK uses.
|
|
158
|
+
|
|
159
|
+
## Development
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
python -m venv .venv && . .venv/bin/activate
|
|
163
|
+
pip install -e ".[test]"
|
|
164
|
+
pytest
|
|
165
|
+
python examples/basic_worker.py # configure with MILVAION_* variables
|
|
166
|
+
```
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# Milvaion Python Worker SDK
|
|
2
|
+
|
|
3
|
+
Run [Milvaion](https://github.com/Milvasoft/milvaion) jobs in Python. A Python worker shows up in the dashboard like a .NET worker. The scheduler dispatches jobs to it, and the dashboard shows its live logs, status, retries, cancellation and timeouts.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install milvaion-worker
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
It requires Python 3.10 or later, and connects to the same RabbitMQ and Redis instances as your Milvaion API.
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from milvaion import JobContext, PermanentJobError, Worker
|
|
15
|
+
|
|
16
|
+
worker = Worker(
|
|
17
|
+
worker_id="report-worker",
|
|
18
|
+
rabbitmq_url="amqp://guest:guest@localhost:5672/",
|
|
19
|
+
redis_url="redis://localhost:6379",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@worker.job("BuildReportJob", max_retries=3, execution_timeout=600)
|
|
24
|
+
async def build_report(ctx: JobContext) -> dict:
|
|
25
|
+
data = ctx.data or {}
|
|
26
|
+
|
|
27
|
+
if "month" not in data:
|
|
28
|
+
raise PermanentJobError('Job data needs "month".') # no retries, straight to the DLQ
|
|
29
|
+
|
|
30
|
+
ctx.log(f"Building the report for {data['month']}")
|
|
31
|
+
rows = await fetch_rows(data["month"]) # cancelled if the job is cancelled or times out
|
|
32
|
+
|
|
33
|
+
return {"rows": len(rows)} # shown as the occurrence result
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@worker.job("CleanupJob")
|
|
37
|
+
def cleanup(ctx: JobContext) -> None: # plain functions run in a thread
|
|
38
|
+
for path in old_files():
|
|
39
|
+
ctx.raise_if_aborted() # threads cannot be stopped from outside; check in long loops
|
|
40
|
+
path.unlink()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
worker.run() # blocks; SIGINT/SIGTERM shut down gracefully
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Then create a job in the dashboard: pick the worker `report-worker` and the job `BuildReportJob`, and set a schedule.
|
|
48
|
+
|
|
49
|
+
To run the worker inside your own event loop, call `await worker.start()` and `await worker.stop()` instead of `run()`.
|
|
50
|
+
|
|
51
|
+
## Jobs
|
|
52
|
+
|
|
53
|
+
`@worker.job(name, **options)` or `worker.add_job(name, handler, JobOptions(...))` registers a handler under the name the dashboard dispatches it by. Names are unique within a worker and matched case-insensitively.
|
|
54
|
+
|
|
55
|
+
Handlers can be `async def` or plain functions. A plain function runs in a thread, so blocking code does not stall other jobs.
|
|
56
|
+
|
|
57
|
+
The handler receives a `JobContext`:
|
|
58
|
+
|
|
59
|
+
| Member | Description |
|
|
60
|
+
|---|---|
|
|
61
|
+
| `ctx.data` | Parsed job data, or `None`. Data that is not valid JSON fails the job permanently. |
|
|
62
|
+
| `ctx.raw_data` | Job data as the raw JSON string. |
|
|
63
|
+
| `ctx.occurrence_id` | Id of this execution. It stays the same across retries and redeliveries, so you can use it as an idempotency key. |
|
|
64
|
+
| `ctx.retry_count` | `0` on the first attempt, then `1`, `2` and so on. |
|
|
65
|
+
| `ctx.is_aborted`, `ctx.abort_reason`, `ctx.raise_if_aborted()` | Whether the job was cancelled, timed out or interrupted by shutdown. |
|
|
66
|
+
| `ctx.log` / `debug` / `warn` / `error(message, error=None, data=None)` | Write to the occurrence log in the dashboard. |
|
|
67
|
+
| `ctx.job` | The scheduled job as the API sent it. |
|
|
68
|
+
|
|
69
|
+
How the handler ends decides the outcome:
|
|
70
|
+
|
|
71
|
+
| The handler... | Outcome |
|
|
72
|
+
|---|---|
|
|
73
|
+
| returns a value | `Completed`. A string is stored as-is; anything else is stored as JSON. |
|
|
74
|
+
| raises | Retried with exponential backoff. After `max_retries`, the occurrence is `Failed` and the job goes to the dead letter queue. |
|
|
75
|
+
| raises `PermanentJobError` | `Failed` without retrying, and the job goes to the dead letter queue. |
|
|
76
|
+
| runs past its timeout | `TimedOut` |
|
|
77
|
+
| is cancelled from the dashboard | `Cancelled` |
|
|
78
|
+
| is still running when the worker stops | The job goes back to the queue and runs again. |
|
|
79
|
+
|
|
80
|
+
When a job is aborted, an `async` handler is cancelled, so it gets `asyncio.CancelledError` at its next `await`. A plain function cannot be stopped from outside. The occurrence still settles right away, but the thread keeps running until the function returns, so check `ctx.raise_if_aborted()` in long loops.
|
|
81
|
+
|
|
82
|
+
### Job options
|
|
83
|
+
|
|
84
|
+
| Option | Default | Description |
|
|
85
|
+
|---|---|---|
|
|
86
|
+
| `max_parallel_jobs` | min(10, worker `max_parallel_jobs`) | Jobs of this type that may run at once on this instance |
|
|
87
|
+
| `execution_timeout` | worker default (3600 s) | Seconds. `0` disables the timeout. A timeout set on the job in the dashboard takes precedence. |
|
|
88
|
+
| `max_retries` | 3 | Retries after the first failure |
|
|
89
|
+
| `base_retry_delay` | 5 | Retry *n* waits `base × 2^(n-1)` seconds |
|
|
90
|
+
| `data_schema` | none | JSON Schema of the job data, as a dict or string. The dashboard builds the job data form from it; with Pydantic, use `Model.model_json_schema()`. |
|
|
91
|
+
| `result_schema` | none | JSON Schema of the result |
|
|
92
|
+
| `routing_pattern` | `{worker_id}.{name without "Job"}.*` | RabbitMQ routing pattern |
|
|
93
|
+
| `consumer_id` | `{name without "Job"}-consumer` | Shown in the dashboard |
|
|
94
|
+
|
|
95
|
+
## Worker options
|
|
96
|
+
|
|
97
|
+
Pass these to `Worker(...)` or `WorkerOptions(...)`:
|
|
98
|
+
|
|
99
|
+
| Option | Default | Description |
|
|
100
|
+
|---|---|---|
|
|
101
|
+
| `worker_id` | required | Identifies the worker application. Use the same value on every replica. |
|
|
102
|
+
| `rabbitmq_url` | `amqp://guest:guest@localhost:5672/` | |
|
|
103
|
+
| `queue_type` | `classic` | Must match the API's `MilvaionConfig:RabbitMQ:QueueType` |
|
|
104
|
+
| `redis_url` | `redis://localhost:6379` | Redis only carries cancellation requests. `None` disables them. |
|
|
105
|
+
| `redis_key_prefix` | `Milvaion:JobScheduler:` | Must match the API's `MilvaionConfig:Redis:KeyPrefix` |
|
|
106
|
+
| `max_parallel_jobs` | CPU count × 2 | Capacity reported for this instance |
|
|
107
|
+
| `execution_timeout` | 3600 | Default job timeout in seconds |
|
|
108
|
+
| `heartbeat_interval` | 30 | |
|
|
109
|
+
| `job_heartbeat_interval` | 60 | Keeps long jobs from being flagged as zombies |
|
|
110
|
+
| `shutdown_timeout` | 30 | How long `stop()` lets running jobs finish |
|
|
111
|
+
| `startup_timeout` | 30 | How long `start()` waits for RabbitMQ and Redis |
|
|
112
|
+
| `version` | `1.0.0` | Shown in the dashboard |
|
|
113
|
+
| `logger` | `logging.getLogger("milvaion")` | |
|
|
114
|
+
|
|
115
|
+
For containers, `Worker(WorkerOptions.from_env())` reads the following variables:
|
|
116
|
+
|
|
117
|
+
- `MILVAION_WORKER_ID`
|
|
118
|
+
- `MILVAION_RABBITMQ_URL`
|
|
119
|
+
- `MILVAION_RABBITMQ_QUEUE_TYPE`
|
|
120
|
+
- `MILVAION_REDIS_URL` (`false` disables Redis)
|
|
121
|
+
- `MILVAION_REDIS_KEY_PREFIX`
|
|
122
|
+
- `MILVAION_MAX_PARALLEL_JOBS`
|
|
123
|
+
- `MILVAION_EXECUTION_TIMEOUT_SECONDS`
|
|
124
|
+
- `MILVAION_HEARTBEAT_INTERVAL_SECONDS`
|
|
125
|
+
|
|
126
|
+
## Reliability
|
|
127
|
+
|
|
128
|
+
- **Reconnects.** If RabbitMQ goes away, the connection is restored with its queues and consumers, and the worker registers again. The Redis listener reconnects and resubscribes.
|
|
129
|
+
- **At-least-once delivery.** A message is acknowledged only after its outcome has been reported, so a crash makes the job run again. Make handlers idempotent; `ctx.occurrence_id` helps with that.
|
|
130
|
+
- **Graceful shutdown.** `stop()` does the following in order:
|
|
131
|
+
1. Stops taking new jobs.
|
|
132
|
+
2. Waits up to `shutdown_timeout` for running jobs to finish.
|
|
133
|
+
3. Returns the jobs that are still running to the queue.
|
|
134
|
+
4. Flushes logs and tells the API the instance has left.
|
|
135
|
+
|
|
136
|
+
## How it works
|
|
137
|
+
|
|
138
|
+
The SDK implements the language-neutral [worker protocol](https://github.com/Milvasoft/milvaion/blob/master/docs/githubdocs/WORKER-PROTOCOL.md): the RabbitMQ queues and message formats the .NET SDK uses.
|
|
139
|
+
|
|
140
|
+
## Development
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
python -m venv .venv && . .venv/bin/activate
|
|
144
|
+
pip install -e ".[test]"
|
|
145
|
+
pytest
|
|
146
|
+
python examples/basic_worker.py # configure with MILVAION_* variables
|
|
147
|
+
```
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Sample Milvaion worker.
|
|
2
|
+
|
|
3
|
+
MILVAION_WORKER_ID=py-sample-worker \
|
|
4
|
+
MILVAION_RABBITMQ_URL=amqp://guest:guest@localhost:5672 \
|
|
5
|
+
MILVAION_REDIS_URL=redis://localhost:6379 \
|
|
6
|
+
python examples/basic_worker.py
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
|
|
15
|
+
from milvaion import JobContext, PermanentJobError, Worker, WorkerOptions
|
|
16
|
+
|
|
17
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
18
|
+
|
|
19
|
+
worker = Worker(WorkerOptions.from_env(
|
|
20
|
+
worker_id=os.environ.get("MILVAION_WORKER_ID", "py-sample-worker"),
|
|
21
|
+
heartbeat_interval=5,
|
|
22
|
+
))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@worker.job(
|
|
26
|
+
"GreetJob",
|
|
27
|
+
execution_timeout=60,
|
|
28
|
+
data_schema={
|
|
29
|
+
"type": "object",
|
|
30
|
+
"properties": {
|
|
31
|
+
"name": {"type": "string", "description": "Who to greet"},
|
|
32
|
+
"delayMs": {"type": "integer", "description": "How long the job takes"},
|
|
33
|
+
},
|
|
34
|
+
"required": ["name"],
|
|
35
|
+
},
|
|
36
|
+
)
|
|
37
|
+
async def greet(ctx: JobContext) -> dict:
|
|
38
|
+
data = ctx.data or {}
|
|
39
|
+
|
|
40
|
+
if not data.get("name"):
|
|
41
|
+
raise PermanentJobError('Job data needs a "name".')
|
|
42
|
+
|
|
43
|
+
ctx.log(f"Greeting {data['name']}", {"attempt": ctx.retry_count + 1})
|
|
44
|
+
|
|
45
|
+
# Cancelled from the dashboard or by a timeout: asyncio cancels this await.
|
|
46
|
+
await asyncio.sleep(data.get("delayMs", 1000) / 1000)
|
|
47
|
+
|
|
48
|
+
return {"greeting": f"Hello, {data['name']}!"}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@worker.job("FlakyJob", max_retries=2, base_retry_delay=2)
|
|
52
|
+
def flaky(ctx: JobContext) -> None:
|
|
53
|
+
# A plain function runs in a thread. Fails on the first attempt to show retries in the dashboard.
|
|
54
|
+
if ctx.retry_count == 0:
|
|
55
|
+
raise RuntimeError("Simulated transient failure")
|
|
56
|
+
|
|
57
|
+
ctx.log("Succeeded on retry")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
if __name__ == "__main__":
|
|
61
|
+
worker.run()
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "milvaion-worker"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Python worker SDK for the Milvaion job scheduler"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
keywords = ["milvaion", "job-scheduler", "worker", "rabbitmq", "cron"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Framework :: AsyncIO",
|
|
16
|
+
"Typing :: Typed",
|
|
17
|
+
]
|
|
18
|
+
dependencies = [
|
|
19
|
+
"aio-pika>=9.4",
|
|
20
|
+
"redis>=5.0",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
test = ["pytest>=8", "pytest-asyncio>=0.23"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/Milvasoft/milvaion"
|
|
28
|
+
Source = "https://github.com/Milvasoft/milvaion/tree/master/sdks/python/milvaion-worker"
|
|
29
|
+
|
|
30
|
+
[tool.hatch.build.targets.wheel]
|
|
31
|
+
packages = ["src/milvaion"]
|
|
32
|
+
|
|
33
|
+
[tool.pytest.ini_options]
|
|
34
|
+
asyncio_mode = "auto"
|
|
35
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
36
|
+
testpaths = ["tests"]
|
|
37
|
+
pythonpath = ["src"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Python worker SDK for the Milvaion job scheduler."""
|
|
2
|
+
|
|
3
|
+
from .context import JobContext
|
|
4
|
+
from .errors import AbortReason, JobAborted, PermanentJobError
|
|
5
|
+
from .options import JobOptions, WorkerOptions, generate_routing_pattern, queue_name_for_pattern
|
|
6
|
+
from .protocol import SDK_VERSION, Status
|
|
7
|
+
from .worker import Worker
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"AbortReason",
|
|
11
|
+
"JobAborted",
|
|
12
|
+
"JobContext",
|
|
13
|
+
"JobOptions",
|
|
14
|
+
"PermanentJobError",
|
|
15
|
+
"Status",
|
|
16
|
+
"Worker",
|
|
17
|
+
"WorkerOptions",
|
|
18
|
+
"generate_routing_pattern",
|
|
19
|
+
"queue_name_for_pattern",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
__version__ = SDK_VERSION
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import contextlib
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from redis import asyncio as aioredis
|
|
10
|
+
|
|
11
|
+
from .processor import CancellationRegistry
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CancellationListener:
|
|
15
|
+
"""Receives the cancellation requests the API publishes on Redis."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, url: str, channel: str, registry: CancellationRegistry, logger: logging.Logger) -> None:
|
|
18
|
+
self._url = url
|
|
19
|
+
self._channel = channel
|
|
20
|
+
self._registry = registry
|
|
21
|
+
self._logger = logger
|
|
22
|
+
self._client: Any = None
|
|
23
|
+
self._task: asyncio.Task[None] | None = None
|
|
24
|
+
|
|
25
|
+
async def start(self) -> None:
|
|
26
|
+
self._client = aioredis.from_url(self._url, decode_responses=True, health_check_interval=30)
|
|
27
|
+
await self._client.ping()
|
|
28
|
+
|
|
29
|
+
pubsub = self._client.pubsub(ignore_subscribe_messages=True)
|
|
30
|
+
await pubsub.subscribe(self._channel)
|
|
31
|
+
|
|
32
|
+
self._task = asyncio.get_running_loop().create_task(self._listen(pubsub))
|
|
33
|
+
self._logger.info("Listening for cancellations on %s", self._channel)
|
|
34
|
+
|
|
35
|
+
async def _listen(self, pubsub: Any) -> None:
|
|
36
|
+
while True:
|
|
37
|
+
try:
|
|
38
|
+
async for message in pubsub.listen():
|
|
39
|
+
if message.get("type") == "message":
|
|
40
|
+
self._on_message(message.get("data"))
|
|
41
|
+
except asyncio.CancelledError:
|
|
42
|
+
with contextlib.suppress(Exception):
|
|
43
|
+
await pubsub.aclose()
|
|
44
|
+
raise
|
|
45
|
+
except Exception as error: # noqa: BLE001 - reconnect and resubscribe
|
|
46
|
+
self._logger.warning("Redis cancellation listener error: %s. Reconnecting...", error)
|
|
47
|
+
await asyncio.sleep(2)
|
|
48
|
+
|
|
49
|
+
with contextlib.suppress(Exception):
|
|
50
|
+
await pubsub.subscribe(self._channel)
|
|
51
|
+
|
|
52
|
+
def _on_message(self, payload: Any) -> None:
|
|
53
|
+
try:
|
|
54
|
+
request = json.loads(payload)
|
|
55
|
+
except (TypeError, ValueError):
|
|
56
|
+
self._logger.debug("Ignored a cancellation message that is not JSON.")
|
|
57
|
+
return
|
|
58
|
+
|
|
59
|
+
if not isinstance(request, dict):
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
occurrence_id = request.get("CorrelationId") or request.get("OccurrenceId")
|
|
63
|
+
|
|
64
|
+
if occurrence_id and self._registry.cancel(occurrence_id, request.get("Reason")):
|
|
65
|
+
self._logger.info("Cancelling occurrence %s: %s", occurrence_id, request.get("Reason") or "no reason given")
|
|
66
|
+
|
|
67
|
+
async def stop(self) -> None:
|
|
68
|
+
if self._task is not None:
|
|
69
|
+
self._task.cancel()
|
|
70
|
+
|
|
71
|
+
with contextlib.suppress(BaseException):
|
|
72
|
+
await self._task
|
|
73
|
+
|
|
74
|
+
if self._client is not None:
|
|
75
|
+
with contextlib.suppress(Exception):
|
|
76
|
+
await self._client.aclose()
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import threading
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .errors import AbortReason, JobAborted, PermanentJobError, _type_name, format_error
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def utc_now() -> str:
|
|
13
|
+
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class _Abort:
|
|
17
|
+
"""Abort state of one execution. Readable from handler threads."""
|
|
18
|
+
|
|
19
|
+
def __init__(self) -> None:
|
|
20
|
+
self._lock = threading.Lock()
|
|
21
|
+
self.reason: AbortReason | None = None
|
|
22
|
+
self.message = ""
|
|
23
|
+
self._listeners: list[Callable[[], None]] = []
|
|
24
|
+
|
|
25
|
+
def trigger(self, reason: AbortReason, message: str) -> bool:
|
|
26
|
+
with self._lock:
|
|
27
|
+
if self.reason is not None:
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
self.reason, self.message = reason, message
|
|
31
|
+
listeners = list(self._listeners)
|
|
32
|
+
|
|
33
|
+
for listener in listeners:
|
|
34
|
+
listener()
|
|
35
|
+
|
|
36
|
+
return True
|
|
37
|
+
|
|
38
|
+
def on_abort(self, listener: Callable[[], None]) -> None:
|
|
39
|
+
with self._lock:
|
|
40
|
+
if self.reason is None:
|
|
41
|
+
self._listeners.append(listener)
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
listener()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class JobContext:
|
|
48
|
+
"""What a job handler receives."""
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
*,
|
|
53
|
+
occurrence_id: str,
|
|
54
|
+
job: dict[str, Any],
|
|
55
|
+
retry_count: int,
|
|
56
|
+
instance_id: str,
|
|
57
|
+
abort: _Abort,
|
|
58
|
+
sink: Callable[[dict[str, Any]], None],
|
|
59
|
+
) -> None:
|
|
60
|
+
self.occurrence_id = occurrence_id
|
|
61
|
+
"""Id of this execution. Stable across retries and redeliveries, so it works as an idempotency key."""
|
|
62
|
+
self.retry_count = retry_count
|
|
63
|
+
"""0 on the first attempt, then 1, 2, ..."""
|
|
64
|
+
self.instance_id = instance_id
|
|
65
|
+
"""Instance id of this worker process."""
|
|
66
|
+
self.job = job
|
|
67
|
+
"""The scheduled job as the API sent it."""
|
|
68
|
+
self._abort = abort
|
|
69
|
+
self._sink = sink
|
|
70
|
+
self._data: Any = None
|
|
71
|
+
self._parsed = False
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def job_id(self) -> str:
|
|
75
|
+
return self.job["id"]
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def job_name(self) -> str:
|
|
79
|
+
return self.job["jobNameInWorker"]
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def raw_data(self) -> str | None:
|
|
83
|
+
"""Job data as the raw JSON string, or None."""
|
|
84
|
+
return self.job.get("jobData")
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def data(self) -> Any:
|
|
88
|
+
"""Parsed job data, or None when the job has none. Invalid JSON fails the job permanently."""
|
|
89
|
+
if not self._parsed:
|
|
90
|
+
raw = (self.raw_data or "").strip()
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
self._data = json.loads(raw) if raw else None
|
|
94
|
+
except json.JSONDecodeError as error:
|
|
95
|
+
raise PermanentJobError("Job data is not valid JSON.") from error
|
|
96
|
+
|
|
97
|
+
self._parsed = True
|
|
98
|
+
|
|
99
|
+
return self._data
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def is_aborted(self) -> bool:
|
|
103
|
+
"""True once the job has been cancelled, has timed out or the worker is shutting down.
|
|
104
|
+
|
|
105
|
+
Async handlers are also cancelled (they get ``asyncio.CancelledError``); synchronous handlers
|
|
106
|
+
run in a thread and should check this in long loops."""
|
|
107
|
+
return self._abort.reason is not None
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def abort_reason(self) -> AbortReason | None:
|
|
111
|
+
return self._abort.reason
|
|
112
|
+
|
|
113
|
+
def raise_if_aborted(self) -> None:
|
|
114
|
+
"""Raises JobAborted if the job has been aborted."""
|
|
115
|
+
if self._abort.reason is not None:
|
|
116
|
+
raise JobAborted(self._abort.reason, self._abort.message)
|
|
117
|
+
|
|
118
|
+
def log(self, message: str, data: dict[str, Any] | None = None) -> None:
|
|
119
|
+
"""Writes an Information entry to the occurrence log."""
|
|
120
|
+
self._write("Information", message, data)
|
|
121
|
+
|
|
122
|
+
def debug(self, message: str, data: dict[str, Any] | None = None) -> None:
|
|
123
|
+
self._write("Debug", message, data)
|
|
124
|
+
|
|
125
|
+
def warn(self, message: str, data: dict[str, Any] | None = None) -> None:
|
|
126
|
+
self._write("Warning", message, data)
|
|
127
|
+
|
|
128
|
+
def error(self, message: str, error: BaseException | None = None, data: dict[str, Any] | None = None) -> None:
|
|
129
|
+
"""Writes an Error entry, with the details of ``error`` when given."""
|
|
130
|
+
details = dict(data or {})
|
|
131
|
+
exception_type = None
|
|
132
|
+
|
|
133
|
+
if error is not None:
|
|
134
|
+
exception_type = _type_name(error)
|
|
135
|
+
details["ExceptionType"] = exception_type
|
|
136
|
+
details["StackTrace"] = format_error(error)
|
|
137
|
+
inner = error.__cause__ or error.__context__
|
|
138
|
+
|
|
139
|
+
if inner is not None:
|
|
140
|
+
details["InnerException"] = str(inner)
|
|
141
|
+
|
|
142
|
+
self._write("Error", message, details, "UserDefined", exception_type)
|
|
143
|
+
|
|
144
|
+
def _write(self, level: str, message: str, data: dict[str, Any] | None, category: str = "UserCode", exception_type: str | None = None) -> None:
|
|
145
|
+
self._sink({
|
|
146
|
+
"timestamp": utc_now(),
|
|
147
|
+
"level": level,
|
|
148
|
+
"message": message,
|
|
149
|
+
"data": data,
|
|
150
|
+
"category": category,
|
|
151
|
+
"exceptionType": exception_type,
|
|
152
|
+
})
|