beanqueue 1.2.0__tar.gz → 2.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- beanqueue-1.2.0/README.md → beanqueue-2.0.0rc1/PKG-INFO +159 -9
- beanqueue-1.2.0/PKG-INFO → beanqueue-2.0.0rc1/README.md +139 -25
- beanqueue-2.0.0rc1/bq/app.py +508 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/create_tables.py +8 -1
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/process.py +3 -1
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/submit.py +20 -10
- beanqueue-2.0.0rc1/bq/config.py +156 -0
- beanqueue-2.0.0rc1/bq/db/session.py +6 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/events.py +2 -0
- beanqueue-2.0.0rc1/bq/metrics.py +176 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/processors/processor.py +63 -12
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/processors/registry.py +7 -5
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/processors/retry_policies.py +8 -5
- beanqueue-2.0.0rc1/bq/services/dispatch.py +135 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/services/worker.py +26 -24
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/pyproject.toml +20 -4
- beanqueue-1.2.0/bq/app.py +0 -564
- beanqueue-1.2.0/bq/config.py +0 -87
- beanqueue-1.2.0/bq/db/session.py +0 -5
- beanqueue-1.2.0/bq/services/dispatch.py +0 -117
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/.gitignore +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/LICENSE +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/__init__.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/__init__.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/cli.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/environment.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/main.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/cmds/utils.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/constants.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/db/__init__.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/db/base.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/models/__init__.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/models/event.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/models/helpers.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/models/task.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/models/worker.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/processors/__init__.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/services/__init__.py +0 -0
- {beanqueue-1.2.0 → beanqueue-2.0.0rc1}/bq/utils.py +0 -0
|
@@ -1,6 +1,26 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: beanqueue
|
|
3
|
+
Version: 2.0.0rc1
|
|
4
|
+
Summary: BeanQueue or BQ for short, PostgreSQL SKIP LOCK and SQLAlchemy based worker queue library
|
|
5
|
+
Author-email: Fang-Pen Lin <fangpen@launchplatform.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: <4,>=3.11
|
|
9
|
+
Requires-Dist: blinker<2,>=1.8.2
|
|
10
|
+
Requires-Dist: click<9,>=8.1.7
|
|
11
|
+
Requires-Dist: psycopg[binary]<4,>=3.1
|
|
12
|
+
Requires-Dist: pydantic-settings<3,>=2.2.1
|
|
13
|
+
Requires-Dist: rich<14,>=13.7.1
|
|
14
|
+
Requires-Dist: sqlalchemy[asyncio]<3,>=2.0.30
|
|
15
|
+
Requires-Dist: venusian<4,>=3.1.0
|
|
16
|
+
Provides-Extra: metrics
|
|
17
|
+
Requires-Dist: starlette<2,>=0.27; extra == 'metrics'
|
|
18
|
+
Requires-Dist: uvicorn<1,>=0.30.0; extra == 'metrics'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
1
21
|
# BeanQueue [](https://dl.circleci.com/status-badge/redirect/gh/LaunchPlatform/bq/tree/master)
|
|
2
22
|
|
|
3
|
-
BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https://www.sqlalchemy.org/), PostgreSQL [SKIP LOCKED queries](https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/) and [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html) / [LISTEN](https://www.postgresql.org/docs/current/sql-listen.html) statements.
|
|
23
|
+
BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https://www.sqlalchemy.org/) 2.0 asyncio, [psycopg3](https://www.psycopg.org/psycopg3/), PostgreSQL [SKIP LOCKED queries](https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/) and [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html) / [LISTEN](https://www.postgresql.org/docs/current/sql-listen.html) statements.
|
|
4
24
|
|
|
5
25
|
**Notice**: Still in its early stage, we built this for [BeanHub](https://beanhub.io)'s internal usage. May change rapidly. Use at your own risk for now.
|
|
6
26
|
|
|
@@ -14,6 +34,7 @@ BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https
|
|
|
14
34
|
- **Retry**: Built-in and customizable retry policies
|
|
15
35
|
- **Schedule**: Schedule tasks to run later
|
|
16
36
|
- **Worker heartbeat and auto-reschedule**: Each worker keeps updating heartbeat, if one is found dead, the others will reschedule the tasks
|
|
37
|
+
- **Custom health checks**: Optional HTTP `/healthz` endpoint with pluggable checks via Blinker signals
|
|
17
38
|
- **Customizable**: Custom Task, Worker and Event models. Use it as a library and build your own work queue
|
|
18
39
|
|
|
19
40
|
## Install
|
|
@@ -22,12 +43,45 @@ BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https
|
|
|
22
43
|
pip install beanqueue
|
|
23
44
|
```
|
|
24
45
|
|
|
46
|
+
To enable the optional metrics HTTP server (currently `/healthz` only), install with the `metrics` extra:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install "beanqueue[metrics]"
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Testing
|
|
53
|
+
|
|
54
|
+
Unit and in-process acceptance tests:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
uv run python -m pytest tests
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
End-to-end tests start PostgreSQL and three worker containers with Docker Compose, enqueue real tasks, apply load, kill a worker, and check graceful shutdown cleanup:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
uv run python -m pytest tests/e2e -svvvv
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Upgrading to 2.0
|
|
67
|
+
|
|
68
|
+
BeanQueue 2.0 is asyncio-first (SQLAlchemy `AsyncSession` + psycopg3) and includes breaking changes:
|
|
69
|
+
|
|
70
|
+
- **Workers use asyncio SQLAlchemy.** `BeanQueue.make_session()` returns an `AsyncSession`, `engine` is an `AsyncEngine`, and `process_tasks()` is `async`. Run it with `asyncio.run(app.process_tasks(...))` or `await` it from your own loop.
|
|
71
|
+
- **Database URLs use the psycopg3 driver.** Prefer `postgresql+psycopg://...`. Bare `postgresql://...` URLs are normalized to `postgresql+psycopg://...`.
|
|
72
|
+
- **`MAX_WORKER_THREADS` is now `MAX_CONCURRENT_TASKS`.** `BQ_MAX_WORKER_THREADS` and the old keyword still work as a deprecated alias.
|
|
73
|
+
- **Processors may be `async def` or `def`.** Async processors receive `db: AsyncSession`. Sync processors that take `db` still receive a sync `Session` via `AsyncSession.run_sync()`.
|
|
74
|
+
- **`METRICS_HTTP_SERVER_ENABLED` defaults to `False`** (it was `True` in 1.x). Set `BQ_METRICS_HTTP_SERVER_ENABLED=true` to turn the server back on.
|
|
75
|
+
- **The metrics server requires optional dependencies.** Install `beanqueue[metrics]` (`starlette` and `uvicorn`). Without them, enabling the server raises an error at startup.
|
|
76
|
+
- **Custom health checks use the `healthz_check` event** (`bq.events.healthz_check`) instead of a `healthz_check` argument on `bq.BeanQueue`. Receivers that query the database should use `AsyncSession` (`await session.execute(...)`).
|
|
77
|
+
|
|
25
78
|
## Usage
|
|
26
79
|
|
|
27
80
|
You can define a basic task processor like this
|
|
28
81
|
|
|
29
82
|
```python
|
|
30
|
-
from sqlalchemy
|
|
83
|
+
from sqlalchemy import select
|
|
84
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
31
85
|
|
|
32
86
|
import bq
|
|
33
87
|
from .. import models
|
|
@@ -35,15 +89,20 @@ from .. import image_utils
|
|
|
35
89
|
|
|
36
90
|
app = bq.BeanQueue()
|
|
37
91
|
|
|
92
|
+
|
|
38
93
|
@app.processor(channel="images")
|
|
39
|
-
def resize_image(db:
|
|
40
|
-
image =
|
|
94
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
95
|
+
image = (
|
|
96
|
+
await db.execute(select(models.Image).where(models.Image.task == task))
|
|
97
|
+
).scalar_one()
|
|
41
98
|
image_utils.resize(image, size=(width, height))
|
|
42
99
|
db.add(image)
|
|
43
100
|
# by default the `processor` decorator has `auto_complete` flag turns on,
|
|
44
101
|
# so it will commit the db changes for us automatically
|
|
45
102
|
```
|
|
46
103
|
|
|
104
|
+
Synchronous processors are still supported. If the function takes a `db` argument, BeanQueue injects the sync `Session` behind the `AsyncSession`. Processors that do not use the database can stay as plain `def` functions and are run in a worker thread so they do not block the event loop.
|
|
105
|
+
|
|
47
106
|
The `db` and `task` keyword arguments are optional.
|
|
48
107
|
If you don't need to access the task object, you can simply define the function without these two parameters.
|
|
49
108
|
We also provide an optional `savepoint` argument in case if you want to rollback database changes you made.
|
|
@@ -137,13 +196,14 @@ To automatically retry a task after failure, you can specify a retry policy to t
|
|
|
137
196
|
```python
|
|
138
197
|
import datetime
|
|
139
198
|
import bq
|
|
140
|
-
from sqlalchemy.
|
|
199
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
141
200
|
|
|
142
201
|
app = bq.BeanQueue()
|
|
143
202
|
delay_retry = bq.DelayRetry(delay=datetime.timedelta(seconds=120))
|
|
144
203
|
|
|
204
|
+
|
|
145
205
|
@app.processor(channel="images", retry_policy=delay_retry)
|
|
146
|
-
def resize_image(db:
|
|
206
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
147
207
|
# resize image here ...
|
|
148
208
|
pass
|
|
149
209
|
```
|
|
@@ -163,8 +223,9 @@ To cap how many attempts are allowed, you can also use `LimitAttempt` like this:
|
|
|
163
223
|
delay_retry = bq.DelayRetry(delay=datetime.timedelta(seconds=120))
|
|
164
224
|
capped_delay_retry = bq.LimitAttempt(3, delay_retry)
|
|
165
225
|
|
|
226
|
+
|
|
166
227
|
@app.processor(channel="images", retry_policy=capped_delay_retry)
|
|
167
|
-
def resize_image(db:
|
|
228
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
168
229
|
# Resize image here ...
|
|
169
230
|
pass
|
|
170
231
|
```
|
|
@@ -177,7 +238,7 @@ You can also retry only for specific exception classes with the `retry_exception
|
|
|
177
238
|
retry_policy=delay_retry,
|
|
178
239
|
retry_exceptions=ValueError,
|
|
179
240
|
)
|
|
180
|
-
def resize_image(db:
|
|
241
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
181
242
|
# resize image here ...
|
|
182
243
|
pass
|
|
183
244
|
```
|
|
@@ -213,9 +274,96 @@ bq -a my_pkgs.bq.app process images
|
|
|
213
274
|
Or if you prefer to define your own process command, you can also call `process_tasks` of the `BeanQueue` object directly like this:
|
|
214
275
|
|
|
215
276
|
```python
|
|
216
|
-
|
|
277
|
+
import asyncio
|
|
278
|
+
|
|
279
|
+
asyncio.run(app.process_tasks(channels=("images",)))
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
### Health check and metrics HTTP server
|
|
283
|
+
|
|
284
|
+
When enabled, each worker starts a small HTTP server (Starlette + Uvicorn) for operational endpoints.
|
|
285
|
+
Today this only exposes `GET /healthz`, which returns `{"status": "ok"}` by default.
|
|
286
|
+
|
|
287
|
+
Enable it with the `metrics` extra installed and configuration:
|
|
288
|
+
|
|
289
|
+
```bash
|
|
290
|
+
pip install "beanqueue[metrics]"
|
|
291
|
+
BQ_METRICS_HTTP_SERVER_ENABLED=true bq process images
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Relevant settings (see [bq/config.py](bq/config.py)):
|
|
295
|
+
|
|
296
|
+
| Setting | Env var | Default |
|
|
297
|
+
| --- | --- | --- |
|
|
298
|
+
| `METRICS_HTTP_SERVER_ENABLED` | `BQ_METRICS_HTTP_SERVER_ENABLED` | `False` |
|
|
299
|
+
| `METRICS_HTTP_SERVER_INTERFACE` | `BQ_METRICS_HTTP_SERVER_INTERFACE` | `""` (all interfaces) |
|
|
300
|
+
| `METRICS_HTTP_SERVER_PORT` | `BQ_METRICS_HTTP_SERVER_PORT` | `8000` |
|
|
301
|
+
| `METRICS_HTTP_SERVER_LOG_LEVEL` | `BQ_METRICS_HTTP_SERVER_LOG_LEVEL` | `30` (`WARNING`) |
|
|
302
|
+
|
|
303
|
+
Access requests are logged at INFO via `uvicorn.access` (visible even when `METRICS_HTTP_SERVER_LOG_LEVEL` is `WARNING`).
|
|
304
|
+
BeanQueue also uses a `metrics_server` logger for its own messages.
|
|
305
|
+
Override the entire logging setup by passing a [logging.config](https://docs.python.org/3/library/logging.config.html#logging-config-dictschema) dict via `METRICS_HTTP_SERVER_LOG_CONFIG` (or `BQ_METRICS_HTTP_SERVER_LOG_CONFIG` as JSON):
|
|
306
|
+
|
|
307
|
+
```python
|
|
308
|
+
import bq
|
|
309
|
+
|
|
310
|
+
config = bq.Config(
|
|
311
|
+
METRICS_HTTP_SERVER_ENABLED=True,
|
|
312
|
+
METRICS_HTTP_SERVER_LOG_CONFIG={
|
|
313
|
+
"version": 1,
|
|
314
|
+
"disable_existing_loggers": False,
|
|
315
|
+
"handlers": {
|
|
316
|
+
"default": {
|
|
317
|
+
"class": "logging.StreamHandler",
|
|
318
|
+
"formatter": "default",
|
|
319
|
+
}
|
|
320
|
+
},
|
|
321
|
+
"formatters": {
|
|
322
|
+
"default": {
|
|
323
|
+
"format": "%(asctime)s %(name)s %(levelname)s %(message)s",
|
|
324
|
+
}
|
|
325
|
+
},
|
|
326
|
+
"loggers": {
|
|
327
|
+
"metrics_server": {"handlers": ["default"], "level": "INFO"},
|
|
328
|
+
"uvicorn.access": {"handlers": ["default"], "level": "INFO"},
|
|
329
|
+
},
|
|
330
|
+
},
|
|
331
|
+
)
|
|
332
|
+
app = bq.BeanQueue(config=config)
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
#### Custom health checks
|
|
336
|
+
|
|
337
|
+
Register additional checks by connecting receivers to `bq.events.healthz_check`.
|
|
338
|
+
If no receivers are connected, `/healthz` returns OK without touching the database.
|
|
339
|
+
|
|
340
|
+
With receivers connected, BeanQueue loads the current worker and passes a database `session` to each check.
|
|
341
|
+
Receivers may be synchronous or asynchronous; both can be mixed on the same signal.
|
|
342
|
+
|
|
343
|
+
```python
|
|
344
|
+
from sqlalchemy import text
|
|
345
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
346
|
+
|
|
347
|
+
import bq
|
|
348
|
+
from bq import events
|
|
349
|
+
|
|
350
|
+
app = bq.BeanQueue()
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@events.healthz_check.connect
|
|
354
|
+
async def check_database(sender: bq.BeanQueue, worker, session: AsyncSession):
|
|
355
|
+
await session.execute(text("SELECT 1"))
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
@events.healthz_check.connect
|
|
359
|
+
async def check_external_service(sender: bq.BeanQueue, worker, session: AsyncSession):
|
|
360
|
+
# async HTTP call, etc.
|
|
361
|
+
...
|
|
217
362
|
```
|
|
218
363
|
|
|
364
|
+
Receiver signature must accept the keyword arguments you need, for example `(sender, worker, session)`, or use `(sender, **kwargs)`.
|
|
365
|
+
If a check raises an exception, `/healthz` responds with HTTP 500 and a JSON body containing the error message.
|
|
366
|
+
|
|
219
367
|
### Define your own tables
|
|
220
368
|
|
|
221
369
|
BeanQueue is designed to be as customizable as much as possible.
|
|
@@ -261,6 +409,7 @@ class Task(bq.TaskModelMixin, Base):
|
|
|
261
409
|
"Worker", back_populates="tasks", uselist=False
|
|
262
410
|
)
|
|
263
411
|
|
|
412
|
+
|
|
264
413
|
listen_events(Task)
|
|
265
414
|
```
|
|
266
415
|
|
|
@@ -293,6 +442,7 @@ With the model class ready, you only need to change the `TASK_MODEL`, `WORKER_MO
|
|
|
293
442
|
|
|
294
443
|
```python
|
|
295
444
|
import bq
|
|
445
|
+
|
|
296
446
|
config = bq.Config(
|
|
297
447
|
TASK_MODEL="my_pkgs.models.Task",
|
|
298
448
|
WORKER_MODEL="my_pkgs.models.Worker",
|
|
@@ -1,22 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: beanqueue
|
|
3
|
-
Version: 1.2.0
|
|
4
|
-
Summary: BeanQueue or BQ for short, PostgreSQL SKIP LOCK and SQLAlchemy based worker queue library
|
|
5
|
-
Author-email: Fang-Pen Lin <fangpen@launchplatform.com>
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
License-File: LICENSE
|
|
8
|
-
Requires-Python: <4,>=3.11
|
|
9
|
-
Requires-Dist: blinker<2,>=1.8.2
|
|
10
|
-
Requires-Dist: click<9,>=8.1.7
|
|
11
|
-
Requires-Dist: pydantic-settings<3,>=2.2.1
|
|
12
|
-
Requires-Dist: rich<14,>=13.7.1
|
|
13
|
-
Requires-Dist: sqlalchemy<3,>=2.0.30
|
|
14
|
-
Requires-Dist: venusian<4,>=3.1.0
|
|
15
|
-
Description-Content-Type: text/markdown
|
|
16
|
-
|
|
17
1
|
# BeanQueue [](https://dl.circleci.com/status-badge/redirect/gh/LaunchPlatform/bq/tree/master)
|
|
18
2
|
|
|
19
|
-
BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https://www.sqlalchemy.org/), PostgreSQL [SKIP LOCKED queries](https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/) and [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html) / [LISTEN](https://www.postgresql.org/docs/current/sql-listen.html) statements.
|
|
3
|
+
BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https://www.sqlalchemy.org/) 2.0 asyncio, [psycopg3](https://www.psycopg.org/psycopg3/), PostgreSQL [SKIP LOCKED queries](https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/) and [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html) / [LISTEN](https://www.postgresql.org/docs/current/sql-listen.html) statements.
|
|
20
4
|
|
|
21
5
|
**Notice**: Still in its early stage, we built this for [BeanHub](https://beanhub.io)'s internal usage. May change rapidly. Use at your own risk for now.
|
|
22
6
|
|
|
@@ -30,6 +14,7 @@ BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https
|
|
|
30
14
|
- **Retry**: Built-in and customizable retry policies
|
|
31
15
|
- **Schedule**: Schedule tasks to run later
|
|
32
16
|
- **Worker heartbeat and auto-reschedule**: Each worker keeps updating heartbeat, if one is found dead, the others will reschedule the tasks
|
|
17
|
+
- **Custom health checks**: Optional HTTP `/healthz` endpoint with pluggable checks via Blinker signals
|
|
33
18
|
- **Customizable**: Custom Task, Worker and Event models. Use it as a library and build your own work queue
|
|
34
19
|
|
|
35
20
|
## Install
|
|
@@ -38,12 +23,45 @@ BeanQueue, a lightweight Python task queue framework based on [SQLAlchemy](https
|
|
|
38
23
|
pip install beanqueue
|
|
39
24
|
```
|
|
40
25
|
|
|
26
|
+
To enable the optional metrics HTTP server (currently `/healthz` only), install with the `metrics` extra:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install "beanqueue[metrics]"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Testing
|
|
33
|
+
|
|
34
|
+
Unit and in-process acceptance tests:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
uv run python -m pytest tests
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
End-to-end tests start PostgreSQL and three worker containers with Docker Compose, enqueue real tasks, apply load, kill a worker, and check graceful shutdown cleanup:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uv run python -m pytest tests/e2e -svvvv
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Upgrading to 2.0
|
|
47
|
+
|
|
48
|
+
BeanQueue 2.0 is asyncio-first (SQLAlchemy `AsyncSession` + psycopg3) and includes breaking changes:
|
|
49
|
+
|
|
50
|
+
- **Workers use asyncio SQLAlchemy.** `BeanQueue.make_session()` returns an `AsyncSession`, `engine` is an `AsyncEngine`, and `process_tasks()` is `async`. Run it with `asyncio.run(app.process_tasks(...))` or `await` it from your own loop.
|
|
51
|
+
- **Database URLs use the psycopg3 driver.** Prefer `postgresql+psycopg://...`. Bare `postgresql://...` URLs are normalized to `postgresql+psycopg://...`.
|
|
52
|
+
- **`MAX_WORKER_THREADS` is now `MAX_CONCURRENT_TASKS`.** `BQ_MAX_WORKER_THREADS` and the old keyword still work as a deprecated alias.
|
|
53
|
+
- **Processors may be `async def` or `def`.** Async processors receive `db: AsyncSession`. Sync processors that take `db` still receive a sync `Session` via `AsyncSession.run_sync()`.
|
|
54
|
+
- **`METRICS_HTTP_SERVER_ENABLED` defaults to `False`** (it was `True` in 1.x). Set `BQ_METRICS_HTTP_SERVER_ENABLED=true` to turn the server back on.
|
|
55
|
+
- **The metrics server requires optional dependencies.** Install `beanqueue[metrics]` (`starlette` and `uvicorn`). Without them, enabling the server raises an error at startup.
|
|
56
|
+
- **Custom health checks use the `healthz_check` event** (`bq.events.healthz_check`) instead of a `healthz_check` argument on `bq.BeanQueue`. Receivers that query the database should use `AsyncSession` (`await session.execute(...)`).
|
|
57
|
+
|
|
41
58
|
## Usage
|
|
42
59
|
|
|
43
60
|
You can define a basic task processor like this
|
|
44
61
|
|
|
45
62
|
```python
|
|
46
|
-
from sqlalchemy
|
|
63
|
+
from sqlalchemy import select
|
|
64
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
47
65
|
|
|
48
66
|
import bq
|
|
49
67
|
from .. import models
|
|
@@ -51,15 +69,20 @@ from .. import image_utils
|
|
|
51
69
|
|
|
52
70
|
app = bq.BeanQueue()
|
|
53
71
|
|
|
72
|
+
|
|
54
73
|
@app.processor(channel="images")
|
|
55
|
-
def resize_image(db:
|
|
56
|
-
image =
|
|
74
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
75
|
+
image = (
|
|
76
|
+
await db.execute(select(models.Image).where(models.Image.task == task))
|
|
77
|
+
).scalar_one()
|
|
57
78
|
image_utils.resize(image, size=(width, height))
|
|
58
79
|
db.add(image)
|
|
59
80
|
# by default the `processor` decorator has `auto_complete` flag turns on,
|
|
60
81
|
# so it will commit the db changes for us automatically
|
|
61
82
|
```
|
|
62
83
|
|
|
84
|
+
Synchronous processors are still supported. If the function takes a `db` argument, BeanQueue injects the sync `Session` behind the `AsyncSession`. Processors that do not use the database can stay as plain `def` functions and are run in a worker thread so they do not block the event loop.
|
|
85
|
+
|
|
63
86
|
The `db` and `task` keyword arguments are optional.
|
|
64
87
|
If you don't need to access the task object, you can simply define the function without these two parameters.
|
|
65
88
|
We also provide an optional `savepoint` argument in case if you want to rollback database changes you made.
|
|
@@ -153,13 +176,14 @@ To automatically retry a task after failure, you can specify a retry policy to t
|
|
|
153
176
|
```python
|
|
154
177
|
import datetime
|
|
155
178
|
import bq
|
|
156
|
-
from sqlalchemy.
|
|
179
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
157
180
|
|
|
158
181
|
app = bq.BeanQueue()
|
|
159
182
|
delay_retry = bq.DelayRetry(delay=datetime.timedelta(seconds=120))
|
|
160
183
|
|
|
184
|
+
|
|
161
185
|
@app.processor(channel="images", retry_policy=delay_retry)
|
|
162
|
-
def resize_image(db:
|
|
186
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
163
187
|
# resize image here ...
|
|
164
188
|
pass
|
|
165
189
|
```
|
|
@@ -179,8 +203,9 @@ To cap how many attempts are allowed, you can also use `LimitAttempt` like this:
|
|
|
179
203
|
delay_retry = bq.DelayRetry(delay=datetime.timedelta(seconds=120))
|
|
180
204
|
capped_delay_retry = bq.LimitAttempt(3, delay_retry)
|
|
181
205
|
|
|
206
|
+
|
|
182
207
|
@app.processor(channel="images", retry_policy=capped_delay_retry)
|
|
183
|
-
def resize_image(db:
|
|
208
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
184
209
|
# Resize image here ...
|
|
185
210
|
pass
|
|
186
211
|
```
|
|
@@ -193,7 +218,7 @@ You can also retry only for specific exception classes with the `retry_exception
|
|
|
193
218
|
retry_policy=delay_retry,
|
|
194
219
|
retry_exceptions=ValueError,
|
|
195
220
|
)
|
|
196
|
-
def resize_image(db:
|
|
221
|
+
async def resize_image(db: AsyncSession, task: bq.Task, width: int, height: int):
|
|
197
222
|
# resize image here ...
|
|
198
223
|
pass
|
|
199
224
|
```
|
|
@@ -229,9 +254,96 @@ bq -a my_pkgs.bq.app process images
|
|
|
229
254
|
Or if you prefer to define your own process command, you can also call `process_tasks` of the `BeanQueue` object directly like this:
|
|
230
255
|
|
|
231
256
|
```python
|
|
232
|
-
|
|
257
|
+
import asyncio
|
|
258
|
+
|
|
259
|
+
asyncio.run(app.process_tasks(channels=("images",)))
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
### Health check and metrics HTTP server
|
|
263
|
+
|
|
264
|
+
When enabled, each worker starts a small HTTP server (Starlette + Uvicorn) for operational endpoints.
|
|
265
|
+
Today this only exposes `GET /healthz`, which returns `{"status": "ok"}` by default.
|
|
266
|
+
|
|
267
|
+
Enable it with the `metrics` extra installed and configuration:
|
|
268
|
+
|
|
269
|
+
```bash
|
|
270
|
+
pip install "beanqueue[metrics]"
|
|
271
|
+
BQ_METRICS_HTTP_SERVER_ENABLED=true bq process images
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
Relevant settings (see [bq/config.py](bq/config.py)):
|
|
275
|
+
|
|
276
|
+
| Setting | Env var | Default |
|
|
277
|
+
| --- | --- | --- |
|
|
278
|
+
| `METRICS_HTTP_SERVER_ENABLED` | `BQ_METRICS_HTTP_SERVER_ENABLED` | `False` |
|
|
279
|
+
| `METRICS_HTTP_SERVER_INTERFACE` | `BQ_METRICS_HTTP_SERVER_INTERFACE` | `""` (all interfaces) |
|
|
280
|
+
| `METRICS_HTTP_SERVER_PORT` | `BQ_METRICS_HTTP_SERVER_PORT` | `8000` |
|
|
281
|
+
| `METRICS_HTTP_SERVER_LOG_LEVEL` | `BQ_METRICS_HTTP_SERVER_LOG_LEVEL` | `30` (`WARNING`) |
|
|
282
|
+
|
|
283
|
+
Access requests are logged at INFO via `uvicorn.access` (visible even when `METRICS_HTTP_SERVER_LOG_LEVEL` is `WARNING`).
|
|
284
|
+
BeanQueue also uses a `metrics_server` logger for its own messages.
|
|
285
|
+
Override the entire logging setup by passing a [logging.config](https://docs.python.org/3/library/logging.config.html#logging-config-dictschema) dict via `METRICS_HTTP_SERVER_LOG_CONFIG` (or `BQ_METRICS_HTTP_SERVER_LOG_CONFIG` as JSON):
|
|
286
|
+
|
|
287
|
+
```python
|
|
288
|
+
import bq
|
|
289
|
+
|
|
290
|
+
config = bq.Config(
|
|
291
|
+
METRICS_HTTP_SERVER_ENABLED=True,
|
|
292
|
+
METRICS_HTTP_SERVER_LOG_CONFIG={
|
|
293
|
+
"version": 1,
|
|
294
|
+
"disable_existing_loggers": False,
|
|
295
|
+
"handlers": {
|
|
296
|
+
"default": {
|
|
297
|
+
"class": "logging.StreamHandler",
|
|
298
|
+
"formatter": "default",
|
|
299
|
+
}
|
|
300
|
+
},
|
|
301
|
+
"formatters": {
|
|
302
|
+
"default": {
|
|
303
|
+
"format": "%(asctime)s %(name)s %(levelname)s %(message)s",
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
"loggers": {
|
|
307
|
+
"metrics_server": {"handlers": ["default"], "level": "INFO"},
|
|
308
|
+
"uvicorn.access": {"handlers": ["default"], "level": "INFO"},
|
|
309
|
+
},
|
|
310
|
+
},
|
|
311
|
+
)
|
|
312
|
+
app = bq.BeanQueue(config=config)
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
#### Custom health checks
|
|
316
|
+
|
|
317
|
+
Register additional checks by connecting receivers to `bq.events.healthz_check`.
|
|
318
|
+
If no receivers are connected, `/healthz` returns OK without touching the database.
|
|
319
|
+
|
|
320
|
+
With receivers connected, BeanQueue loads the current worker and passes a database `session` to each check.
|
|
321
|
+
Receivers may be synchronous or asynchronous; both can be mixed on the same signal.
|
|
322
|
+
|
|
323
|
+
```python
|
|
324
|
+
from sqlalchemy import text
|
|
325
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
326
|
+
|
|
327
|
+
import bq
|
|
328
|
+
from bq import events
|
|
329
|
+
|
|
330
|
+
app = bq.BeanQueue()
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
@events.healthz_check.connect
|
|
334
|
+
async def check_database(sender: bq.BeanQueue, worker, session: AsyncSession):
|
|
335
|
+
await session.execute(text("SELECT 1"))
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@events.healthz_check.connect
|
|
339
|
+
async def check_external_service(sender: bq.BeanQueue, worker, session: AsyncSession):
|
|
340
|
+
# async HTTP call, etc.
|
|
341
|
+
...
|
|
233
342
|
```
|
|
234
343
|
|
|
344
|
+
Receiver signature must accept the keyword arguments you need, for example `(sender, worker, session)`, or use `(sender, **kwargs)`.
|
|
345
|
+
If a check raises an exception, `/healthz` responds with HTTP 500 and a JSON body containing the error message.
|
|
346
|
+
|
|
235
347
|
### Define your own tables
|
|
236
348
|
|
|
237
349
|
BeanQueue is designed to be as customizable as much as possible.
|
|
@@ -277,6 +389,7 @@ class Task(bq.TaskModelMixin, Base):
|
|
|
277
389
|
"Worker", back_populates="tasks", uselist=False
|
|
278
390
|
)
|
|
279
391
|
|
|
392
|
+
|
|
280
393
|
listen_events(Task)
|
|
281
394
|
```
|
|
282
395
|
|
|
@@ -309,6 +422,7 @@ With the model class ready, you only need to change the `TASK_MODEL`, `WORKER_MO
|
|
|
309
422
|
|
|
310
423
|
```python
|
|
311
424
|
import bq
|
|
425
|
+
|
|
312
426
|
config = bq.Config(
|
|
313
427
|
TASK_MODEL="my_pkgs.models.Task",
|
|
314
428
|
WORKER_MODEL="my_pkgs.models.Worker",
|