potatoq 26.1a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- potatoq-26.1a1/LICENSE +21 -0
- potatoq-26.1a1/PKG-INFO +347 -0
- potatoq-26.1a1/README.md +306 -0
- potatoq-26.1a1/pyproject.toml +153 -0
- potatoq-26.1a1/pyproject.toml.orig +120 -0
- potatoq-26.1a1/src/potatoq/__init__.py +45 -0
- potatoq-26.1a1/src/potatoq/__main__.py +5 -0
- potatoq-26.1a1/src/potatoq/app.py +786 -0
- potatoq-26.1a1/src/potatoq/brokers/__init__.py +51 -0
- potatoq-26.1a1/src/potatoq/brokers/base.py +279 -0
- potatoq-26.1a1/src/potatoq/brokers/memory.py +275 -0
- potatoq-26.1a1/src/potatoq/brokers/postgres.py +803 -0
- potatoq-26.1a1/src/potatoq/brokers/rabbitmq.py +701 -0
- potatoq-26.1a1/src/potatoq/brokers/redis.py +661 -0
- potatoq-26.1a1/src/potatoq/brokers/sqlite.py +572 -0
- potatoq-26.1a1/src/potatoq/canvas.py +730 -0
- potatoq-26.1a1/src/potatoq/cli.py +623 -0
- potatoq-26.1a1/src/potatoq/config.py +253 -0
- potatoq-26.1a1/src/potatoq/console.py +159 -0
- potatoq-26.1a1/src/potatoq/contrib/__init__.py +0 -0
- potatoq-26.1a1/src/potatoq/contrib/django/__init__.py +333 -0
- potatoq-26.1a1/src/potatoq/contrib/django/apps.py +17 -0
- potatoq-26.1a1/src/potatoq/contrib/django/management/__init__.py +0 -0
- potatoq-26.1a1/src/potatoq/contrib/django/management/commands/__init__.py +0 -0
- potatoq-26.1a1/src/potatoq/contrib/django/management/commands/potatoq.py +24 -0
- potatoq-26.1a1/src/potatoq/contrib/django/tasks.py +385 -0
- potatoq-26.1a1/src/potatoq/contrib/fastapi.py +48 -0
- potatoq-26.1a1/src/potatoq/contrib/flask.py +62 -0
- potatoq-26.1a1/src/potatoq/contrib/sqlalchemy.py +215 -0
- potatoq-26.1a1/src/potatoq/control.py +136 -0
- potatoq-26.1a1/src/potatoq/exceptions.py +157 -0
- potatoq-26.1a1/src/potatoq/log.py +209 -0
- potatoq-26.1a1/src/potatoq/message.py +125 -0
- potatoq-26.1a1/src/potatoq/py.typed +0 -0
- potatoq-26.1a1/src/potatoq/result.py +417 -0
- potatoq-26.1a1/src/potatoq/schedules.py +224 -0
- potatoq-26.1a1/src/potatoq/serialization.py +163 -0
- potatoq-26.1a1/src/potatoq/signals.py +152 -0
- potatoq-26.1a1/src/potatoq/states.py +18 -0
- potatoq-26.1a1/src/potatoq/task.py +608 -0
- potatoq-26.1a1/src/potatoq/testing.py +99 -0
- potatoq-26.1a1/src/potatoq/utils/__init__.py +1 -0
- potatoq-26.1a1/src/potatoq/utils/log.py +5 -0
- potatoq-26.1a1/src/potatoq/worker/child.py +363 -0
- potatoq-26.1a1/src/potatoq/worker/executor.py +592 -0
- potatoq-26.1a1/src/potatoq/worker/scheduler.py +173 -0
- potatoq-26.1a1/src/potatoq/worker/supervisor.py +569 -0
potatoq-26.1a1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anže Pečar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
potatoq-26.1a1/PKG-INFO
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: potatoq
|
|
3
|
+
Version: 26.1a1
|
|
4
|
+
Summary: A Celery-compatible task queue with production-ready defaults and backend-native Redis, RabbitMQ, Postgres and SQLite brokers.
|
|
5
|
+
Keywords: celery,task queue,jobs,background tasks,redis,rabbitmq,postgres,sqlite,django
|
|
6
|
+
Author: Anže Pečar
|
|
7
|
+
Author-email: Anže Pečar <anze@pecar.me>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Framework :: Django
|
|
12
|
+
Classifier: Framework :: Flask
|
|
13
|
+
Classifier: Framework :: FastAPI
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
24
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
25
|
+
Requires-Dist: redis>=5.0 ; extra == 'all'
|
|
26
|
+
Requires-Dist: psycopg[binary]>=3.2 ; extra == 'all'
|
|
27
|
+
Requires-Dist: pika>=1.3 ; extra == 'all'
|
|
28
|
+
Requires-Dist: psycopg[binary]>=3.2 ; extra == 'postgres'
|
|
29
|
+
Requires-Dist: pika>=1.3 ; extra == 'rabbitmq'
|
|
30
|
+
Requires-Dist: redis>=5.0 ; extra == 'redis'
|
|
31
|
+
Requires-Python: >=3.11
|
|
32
|
+
Project-URL: Homepage, https://github.com/anze3db/potatoq
|
|
33
|
+
Project-URL: Documentation, https://anze3db.github.io/potatoq/
|
|
34
|
+
Project-URL: Changelog, https://github.com/anze3db/potatoq/blob/main/CHANGELOG.md
|
|
35
|
+
Project-URL: Issues, https://github.com/anze3db/potatoq/issues
|
|
36
|
+
Provides-Extra: all
|
|
37
|
+
Provides-Extra: postgres
|
|
38
|
+
Provides-Extra: rabbitmq
|
|
39
|
+
Provides-Extra: redis
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# 🥔 potatoq
|
|
43
|
+
|
|
44
|
+
> ⚠️ **potatoq is alpha software** in limited production use. Use it in
|
|
45
|
+
> production **at your own risk** for now. APIs and defaults may still change between
|
|
46
|
+
> releases. Please [report bugs on GitHub](https://github.com/anze3db/potatoq/issues).
|
|
47
|
+
|
|
48
|
+
**A Celery-compatible task queue with production-ready defaults.**
|
|
49
|
+
|
|
50
|
+
- 🧵 **Free-threaded Python** (3.14t, 3.15t): tested in CI, and potatoq never turns the
|
|
51
|
+
GIL back on, so `--threads` runs CPU-bound tasks in parallel in one process: 3.4×
|
|
52
|
+
with 4 threads, in half the memory of 4 processes ([numbers](https://github.com/anze3db/potatoq#free-threaded-python)).
|
|
53
|
+
- 🎸 **Django tasks**: a backend for Django 6's built-in
|
|
54
|
+
[`django.tasks`](https://anze3db.github.io/potatoq/integrations/django-tasks/), sharing workers with
|
|
55
|
+
`@shared_task`. The classic setup works too: add `"potatoq.contrib.django"` to
|
|
56
|
+
`INSTALLED_APPS` (no `celery.py` needed), and `.delay()` inside `atomic()` is sent on
|
|
57
|
+
commit.
|
|
58
|
+
- 🔁 **Switching from Celery** is meant to be a find-and-replace: the same decorators,
|
|
59
|
+
the same `delay()`/`apply_async()`, the same `chain`/`group`/`chord`, the same
|
|
60
|
+
settings names. What changes is everything you used to have to know to run Celery
|
|
61
|
+
safely in production. The defaults come from years of community post-mortems, the
|
|
62
|
+
Ruby job-queue world (Sidekiq, Solid Queue, GoodJob), and the best Python
|
|
63
|
+
alternatives (Dramatiq, RQ, arq, Procrastinate, Oban/River designs). Each backend is
|
|
64
|
+
implemented with its own native primitives instead of a lowest-common-denominator
|
|
65
|
+
abstraction.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from potatoq import Potatoq # or: from potatoq import Celery
|
|
69
|
+
|
|
70
|
+
app = Potatoq("proj") # broker: $POTATOQ_BROKER_URL, your Django DB, or ./potatoq.sqlite3
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@app.task
|
|
74
|
+
def add(x, y):
|
|
75
|
+
return x + y
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
add.delay(2, 2)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
```console
|
|
82
|
+
$ potatoq -A proj worker
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Why
|
|
86
|
+
|
|
87
|
+
Every Celery deployment eventually learns these the hard way. In Potatoq they are the defaults:
|
|
88
|
+
|
|
89
|
+
| | Celery default | Potatoq default |
|
|
90
|
+
|---|---|---|
|
|
91
|
+
| Acknowledgement | before the task runs (crash = lost task) | **after it finishes** (at-least-once) |
|
|
92
|
+
| Worker process killed (OOM, segfault) | task acked and lost | **requeued**, dead-lettered after 5 crashes (poison-message guard) |
|
|
93
|
+
| Prefetch | 4 × concurrency (short tasks wait behind long ones) | **one task per idle process** |
|
|
94
|
+
| ETA / countdown | held in worker RAM; Redis `visibility_timeout` re-runs them; RabbitMQ's 30 min timeout kills them | **stored by the broker** (sorted set, `run_at` column, TTL cascade) |
|
|
95
|
+
| Long tasks on Redis | redelivered every hour (`visibility_timeout`) | **leases renewed** by the worker while the task runs |
|
|
96
|
+
| Time limits | none | **30 min hard, soft 30 s earlier** |
|
|
97
|
+
| Failed tasks | gone | **dead-letter store** you can list and replay (`potatoq dead list/retry`) |
|
|
98
|
+
| Retries | fixed 180 s | **exponential backoff with jitter** |
|
|
99
|
+
| Memory leaks | processes live forever | **recycled every 1000 tasks** (`max_memory_per_child="512MB"` available) |
|
|
100
|
+
| Concurrency | host CPU count (over-subscribes containers) | **CPUs actually available** (affinity + cgroup quota) |
|
|
101
|
+
| SIGTERM | waits forever, then Kubernetes SIGKILLs | **25 s grace, then requeue** unfinished tasks |
|
|
102
|
+
| `celery beat` | separate single process; two of them = everything twice | **every worker runs the scheduler, deduplicated** |
|
|
103
|
+
| Django `.delay()` inside `atomic()` | sent immediately (task runs before COMMIT) | **sent on commit**, dropped on rollback |
|
|
104
|
+
| RabbitMQ | classic queues, no publisher confirms | **quorum queues + publisher confirms** |
|
|
105
|
+
| Redis | no key prefix, eviction can eat your queue | **prefixed keys**, warns unless `noeviction` |
|
|
106
|
+
| Root logger | hijacked | **left alone** |
|
|
107
|
+
| `async def` tasks | not supported | **supported** |
|
|
108
|
+
| Gossip/mingle/heartbeat chatter | on | **doesn't exist** |
|
|
109
|
+
|
|
110
|
+
Why each one, with sources: [the defaults, explained](https://anze3db.github.io/potatoq/design/defaults/).
|
|
111
|
+
|
|
112
|
+
## Install
|
|
113
|
+
|
|
114
|
+
```console
|
|
115
|
+
$ pip install potatoq # SQLite broker, no dependencies
|
|
116
|
+
$ pip install "potatoq[redis]" # + Redis / Valkey
|
|
117
|
+
$ pip install "potatoq[postgres]" # + PostgreSQL (psycopg 3)
|
|
118
|
+
$ pip install "potatoq[rabbitmq]" # + RabbitMQ (pika)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Python 3.11+ (including free-threaded builds) on **Linux and macOS**. Windows isn't
|
|
122
|
+
supported: the worker relies on `fork()` and POSIX signals. Use WSL on Windows.
|
|
123
|
+
|
|
124
|
+
## Brokers
|
|
125
|
+
|
|
126
|
+
| URL | Best for | How it works |
|
|
127
|
+
|---|---|---|
|
|
128
|
+
| `postgresql://…` | Most apps: zero extra infrastructure, **transactional enqueue** | `FOR UPDATE SKIP LOCKED` claims, `LISTEN/NOTIFY` wake-ups, heartbeat-based recovery, results in the same transaction as the ack |
|
|
129
|
+
| `sqlite:///path.db` | Development, single-host apps | WAL, `BEGIN IMMEDIATE`, `UPDATE … RETURNING` claims, `PRAGMA data_version` wake-ups |
|
|
130
|
+
| `redis://…` / `valkey://…` | High throughput | Atomic Lua scripts, priority sorted sets, leases + fencing tokens, `BZPOPMIN` wake-ups |
|
|
131
|
+
| `amqp://…` | Existing RabbitMQ shops | Quorum queues, publisher confirms, 28-level TTL delay cascade, dead-letter queues |
|
|
132
|
+
| `memory://` | Unit tests | In-process, with `potatoq.testing.drain()` |
|
|
133
|
+
|
|
134
|
+
The design and research behind each backend are in [How it works](https://anze3db.github.io/potatoq/design/internals/).
|
|
135
|
+
|
|
136
|
+
Results are stored by default on the database brokers, where writing a result is part of
|
|
137
|
+
the same transaction as the acknowledgement and costs nothing. On Redis and RabbitMQ they
|
|
138
|
+
are opt-in, exactly like Celery: set `result_backend` (`"broker"`, `redis://…`,
|
|
139
|
+
`postgresql://…`). Calling `.get()` on a task whose result is ignored raises immediately
|
|
140
|
+
instead of hanging forever.
|
|
141
|
+
|
|
142
|
+
## Migrating from Celery
|
|
143
|
+
|
|
144
|
+
```diff
|
|
145
|
+
-from celery import Celery, shared_task
|
|
146
|
+
-from celery.schedules import crontab
|
|
147
|
+
+from potatoq import Celery, shared_task
|
|
148
|
+
+from potatoq.schedules import crontab
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
```console
|
|
152
|
+
-$ celery -A proj worker -l info -Q default -c 8 -B --without-gossip --without-mingle
|
|
153
|
+
+$ potatoq -A proj worker -l info -Q default -c 8
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Your `CELERY_*` Django settings, `config_from_object(..., namespace="CELERY")`,
|
|
157
|
+
`beat_schedule`, `task_routes`, `autoretry_for`, `bind=True` + `self.retry()`, canvas,
|
|
158
|
+
signals, `AsyncResult` and custom `Task` base classes keep working. Then delete the
|
|
159
|
+
settings you only had because of Celery's defaults (`task_acks_late`,
|
|
160
|
+
`worker_prefetch_multiplier`, `broker_transport_options={"visibility_timeout": …}`,
|
|
161
|
+
`task_reject_on_worker_lost`, `broker_connection_retry_on_startup`, …).
|
|
162
|
+
See [Migrating from Celery](https://anze3db.github.io/potatoq/migrating-from-celery/) for the full list,
|
|
163
|
+
including the few deliberate differences.
|
|
164
|
+
|
|
165
|
+
## Django
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
INSTALLED_APPS = [..., "potatoq.contrib.django"]
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
That's it. No `celery.py`: `@shared_task` works, every app's `tasks.py` is discovered,
|
|
172
|
+
settings come from `POTATOQ = {...}` or your existing `CELERY_*` settings, and without a
|
|
173
|
+
broker setting **your default database is the broker**. Postgres and SQLite both work, so
|
|
174
|
+
you need no Redis to get started.
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
from django.db import transaction
|
|
178
|
+
from potatoq import shared_task
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@shared_task
|
|
182
|
+
def send_receipt(order_id): ...
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
with transaction.atomic():
|
|
186
|
+
order = Order.objects.create(...)
|
|
187
|
+
send_receipt.delay(order.id) # runs only after COMMIT; never if the transaction rolls back
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
When the broker is your database, the task row is written inside your transaction:
|
|
191
|
+
the same semantics as `on_commit`, without the window where a crash between COMMIT and
|
|
192
|
+
publishing loses the task. `delay_on_commit()` (Celery 5.4) is supported too, and
|
|
193
|
+
`@shared_task(enqueue_on_commit=False)` opts a task out.
|
|
194
|
+
|
|
195
|
+
Using Django 6's built-in `django.tasks`? potatoq is a backend for it, with
|
|
196
|
+
priorities, `run_after`, async tasks, `get_result()` and signals. Set
|
|
197
|
+
`TASKS = {"default": {"BACKEND": "potatoq.contrib.django.tasks.PotatoqBackend"}}`.
|
|
198
|
+
|
|
199
|
+
Run workers with `potatoq worker` (it reads `DJANGO_SETTINGS_MODULE`) or `python manage.py potatoq worker`.
|
|
200
|
+
|
|
201
|
+
## Flask, FastAPI, SQLAlchemy
|
|
202
|
+
|
|
203
|
+
```python
|
|
204
|
+
# Flask: config from app.config, tasks run inside app.app_context()
|
|
205
|
+
from potatoq.contrib.flask import init_app
|
|
206
|
+
|
|
207
|
+
potatoq = init_app(flask_app)
|
|
208
|
+
|
|
209
|
+
# FastAPI: non-blocking enqueue and result polling
|
|
210
|
+
result = await send_welcome.adelay(user.id)
|
|
211
|
+
value = await result.aget(timeout=10)
|
|
212
|
+
|
|
213
|
+
# SQLAlchemy: .delay() inside a session transaction is sent on commit
|
|
214
|
+
# (or written in the transaction when the broker is the same database)
|
|
215
|
+
from potatoq.contrib.sqlalchemy import install
|
|
216
|
+
|
|
217
|
+
install(app)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
## Periodic tasks
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
app.conf.beat_schedule = {
|
|
224
|
+
"nightly-report": {"task": "reports.build", "schedule": crontab(hour=3, minute=0)},
|
|
225
|
+
"heartbeat": {"task": "monitoring.ping", "schedule": 30.0},
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Every worker runs the scheduler. Fire times are deterministic, and each one is claimed
|
|
230
|
+
exactly once through the broker: a unique row on Postgres/SQLite, `SET NX` on Redis, a
|
|
231
|
+
single-active-consumer token on RabbitMQ. You no longer need a `beat` process, and
|
|
232
|
+
running two of them can't double-schedule anything. A periodic run that couldn't start
|
|
233
|
+
before the next one is due expires instead of piling up.
|
|
234
|
+
|
|
235
|
+
## Operations
|
|
236
|
+
|
|
237
|
+
```console
|
|
238
|
+
$ potatoq -A proj status # live workers and what they are running
|
|
239
|
+
$ potatoq -A proj queues # waiting tasks per queue
|
|
240
|
+
$ potatoq -A proj dead list # dead-lettered tasks, with the reason
|
|
241
|
+
$ potatoq -A proj dead retry <task-id> # replay one
|
|
242
|
+
$ potatoq -A proj inspect active # Celery-style inspect
|
|
243
|
+
$ potatoq -A proj call proj.tasks.add -a '[2, 2]'
|
|
244
|
+
$ potatoq -A proj worker -P solo # run in-process, one task at a time (pdb-friendly)
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Testing
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
from potatoq.testing import drain
|
|
251
|
+
|
|
252
|
+
app = Potatoq("tests", broker="memory://", result_backend="broker")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def test_signup():
|
|
256
|
+
signup("ann@example.com") # calls send_welcome.delay(...)
|
|
257
|
+
[task] = drain(app) # runs what was enqueued: serialization, retries and chords included
|
|
258
|
+
assert task.state == "SUCCESS"
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
`task_always_eager` works as in Celery, and arguments are still round-tripped through
|
|
262
|
+
the serializer so eager tests catch what production would.
|
|
263
|
+
|
|
264
|
+
## Performance
|
|
265
|
+
|
|
266
|
+
`benchmarks/throughput.py`: 5000 no-op tasks, 4 worker processes, all brokers on
|
|
267
|
+
localhost (Apple M3 Max, Python 3.13, median of 3 runs). Run it yourself with
|
|
268
|
+
`uv run --with 'celery[redis]' python benchmarks/throughput.py`.
|
|
269
|
+
|
|
270
|
+
| Library | Broker | Enqueue (tasks/s) | Process (tasks/s) |
|
|
271
|
+
|---|---|---:|---:|
|
|
272
|
+
| **Potatoq** | Redis | 10,200 | **11,900** |
|
|
273
|
+
| Celery 5.6 | Redis | 5,400 | 2,700 |
|
|
274
|
+
| **Potatoq** | RabbitMQ | 2,700 ¹ | 6,800 |
|
|
275
|
+
| Celery 5.6 | RabbitMQ | 8,300 ¹ | 8,000 ² |
|
|
276
|
+
| **Potatoq** | Postgres | 5,500 | 7,500 |
|
|
277
|
+
| **Potatoq** | SQLite | 14,700 | 5,700 |
|
|
278
|
+
|
|
279
|
+
¹ Potatoq waits for RabbitMQ publisher confirms on every publish, so an enqueued task
|
|
280
|
+
is replicated before `delay()` returns. Celery doesn't wait, which is how its publishes
|
|
281
|
+
can disappear silently ([#5410](https://github.com/celery/celery/issues/5410)).
|
|
282
|
+
² Celery acks before running the task (a crash loses it) and prefetches 16 messages.
|
|
283
|
+
Potatoq acks after the task finishes and only takes work for idle processes.
|
|
284
|
+
|
|
285
|
+
Getting Celery to run at all for this comparison took two workarounds:
|
|
286
|
+
`FORKED_BY_MULTIPROCESSING=1`, because the prefork pool fails on macOS with Python 3.13,
|
|
287
|
+
and disabling remote control, because RabbitMQ 4.3 rejects Celery's transient pidbox
|
|
288
|
+
queues. Potatoq needed none. Redis, Postgres and SQLite numbers vary by about ±10% between runs. RabbitMQ numbers swing up to 2× with machine load, because every publish waits for the broker to confirm it.
|
|
289
|
+
|
|
290
|
+
### Free-threaded Python
|
|
291
|
+
|
|
292
|
+
`benchmarks/cpu.py`: 400 CPU-bound tasks (a pure-Python loop, about 40 ms each) on
|
|
293
|
+
Redis, median of 2 runs. Speedup is over one process on the same interpreter; memory is
|
|
294
|
+
the RSS of the whole worker (supervisor and children).
|
|
295
|
+
|
|
296
|
+
| Python | 1 process | 4 processes (`-c 4`) | 4 threads in 1 process (`-c 1 -t 4`) |
|
|
297
|
+
|---|---:|---:|---:|
|
|
298
|
+
| 3.14, with the GIL | 25 tasks/s, 69 MB | 99 tasks/s (4.0×), 146 MB | 26 tasks/s (1.1×), 70 MB |
|
|
299
|
+
| **3.14t, free-threaded** | 27 tasks/s, 78 MB | 107 tasks/s (4.0×), 166 MB | **91 tasks/s (3.4×), 87 MB** |
|
|
300
|
+
| 3.15 rc3, with the GIL | 19 tasks/s, 70 MB | 74 tasks/s (3.9×), 151 MB | 19 tasks/s (1.0×), 72 MB |
|
|
301
|
+
| **3.15t rc3, free-threaded** | 20 tasks/s, 80 MB | 79 tasks/s (4.0×), 172 MB | **67 tasks/s (3.4×), 90 MB** |
|
|
302
|
+
|
|
303
|
+
With the GIL, threads only help I/O-bound tasks. On a free-threaded build, 4 threads
|
|
304
|
+
run CPU-bound tasks 3.4× faster than one process, close to 4 processes' 4.0×, in
|
|
305
|
+
about half the memory. A bare `ThreadPoolExecutor` running the same loop scales about
|
|
306
|
+
as well, so the remaining gap is the interpreter's, not potatoq's
|
|
307
|
+
([details](https://anze3db.github.io/potatoq/guide/workers/#free-threaded-python)).
|
|
308
|
+
|
|
309
|
+
There is no Rust in the hot path, and that's deliberate. The research
|
|
310
|
+
([why not Rust](https://anze3db.github.io/potatoq/design/internals/#rust)) found that per-task overhead is
|
|
311
|
+
dominated by broker round trips. Potatoq minimizes those: one Lua call or one SQL
|
|
312
|
+
statement per state change, with the result, ack and follow-up tasks committed together.
|
|
313
|
+
A Rust core measured +0–10% on realistic tasks, while costing fork-safety (prefork
|
|
314
|
+
workers) and a much larger wheel matrix. The serialization seam is kept so a native
|
|
315
|
+
codec can be added later as an optional extra.
|
|
316
|
+
|
|
317
|
+
## Documentation
|
|
318
|
+
|
|
319
|
+
Full docs are at **[anze3db.github.io/potatoq](https://anze3db.github.io/potatoq/)**. They live in [`docs/`](https://github.com/anze3db/potatoq/tree/main/docs) and build with [Zensical](https://zensical.org):
|
|
320
|
+
|
|
321
|
+
```console
|
|
322
|
+
$ uv run --group docs zensical serve # http://localhost:8000
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
## Development
|
|
326
|
+
|
|
327
|
+
```console
|
|
328
|
+
$ uv sync
|
|
329
|
+
$ uv run pytest # needs local Redis, Postgres and RabbitMQ; skips what isn't running
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
## Roadmap
|
|
333
|
+
|
|
334
|
+
Known gaps and planned features live in the [wishlist](https://anze3db.github.io/potatoq/wishlist/). Pick one up!
|
|
335
|
+
|
|
336
|
+
## Releases
|
|
337
|
+
|
|
338
|
+
potatoq uses CalVer (`26.1`, `26.2`, …). See [CHANGELOG.md](https://github.com/anze3db/potatoq/blob/main/CHANGELOG.md) for what
|
|
339
|
+
changed and [RELEASING.md](https://github.com/anze3db/potatoq/blob/main/RELEASING.md) for how releases are cut: a release PR,
|
|
340
|
+
trusted publishing to PyPI, and a GitHub release with generated notes.
|
|
341
|
+
|
|
342
|
+
## License
|
|
343
|
+
|
|
344
|
+
MIT
|
|
345
|
+
|
|
346
|
+
> 🤖 potatoq was generated with **Claude Opus 5.5** (high effort): the research, design,
|
|
347
|
+
> code, tests and docs.
|
potatoq-26.1a1/README.md
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
# 🥔 potatoq
|
|
2
|
+
|
|
3
|
+
> ⚠️ **potatoq is alpha software** in limited production use. Use it in
|
|
4
|
+
> production **at your own risk** for now. APIs and defaults may still change between
|
|
5
|
+
> releases. Please [report bugs on GitHub](https://github.com/anze3db/potatoq/issues).
|
|
6
|
+
|
|
7
|
+
**A Celery-compatible task queue with production-ready defaults.**
|
|
8
|
+
|
|
9
|
+
- 🧵 **Free-threaded Python** (3.14t, 3.15t): tested in CI, and potatoq never turns the
|
|
10
|
+
GIL back on, so `--threads` runs CPU-bound tasks in parallel in one process: 3.4×
|
|
11
|
+
with 4 threads, in half the memory of 4 processes ([numbers](https://github.com/anze3db/potatoq#free-threaded-python)).
|
|
12
|
+
- 🎸 **Django tasks**: a backend for Django 6's built-in
|
|
13
|
+
[`django.tasks`](https://anze3db.github.io/potatoq/integrations/django-tasks/), sharing workers with
|
|
14
|
+
`@shared_task`. The classic setup works too: add `"potatoq.contrib.django"` to
|
|
15
|
+
`INSTALLED_APPS` (no `celery.py` needed), and `.delay()` inside `atomic()` is sent on
|
|
16
|
+
commit.
|
|
17
|
+
- 🔁 **Switching from Celery** is meant to be a find-and-replace: the same decorators,
|
|
18
|
+
the same `delay()`/`apply_async()`, the same `chain`/`group`/`chord`, the same
|
|
19
|
+
settings names. What changes is everything you used to have to know to run Celery
|
|
20
|
+
safely in production. The defaults come from years of community post-mortems, the
|
|
21
|
+
Ruby job-queue world (Sidekiq, Solid Queue, GoodJob), and the best Python
|
|
22
|
+
alternatives (Dramatiq, RQ, arq, Procrastinate, Oban/River designs). Each backend is
|
|
23
|
+
implemented with its own native primitives instead of a lowest-common-denominator
|
|
24
|
+
abstraction.
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
from potatoq import Potatoq # or: from potatoq import Celery
|
|
28
|
+
|
|
29
|
+
app = Potatoq("proj") # broker: $POTATOQ_BROKER_URL, your Django DB, or ./potatoq.sqlite3
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@app.task
|
|
33
|
+
def add(x, y):
|
|
34
|
+
return x + y
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
add.delay(2, 2)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```console
|
|
41
|
+
$ potatoq -A proj worker
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Why
|
|
45
|
+
|
|
46
|
+
Every Celery deployment eventually learns these the hard way. In Potatoq they are the defaults:
|
|
47
|
+
|
|
48
|
+
| | Celery default | Potatoq default |
|
|
49
|
+
|---|---|---|
|
|
50
|
+
| Acknowledgement | before the task runs (crash = lost task) | **after it finishes** (at-least-once) |
|
|
51
|
+
| Worker process killed (OOM, segfault) | task acked and lost | **requeued**, dead-lettered after 5 crashes (poison-message guard) |
|
|
52
|
+
| Prefetch | 4 × concurrency (short tasks wait behind long ones) | **one task per idle process** |
|
|
53
|
+
| ETA / countdown | held in worker RAM; Redis `visibility_timeout` re-runs them; RabbitMQ's 30 min timeout kills them | **stored by the broker** (sorted set, `run_at` column, TTL cascade) |
|
|
54
|
+
| Long tasks on Redis | redelivered every hour (`visibility_timeout`) | **leases renewed** by the worker while the task runs |
|
|
55
|
+
| Time limits | none | **30 min hard, soft 30 s earlier** |
|
|
56
|
+
| Failed tasks | gone | **dead-letter store** you can list and replay (`potatoq dead list/retry`) |
|
|
57
|
+
| Retries | fixed 180 s | **exponential backoff with jitter** |
|
|
58
|
+
| Memory leaks | processes live forever | **recycled every 1000 tasks** (`max_memory_per_child="512MB"` available) |
|
|
59
|
+
| Concurrency | host CPU count (over-subscribes containers) | **CPUs actually available** (affinity + cgroup quota) |
|
|
60
|
+
| SIGTERM | waits forever, then Kubernetes SIGKILLs | **25 s grace, then requeue** unfinished tasks |
|
|
61
|
+
| `celery beat` | separate single process; two of them = everything twice | **every worker runs the scheduler, deduplicated** |
|
|
62
|
+
| Django `.delay()` inside `atomic()` | sent immediately (task runs before COMMIT) | **sent on commit**, dropped on rollback |
|
|
63
|
+
| RabbitMQ | classic queues, no publisher confirms | **quorum queues + publisher confirms** |
|
|
64
|
+
| Redis | no key prefix, eviction can eat your queue | **prefixed keys**, warns unless `noeviction` |
|
|
65
|
+
| Root logger | hijacked | **left alone** |
|
|
66
|
+
| `async def` tasks | not supported | **supported** |
|
|
67
|
+
| Gossip/mingle/heartbeat chatter | on | **doesn't exist** |
|
|
68
|
+
|
|
69
|
+
Why each one, with sources: [the defaults, explained](https://anze3db.github.io/potatoq/design/defaults/).
|
|
70
|
+
|
|
71
|
+
## Install
|
|
72
|
+
|
|
73
|
+
```console
|
|
74
|
+
$ pip install potatoq # SQLite broker, no dependencies
|
|
75
|
+
$ pip install "potatoq[redis]" # + Redis / Valkey
|
|
76
|
+
$ pip install "potatoq[postgres]" # + PostgreSQL (psycopg 3)
|
|
77
|
+
$ pip install "potatoq[rabbitmq]" # + RabbitMQ (pika)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Python 3.11+ (including free-threaded builds) on **Linux and macOS**. Windows isn't
|
|
81
|
+
supported: the worker relies on `fork()` and POSIX signals. Use WSL on Windows.
|
|
82
|
+
|
|
83
|
+
## Brokers
|
|
84
|
+
|
|
85
|
+
| URL | Best for | How it works |
|
|
86
|
+
|---|---|---|
|
|
87
|
+
| `postgresql://…` | Most apps: zero extra infrastructure, **transactional enqueue** | `FOR UPDATE SKIP LOCKED` claims, `LISTEN/NOTIFY` wake-ups, heartbeat-based recovery, results in the same transaction as the ack |
|
|
88
|
+
| `sqlite:///path.db` | Development, single-host apps | WAL, `BEGIN IMMEDIATE`, `UPDATE … RETURNING` claims, `PRAGMA data_version` wake-ups |
|
|
89
|
+
| `redis://…` / `valkey://…` | High throughput | Atomic Lua scripts, priority sorted sets, leases + fencing tokens, `BZPOPMIN` wake-ups |
|
|
90
|
+
| `amqp://…` | Existing RabbitMQ shops | Quorum queues, publisher confirms, 28-level TTL delay cascade, dead-letter queues |
|
|
91
|
+
| `memory://` | Unit tests | In-process, with `potatoq.testing.drain()` |
|
|
92
|
+
|
|
93
|
+
The design and research behind each backend are in [How it works](https://anze3db.github.io/potatoq/design/internals/).
|
|
94
|
+
|
|
95
|
+
Results are stored by default on the database brokers, where writing a result is part of
|
|
96
|
+
the same transaction as the acknowledgement and costs nothing. On Redis and RabbitMQ they
|
|
97
|
+
are opt-in, exactly like Celery: set `result_backend` (`"broker"`, `redis://…`,
|
|
98
|
+
`postgresql://…`). Calling `.get()` on a task whose result is ignored raises immediately
|
|
99
|
+
instead of hanging forever.
|
|
100
|
+
|
|
101
|
+
## Migrating from Celery
|
|
102
|
+
|
|
103
|
+
```diff
|
|
104
|
+
-from celery import Celery, shared_task
|
|
105
|
+
-from celery.schedules import crontab
|
|
106
|
+
+from potatoq import Celery, shared_task
|
|
107
|
+
+from potatoq.schedules import crontab
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
```console
|
|
111
|
+
-$ celery -A proj worker -l info -Q default -c 8 -B --without-gossip --without-mingle
|
|
112
|
+
+$ potatoq -A proj worker -l info -Q default -c 8
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Your `CELERY_*` Django settings, `config_from_object(..., namespace="CELERY")`,
|
|
116
|
+
`beat_schedule`, `task_routes`, `autoretry_for`, `bind=True` + `self.retry()`, canvas,
|
|
117
|
+
signals, `AsyncResult` and custom `Task` base classes keep working. Then delete the
|
|
118
|
+
settings you only had because of Celery's defaults (`task_acks_late`,
|
|
119
|
+
`worker_prefetch_multiplier`, `broker_transport_options={"visibility_timeout": …}`,
|
|
120
|
+
`task_reject_on_worker_lost`, `broker_connection_retry_on_startup`, …).
|
|
121
|
+
See [Migrating from Celery](https://anze3db.github.io/potatoq/migrating-from-celery/) for the full list,
|
|
122
|
+
including the few deliberate differences.
|
|
123
|
+
|
|
124
|
+
## Django
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
INSTALLED_APPS = [..., "potatoq.contrib.django"]
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
That's it. No `celery.py`: `@shared_task` works, every app's `tasks.py` is discovered,
|
|
131
|
+
settings come from `POTATOQ = {...}` or your existing `CELERY_*` settings, and without a
|
|
132
|
+
broker setting **your default database is the broker**. Postgres and SQLite both work, so
|
|
133
|
+
you need no Redis to get started.
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from django.db import transaction
|
|
137
|
+
from potatoq import shared_task
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@shared_task
|
|
141
|
+
def send_receipt(order_id): ...
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
with transaction.atomic():
|
|
145
|
+
order = Order.objects.create(...)
|
|
146
|
+
send_receipt.delay(order.id) # runs only after COMMIT; never if the transaction rolls back
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
When the broker is your database, the task row is written inside your transaction:
|
|
150
|
+
the same semantics as `on_commit`, without the window where a crash between COMMIT and
|
|
151
|
+
publishing loses the task. `delay_on_commit()` (Celery 5.4) is supported too, and
|
|
152
|
+
`@shared_task(enqueue_on_commit=False)` opts a task out.
|
|
153
|
+
|
|
154
|
+
Using Django 6's built-in `django.tasks`? potatoq is a backend for it, with
|
|
155
|
+
priorities, `run_after`, async tasks, `get_result()` and signals. Set
|
|
156
|
+
`TASKS = {"default": {"BACKEND": "potatoq.contrib.django.tasks.PotatoqBackend"}}`.
|
|
157
|
+
|
|
158
|
+
Run workers with `potatoq worker` (it reads `DJANGO_SETTINGS_MODULE`) or `python manage.py potatoq worker`.
|
|
159
|
+
|
|
160
|
+
## Flask, FastAPI, SQLAlchemy
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
# Flask: config from app.config, tasks run inside app.app_context()
|
|
164
|
+
from potatoq.contrib.flask import init_app
|
|
165
|
+
|
|
166
|
+
potatoq = init_app(flask_app)
|
|
167
|
+
|
|
168
|
+
# FastAPI: non-blocking enqueue and result polling
|
|
169
|
+
result = await send_welcome.adelay(user.id)
|
|
170
|
+
value = await result.aget(timeout=10)
|
|
171
|
+
|
|
172
|
+
# SQLAlchemy: .delay() inside a session transaction is sent on commit
|
|
173
|
+
# (or written in the transaction when the broker is the same database)
|
|
174
|
+
from potatoq.contrib.sqlalchemy import install
|
|
175
|
+
|
|
176
|
+
install(app)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Periodic tasks
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
app.conf.beat_schedule = {
|
|
183
|
+
"nightly-report": {"task": "reports.build", "schedule": crontab(hour=3, minute=0)},
|
|
184
|
+
"heartbeat": {"task": "monitoring.ping", "schedule": 30.0},
|
|
185
|
+
}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Every worker runs the scheduler. Fire times are deterministic, and each one is claimed
|
|
189
|
+
exactly once through the broker: a unique row on Postgres/SQLite, `SET NX` on Redis, a
|
|
190
|
+
single-active-consumer token on RabbitMQ. You no longer need a `beat` process, and
|
|
191
|
+
running two of them can't double-schedule anything. A periodic run that couldn't start
|
|
192
|
+
before the next one is due expires instead of piling up.
|
|
193
|
+
|
|
194
|
+
## Operations
|
|
195
|
+
|
|
196
|
+
```console
|
|
197
|
+
$ potatoq -A proj status # live workers and what they are running
|
|
198
|
+
$ potatoq -A proj queues # waiting tasks per queue
|
|
199
|
+
$ potatoq -A proj dead list # dead-lettered tasks, with the reason
|
|
200
|
+
$ potatoq -A proj dead retry <task-id> # replay one
|
|
201
|
+
$ potatoq -A proj inspect active # Celery-style inspect
|
|
202
|
+
$ potatoq -A proj call proj.tasks.add -a '[2, 2]'
|
|
203
|
+
$ potatoq -A proj worker -P solo # run in-process, one task at a time (pdb-friendly)
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
## Testing
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from potatoq.testing import drain
|
|
210
|
+
|
|
211
|
+
app = Potatoq("tests", broker="memory://", result_backend="broker")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def test_signup():
|
|
215
|
+
signup("ann@example.com") # calls send_welcome.delay(...)
|
|
216
|
+
[task] = drain(app) # runs what was enqueued: serialization, retries and chords included
|
|
217
|
+
assert task.state == "SUCCESS"
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
`task_always_eager` works as in Celery, and arguments are still round-tripped through
|
|
221
|
+
the serializer so eager tests catch what production would.
|
|
222
|
+
|
|
223
|
+
## Performance
|
|
224
|
+
|
|
225
|
+
`benchmarks/throughput.py`: 5000 no-op tasks, 4 worker processes, all brokers on
|
|
226
|
+
localhost (Apple M3 Max, Python 3.13, median of 3 runs). Run it yourself with
|
|
227
|
+
`uv run --with 'celery[redis]' python benchmarks/throughput.py`.
|
|
228
|
+
|
|
229
|
+
| Library | Broker | Enqueue (tasks/s) | Process (tasks/s) |
|
|
230
|
+
|---|---|---:|---:|
|
|
231
|
+
| **Potatoq** | Redis | 10,200 | **11,900** |
|
|
232
|
+
| Celery 5.6 | Redis | 5,400 | 2,700 |
|
|
233
|
+
| **Potatoq** | RabbitMQ | 2,700 ¹ | 6,800 |
|
|
234
|
+
| Celery 5.6 | RabbitMQ | 8,300 ¹ | 8,000 ² |
|
|
235
|
+
| **Potatoq** | Postgres | 5,500 | 7,500 |
|
|
236
|
+
| **Potatoq** | SQLite | 14,700 | 5,700 |
|
|
237
|
+
|
|
238
|
+
¹ Potatoq waits for RabbitMQ publisher confirms on every publish, so an enqueued task
|
|
239
|
+
is replicated before `delay()` returns. Celery doesn't wait, which is how its publishes
|
|
240
|
+
can disappear silently ([#5410](https://github.com/celery/celery/issues/5410)).
|
|
241
|
+
² Celery acks before running the task (a crash loses it) and prefetches 16 messages.
|
|
242
|
+
Potatoq acks after the task finishes and only takes work for idle processes.
|
|
243
|
+
|
|
244
|
+
Getting Celery to run at all for this comparison took two workarounds:
|
|
245
|
+
`FORKED_BY_MULTIPROCESSING=1`, because the prefork pool fails on macOS with Python 3.13,
|
|
246
|
+
and disabling remote control, because RabbitMQ 4.3 rejects Celery's transient pidbox
|
|
247
|
+
queues. Potatoq needed none. Redis, Postgres and SQLite numbers vary by about ±10% between runs. RabbitMQ numbers swing up to 2× with machine load, because every publish waits for the broker to confirm it.
|
|
248
|
+
|
|
249
|
+
### Free-threaded Python
|
|
250
|
+
|
|
251
|
+
`benchmarks/cpu.py`: 400 CPU-bound tasks (a pure-Python loop, about 40 ms each) on
|
|
252
|
+
Redis, median of 2 runs. Speedup is over one process on the same interpreter; memory is
|
|
253
|
+
the RSS of the whole worker (supervisor and children).
|
|
254
|
+
|
|
255
|
+
| Python | 1 process | 4 processes (`-c 4`) | 4 threads in 1 process (`-c 1 -t 4`) |
|
|
256
|
+
|---|---:|---:|---:|
|
|
257
|
+
| 3.14, with the GIL | 25 tasks/s, 69 MB | 99 tasks/s (4.0×), 146 MB | 26 tasks/s (1.1×), 70 MB |
|
|
258
|
+
| **3.14t, free-threaded** | 27 tasks/s, 78 MB | 107 tasks/s (4.0×), 166 MB | **91 tasks/s (3.4×), 87 MB** |
|
|
259
|
+
| 3.15 rc3, with the GIL | 19 tasks/s, 70 MB | 74 tasks/s (3.9×), 151 MB | 19 tasks/s (1.0×), 72 MB |
|
|
260
|
+
| **3.15t rc3, free-threaded** | 20 tasks/s, 80 MB | 79 tasks/s (4.0×), 172 MB | **67 tasks/s (3.4×), 90 MB** |
|
|
261
|
+
|
|
262
|
+
With the GIL, threads only help I/O-bound tasks. On a free-threaded build, 4 threads
|
|
263
|
+
run CPU-bound tasks 3.4× faster than one process, close to 4 processes' 4.0×, in
|
|
264
|
+
about half the memory. A bare `ThreadPoolExecutor` running the same loop scales about
|
|
265
|
+
as well, so the remaining gap is the interpreter's, not potatoq's
|
|
266
|
+
([details](https://anze3db.github.io/potatoq/guide/workers/#free-threaded-python)).
|
|
267
|
+
|
|
268
|
+
There is no Rust in the hot path, and that's deliberate. The research
|
|
269
|
+
([why not Rust](https://anze3db.github.io/potatoq/design/internals/#rust)) found that per-task overhead is
|
|
270
|
+
dominated by broker round trips. Potatoq minimizes those: one Lua call or one SQL
|
|
271
|
+
statement per state change, with the result, ack and follow-up tasks committed together.
|
|
272
|
+
A Rust core measured +0–10% on realistic tasks, while costing fork-safety (prefork
|
|
273
|
+
workers) and a much larger wheel matrix. The serialization seam is kept so a native
|
|
274
|
+
codec can be added later as an optional extra.
|
|
275
|
+
|
|
276
|
+
## Documentation
|
|
277
|
+
|
|
278
|
+
Full docs are at **[anze3db.github.io/potatoq](https://anze3db.github.io/potatoq/)**. They live in [`docs/`](https://github.com/anze3db/potatoq/tree/main/docs) and build with [Zensical](https://zensical.org):
|
|
279
|
+
|
|
280
|
+
```console
|
|
281
|
+
$ uv run --group docs zensical serve # http://localhost:8000
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
## Development
|
|
285
|
+
|
|
286
|
+
```console
|
|
287
|
+
$ uv sync
|
|
288
|
+
$ uv run pytest # needs local Redis, Postgres and RabbitMQ; skips what isn't running
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
## Roadmap
|
|
292
|
+
|
|
293
|
+
Known gaps and planned features live in the [wishlist](https://anze3db.github.io/potatoq/wishlist/). Pick one up!
|
|
294
|
+
|
|
295
|
+
## Releases
|
|
296
|
+
|
|
297
|
+
potatoq uses CalVer (`26.1`, `26.2`, …). See [CHANGELOG.md](https://github.com/anze3db/potatoq/blob/main/CHANGELOG.md) for what
|
|
298
|
+
changed and [RELEASING.md](https://github.com/anze3db/potatoq/blob/main/RELEASING.md) for how releases are cut: a release PR,
|
|
299
|
+
trusted publishing to PyPI, and a GitHub release with generated notes.
|
|
300
|
+
|
|
301
|
+
## License
|
|
302
|
+
|
|
303
|
+
MIT
|
|
304
|
+
|
|
305
|
+
> 🤖 potatoq was generated with **Claude Opus 5.5** (high effort): the research, design,
|
|
306
|
+
> code, tests and docs.
|