potatoq 26.1a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. potatoq-26.1a1/LICENSE +21 -0
  2. potatoq-26.1a1/PKG-INFO +347 -0
  3. potatoq-26.1a1/README.md +306 -0
  4. potatoq-26.1a1/pyproject.toml +153 -0
  5. potatoq-26.1a1/pyproject.toml.orig +120 -0
  6. potatoq-26.1a1/src/potatoq/__init__.py +45 -0
  7. potatoq-26.1a1/src/potatoq/__main__.py +5 -0
  8. potatoq-26.1a1/src/potatoq/app.py +786 -0
  9. potatoq-26.1a1/src/potatoq/brokers/__init__.py +51 -0
  10. potatoq-26.1a1/src/potatoq/brokers/base.py +279 -0
  11. potatoq-26.1a1/src/potatoq/brokers/memory.py +275 -0
  12. potatoq-26.1a1/src/potatoq/brokers/postgres.py +803 -0
  13. potatoq-26.1a1/src/potatoq/brokers/rabbitmq.py +701 -0
  14. potatoq-26.1a1/src/potatoq/brokers/redis.py +661 -0
  15. potatoq-26.1a1/src/potatoq/brokers/sqlite.py +572 -0
  16. potatoq-26.1a1/src/potatoq/canvas.py +730 -0
  17. potatoq-26.1a1/src/potatoq/cli.py +623 -0
  18. potatoq-26.1a1/src/potatoq/config.py +253 -0
  19. potatoq-26.1a1/src/potatoq/console.py +159 -0
  20. potatoq-26.1a1/src/potatoq/contrib/__init__.py +0 -0
  21. potatoq-26.1a1/src/potatoq/contrib/django/__init__.py +333 -0
  22. potatoq-26.1a1/src/potatoq/contrib/django/apps.py +17 -0
  23. potatoq-26.1a1/src/potatoq/contrib/django/management/__init__.py +0 -0
  24. potatoq-26.1a1/src/potatoq/contrib/django/management/commands/__init__.py +0 -0
  25. potatoq-26.1a1/src/potatoq/contrib/django/management/commands/potatoq.py +24 -0
  26. potatoq-26.1a1/src/potatoq/contrib/django/tasks.py +385 -0
  27. potatoq-26.1a1/src/potatoq/contrib/fastapi.py +48 -0
  28. potatoq-26.1a1/src/potatoq/contrib/flask.py +62 -0
  29. potatoq-26.1a1/src/potatoq/contrib/sqlalchemy.py +215 -0
  30. potatoq-26.1a1/src/potatoq/control.py +136 -0
  31. potatoq-26.1a1/src/potatoq/exceptions.py +157 -0
  32. potatoq-26.1a1/src/potatoq/log.py +209 -0
  33. potatoq-26.1a1/src/potatoq/message.py +125 -0
  34. potatoq-26.1a1/src/potatoq/py.typed +0 -0
  35. potatoq-26.1a1/src/potatoq/result.py +417 -0
  36. potatoq-26.1a1/src/potatoq/schedules.py +224 -0
  37. potatoq-26.1a1/src/potatoq/serialization.py +163 -0
  38. potatoq-26.1a1/src/potatoq/signals.py +152 -0
  39. potatoq-26.1a1/src/potatoq/states.py +18 -0
  40. potatoq-26.1a1/src/potatoq/task.py +608 -0
  41. potatoq-26.1a1/src/potatoq/testing.py +99 -0
  42. potatoq-26.1a1/src/potatoq/utils/__init__.py +1 -0
  43. potatoq-26.1a1/src/potatoq/utils/log.py +5 -0
  44. potatoq-26.1a1/src/potatoq/worker/child.py +363 -0
  45. potatoq-26.1a1/src/potatoq/worker/executor.py +592 -0
  46. potatoq-26.1a1/src/potatoq/worker/scheduler.py +173 -0
  47. potatoq-26.1a1/src/potatoq/worker/supervisor.py +569 -0
potatoq-26.1a1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Anže Pečar
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,347 @@
1
+ Metadata-Version: 2.4
2
+ Name: potatoq
3
+ Version: 26.1a1
4
+ Summary: A Celery-compatible task queue with production-ready defaults and backend-native Redis, RabbitMQ, Postgres and SQLite brokers.
5
+ Keywords: celery,task queue,jobs,background tasks,redis,rabbitmq,postgres,sqlite,django
6
+ Author: Anže Pečar
7
+ Author-email: Anže Pečar <anze@pecar.me>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Framework :: Django
12
+ Classifier: Framework :: Flask
13
+ Classifier: Framework :: FastAPI
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Classifier: Operating System :: MacOS
17
+ Classifier: Typing :: Typed
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Programming Language :: Python :: 3.14
23
+ Classifier: Programming Language :: Python :: 3.15
24
+ Classifier: Topic :: System :: Distributed Computing
25
+ Requires-Dist: redis>=5.0 ; extra == 'all'
26
+ Requires-Dist: psycopg[binary]>=3.2 ; extra == 'all'
27
+ Requires-Dist: pika>=1.3 ; extra == 'all'
28
+ Requires-Dist: psycopg[binary]>=3.2 ; extra == 'postgres'
29
+ Requires-Dist: pika>=1.3 ; extra == 'rabbitmq'
30
+ Requires-Dist: redis>=5.0 ; extra == 'redis'
31
+ Requires-Python: >=3.11
32
+ Project-URL: Homepage, https://github.com/anze3db/potatoq
33
+ Project-URL: Documentation, https://anze3db.github.io/potatoq/
34
+ Project-URL: Changelog, https://github.com/anze3db/potatoq/blob/main/CHANGELOG.md
35
+ Project-URL: Issues, https://github.com/anze3db/potatoq/issues
36
+ Provides-Extra: all
37
+ Provides-Extra: postgres
38
+ Provides-Extra: rabbitmq
39
+ Provides-Extra: redis
40
+ Description-Content-Type: text/markdown
41
+
42
+ # 🥔 potatoq
43
+
44
+ > ⚠️ **potatoq is alpha software** in limited production use. Use it in
45
+ > production **at your own risk** for now. APIs and defaults may still change between
46
+ > releases. Please [report bugs on GitHub](https://github.com/anze3db/potatoq/issues).
47
+
48
+ **A Celery-compatible task queue with production-ready defaults.**
49
+
50
+ - 🧵 **Free-threaded Python** (3.14t, 3.15t): tested in CI, and potatoq never turns the
51
+ GIL back on, so `--threads` runs CPU-bound tasks in parallel in one process: 3.4×
52
+ with 4 threads, in half the memory of 4 processes ([numbers](https://github.com/anze3db/potatoq#free-threaded-python)).
53
+ - 🎸 **Django tasks**: a backend for Django 6's built-in
54
+ [`django.tasks`](https://anze3db.github.io/potatoq/integrations/django-tasks/), sharing workers with
55
+ `@shared_task`. The classic setup works too: add `"potatoq.contrib.django"` to
56
+ `INSTALLED_APPS` (no `celery.py` needed), and `.delay()` inside `atomic()` is sent on
57
+ commit.
58
+ - 🔁 **Switching from Celery** is meant to be a find-and-replace: the same decorators,
59
+ the same `delay()`/`apply_async()`, the same `chain`/`group`/`chord`, the same
60
+ settings names. What changes is everything you used to have to know to run Celery
61
+ safely in production. The defaults come from years of community post-mortems, the
62
+ Ruby job-queue world (Sidekiq, Solid Queue, GoodJob), and the best Python
63
+ alternatives (Dramatiq, RQ, arq, Procrastinate, Oban/River designs). Each backend is
64
+ implemented with its own native primitives instead of a lowest-common-denominator
65
+ abstraction.
66
+
67
+ ```python
68
+ from potatoq import Potatoq # or: from potatoq import Celery
69
+
70
+ app = Potatoq("proj") # broker: $POTATOQ_BROKER_URL, your Django DB, or ./potatoq.sqlite3
71
+
72
+
73
+ @app.task
74
+ def add(x, y):
75
+ return x + y
76
+
77
+
78
+ add.delay(2, 2)
79
+ ```
80
+
81
+ ```console
82
+ $ potatoq -A proj worker
83
+ ```
84
+
85
+ ## Why
86
+
87
+ Every Celery deployment eventually learns these the hard way. In Potatoq they are the defaults:
88
+
89
+ | | Celery default | Potatoq default |
90
+ |---|---|---|
91
+ | Acknowledgement | before the task runs (crash = lost task) | **after it finishes** (at-least-once) |
92
+ | Worker process killed (OOM, segfault) | task acked and lost | **requeued**, dead-lettered after 5 crashes (poison-message guard) |
93
+ | Prefetch | 4 × concurrency (short tasks wait behind long ones) | **one task per idle process** |
94
+ | ETA / countdown | held in worker RAM; Redis `visibility_timeout` re-runs them; RabbitMQ's 30 min timeout kills them | **stored by the broker** (sorted set, `run_at` column, TTL cascade) |
95
+ | Long tasks on Redis | redelivered every hour (`visibility_timeout`) | **leases renewed** by the worker while the task runs |
96
+ | Time limits | none | **30 min hard, soft 30 s earlier** |
97
+ | Failed tasks | gone | **dead-letter store** you can list and replay (`potatoq dead list/retry`) |
98
+ | Retries | fixed 180 s | **exponential backoff with jitter** |
99
+ | Memory leaks | processes live forever | **recycled every 1000 tasks** (`max_memory_per_child="512MB"` available) |
100
+ | Concurrency | host CPU count (over-subscribes containers) | **CPUs actually available** (affinity + cgroup quota) |
101
+ | SIGTERM | waits forever, then Kubernetes SIGKILLs | **25 s grace, then requeue** unfinished tasks |
102
+ | `celery beat` | separate single process; two of them = everything twice | **every worker runs the scheduler, deduplicated** |
103
+ | Django `.delay()` inside `atomic()` | sent immediately (task runs before COMMIT) | **sent on commit**, dropped on rollback |
104
+ | RabbitMQ | classic queues, no publisher confirms | **quorum queues + publisher confirms** |
105
+ | Redis | no key prefix, eviction can eat your queue | **prefixed keys**, warns unless `noeviction` |
106
+ | Root logger | hijacked | **left alone** |
107
+ | `async def` tasks | not supported | **supported** |
108
+ | Gossip/mingle/heartbeat chatter | on | **doesn't exist** |
109
+
110
+ Why each one, with sources: [the defaults, explained](https://anze3db.github.io/potatoq/design/defaults/).
111
+
112
+ ## Install
113
+
114
+ ```console
115
+ $ pip install potatoq # SQLite broker, no dependencies
116
+ $ pip install "potatoq[redis]" # + Redis / Valkey
117
+ $ pip install "potatoq[postgres]" # + PostgreSQL (psycopg 3)
118
+ $ pip install "potatoq[rabbitmq]" # + RabbitMQ (pika)
119
+ ```
120
+
121
+ Python 3.11+ (including free-threaded builds) on **Linux and macOS**. Windows isn't
122
+ supported: the worker relies on `fork()` and POSIX signals. Use WSL on Windows.
123
+
124
+ ## Brokers
125
+
126
+ | URL | Best for | How it works |
127
+ |---|---|---|
128
+ | `postgresql://…` | Most apps: zero extra infrastructure, **transactional enqueue** | `FOR UPDATE SKIP LOCKED` claims, `LISTEN/NOTIFY` wake-ups, heartbeat-based recovery, results in the same transaction as the ack |
129
+ | `sqlite:///path.db` | Development, single-host apps | WAL, `BEGIN IMMEDIATE`, `UPDATE … RETURNING` claims, `PRAGMA data_version` wake-ups |
130
+ | `redis://…` / `valkey://…` | High throughput | Atomic Lua scripts, priority sorted sets, leases + fencing tokens, `BZPOPMIN` wake-ups |
131
+ | `amqp://…` | Existing RabbitMQ shops | Quorum queues, publisher confirms, 28-level TTL delay cascade, dead-letter queues |
132
+ | `memory://` | Unit tests | In-process, with `potatoq.testing.drain()` |
133
+
134
+ The design and research behind each backend are in [How it works](https://anze3db.github.io/potatoq/design/internals/).
135
+
136
+ Results are stored by default on the database brokers, where writing a result is part of
137
+ the same transaction as the acknowledgement and costs nothing. On Redis and RabbitMQ they
138
+ are opt-in, exactly like Celery: set `result_backend` (`"broker"`, `redis://…`,
139
+ `postgresql://…`). Calling `.get()` on a task whose result is ignored raises immediately
140
+ instead of hanging forever.
141
+
142
+ ## Migrating from Celery
143
+
144
+ ```diff
145
+ -from celery import Celery, shared_task
146
+ -from celery.schedules import crontab
147
+ +from potatoq import Celery, shared_task
148
+ +from potatoq.schedules import crontab
149
+ ```
150
+
151
+ ```console
152
+ -$ celery -A proj worker -l info -Q default -c 8 -B --without-gossip --without-mingle
153
+ +$ potatoq -A proj worker -l info -Q default -c 8
154
+ ```
155
+
156
+ Your `CELERY_*` Django settings, `config_from_object(..., namespace="CELERY")`,
157
+ `beat_schedule`, `task_routes`, `autoretry_for`, `bind=True` + `self.retry()`, canvas,
158
+ signals, `AsyncResult` and custom `Task` base classes keep working. Then delete the
159
+ settings you only had because of Celery's defaults (`task_acks_late`,
160
+ `worker_prefetch_multiplier`, `broker_transport_options={"visibility_timeout": …}`,
161
+ `task_reject_on_worker_lost`, `broker_connection_retry_on_startup`, …).
162
+ See [Migrating from Celery](https://anze3db.github.io/potatoq/migrating-from-celery/) for the full list,
163
+ including the few deliberate differences.
164
+
165
+ ## Django
166
+
167
+ ```python
168
+ INSTALLED_APPS = [..., "potatoq.contrib.django"]
169
+ ```
170
+
171
+ That's it. No `celery.py`: `@shared_task` works, every app's `tasks.py` is discovered,
172
+ settings come from `POTATOQ = {...}` or your existing `CELERY_*` settings, and without a
173
+ broker setting **your default database is the broker**. Postgres and SQLite both work, so
174
+ you need no Redis to get started.
175
+
176
+ ```python
177
+ from django.db import transaction
178
+ from potatoq import shared_task
179
+
180
+
181
+ @shared_task
182
+ def send_receipt(order_id): ...
183
+
184
+
185
+ with transaction.atomic():
186
+ order = Order.objects.create(...)
187
+ send_receipt.delay(order.id) # runs only after COMMIT; never if the transaction rolls back
188
+ ```
189
+
190
+ When the broker is your database, the task row is written inside your transaction:
191
+ the same semantics as `on_commit`, without the window where a crash between COMMIT and
192
+ publishing loses the task. `delay_on_commit()` (Celery 5.4) is supported too, and
193
+ `@shared_task(enqueue_on_commit=False)` opts a task out.
194
+
195
+ Using Django 6's built-in `django.tasks`? potatoq is a backend for it, with
196
+ priorities, `run_after`, async tasks, `get_result()` and signals. Set
197
+ `TASKS = {"default": {"BACKEND": "potatoq.contrib.django.tasks.PotatoqBackend"}}`.
198
+
199
+ Run workers with `potatoq worker` (it reads `DJANGO_SETTINGS_MODULE`) or `python manage.py potatoq worker`.
200
+
201
+ ## Flask, FastAPI, SQLAlchemy
202
+
203
+ ```python
204
+ # Flask: config from app.config, tasks run inside app.app_context()
205
+ from potatoq.contrib.flask import init_app
206
+
207
+ potatoq = init_app(flask_app)
208
+
209
+ # FastAPI: non-blocking enqueue and result polling
210
+ result = await send_welcome.adelay(user.id)
211
+ value = await result.aget(timeout=10)
212
+
213
+ # SQLAlchemy: .delay() inside a session transaction is sent on commit
214
+ # (or written in the transaction when the broker is the same database)
215
+ from potatoq.contrib.sqlalchemy import install
216
+
217
+ install(app)
218
+ ```
219
+
220
+ ## Periodic tasks
221
+
222
+ ```python
223
+ app.conf.beat_schedule = {
224
+ "nightly-report": {"task": "reports.build", "schedule": crontab(hour=3, minute=0)},
225
+ "heartbeat": {"task": "monitoring.ping", "schedule": 30.0},
226
+ }
227
+ ```
228
+
229
+ Every worker runs the scheduler. Fire times are deterministic, and each one is claimed
230
+ exactly once through the broker: a unique row on Postgres/SQLite, `SET NX` on Redis, a
231
+ single-active-consumer token on RabbitMQ. You no longer need a `beat` process, and
232
+ running two of them can't double-schedule anything. A periodic run that couldn't start
233
+ before the next one is due expires instead of piling up.
234
+
235
+ ## Operations
236
+
237
+ ```console
238
+ $ potatoq -A proj status # live workers and what they are running
239
+ $ potatoq -A proj queues # waiting tasks per queue
240
+ $ potatoq -A proj dead list # dead-lettered tasks, with the reason
241
+ $ potatoq -A proj dead retry <task-id> # replay one
242
+ $ potatoq -A proj inspect active # Celery-style inspect
243
+ $ potatoq -A proj call proj.tasks.add -a '[2, 2]'
244
+ $ potatoq -A proj worker -P solo # run in-process, one task at a time (pdb-friendly)
245
+ ```
246
+
247
+ ## Testing
248
+
249
+ ```python
250
+ from potatoq.testing import drain
251
+
252
+ app = Potatoq("tests", broker="memory://", result_backend="broker")
253
+
254
+
255
+ def test_signup():
256
+ signup("ann@example.com") # calls send_welcome.delay(...)
257
+ [task] = drain(app) # runs what was enqueued: serialization, retries and chords included
258
+ assert task.state == "SUCCESS"
259
+ ```
260
+
261
+ `task_always_eager` works as in Celery, and arguments are still round-tripped through
262
+ the serializer so eager tests catch what production would.
263
+
264
+ ## Performance
265
+
266
+ `benchmarks/throughput.py`: 5000 no-op tasks, 4 worker processes, all brokers on
267
+ localhost (Apple M3 Max, Python 3.13, median of 3 runs). Run it yourself with
268
+ `uv run --with 'celery[redis]' python benchmarks/throughput.py`.
269
+
270
+ | Library | Broker | Enqueue (tasks/s) | Process (tasks/s) |
271
+ |---|---|---:|---:|
272
+ | **Potatoq** | Redis | 10,200 | **11,900** |
273
+ | Celery 5.6 | Redis | 5,400 | 2,700 |
274
+ | **Potatoq** | RabbitMQ | 2,700 ¹ | 6,800 |
275
+ | Celery 5.6 | RabbitMQ | 8,300 ¹ | 8,000 ² |
276
+ | **Potatoq** | Postgres | 5,500 | 7,500 |
277
+ | **Potatoq** | SQLite | 14,700 | 5,700 |
278
+
279
+ ¹ Potatoq waits for RabbitMQ publisher confirms on every publish, so an enqueued task
280
+ is replicated before `delay()` returns. Celery doesn't wait, which is how its publishes
281
+ can disappear silently ([#5410](https://github.com/celery/celery/issues/5410)).
282
+ ² Celery acks before running the task (a crash loses it) and prefetches 16 messages.
283
+ Potatoq acks after the task finishes and only takes work for idle processes.
284
+
285
+ Getting Celery to run at all for this comparison took two workarounds:
286
+ `FORKED_BY_MULTIPROCESSING=1`, because the prefork pool fails on macOS with Python 3.13,
287
+ and disabling remote control, because RabbitMQ 4.3 rejects Celery's transient pidbox
288
+ queues. Potatoq needed none. Redis, Postgres and SQLite numbers vary by about ±10% between runs. RabbitMQ numbers swing up to 2× with machine load, because every publish waits for the broker to confirm it.
289
+
290
+ ### Free-threaded Python
291
+
292
+ `benchmarks/cpu.py`: 400 CPU-bound tasks (a pure-Python loop, about 40 ms each) on
293
+ Redis, median of 2 runs. Speedup is over one process on the same interpreter; memory is
294
+ the RSS of the whole worker (supervisor and children).
295
+
296
+ | Python | 1 process | 4 processes (`-c 4`) | 4 threads in 1 process (`-c 1 -t 4`) |
297
+ |---|---:|---:|---:|
298
+ | 3.14, with the GIL | 25 tasks/s, 69 MB | 99 tasks/s (4.0×), 146 MB | 26 tasks/s (1.1×), 70 MB |
299
+ | **3.14t, free-threaded** | 27 tasks/s, 78 MB | 107 tasks/s (4.0×), 166 MB | **91 tasks/s (3.4×), 87 MB** |
300
+ | 3.15 rc3, with the GIL | 19 tasks/s, 70 MB | 74 tasks/s (3.9×), 151 MB | 19 tasks/s (1.0×), 72 MB |
301
+ | **3.15t rc3, free-threaded** | 20 tasks/s, 80 MB | 79 tasks/s (4.0×), 172 MB | **67 tasks/s (3.4×), 90 MB** |
302
+
303
+ With the GIL, threads only help I/O-bound tasks. On a free-threaded build, 4 threads
304
+ run CPU-bound tasks 3.4× faster than one process, close to 4 processes' 4.0×, in
305
+ about half the memory. A bare `ThreadPoolExecutor` running the same loop scales about
306
+ as well, so the remaining gap is the interpreter's, not potatoq's
307
+ ([details](https://anze3db.github.io/potatoq/guide/workers/#free-threaded-python)).
308
+
309
+ There is no Rust in the hot path, and that's deliberate. The research
310
+ ([why not Rust](https://anze3db.github.io/potatoq/design/internals/#rust)) found that per-task overhead is
311
+ dominated by broker round trips. Potatoq minimizes those: one Lua call or one SQL
312
+ statement per state change, with the result, ack and follow-up tasks committed together.
313
+ A Rust core measured +0–10% on realistic tasks, while costing fork-safety (prefork
314
+ workers) and a much larger wheel matrix. The serialization seam is kept so a native
315
+ codec can be added later as an optional extra.
316
+
317
+ ## Documentation
318
+
319
+ Full docs are at **[anze3db.github.io/potatoq](https://anze3db.github.io/potatoq/)**. They live in [`docs/`](https://github.com/anze3db/potatoq/tree/main/docs) and build with [Zensical](https://zensical.org):
320
+
321
+ ```console
322
+ $ uv run --group docs zensical serve # http://localhost:8000
323
+ ```
324
+
325
+ ## Development
326
+
327
+ ```console
328
+ $ uv sync
329
+ $ uv run pytest # needs local Redis, Postgres and RabbitMQ; skips what isn't running
330
+ ```
331
+
332
+ ## Roadmap
333
+
334
+ Known gaps and planned features live in the [wishlist](https://anze3db.github.io/potatoq/wishlist/). Pick one up!
335
+
336
+ ## Releases
337
+
338
+ potatoq uses CalVer (`26.1`, `26.2`, …). See [CHANGELOG.md](https://github.com/anze3db/potatoq/blob/main/CHANGELOG.md) for what
339
+ changed and [RELEASING.md](https://github.com/anze3db/potatoq/blob/main/RELEASING.md) for how releases are cut: a release PR,
340
+ trusted publishing to PyPI, and a GitHub release with generated notes.
341
+
342
+ ## License
343
+
344
+ MIT
345
+
346
+ > 🤖 potatoq was generated with **Claude Opus 5.5** (high effort): the research, design,
347
+ > code, tests and docs.
@@ -0,0 +1,306 @@
1
+ # 🥔 potatoq
2
+
3
+ > ⚠️ **potatoq is alpha software** in limited production use. Use it in
4
+ > production **at your own risk** for now. APIs and defaults may still change between
5
+ > releases. Please [report bugs on GitHub](https://github.com/anze3db/potatoq/issues).
6
+
7
+ **A Celery-compatible task queue with production-ready defaults.**
8
+
9
+ - 🧵 **Free-threaded Python** (3.14t, 3.15t): tested in CI, and potatoq never turns the
10
+ GIL back on, so `--threads` runs CPU-bound tasks in parallel in one process: 3.4×
11
+ with 4 threads, in half the memory of 4 processes ([numbers](https://github.com/anze3db/potatoq#free-threaded-python)).
12
+ - 🎸 **Django tasks**: a backend for Django 6's built-in
13
+ [`django.tasks`](https://anze3db.github.io/potatoq/integrations/django-tasks/), sharing workers with
14
+ `@shared_task`. The classic setup works too: add `"potatoq.contrib.django"` to
15
+ `INSTALLED_APPS` (no `celery.py` needed), and `.delay()` inside `atomic()` is sent on
16
+ commit.
17
+ - 🔁 **Switching from Celery** is meant to be a find-and-replace: the same decorators,
18
+ the same `delay()`/`apply_async()`, the same `chain`/`group`/`chord`, the same
19
+ settings names. What changes is everything you used to have to know to run Celery
20
+ safely in production. The defaults come from years of community post-mortems, the
21
+ Ruby job-queue world (Sidekiq, Solid Queue, GoodJob), and the best Python
22
+ alternatives (Dramatiq, RQ, arq, Procrastinate, Oban/River designs). Each backend is
23
+ implemented with its own native primitives instead of a lowest-common-denominator
24
+ abstraction.
25
+
26
+ ```python
27
+ from potatoq import Potatoq # or: from potatoq import Celery
28
+
29
+ app = Potatoq("proj") # broker: $POTATOQ_BROKER_URL, your Django DB, or ./potatoq.sqlite3
30
+
31
+
32
+ @app.task
33
+ def add(x, y):
34
+ return x + y
35
+
36
+
37
+ add.delay(2, 2)
38
+ ```
39
+
40
+ ```console
41
+ $ potatoq -A proj worker
42
+ ```
43
+
44
+ ## Why
45
+
46
+ Every Celery deployment eventually learns these the hard way. In Potatoq they are the defaults:
47
+
48
+ | | Celery default | Potatoq default |
49
+ |---|---|---|
50
+ | Acknowledgement | before the task runs (crash = lost task) | **after it finishes** (at-least-once) |
51
+ | Worker process killed (OOM, segfault) | task acked and lost | **requeued**, dead-lettered after 5 crashes (poison-message guard) |
52
+ | Prefetch | 4 × concurrency (short tasks wait behind long ones) | **one task per idle process** |
53
+ | ETA / countdown | held in worker RAM; Redis `visibility_timeout` re-runs them; RabbitMQ's 30 min timeout kills them | **stored by the broker** (sorted set, `run_at` column, TTL cascade) |
54
+ | Long tasks on Redis | redelivered every hour (`visibility_timeout`) | **leases renewed** by the worker while the task runs |
55
+ | Time limits | none | **30 min hard, soft 30 s earlier** |
56
+ | Failed tasks | gone | **dead-letter store** you can list and replay (`potatoq dead list/retry`) |
57
+ | Retries | fixed 180 s | **exponential backoff with jitter** |
58
+ | Memory leaks | processes live forever | **recycled every 1000 tasks** (`max_memory_per_child="512MB"` available) |
59
+ | Concurrency | host CPU count (over-subscribes containers) | **CPUs actually available** (affinity + cgroup quota) |
60
+ | SIGTERM | waits forever, then Kubernetes SIGKILLs | **25 s grace, then requeue** unfinished tasks |
61
+ | `celery beat` | separate single process; two of them = everything twice | **every worker runs the scheduler, deduplicated** |
62
+ | Django `.delay()` inside `atomic()` | sent immediately (task runs before COMMIT) | **sent on commit**, dropped on rollback |
63
+ | RabbitMQ | classic queues, no publisher confirms | **quorum queues + publisher confirms** |
64
+ | Redis | no key prefix, eviction can eat your queue | **prefixed keys**, warns unless `noeviction` |
65
+ | Root logger | hijacked | **left alone** |
66
+ | `async def` tasks | not supported | **supported** |
67
+ | Gossip/mingle/heartbeat chatter | on | **doesn't exist** |
68
+
69
+ Why each one, with sources: [the defaults, explained](https://anze3db.github.io/potatoq/design/defaults/).
70
+
71
+ ## Install
72
+
73
+ ```console
74
+ $ pip install potatoq # SQLite broker, no dependencies
75
+ $ pip install "potatoq[redis]" # + Redis / Valkey
76
+ $ pip install "potatoq[postgres]" # + PostgreSQL (psycopg 3)
77
+ $ pip install "potatoq[rabbitmq]" # + RabbitMQ (pika)
78
+ ```
79
+
80
+ Python 3.11+ (including free-threaded builds) on **Linux and macOS**. Windows isn't
81
+ supported: the worker relies on `fork()` and POSIX signals. Use WSL on Windows.
82
+
83
+ ## Brokers
84
+
85
+ | URL | Best for | How it works |
86
+ |---|---|---|
87
+ | `postgresql://…` | Most apps: zero extra infrastructure, **transactional enqueue** | `FOR UPDATE SKIP LOCKED` claims, `LISTEN/NOTIFY` wake-ups, heartbeat-based recovery, results in the same transaction as the ack |
88
+ | `sqlite:///path.db` | Development, single-host apps | WAL, `BEGIN IMMEDIATE`, `UPDATE … RETURNING` claims, `PRAGMA data_version` wake-ups |
89
+ | `redis://…` / `valkey://…` | High throughput | Atomic Lua scripts, priority sorted sets, leases + fencing tokens, `BZPOPMIN` wake-ups |
90
+ | `amqp://…` | Existing RabbitMQ shops | Quorum queues, publisher confirms, 28-level TTL delay cascade, dead-letter queues |
91
+ | `memory://` | Unit tests | In-process, with `potatoq.testing.drain()` |
92
+
93
+ The design and research behind each backend are in [How it works](https://anze3db.github.io/potatoq/design/internals/).
94
+
95
+ Results are stored by default on the database brokers, where writing a result is part of
96
+ the same transaction as the acknowledgement and costs nothing. On Redis and RabbitMQ they
97
+ are opt-in, exactly like Celery: set `result_backend` (`"broker"`, `redis://…`,
98
+ `postgresql://…`). Calling `.get()` on a task whose result is ignored raises immediately
99
+ instead of hanging forever.
100
+
101
+ ## Migrating from Celery
102
+
103
+ ```diff
104
+ -from celery import Celery, shared_task
105
+ -from celery.schedules import crontab
106
+ +from potatoq import Celery, shared_task
107
+ +from potatoq.schedules import crontab
108
+ ```
109
+
110
+ ```console
111
+ -$ celery -A proj worker -l info -Q default -c 8 -B --without-gossip --without-mingle
112
+ +$ potatoq -A proj worker -l info -Q default -c 8
113
+ ```
114
+
115
+ Your `CELERY_*` Django settings, `config_from_object(..., namespace="CELERY")`,
116
+ `beat_schedule`, `task_routes`, `autoretry_for`, `bind=True` + `self.retry()`, canvas,
117
+ signals, `AsyncResult` and custom `Task` base classes keep working. Then delete the
118
+ settings you only had because of Celery's defaults (`task_acks_late`,
119
+ `worker_prefetch_multiplier`, `broker_transport_options={"visibility_timeout": …}`,
120
+ `task_reject_on_worker_lost`, `broker_connection_retry_on_startup`, …).
121
+ See [Migrating from Celery](https://anze3db.github.io/potatoq/migrating-from-celery/) for the full list,
122
+ including the few deliberate differences.
123
+
124
+ ## Django
125
+
126
+ ```python
127
+ INSTALLED_APPS = [..., "potatoq.contrib.django"]
128
+ ```
129
+
130
+ That's it. No `celery.py`: `@shared_task` works, every app's `tasks.py` is discovered,
131
+ settings come from `POTATOQ = {...}` or your existing `CELERY_*` settings, and without a
132
+ broker setting **your default database is the broker**. Postgres and SQLite both work, so
133
+ you need no Redis to get started.
134
+
135
+ ```python
136
+ from django.db import transaction
137
+ from potatoq import shared_task
138
+
139
+
140
+ @shared_task
141
+ def send_receipt(order_id): ...
142
+
143
+
144
+ with transaction.atomic():
145
+ order = Order.objects.create(...)
146
+ send_receipt.delay(order.id) # runs only after COMMIT; never if the transaction rolls back
147
+ ```
148
+
149
+ When the broker is your database, the task row is written inside your transaction:
150
+ the same semantics as `on_commit`, without the window where a crash between COMMIT and
151
+ publishing loses the task. `delay_on_commit()` (Celery 5.4) is supported too, and
152
+ `@shared_task(enqueue_on_commit=False)` opts a task out.
153
+
154
+ Using Django 6's built-in `django.tasks`? potatoq is a backend for it, with
155
+ priorities, `run_after`, async tasks, `get_result()` and signals. Set
156
+ `TASKS = {"default": {"BACKEND": "potatoq.contrib.django.tasks.PotatoqBackend"}}`.
157
+
158
+ Run workers with `potatoq worker` (it reads `DJANGO_SETTINGS_MODULE`) or `python manage.py potatoq worker`.
159
+
160
+ ## Flask, FastAPI, SQLAlchemy
161
+
162
+ ```python
163
+ # Flask: config from app.config, tasks run inside app.app_context()
164
+ from potatoq.contrib.flask import init_app
165
+
166
+ potatoq = init_app(flask_app)
167
+
168
+ # FastAPI: non-blocking enqueue and result polling
169
+ result = await send_welcome.adelay(user.id)
170
+ value = await result.aget(timeout=10)
171
+
172
+ # SQLAlchemy: .delay() inside a session transaction is sent on commit
173
+ # (or written in the transaction when the broker is the same database)
174
+ from potatoq.contrib.sqlalchemy import install
175
+
176
+ install(app)
177
+ ```
178
+
179
+ ## Periodic tasks
180
+
181
+ ```python
182
+ app.conf.beat_schedule = {
183
+ "nightly-report": {"task": "reports.build", "schedule": crontab(hour=3, minute=0)},
184
+ "heartbeat": {"task": "monitoring.ping", "schedule": 30.0},
185
+ }
186
+ ```
187
+
188
+ Every worker runs the scheduler. Fire times are deterministic, and each one is claimed
189
+ exactly once through the broker: a unique row on Postgres/SQLite, `SET NX` on Redis, a
190
+ single-active-consumer token on RabbitMQ. You no longer need a `beat` process, and
191
+ running two of them can't double-schedule anything. A periodic run that couldn't start
192
+ before the next one is due expires instead of piling up.
193
+
194
+ ## Operations
195
+
196
+ ```console
197
+ $ potatoq -A proj status # live workers and what they are running
198
+ $ potatoq -A proj queues # waiting tasks per queue
199
+ $ potatoq -A proj dead list # dead-lettered tasks, with the reason
200
+ $ potatoq -A proj dead retry <task-id> # replay one
201
+ $ potatoq -A proj inspect active # Celery-style inspect
202
+ $ potatoq -A proj call proj.tasks.add -a '[2, 2]'
203
+ $ potatoq -A proj worker -P solo # run in-process, one task at a time (pdb-friendly)
204
+ ```
205
+
206
+ ## Testing
207
+
208
+ ```python
209
+ from potatoq.testing import drain
210
+
211
+ app = Potatoq("tests", broker="memory://", result_backend="broker")
212
+
213
+
214
+ def test_signup():
215
+ signup("ann@example.com") # calls send_welcome.delay(...)
216
+ [task] = drain(app) # runs what was enqueued: serialization, retries and chords included
217
+ assert task.state == "SUCCESS"
218
+ ```
219
+
220
+ `task_always_eager` works as in Celery, and arguments are still round-tripped through
221
+ the serializer so eager tests catch what production would.
222
+
223
+ ## Performance
224
+
225
+ `benchmarks/throughput.py`: 5000 no-op tasks, 4 worker processes, all brokers on
226
+ localhost (Apple M3 Max, Python 3.13, median of 3 runs). Run it yourself with
227
+ `uv run --with 'celery[redis]' python benchmarks/throughput.py`.
228
+
229
+ | Library | Broker | Enqueue (tasks/s) | Process (tasks/s) |
230
+ |---|---|---:|---:|
231
+ | **Potatoq** | Redis | 10,200 | **11,900** |
232
+ | Celery 5.6 | Redis | 5,400 | 2,700 |
233
+ | **Potatoq** | RabbitMQ | 2,700 ¹ | 6,800 |
234
+ | Celery 5.6 | RabbitMQ | 8,300 ¹ | 8,000 ² |
235
+ | **Potatoq** | Postgres | 5,500 | 7,500 |
236
+ | **Potatoq** | SQLite | 14,700 | 5,700 |
237
+
238
+ ¹ Potatoq waits for RabbitMQ publisher confirms on every publish, so an enqueued task
239
+ is replicated before `delay()` returns. Celery doesn't wait, which is how its publishes
240
+ can disappear silently ([#5410](https://github.com/celery/celery/issues/5410)).
241
+ ² Celery acks before running the task (a crash loses it) and prefetches 16 messages.
242
+ Potatoq acks after the task finishes and only takes work for idle processes.
243
+
244
+ Getting Celery to run at all for this comparison took two workarounds:
245
+ `FORKED_BY_MULTIPROCESSING=1`, because the prefork pool fails on macOS with Python 3.13,
246
+ and disabling remote control, because RabbitMQ 4.3 rejects Celery's transient pidbox
247
+ queues. Potatoq needed none. Redis, Postgres and SQLite numbers vary by about ±10% between runs. RabbitMQ numbers swing up to 2× with machine load, because every publish waits for the broker to confirm it.
248
+
249
+ ### Free-threaded Python
250
+
251
+ `benchmarks/cpu.py`: 400 CPU-bound tasks (a pure-Python loop, about 40 ms each) on
252
+ Redis, median of 2 runs. Speedup is over one process on the same interpreter; memory is
253
+ the RSS of the whole worker (supervisor and children).
254
+
255
+ | Python | 1 process | 4 processes (`-c 4`) | 4 threads in 1 process (`-c 1 -t 4`) |
256
+ |---|---:|---:|---:|
257
+ | 3.14, with the GIL | 25 tasks/s, 69 MB | 99 tasks/s (4.0×), 146 MB | 26 tasks/s (1.1×), 70 MB |
258
+ | **3.14t, free-threaded** | 27 tasks/s, 78 MB | 107 tasks/s (4.0×), 166 MB | **91 tasks/s (3.4×), 87 MB** |
259
+ | 3.15 rc3, with the GIL | 19 tasks/s, 70 MB | 74 tasks/s (3.9×), 151 MB | 19 tasks/s (1.0×), 72 MB |
260
+ | **3.15t rc3, free-threaded** | 20 tasks/s, 80 MB | 79 tasks/s (4.0×), 172 MB | **67 tasks/s (3.4×), 90 MB** |
261
+
262
+ With the GIL, threads only help I/O-bound tasks. On a free-threaded build, 4 threads
263
+ run CPU-bound tasks 3.4× faster than one process, close to 4 processes' 4.0×, in
264
+ about half the memory. A bare `ThreadPoolExecutor` running the same loop scales about
265
+ as well, so the remaining gap is the interpreter's, not potatoq's
266
+ ([details](https://anze3db.github.io/potatoq/guide/workers/#free-threaded-python)).
267
+
268
+ There is no Rust in the hot path, and that's deliberate. The research
269
+ ([why not Rust](https://anze3db.github.io/potatoq/design/internals/#rust)) found that per-task overhead is
270
+ dominated by broker round trips. Potatoq minimizes those: one Lua call or one SQL
271
+ statement per state change, with the result, ack and follow-up tasks committed together.
272
+ A Rust core measured +0–10% on realistic tasks, while costing fork-safety (prefork
273
+ workers) and a much larger wheel matrix. The serialization seam is kept so a native
274
+ codec can be added later as an optional extra.
275
+
276
+ ## Documentation
277
+
278
+ Full docs are at **[anze3db.github.io/potatoq](https://anze3db.github.io/potatoq/)**. They live in [`docs/`](https://github.com/anze3db/potatoq/tree/main/docs) and build with [Zensical](https://zensical.org):
279
+
280
+ ```console
281
+ $ uv run --group docs zensical serve # http://localhost:8000
282
+ ```
283
+
284
+ ## Development
285
+
286
+ ```console
287
+ $ uv sync
288
+ $ uv run pytest # needs local Redis, Postgres and RabbitMQ; skips what isn't running
289
+ ```
290
+
291
+ ## Roadmap
292
+
293
+ Known gaps and planned features live in the [wishlist](https://anze3db.github.io/potatoq/wishlist/). Pick one up!
294
+
295
+ ## Releases
296
+
297
+ potatoq uses CalVer (`26.1`, `26.2`, …). See [CHANGELOG.md](https://github.com/anze3db/potatoq/blob/main/CHANGELOG.md) for what
298
+ changed and [RELEASING.md](https://github.com/anze3db/potatoq/blob/main/RELEASING.md) for how releases are cut: a release PR,
299
+ trusted publishing to PyPI, and a GitHub release with generated notes.
300
+
301
+ ## License
302
+
303
+ MIT
304
+
305
+ > 🤖 potatoq was generated with **Claude Opus 5.5** (high effort): the research, design,
306
+ > code, tests and docs.