pytaskq 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytaskq-0.3.0/LICENSE +21 -0
- pytaskq-0.3.0/PKG-INFO +262 -0
- pytaskq-0.3.0/README.md +211 -0
- pytaskq-0.3.0/pyproject.toml +73 -0
- pytaskq-0.3.0/pytaskq/__init__.py +6 -0
- pytaskq-0.3.0/pytaskq/__main__.py +4 -0
- pytaskq-0.3.0/pytaskq/api/router.py +15 -0
- pytaskq-0.3.0/pytaskq/api/v1/auth.py +235 -0
- pytaskq-0.3.0/pytaskq/api/v1/dashboard.py +99 -0
- pytaskq-0.3.0/pytaskq/api/v1/dlq.py +114 -0
- pytaskq-0.3.0/pytaskq/api/v1/system.py +68 -0
- pytaskq-0.3.0/pytaskq/api/v1/tasks.py +99 -0
- pytaskq-0.3.0/pytaskq/app.py +324 -0
- pytaskq-0.3.0/pytaskq/cli.py +250 -0
- pytaskq-0.3.0/pytaskq/core/auth.py +535 -0
- pytaskq-0.3.0/pytaskq/core/config.py +52 -0
- pytaskq-0.3.0/pytaskq/core/dependencies.py +106 -0
- pytaskq-0.3.0/pytaskq/core/exceptions.py +12 -0
- pytaskq-0.3.0/pytaskq/core/idempotency.py +45 -0
- pytaskq-0.3.0/pytaskq/core/queueing.py +282 -0
- pytaskq-0.3.0/pytaskq/core/redis_client.py +19 -0
- pytaskq-0.3.0/pytaskq/main.py +124 -0
- pytaskq-0.3.0/pytaskq/net.py +67 -0
- pytaskq-0.3.0/pytaskq/schemas/requests.py +32 -0
- pytaskq-0.3.0/pytaskq/services/dashboard_service.py +42 -0
- pytaskq-0.3.0/pytaskq/static/assets/index-Biu-vuAP.js +9 -0
- pytaskq-0.3.0/pytaskq/static/assets/index-mgpfRVrS.css +1 -0
- pytaskq-0.3.0/pytaskq/static/favicon.svg +1 -0
- pytaskq-0.3.0/pytaskq/static/index.html +15 -0
- pytaskq-0.3.0/pytaskq/task_registery.py +144 -0
- pytaskq-0.3.0/pytaskq/tracing.py +98 -0
- pytaskq-0.3.0/pytaskq/worker.py +636 -0
- pytaskq-0.3.0/pytaskq.egg-info/PKG-INFO +262 -0
- pytaskq-0.3.0/pytaskq.egg-info/SOURCES.txt +48 -0
- pytaskq-0.3.0/pytaskq.egg-info/dependency_links.txt +1 -0
- pytaskq-0.3.0/pytaskq.egg-info/entry_points.txt +2 -0
- pytaskq-0.3.0/pytaskq.egg-info/requires.txt +25 -0
- pytaskq-0.3.0/pytaskq.egg-info/top_level.txt +1 -0
- pytaskq-0.3.0/setup.cfg +4 -0
- pytaskq-0.3.0/tests/test_api.py +268 -0
- pytaskq-0.3.0/tests/test_auth.py +454 -0
- pytaskq-0.3.0/tests/test_cli.py +79 -0
- pytaskq-0.3.0/tests/test_crash_recovery.py +222 -0
- pytaskq-0.3.0/tests/test_dlq_endpoints.py +275 -0
- pytaskq-0.3.0/tests/test_graceful_shutdown.py +242 -0
- pytaskq-0.3.0/tests/test_sdk.py +284 -0
- pytaskq-0.3.0/tests/test_ssrf_guard.py +126 -0
- pytaskq-0.3.0/tests/test_task_queuing.py +303 -0
- pytaskq-0.3.0/tests/test_worker_features.py +286 -0
- pytaskq-0.3.0/tests/test_worker_handle_task.py +375 -0
pytaskq-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vraj Patel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pytaskq-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytaskq
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Background jobs for Python — simple to start, safe in production. Async-first task queue with a live dashboard, built on Redis.
|
|
5
|
+
Author: Vraj Patel
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/vrajpatelm/PyTaskQ
|
|
8
|
+
Project-URL: Documentation, https://github.com/vrajpatelm/PyTaskQ#readme
|
|
9
|
+
Project-URL: Issues, https://github.com/vrajpatelm/PyTaskQ/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/vrajpatelm/PyTaskQ/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: task queue,background jobs,asyncio,redis,fastapi,worker,cron,celery alternative
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Framework :: AsyncIO
|
|
14
|
+
Classifier: Framework :: FastAPI
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: fastapi>=0.100.0
|
|
29
|
+
Requires-Dist: uvicorn[standard]>=0.23.0
|
|
30
|
+
Requires-Dist: redis>=5.0.0
|
|
31
|
+
Requires-Dist: pydantic>=2.0.0
|
|
32
|
+
Requires-Dist: pydantic-settings>=2.0.0
|
|
33
|
+
Requires-Dist: croniter>=2.0.0
|
|
34
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
35
|
+
Requires-Dist: opentelemetry-api>=1.20
|
|
36
|
+
Provides-Extra: tracing
|
|
37
|
+
Requires-Dist: opentelemetry-sdk>=1.20; extra == "tracing"
|
|
38
|
+
Requires-Dist: opentelemetry-exporter-otlp>=1.20; extra == "tracing"
|
|
39
|
+
Provides-Extra: demo
|
|
40
|
+
Requires-Dist: numpy>=1.26.0; extra == "demo"
|
|
41
|
+
Requires-Dist: Pillow>=10.0.1; extra == "demo"
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
44
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
|
|
45
|
+
Requires-Dist: fakeredis[lua]>=2.20.0; extra == "dev"
|
|
46
|
+
Requires-Dist: httpx>=0.26.0; extra == "dev"
|
|
47
|
+
Requires-Dist: ruff; extra == "dev"
|
|
48
|
+
Requires-Dist: build; extra == "dev"
|
|
49
|
+
Requires-Dist: twine; extra == "dev"
|
|
50
|
+
Dynamic: license-file
|
|
51
|
+
|
|
52
|
+
<div align="center">
|
|
53
|
+
|
|
54
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/pytaskq/static/favicon.svg" width="72" alt="PyTaskQ logo" />
|
|
55
|
+
|
|
56
|
+
# PyTaskQ
|
|
57
|
+
|
|
58
|
+
**Background jobs for Python — simple to start, safe in production.**
|
|
59
|
+
|
|
60
|
+
Decorate a function, call `.delay()`, and watch it run in a live dashboard.<br/>
|
|
61
|
+
Async-first, self-hosted, built on FastAPI and Redis.
|
|
62
|
+
|
|
63
|
+
[](https://pypi.org/project/pytaskq/)
|
|
64
|
+
[](https://pypi.org/project/pytaskq/)
|
|
65
|
+
[](https://github.com/vrajpatelm/PyTaskQ/actions/workflows/ci.yml)
|
|
66
|
+
[](https://github.com/vrajpatelm/PyTaskQ/blob/main/LICENSE)
|
|
67
|
+
|
|
68
|
+
[Quick start](#quick-start) · [Dashboard](#the-dashboard) · [Accounts](#accounts) · [Configuration](#configuration) · [How it works](#how-it-works) · [API](#rest-api) · [Docker](#docker)
|
|
69
|
+
|
|
70
|
+
<br/>
|
|
71
|
+
|
|
72
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/hero.png" alt="The PyTaskQ dashboard in light and dark themes" width="100%" />
|
|
73
|
+
|
|
74
|
+
</div>
|
|
75
|
+
|
|
76
|
+
## Why PyTaskQ
|
|
77
|
+
|
|
78
|
+
- **Plain Python.** One decorator. Sync, `async def`, CPU-heavy and I/O-bound tasks each run in the right pool automatically.
|
|
79
|
+
- **See everything.** A live dashboard shows every job's arguments, timing, result and error, and lets you run, cancel and replay jobs.
|
|
80
|
+
- **Built for failure.** Retries with backoff, a dead-letter queue, crash recovery, fencing tokens and graceful shutdown come built in.
|
|
81
|
+
- **Self-hosted.** Accounts, roles and invite links are included. Redis is the only thing you need to run.
|
|
82
|
+
|
|
83
|
+
## Quick start
|
|
84
|
+
|
|
85
|
+
You need Python 3.10+ and Redis (`docker run -p 6379:6379 redis:7` is enough).
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
pip install pytaskq
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
**1. Write tasks**
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
# tasks.py
|
|
95
|
+
from pytaskq import PyTaskQ
|
|
96
|
+
|
|
97
|
+
queue = PyTaskQ()
|
|
98
|
+
|
|
99
|
+
@queue.task(type="io", priority="high", max_retries=5)
|
|
100
|
+
def send_order_confirmation(order_id: int, email: str):
|
|
101
|
+
"""Email the order confirmation and receipt."""
|
|
102
|
+
...
|
|
103
|
+
|
|
104
|
+
@queue.task # async def is detected automatically
|
|
105
|
+
async def sync_inventory(sku: str):
|
|
106
|
+
...
|
|
107
|
+
|
|
108
|
+
@queue.task(cron="0 2 * * *") # every night at 02:00, once across all workers
|
|
109
|
+
def nightly_sales_report():
|
|
110
|
+
...
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
**2. Start a worker and the dashboard**
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pytaskq worker --app tasks:queue
|
|
117
|
+
pytaskq dashboard --app tasks:queue # → http://localhost:8000
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
On first start, the dashboard prints a **setup code** in its log. Open the dashboard, enter the code and create your admin account.
|
|
121
|
+
|
|
122
|
+
**3. Queue jobs**
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
await send_order_confirmation.delay(10481, email="ada@example.com") # from async code
|
|
126
|
+
send_order_confirmation.delay_sync(10482, email="lin@example.com") # from sync code
|
|
127
|
+
|
|
128
|
+
result = await queue.wait(task_id, timeout=30) # {"status": "Success", "result": ..., "duration_ms": ...}
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
You can also queue jobs from the dashboard or over HTTP. Want to try it without writing code? Run `pytaskq worker --app pytaskq.task_registery:queue` and use the built-in demo tasks (`pip install "pytaskq[demo]"`).
|
|
132
|
+
|
|
133
|
+
## The dashboard
|
|
134
|
+
|
|
135
|
+
Live updates, light and dark themes, and a layout that works on phones.
|
|
136
|
+
|
|
137
|
+
**Jobs.** The lifecycle strip shows where every job is: scheduled, queued, running, succeeded or dead. Below it, filter recent jobs by status or task, or search by ID.
|
|
138
|
+
|
|
139
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/dashboard-dark.png" alt="Jobs view with the lifecycle strip and a live job list" width="100%" />
|
|
140
|
+
|
|
141
|
+
**Job details.** Timings, the worker that ran it, attempts, arguments, and the result or error. Waiting jobs can be cancelled.
|
|
142
|
+
|
|
143
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/job-details.png" alt="Job detail panel with timings, arguments and result" width="100%" />
|
|
144
|
+
|
|
145
|
+
**Run a task.** Each task's form is built from its function signature: required fields, types, defaults, priority and delay.
|
|
146
|
+
|
|
147
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/run-task.png" alt="Run a task form generated from a function signature" width="100%" />
|
|
148
|
+
|
|
149
|
+
**Dead letters.** Jobs that failed every retry, with their last error. Replay them once the cause is fixed.
|
|
150
|
+
|
|
151
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/dead-letters.png" alt="Dead letters with their errors and replay buttons" width="100%" />
|
|
152
|
+
|
|
153
|
+
**Workers.** Host, load, jobs done and failed, pool sizes, uptime and heartbeat health.
|
|
154
|
+
|
|
155
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/workers.png" alt="Worker fleet with health and throughput" width="100%" />
|
|
156
|
+
|
|
157
|
+
## Accounts
|
|
158
|
+
|
|
159
|
+
PyTaskQ has its own accounts, so you can expose it safely without an external service.
|
|
160
|
+
|
|
161
|
+
<table>
|
|
162
|
+
<tr>
|
|
163
|
+
<td width="60%" valign="middle">
|
|
164
|
+
|
|
165
|
+
**First start.** A one-time setup code is printed in the server log. Only someone with access to the server can create the first admin.
|
|
166
|
+
|
|
167
|
+
Lost it? Run `pytaskq users setup-code`. For automated installs, set `PYTASKQ_ADMIN_USER` and `PYTASKQ_ADMIN_PASSWORD` instead.
|
|
168
|
+
|
|
169
|
+
</td>
|
|
170
|
+
<td width="40%" align="center">
|
|
171
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/setup.png" alt="First-run setup screen" width="300" />
|
|
172
|
+
</td>
|
|
173
|
+
</tr>
|
|
174
|
+
</table>
|
|
175
|
+
|
|
176
|
+
**Your team.** Admins send single-use invite links (valid 7 days), change roles and disable accounts from *Settings → Team*.
|
|
177
|
+
|
|
178
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/team.png" alt="Team settings with roles and an invite link" width="100%" />
|
|
179
|
+
|
|
180
|
+
| Role | Can |
|
|
181
|
+
|---|---|
|
|
182
|
+
| **Viewer** | See jobs, workers and dead letters |
|
|
183
|
+
| **Operator** | Also run, cancel, replay and delete jobs |
|
|
184
|
+
| **Admin** | Also invite people, change roles and disable accounts |
|
|
185
|
+
|
|
186
|
+
- **Scripts** use a personal API token from *Settings → API tokens*: `Authorization: Bearer ptq_…`. The token has its owner's role.
|
|
187
|
+
- **Locked out?** Run `pytaskq users reset-password NAME` on the server. `pytaskq users list | create | set-role | delete | setup-code` cover the rest.
|
|
188
|
+
- **Security.** Passwords are hashed with scrypt. Only SHA-256 hashes of sessions, invites and tokens are stored. Cookies are `HttpOnly` and `SameSite`, cross-site writes are blocked, and failed sign-ins are throttled. Serve the dashboard over HTTPS in production.
|
|
189
|
+
- **Local development:** `PYTASKQ_AUTH=off` turns sign-in off. Never use it on a server others can reach.
|
|
190
|
+
|
|
191
|
+
## Configuration
|
|
192
|
+
|
|
193
|
+
| Task option | Default | What it does |
|
|
194
|
+
|---|---|---|
|
|
195
|
+
| `type` | `"async"` for `async def`, else `"cpu"` | `"cpu"` process pool · `"io"` thread pool · `"async"` event loop |
|
|
196
|
+
| `priority` | `"default"` | `"high"`, `"default"` or `"low"` queue |
|
|
197
|
+
| `max_retries` | `3` | Retries with jittered backoff before dead-lettering |
|
|
198
|
+
| `timeout` | `300` s | Abandon and retry an attempt after this long |
|
|
199
|
+
| `cron` | — | 5-field cron schedule |
|
|
200
|
+
| `name` | function name | Registered task name |
|
|
201
|
+
|
|
202
|
+
App-wide defaults: `PyTaskQ(default_priority="low", max_retries=5, timeout=60)`.
|
|
203
|
+
|
|
204
|
+
Every server setting is an environment variable or a line in `.env`, and all of them are documented in [`.env.example`](https://github.com/vrajpatelm/PyTaskQ/blob/main/.env.example). The ones you'll most likely change are `REDIS_URL`, `MAX_RETRIES`, `TASK_TIMEOUT` and `QUEUE_CAPACITY`.
|
|
205
|
+
|
|
206
|
+
| Command | Purpose |
|
|
207
|
+
|---|---|
|
|
208
|
+
| `pytaskq worker --app m:queue [--cpu-workers N --io-workers N]` | Run a worker |
|
|
209
|
+
| `pytaskq dashboard [--app m:queue] [--port 8000]` | Dashboard and REST API |
|
|
210
|
+
| `pytaskq tasks --app m:queue` | List tasks and their settings |
|
|
211
|
+
| `pytaskq users …` | Manage accounts from the server |
|
|
212
|
+
| `pytaskq ping` | Check Redis and count workers |
|
|
213
|
+
|
|
214
|
+
## How it works
|
|
215
|
+
|
|
216
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/architecture.svg" alt="PyTaskQ architecture" width="100%" />
|
|
217
|
+
|
|
218
|
+
1. **Reliable pop.** A worker atomically moves a job into its own processing list, so a crash never loses it.
|
|
219
|
+
2. **Crash recovery.** Workers send a heartbeat every 10 s. An elected leader re-queues work from any worker that has been silent for 30 s.
|
|
220
|
+
3. **Fencing tokens.** A Lua script only accepts results carrying the current token, so a late result from a hung worker can't overwrite a newer one.
|
|
221
|
+
4. **Retries, then dead letters.** Failures back off with jitter. After `max_retries`, the job is dead-lettered with its last error.
|
|
222
|
+
5. **Graceful shutdown.** Ctrl+C finishes running jobs before exiting.
|
|
223
|
+
6. **Safe outbound calls.** Webhooks (`on_success_url=`) are signed with HMAC and retried. Use `pytaskq.net.assert_safe_url` to block requests to private networks.
|
|
224
|
+
|
|
225
|
+
## REST API
|
|
226
|
+
|
|
227
|
+
Interactive docs are served at `/docs`, and the full reference is in [docs/API.md](https://github.com/vrajpatelm/PyTaskQ/blob/main/docs/API.md). Every route except `/health` and sign-in needs a session or API token.
|
|
228
|
+
|
|
229
|
+
| Method | Route | |
|
|
230
|
+
|---|---|---|
|
|
231
|
+
| `POST` | `/task/enqueue` | Queue a job (`task_name`, `args`, `kwargs`, `priority`, `delay_seconds`) |
|
|
232
|
+
| `GET` | `/task/{id}` | Status, timings, result or error |
|
|
233
|
+
| `POST` | `/task/{id}/cancel` | Cancel a waiting job |
|
|
234
|
+
| `GET` | `/tasks` | Recent jobs, filterable by status and task |
|
|
235
|
+
| `GET` | `/metrics`, `/system/workers`, `/dlq` | Queue, worker and dead-letter state |
|
|
236
|
+
| `POST` | `/dlq/replay/{id}`, `/dlq/retry_all` | Replay dead letters |
|
|
237
|
+
|
|
238
|
+
## Docker
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
docker compose up --build # Redis + dashboard + worker + Jaeger
|
|
242
|
+
docker compose logs web | grep -A4 "setup code"
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Point `PYTASKQ_APP` and the worker `command:` in `docker-compose.yml` at your own module. Accounts and jobs persist in the `redis-data` volume.
|
|
246
|
+
|
|
247
|
+
## Development
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
pip install -e ".[dev,demo]"
|
|
251
|
+
python -m pytest # in-memory via fakeredis, no Redis needed
|
|
252
|
+
|
|
253
|
+
cd frontend && npm install
|
|
254
|
+
npm run dev # dashboard with hot reload
|
|
255
|
+
npm run build # writes the bundle into pytaskq/static/
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
See [CONTRIBUTING](https://github.com/vrajpatelm/PyTaskQ/blob/main/docs/CONTRIBUTING.md), [RELEASING](https://github.com/vrajpatelm/PyTaskQ/blob/main/docs/RELEASING.md) and the [CHANGELOG](https://github.com/vrajpatelm/PyTaskQ/blob/main/CHANGELOG.md).
|
|
259
|
+
|
|
260
|
+
## License
|
|
261
|
+
|
|
262
|
+
MIT
|
pytaskq-0.3.0/README.md
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/pytaskq/static/favicon.svg" width="72" alt="PyTaskQ logo" />
|
|
4
|
+
|
|
5
|
+
# PyTaskQ
|
|
6
|
+
|
|
7
|
+
**Background jobs for Python — simple to start, safe in production.**
|
|
8
|
+
|
|
9
|
+
Decorate a function, call `.delay()`, and watch it run in a live dashboard.<br/>
|
|
10
|
+
Async-first, self-hosted, built on FastAPI and Redis.
|
|
11
|
+
|
|
12
|
+
[](https://pypi.org/project/pytaskq/)
|
|
13
|
+
[](https://pypi.org/project/pytaskq/)
|
|
14
|
+
[](https://github.com/vrajpatelm/PyTaskQ/actions/workflows/ci.yml)
|
|
15
|
+
[](https://github.com/vrajpatelm/PyTaskQ/blob/main/LICENSE)
|
|
16
|
+
|
|
17
|
+
[Quick start](#quick-start) · [Dashboard](#the-dashboard) · [Accounts](#accounts) · [Configuration](#configuration) · [How it works](#how-it-works) · [API](#rest-api) · [Docker](#docker)
|
|
18
|
+
|
|
19
|
+
<br/>
|
|
20
|
+
|
|
21
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/hero.png" alt="The PyTaskQ dashboard in light and dark themes" width="100%" />
|
|
22
|
+
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
## Why PyTaskQ
|
|
26
|
+
|
|
27
|
+
- **Plain Python.** One decorator. Sync, `async def`, CPU-heavy and I/O-bound tasks each run in the right pool automatically.
|
|
28
|
+
- **See everything.** A live dashboard shows every job's arguments, timing, result and error, and lets you run, cancel and replay jobs.
|
|
29
|
+
- **Built for failure.** Retries with backoff, a dead-letter queue, crash recovery, fencing tokens and graceful shutdown come built in.
|
|
30
|
+
- **Self-hosted.** Accounts, roles and invite links are included. Redis is the only thing you need to run.
|
|
31
|
+
|
|
32
|
+
## Quick start
|
|
33
|
+
|
|
34
|
+
You need Python 3.10+ and Redis (`docker run -p 6379:6379 redis:7` is enough).
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install pytaskq
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
**1. Write tasks**
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
# tasks.py
|
|
44
|
+
from pytaskq import PyTaskQ
|
|
45
|
+
|
|
46
|
+
queue = PyTaskQ()
|
|
47
|
+
|
|
48
|
+
@queue.task(type="io", priority="high", max_retries=5)
|
|
49
|
+
def send_order_confirmation(order_id: int, email: str):
|
|
50
|
+
"""Email the order confirmation and receipt."""
|
|
51
|
+
...
|
|
52
|
+
|
|
53
|
+
@queue.task # async def is detected automatically
|
|
54
|
+
async def sync_inventory(sku: str):
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
@queue.task(cron="0 2 * * *") # every night at 02:00, once across all workers
|
|
58
|
+
def nightly_sales_report():
|
|
59
|
+
...
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
**2. Start a worker and the dashboard**
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pytaskq worker --app tasks:queue
|
|
66
|
+
pytaskq dashboard --app tasks:queue # → http://localhost:8000
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
On first start, the dashboard prints a **setup code** in its log. Open the dashboard, enter the code and create your admin account.
|
|
70
|
+
|
|
71
|
+
**3. Queue jobs**
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
await send_order_confirmation.delay(10481, email="ada@example.com") # from async code
|
|
75
|
+
send_order_confirmation.delay_sync(10482, email="lin@example.com") # from sync code
|
|
76
|
+
|
|
77
|
+
result = await queue.wait(task_id, timeout=30) # {"status": "Success", "result": ..., "duration_ms": ...}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
You can also queue jobs from the dashboard or over HTTP. Want to try it without writing code? Run `pytaskq worker --app pytaskq.task_registery:queue` and use the built-in demo tasks (`pip install "pytaskq[demo]"`).
|
|
81
|
+
|
|
82
|
+
## The dashboard
|
|
83
|
+
|
|
84
|
+
Live updates, light and dark themes, and a layout that works on phones.
|
|
85
|
+
|
|
86
|
+
**Jobs.** The lifecycle strip shows where every job is: scheduled, queued, running, succeeded or dead. Below it, filter recent jobs by status or task, or search by ID.
|
|
87
|
+
|
|
88
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/dashboard-dark.png" alt="Jobs view with the lifecycle strip and a live job list" width="100%" />
|
|
89
|
+
|
|
90
|
+
**Job details.** Timings, the worker that ran it, attempts, arguments, and the result or error. Waiting jobs can be cancelled.
|
|
91
|
+
|
|
92
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/job-details.png" alt="Job detail panel with timings, arguments and result" width="100%" />
|
|
93
|
+
|
|
94
|
+
**Run a task.** Each task's form is built from its function signature: required fields, types, defaults, priority and delay.
|
|
95
|
+
|
|
96
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/run-task.png" alt="Run a task form generated from a function signature" width="100%" />
|
|
97
|
+
|
|
98
|
+
**Dead letters.** Jobs that failed every retry, with their last error. Replay them once the cause is fixed.
|
|
99
|
+
|
|
100
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/dead-letters.png" alt="Dead letters with their errors and replay buttons" width="100%" />
|
|
101
|
+
|
|
102
|
+
**Workers.** Host, load, jobs done and failed, pool sizes, uptime and heartbeat health.
|
|
103
|
+
|
|
104
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/workers.png" alt="Worker fleet with health and throughput" width="100%" />
|
|
105
|
+
|
|
106
|
+
## Accounts
|
|
107
|
+
|
|
108
|
+
PyTaskQ has its own accounts, so you can expose it safely without an external service.
|
|
109
|
+
|
|
110
|
+
<table>
|
|
111
|
+
<tr>
|
|
112
|
+
<td width="60%" valign="middle">
|
|
113
|
+
|
|
114
|
+
**First start.** A one-time setup code is printed in the server log. Only someone with access to the server can create the first admin.
|
|
115
|
+
|
|
116
|
+
Lost it? Run `pytaskq users setup-code`. For automated installs, set `PYTASKQ_ADMIN_USER` and `PYTASKQ_ADMIN_PASSWORD` instead.
|
|
117
|
+
|
|
118
|
+
</td>
|
|
119
|
+
<td width="40%" align="center">
|
|
120
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/setup.png" alt="First-run setup screen" width="300" />
|
|
121
|
+
</td>
|
|
122
|
+
</tr>
|
|
123
|
+
</table>
|
|
124
|
+
|
|
125
|
+
**Your team.** Admins send single-use invite links (valid 7 days), change roles and disable accounts from *Settings → Team*.
|
|
126
|
+
|
|
127
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/team.png" alt="Team settings with roles and an invite link" width="100%" />
|
|
128
|
+
|
|
129
|
+
| Role | Can |
|
|
130
|
+
|---|---|
|
|
131
|
+
| **Viewer** | See jobs, workers and dead letters |
|
|
132
|
+
| **Operator** | Also run, cancel, replay and delete jobs |
|
|
133
|
+
| **Admin** | Also invite people, change roles and disable accounts |
|
|
134
|
+
|
|
135
|
+
- **Scripts** use a personal API token from *Settings → API tokens*: `Authorization: Bearer ptq_…`. The token has its owner's role.
|
|
136
|
+
- **Locked out?** Run `pytaskq users reset-password NAME` on the server. `pytaskq users list | create | set-role | delete | setup-code` cover the rest.
|
|
137
|
+
- **Security.** Passwords are hashed with scrypt. Only SHA-256 hashes of sessions, invites and tokens are stored. Cookies are `HttpOnly` and `SameSite`, cross-site writes are blocked, and failed sign-ins are throttled. Serve the dashboard over HTTPS in production.
|
|
138
|
+
- **Local development:** `PYTASKQ_AUTH=off` turns sign-in off. Never use it on a server others can reach.
|
|
139
|
+
|
|
140
|
+
## Configuration
|
|
141
|
+
|
|
142
|
+
| Task option | Default | What it does |
|
|
143
|
+
|---|---|---|
|
|
144
|
+
| `type` | `"async"` for `async def`, else `"cpu"` | `"cpu"` process pool · `"io"` thread pool · `"async"` event loop |
|
|
145
|
+
| `priority` | `"default"` | `"high"`, `"default"` or `"low"` queue |
|
|
146
|
+
| `max_retries` | `3` | Retries with jittered backoff before dead-lettering |
|
|
147
|
+
| `timeout` | `300` s | Abandon and retry an attempt after this long |
|
|
148
|
+
| `cron` | — | 5-field cron schedule |
|
|
149
|
+
| `name` | function name | Registered task name |
|
|
150
|
+
|
|
151
|
+
App-wide defaults: `PyTaskQ(default_priority="low", max_retries=5, timeout=60)`.
|
|
152
|
+
|
|
153
|
+
Every server setting is an environment variable or a line in `.env`, and all of them are documented in [`.env.example`](https://github.com/vrajpatelm/PyTaskQ/blob/main/.env.example). The ones you'll most likely change are `REDIS_URL`, `MAX_RETRIES`, `TASK_TIMEOUT` and `QUEUE_CAPACITY`.
|
|
154
|
+
|
|
155
|
+
| Command | Purpose |
|
|
156
|
+
|---|---|
|
|
157
|
+
| `pytaskq worker --app m:queue [--cpu-workers N --io-workers N]` | Run a worker |
|
|
158
|
+
| `pytaskq dashboard [--app m:queue] [--port 8000]` | Dashboard and REST API |
|
|
159
|
+
| `pytaskq tasks --app m:queue` | List tasks and their settings |
|
|
160
|
+
| `pytaskq users …` | Manage accounts from the server |
|
|
161
|
+
| `pytaskq ping` | Check Redis and count workers |
|
|
162
|
+
|
|
163
|
+
## How it works
|
|
164
|
+
|
|
165
|
+
<img src="https://raw.githubusercontent.com/vrajpatelm/PyTaskQ/main/docs/images/architecture.svg" alt="PyTaskQ architecture" width="100%" />
|
|
166
|
+
|
|
167
|
+
1. **Reliable pop.** A worker atomically moves a job into its own processing list, so a crash never loses it.
|
|
168
|
+
2. **Crash recovery.** Workers send a heartbeat every 10 s. An elected leader re-queues work from any worker that has been silent for 30 s.
|
|
169
|
+
3. **Fencing tokens.** A Lua script only accepts results carrying the current token, so a late result from a hung worker can't overwrite a newer one.
|
|
170
|
+
4. **Retries, then dead letters.** Failures back off with jitter. After `max_retries`, the job is dead-lettered with its last error.
|
|
171
|
+
5. **Graceful shutdown.** Ctrl+C finishes running jobs before exiting.
|
|
172
|
+
6. **Safe outbound calls.** Webhooks (`on_success_url=`) are signed with HMAC and retried. Use `pytaskq.net.assert_safe_url` to block requests to private networks.
|
|
173
|
+
|
|
174
|
+
## REST API
|
|
175
|
+
|
|
176
|
+
Interactive docs are served at `/docs`, and the full reference is in [docs/API.md](https://github.com/vrajpatelm/PyTaskQ/blob/main/docs/API.md). Every route except `/health` and sign-in needs a session or API token.
|
|
177
|
+
|
|
178
|
+
| Method | Route | |
|
|
179
|
+
|---|---|---|
|
|
180
|
+
| `POST` | `/task/enqueue` | Queue a job (`task_name`, `args`, `kwargs`, `priority`, `delay_seconds`) |
|
|
181
|
+
| `GET` | `/task/{id}` | Status, timings, result or error |
|
|
182
|
+
| `POST` | `/task/{id}/cancel` | Cancel a waiting job |
|
|
183
|
+
| `GET` | `/tasks` | Recent jobs, filterable by status and task |
|
|
184
|
+
| `GET` | `/metrics`, `/system/workers`, `/dlq` | Queue, worker and dead-letter state |
|
|
185
|
+
| `POST` | `/dlq/replay/{id}`, `/dlq/retry_all` | Replay dead letters |
|
|
186
|
+
|
|
187
|
+
## Docker
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
docker compose up --build # Redis + dashboard + worker + Jaeger
|
|
191
|
+
docker compose logs web | grep -A4 "setup code"
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Point `PYTASKQ_APP` and the worker `command:` in `docker-compose.yml` at your own module. Accounts and jobs persist in the `redis-data` volume.
|
|
195
|
+
|
|
196
|
+
## Development
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
pip install -e ".[dev,demo]"
|
|
200
|
+
python -m pytest # in-memory via fakeredis, no Redis needed
|
|
201
|
+
|
|
202
|
+
cd frontend && npm install
|
|
203
|
+
npm run dev # dashboard with hot reload
|
|
204
|
+
npm run build # writes the bundle into pytaskq/static/
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
See [CONTRIBUTING](https://github.com/vrajpatelm/PyTaskQ/blob/main/docs/CONTRIBUTING.md), [RELEASING](https://github.com/vrajpatelm/PyTaskQ/blob/main/docs/RELEASING.md) and the [CHANGELOG](https://github.com/vrajpatelm/PyTaskQ/blob/main/CHANGELOG.md).
|
|
208
|
+
|
|
209
|
+
## License
|
|
210
|
+
|
|
211
|
+
MIT
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pytaskq"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Background jobs for Python — simple to start, safe in production. Async-first task queue with a live dashboard, built on Redis."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
authors = [{ name = "Vraj Patel" }]
|
|
14
|
+
keywords = ["task queue", "background jobs", "asyncio", "redis", "fastapi", "worker", "cron", "celery alternative"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Framework :: AsyncIO",
|
|
18
|
+
"Framework :: FastAPI",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Topic :: System :: Distributed Computing",
|
|
28
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
# Everything needed to run workers, the API and the dashboard.
|
|
32
|
+
dependencies = [
|
|
33
|
+
"fastapi>=0.100.0",
|
|
34
|
+
"uvicorn[standard]>=0.23.0", # [standard] brings WebSocket support for the live dashboard
|
|
35
|
+
"redis>=5.0.0",
|
|
36
|
+
"pydantic>=2.0.0",
|
|
37
|
+
"pydantic-settings>=2.0.0",
|
|
38
|
+
"croniter>=2.0.0",
|
|
39
|
+
"python-dotenv>=1.0.0",
|
|
40
|
+
"opentelemetry-api>=1.20",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
tracing = ["opentelemetry-sdk>=1.20", "opentelemetry-exporter-otlp>=1.20"]
|
|
45
|
+
demo = ["numpy>=1.26.0", "Pillow>=10.0.1"]
|
|
46
|
+
dev = [
|
|
47
|
+
"pytest>=8.0.0",
|
|
48
|
+
"pytest-asyncio>=0.23.0",
|
|
49
|
+
"fakeredis[lua]>=2.20.0",
|
|
50
|
+
"httpx>=0.26.0",
|
|
51
|
+
"ruff",
|
|
52
|
+
"build",
|
|
53
|
+
"twine",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[project.scripts]
|
|
57
|
+
pytaskq = "pytaskq.cli:main"
|
|
58
|
+
|
|
59
|
+
[project.urls]
|
|
60
|
+
Homepage = "https://github.com/vrajpatelm/PyTaskQ"
|
|
61
|
+
Documentation = "https://github.com/vrajpatelm/PyTaskQ#readme"
|
|
62
|
+
Issues = "https://github.com/vrajpatelm/PyTaskQ/issues"
|
|
63
|
+
Changelog = "https://github.com/vrajpatelm/PyTaskQ/blob/main/CHANGELOG.md"
|
|
64
|
+
|
|
65
|
+
[tool.setuptools.dynamic]
|
|
66
|
+
version = { attr = "pytaskq.__version__" }
|
|
67
|
+
|
|
68
|
+
[tool.setuptools.packages.find]
|
|
69
|
+
include = ["pytaskq", "pytaskq.*"]
|
|
70
|
+
|
|
71
|
+
[tool.setuptools.package-data]
|
|
72
|
+
# The built dashboard ships inside the package so `pytaskq dashboard` works after pip install
|
|
73
|
+
pytaskq = ["static/*", "static/assets/*"]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from fastapi import APIRouter
|
|
2
|
+
from pytaskq.api.v1.tasks import router as tasks_router, list_router as task_list_router
|
|
3
|
+
from pytaskq.api.v1.dlq import router as dlq_router
|
|
4
|
+
from pytaskq.api.v1.dashboard import router as dashboard_router
|
|
5
|
+
from pytaskq.api.v1.system import router as system_router
|
|
6
|
+
from pytaskq.api.v1.auth import router as auth_router
|
|
7
|
+
|
|
8
|
+
api_router = APIRouter()
|
|
9
|
+
|
|
10
|
+
api_router.include_router(tasks_router)
|
|
11
|
+
api_router.include_router(task_list_router)
|
|
12
|
+
api_router.include_router(dlq_router)
|
|
13
|
+
api_router.include_router(dashboard_router)
|
|
14
|
+
api_router.include_router(system_router)
|
|
15
|
+
api_router.include_router(auth_router)
|