python-pq 0.1.1__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {python_pq-0.1.1 → python_pq-0.1.4}/PKG-INFO +115 -111
- {python_pq-0.1.1 → python_pq-0.1.4}/README.md +110 -109
- {python_pq-0.1.1 → python_pq-0.1.4}/pyproject.toml +8 -2
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/client.py +12 -3
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/worker.py +47 -1
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/__init__.py +0 -0
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/config.py +0 -0
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/logging.py +0 -0
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/models.py +0 -0
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/priority.py +0 -0
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/registry.py +0 -0
- {python_pq-0.1.1 → python_pq-0.1.4}/src/pq/serialization.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: python-pq
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Postgres-backed job queue for Python
|
|
5
5
|
Author: ricwo
|
|
6
6
|
Author-email: ricwo <r@cogram.com>
|
|
@@ -13,21 +13,26 @@ Requires-Dist: psycopg2-binary>=2.9.11
|
|
|
13
13
|
Requires-Dist: pydantic>=2.12.5
|
|
14
14
|
Requires-Dist: pydantic-settings>=2.12.0
|
|
15
15
|
Requires-Dist: sqlalchemy>=2.0.45
|
|
16
|
-
Requires-
|
|
16
|
+
Requires-Dist: mkdocs-material>=9.6 ; extra == 'docs'
|
|
17
|
+
Requires-Dist: mkdocstrings[python]>=0.29 ; extra == 'docs'
|
|
18
|
+
Requires-Python: >=3.13, <3.15
|
|
19
|
+
Provides-Extra: docs
|
|
17
20
|
Description-Content-Type: text/markdown
|
|
18
21
|
|
|
19
22
|
# pq
|
|
20
23
|
|
|
21
24
|
Postgres-backed job queue for Python with fork-based worker isolation.
|
|
22
25
|
|
|
26
|
+
**[Documentation](https://ricwo.github.io/pq/)**
|
|
27
|
+
|
|
23
28
|
## Features
|
|
24
29
|
|
|
25
30
|
- **Fork isolation** - Each task runs in a forked process. OOM or crashes don't affect the worker.
|
|
26
|
-
- **Natural Python API** - Pass `*args, **kwargs
|
|
31
|
+
- **Natural Python API** - Pass functions directly with `*args, **kwargs`.
|
|
27
32
|
- **Periodic tasks** - Schedule with intervals or cron expressions.
|
|
28
|
-
- **Priority queues** - Five
|
|
33
|
+
- **Priority queues** - Five levels, higher priority tasks run first.
|
|
29
34
|
- **Async support** - Async handlers work seamlessly.
|
|
30
|
-
- **Concurrent workers** -
|
|
35
|
+
- **Concurrent workers** - `FOR UPDATE SKIP LOCKED` prevents duplicate processing.
|
|
31
36
|
|
|
32
37
|
## Installation
|
|
33
38
|
|
|
@@ -35,7 +40,7 @@ Postgres-backed job queue for Python with fork-based worker isolation.
|
|
|
35
40
|
uv add pq
|
|
36
41
|
```
|
|
37
42
|
|
|
38
|
-
Requires PostgreSQL and Python 3.
|
|
43
|
+
Requires PostgreSQL and Python 3.13+.
|
|
39
44
|
|
|
40
45
|
## Quick Start
|
|
41
46
|
|
|
@@ -45,14 +50,16 @@ from pq import PQ
|
|
|
45
50
|
pq = PQ("postgresql://localhost/mydb")
|
|
46
51
|
pq.create_tables()
|
|
47
52
|
|
|
48
|
-
def send_email(to: str, subject: str
|
|
49
|
-
print(f"Sending
|
|
53
|
+
def send_email(to: str, subject: str) -> None:
|
|
54
|
+
print(f"Sending to {to}: {subject}")
|
|
50
55
|
|
|
51
|
-
pq.enqueue(send_email, to="user@example.com", subject="Hello"
|
|
56
|
+
pq.enqueue(send_email, to="user@example.com", subject="Hello")
|
|
52
57
|
pq.run_worker()
|
|
53
58
|
```
|
|
54
59
|
|
|
55
|
-
##
|
|
60
|
+
## Tasks
|
|
61
|
+
|
|
62
|
+
### Enqueueing
|
|
56
63
|
|
|
57
64
|
```python
|
|
58
65
|
def greet(name: str) -> None:
|
|
@@ -60,45 +67,108 @@ def greet(name: str) -> None:
|
|
|
60
67
|
|
|
61
68
|
pq.enqueue(greet, name="World")
|
|
62
69
|
pq.enqueue(greet, "World") # Positional args work too
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Delayed Execution
|
|
63
73
|
|
|
64
|
-
|
|
74
|
+
```python
|
|
65
75
|
from datetime import datetime, timedelta, UTC
|
|
76
|
+
|
|
66
77
|
pq.enqueue(greet, "World", run_at=datetime.now(UTC) + timedelta(hours=1))
|
|
78
|
+
```
|
|
67
79
|
|
|
68
|
-
|
|
80
|
+
### Priority
|
|
81
|
+
|
|
82
|
+
```python
|
|
69
83
|
from pq import Priority
|
|
70
|
-
|
|
71
|
-
pq.enqueue(
|
|
72
|
-
pq.enqueue(
|
|
73
|
-
pq.enqueue(
|
|
74
|
-
pq.enqueue(
|
|
84
|
+
|
|
85
|
+
pq.enqueue(task, priority=Priority.CRITICAL) # 100 - highest
|
|
86
|
+
pq.enqueue(task, priority=Priority.HIGH) # 75
|
|
87
|
+
pq.enqueue(task, priority=Priority.NORMAL) # 50 (default)
|
|
88
|
+
pq.enqueue(task, priority=Priority.LOW) # 25
|
|
89
|
+
pq.enqueue(task, priority=Priority.BATCH) # 0 - lowest
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Cancellation
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
task_id = pq.enqueue(my_task)
|
|
96
|
+
pq.cancel(task_id) # Returns True if found and cancelled
|
|
75
97
|
```
|
|
76
98
|
|
|
77
99
|
## Periodic Tasks
|
|
78
100
|
|
|
101
|
+
### Intervals
|
|
102
|
+
|
|
79
103
|
```python
|
|
80
104
|
from datetime import timedelta
|
|
81
105
|
|
|
82
106
|
def heartbeat() -> None:
|
|
83
107
|
print("alive")
|
|
84
108
|
|
|
109
|
+
pq.schedule(heartbeat, run_every=timedelta(minutes=5))
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Cron Expressions
|
|
113
|
+
|
|
114
|
+
```python
|
|
85
115
|
def weekly_report() -> None:
|
|
86
116
|
print("generating report...")
|
|
87
117
|
|
|
88
|
-
#
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
# Cron expression (Monday 9am)
|
|
92
|
-
pq.schedule(weekly_report, cron="0 9 * * 1")
|
|
118
|
+
pq.schedule(weekly_report, cron="0 9 * * 1") # Monday 9am
|
|
119
|
+
```
|
|
93
120
|
|
|
94
|
-
|
|
95
|
-
def report(report_type: str) -> None:
|
|
96
|
-
print(f"generating {report_type} report...")
|
|
121
|
+
### With Arguments
|
|
97
122
|
|
|
123
|
+
```python
|
|
98
124
|
pq.schedule(report, run_every=timedelta(hours=1), report_type="hourly")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Unscheduling
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
pq.unschedule(heartbeat) # Returns True if found
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Workers
|
|
134
|
+
|
|
135
|
+
### Running
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
# Run forever, poll every second when idle
|
|
139
|
+
pq.run_worker(poll_interval=1.0)
|
|
140
|
+
|
|
141
|
+
# Process single task (for testing)
|
|
142
|
+
processed = pq.run_worker_once()
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Timeout
|
|
146
|
+
|
|
147
|
+
Kill tasks that run too long:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
pq.run_worker(max_runtime=300) # 5 minute timeout
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Priority-Dedicated Workers
|
|
154
|
+
|
|
155
|
+
Reserve workers for high-priority tasks:
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
from pq import Priority
|
|
159
|
+
|
|
160
|
+
# This worker only processes CRITICAL and HIGH
|
|
161
|
+
pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Run multiple workers in separate terminals:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
# Terminal 1: High-priority only
|
|
168
|
+
python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
|
|
99
169
|
|
|
100
|
-
#
|
|
101
|
-
pq.
|
|
170
|
+
# Terminal 2-3: All priorities
|
|
171
|
+
python -c "from myapp import pq; pq.run_worker()"
|
|
102
172
|
```
|
|
103
173
|
|
|
104
174
|
## Serialization
|
|
@@ -109,20 +179,20 @@ Arguments are serialized automatically:
|
|
|
109
179
|
|------|--------|
|
|
110
180
|
| JSON-serializable (str, int, list, dict) | JSON |
|
|
111
181
|
| Pydantic models | `model_dump()` → JSON |
|
|
112
|
-
| Custom objects,
|
|
182
|
+
| Custom objects, lambdas | dill (pickle) |
|
|
113
183
|
|
|
114
184
|
```python
|
|
185
|
+
from typing import Callable
|
|
115
186
|
from pydantic import BaseModel
|
|
116
187
|
|
|
117
188
|
class User(BaseModel):
|
|
118
189
|
id: int
|
|
119
190
|
email: str
|
|
120
191
|
|
|
121
|
-
def process(user:
|
|
122
|
-
print(transform(user))
|
|
192
|
+
def process(user: User, transform: Callable[[int], int]) -> None:
|
|
193
|
+
print(transform(user.id))
|
|
123
194
|
|
|
124
|
-
|
|
125
|
-
pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x["id"] * 2)
|
|
195
|
+
pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x * 2)
|
|
126
196
|
```
|
|
127
197
|
|
|
128
198
|
## Async Tasks
|
|
@@ -138,63 +208,22 @@ async def fetch(url: str) -> None:
|
|
|
138
208
|
pq.enqueue(fetch, "https://example.com")
|
|
139
209
|
```
|
|
140
210
|
|
|
141
|
-
##
|
|
142
|
-
|
|
143
|
-
```python
|
|
144
|
-
# Run forever (poll every second when idle)
|
|
145
|
-
pq.run_worker(poll_interval=1.0)
|
|
146
|
-
|
|
147
|
-
# Process single task
|
|
148
|
-
if pq.run_worker_once():
|
|
149
|
-
print("Processed a task")
|
|
150
|
-
|
|
151
|
-
# Timeout (kill tasks running longer than 5 minutes)
|
|
152
|
-
pq.run_worker(max_runtime=300)
|
|
153
|
-
|
|
154
|
-
# Dedicated worker for specific priorities
|
|
155
|
-
from pq import Priority
|
|
156
|
-
pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
|
|
157
|
-
```
|
|
158
|
-
|
|
159
|
-
## Dedicated Priority Workers
|
|
160
|
-
|
|
161
|
-
Run separate workers for different priority tiers to ensure high-priority tasks aren't blocked:
|
|
162
|
-
|
|
163
|
-
```bash
|
|
164
|
-
# Terminal 1: High-priority worker (CRITICAL + HIGH only)
|
|
165
|
-
python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
|
|
166
|
-
|
|
167
|
-
# Terminal 2-3: General workers (all priorities)
|
|
168
|
-
python -c "from myapp import pq; pq.run_worker()"
|
|
169
|
-
```
|
|
170
|
-
|
|
171
|
-
This ensures critical tasks get processed immediately even when the queue is busy.
|
|
211
|
+
## Error Handling
|
|
172
212
|
|
|
173
|
-
|
|
213
|
+
Failed tasks are marked with status `FAILED`:
|
|
174
214
|
|
|
175
215
|
```python
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
# Cancel a pending task
|
|
180
|
-
task_id = pq.enqueue(my_task)
|
|
181
|
-
pq.cancel(task_id)
|
|
182
|
-
|
|
183
|
-
# Counts
|
|
184
|
-
pq.pending_count()
|
|
185
|
-
pq.periodic_count()
|
|
186
|
-
|
|
187
|
-
# List failed/completed
|
|
188
|
-
pq.list_failed(limit=10)
|
|
189
|
-
pq.list_completed(limit=10)
|
|
216
|
+
for task in pq.list_failed():
|
|
217
|
+
print(f"{task.name}: {task.error}")
|
|
190
218
|
|
|
191
|
-
#
|
|
219
|
+
# Cleanup
|
|
192
220
|
pq.clear_failed(before=datetime.now(UTC) - timedelta(days=7))
|
|
193
221
|
pq.clear_completed(before=datetime.now(UTC) - timedelta(days=1))
|
|
194
|
-
pq.clear_all()
|
|
195
222
|
```
|
|
196
223
|
|
|
197
|
-
##
|
|
224
|
+
## Architecture
|
|
225
|
+
|
|
226
|
+
### Fork Isolation
|
|
198
227
|
|
|
199
228
|
Every task runs in a forked child process:
|
|
200
229
|
|
|
@@ -208,45 +237,20 @@ Worker (parent)
|
|
|
208
237
|
```
|
|
209
238
|
|
|
210
239
|
The parent monitors via `os.wait4()` and detects:
|
|
240
|
+
|
|
211
241
|
- **Timeout** - Task exceeded `max_runtime`
|
|
212
242
|
- **OOM** - Killed by SIGKILL (OOM killer)
|
|
213
243
|
- **Signals** - Killed by other signals
|
|
214
244
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
Run multiple workers for parallel processing:
|
|
218
|
-
|
|
219
|
-
```bash
|
|
220
|
-
# Terminal 1
|
|
221
|
-
python -c "from myapp import pq; pq.run_worker()"
|
|
222
|
-
|
|
223
|
-
# Terminal 2
|
|
224
|
-
python -c "from myapp import pq; pq.run_worker()"
|
|
225
|
-
```
|
|
226
|
-
|
|
227
|
-
Tasks are claimed with `FOR UPDATE SKIP LOCKED` - each task runs exactly once.
|
|
245
|
+
### Concurrent Workers
|
|
228
246
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
Failed tasks are marked with status `FAILED`:
|
|
232
|
-
|
|
233
|
-
```python
|
|
234
|
-
for task in pq.list_failed():
|
|
235
|
-
print(f"{task.name}: {task.error}")
|
|
236
|
-
```
|
|
247
|
+
Multiple workers can run in parallel. Tasks are claimed with `FOR UPDATE SKIP LOCKED`, ensuring each task runs exactly once.
|
|
237
248
|
|
|
238
249
|
## Development
|
|
239
250
|
|
|
240
|
-
Start Postgres:
|
|
241
|
-
|
|
242
|
-
```bash
|
|
243
|
-
make dev
|
|
244
|
-
```
|
|
245
|
-
|
|
246
|
-
Run tests:
|
|
247
|
-
|
|
248
251
|
```bash
|
|
249
|
-
|
|
252
|
+
make dev # Start Postgres
|
|
253
|
+
uv run pytest # Run tests
|
|
250
254
|
```
|
|
251
255
|
|
|
252
256
|
See [CLAUDE.md](CLAUDE.md) for full development instructions.
|
|
@@ -2,14 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
Postgres-backed job queue for Python with fork-based worker isolation.
|
|
4
4
|
|
|
5
|
+
**[Documentation](https://ricwo.github.io/pq/)**
|
|
6
|
+
|
|
5
7
|
## Features
|
|
6
8
|
|
|
7
9
|
- **Fork isolation** - Each task runs in a forked process. OOM or crashes don't affect the worker.
|
|
8
|
-
- **Natural Python API** - Pass `*args, **kwargs
|
|
10
|
+
- **Natural Python API** - Pass functions directly with `*args, **kwargs`.
|
|
9
11
|
- **Periodic tasks** - Schedule with intervals or cron expressions.
|
|
10
|
-
- **Priority queues** - Five
|
|
12
|
+
- **Priority queues** - Five levels, higher priority tasks run first.
|
|
11
13
|
- **Async support** - Async handlers work seamlessly.
|
|
12
|
-
- **Concurrent workers** -
|
|
14
|
+
- **Concurrent workers** - `FOR UPDATE SKIP LOCKED` prevents duplicate processing.
|
|
13
15
|
|
|
14
16
|
## Installation
|
|
15
17
|
|
|
@@ -17,7 +19,7 @@ Postgres-backed job queue for Python with fork-based worker isolation.
|
|
|
17
19
|
uv add pq
|
|
18
20
|
```
|
|
19
21
|
|
|
20
|
-
Requires PostgreSQL and Python 3.
|
|
22
|
+
Requires PostgreSQL and Python 3.13+.
|
|
21
23
|
|
|
22
24
|
## Quick Start
|
|
23
25
|
|
|
@@ -27,14 +29,16 @@ from pq import PQ
|
|
|
27
29
|
pq = PQ("postgresql://localhost/mydb")
|
|
28
30
|
pq.create_tables()
|
|
29
31
|
|
|
30
|
-
def send_email(to: str, subject: str
|
|
31
|
-
print(f"Sending
|
|
32
|
+
def send_email(to: str, subject: str) -> None:
|
|
33
|
+
print(f"Sending to {to}: {subject}")
|
|
32
34
|
|
|
33
|
-
pq.enqueue(send_email, to="user@example.com", subject="Hello"
|
|
35
|
+
pq.enqueue(send_email, to="user@example.com", subject="Hello")
|
|
34
36
|
pq.run_worker()
|
|
35
37
|
```
|
|
36
38
|
|
|
37
|
-
##
|
|
39
|
+
## Tasks
|
|
40
|
+
|
|
41
|
+
### Enqueueing
|
|
38
42
|
|
|
39
43
|
```python
|
|
40
44
|
def greet(name: str) -> None:
|
|
@@ -42,45 +46,108 @@ def greet(name: str) -> None:
|
|
|
42
46
|
|
|
43
47
|
pq.enqueue(greet, name="World")
|
|
44
48
|
pq.enqueue(greet, "World") # Positional args work too
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Delayed Execution
|
|
45
52
|
|
|
46
|
-
|
|
53
|
+
```python
|
|
47
54
|
from datetime import datetime, timedelta, UTC
|
|
55
|
+
|
|
48
56
|
pq.enqueue(greet, "World", run_at=datetime.now(UTC) + timedelta(hours=1))
|
|
57
|
+
```
|
|
49
58
|
|
|
50
|
-
|
|
59
|
+
### Priority
|
|
60
|
+
|
|
61
|
+
```python
|
|
51
62
|
from pq import Priority
|
|
52
|
-
|
|
53
|
-
pq.enqueue(
|
|
54
|
-
pq.enqueue(
|
|
55
|
-
pq.enqueue(
|
|
56
|
-
pq.enqueue(
|
|
63
|
+
|
|
64
|
+
pq.enqueue(task, priority=Priority.CRITICAL) # 100 - highest
|
|
65
|
+
pq.enqueue(task, priority=Priority.HIGH) # 75
|
|
66
|
+
pq.enqueue(task, priority=Priority.NORMAL) # 50 (default)
|
|
67
|
+
pq.enqueue(task, priority=Priority.LOW) # 25
|
|
68
|
+
pq.enqueue(task, priority=Priority.BATCH) # 0 - lowest
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### Cancellation
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
task_id = pq.enqueue(my_task)
|
|
75
|
+
pq.cancel(task_id) # Returns True if found and cancelled
|
|
57
76
|
```
|
|
58
77
|
|
|
59
78
|
## Periodic Tasks
|
|
60
79
|
|
|
80
|
+
### Intervals
|
|
81
|
+
|
|
61
82
|
```python
|
|
62
83
|
from datetime import timedelta
|
|
63
84
|
|
|
64
85
|
def heartbeat() -> None:
|
|
65
86
|
print("alive")
|
|
66
87
|
|
|
88
|
+
pq.schedule(heartbeat, run_every=timedelta(minutes=5))
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Cron Expressions
|
|
92
|
+
|
|
93
|
+
```python
|
|
67
94
|
def weekly_report() -> None:
|
|
68
95
|
print("generating report...")
|
|
69
96
|
|
|
70
|
-
#
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
# Cron expression (Monday 9am)
|
|
74
|
-
pq.schedule(weekly_report, cron="0 9 * * 1")
|
|
97
|
+
pq.schedule(weekly_report, cron="0 9 * * 1") # Monday 9am
|
|
98
|
+
```
|
|
75
99
|
|
|
76
|
-
|
|
77
|
-
def report(report_type: str) -> None:
|
|
78
|
-
print(f"generating {report_type} report...")
|
|
100
|
+
### With Arguments
|
|
79
101
|
|
|
102
|
+
```python
|
|
80
103
|
pq.schedule(report, run_every=timedelta(hours=1), report_type="hourly")
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Unscheduling
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
pq.unschedule(heartbeat) # Returns True if found
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Workers
|
|
113
|
+
|
|
114
|
+
### Running
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
# Run forever, poll every second when idle
|
|
118
|
+
pq.run_worker(poll_interval=1.0)
|
|
119
|
+
|
|
120
|
+
# Process single task (for testing)
|
|
121
|
+
processed = pq.run_worker_once()
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### Timeout
|
|
125
|
+
|
|
126
|
+
Kill tasks that run too long:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
pq.run_worker(max_runtime=300) # 5 minute timeout
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### Priority-Dedicated Workers
|
|
133
|
+
|
|
134
|
+
Reserve workers for high-priority tasks:
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from pq import Priority
|
|
138
|
+
|
|
139
|
+
# This worker only processes CRITICAL and HIGH
|
|
140
|
+
pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Run multiple workers in separate terminals:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
# Terminal 1: High-priority only
|
|
147
|
+
python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
|
|
81
148
|
|
|
82
|
-
#
|
|
83
|
-
pq.
|
|
149
|
+
# Terminal 2-3: All priorities
|
|
150
|
+
python -c "from myapp import pq; pq.run_worker()"
|
|
84
151
|
```
|
|
85
152
|
|
|
86
153
|
## Serialization
|
|
@@ -91,20 +158,20 @@ Arguments are serialized automatically:
|
|
|
91
158
|
|------|--------|
|
|
92
159
|
| JSON-serializable (str, int, list, dict) | JSON |
|
|
93
160
|
| Pydantic models | `model_dump()` → JSON |
|
|
94
|
-
| Custom objects,
|
|
161
|
+
| Custom objects, lambdas | dill (pickle) |
|
|
95
162
|
|
|
96
163
|
```python
|
|
164
|
+
from typing import Callable
|
|
97
165
|
from pydantic import BaseModel
|
|
98
166
|
|
|
99
167
|
class User(BaseModel):
|
|
100
168
|
id: int
|
|
101
169
|
email: str
|
|
102
170
|
|
|
103
|
-
def process(user:
|
|
104
|
-
print(transform(user))
|
|
171
|
+
def process(user: User, transform: Callable[[int], int]) -> None:
|
|
172
|
+
print(transform(user.id))
|
|
105
173
|
|
|
106
|
-
|
|
107
|
-
pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x["id"] * 2)
|
|
174
|
+
pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x * 2)
|
|
108
175
|
```
|
|
109
176
|
|
|
110
177
|
## Async Tasks
|
|
@@ -120,63 +187,22 @@ async def fetch(url: str) -> None:
|
|
|
120
187
|
pq.enqueue(fetch, "https://example.com")
|
|
121
188
|
```
|
|
122
189
|
|
|
123
|
-
##
|
|
124
|
-
|
|
125
|
-
```python
|
|
126
|
-
# Run forever (poll every second when idle)
|
|
127
|
-
pq.run_worker(poll_interval=1.0)
|
|
128
|
-
|
|
129
|
-
# Process single task
|
|
130
|
-
if pq.run_worker_once():
|
|
131
|
-
print("Processed a task")
|
|
132
|
-
|
|
133
|
-
# Timeout (kill tasks running longer than 5 minutes)
|
|
134
|
-
pq.run_worker(max_runtime=300)
|
|
135
|
-
|
|
136
|
-
# Dedicated worker for specific priorities
|
|
137
|
-
from pq import Priority
|
|
138
|
-
pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
|
|
139
|
-
```
|
|
140
|
-
|
|
141
|
-
## Dedicated Priority Workers
|
|
142
|
-
|
|
143
|
-
Run separate workers for different priority tiers to ensure high-priority tasks aren't blocked:
|
|
144
|
-
|
|
145
|
-
```bash
|
|
146
|
-
# Terminal 1: High-priority worker (CRITICAL + HIGH only)
|
|
147
|
-
python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
|
|
148
|
-
|
|
149
|
-
# Terminal 2-3: General workers (all priorities)
|
|
150
|
-
python -c "from myapp import pq; pq.run_worker()"
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
This ensures critical tasks get processed immediately even when the queue is busy.
|
|
190
|
+
## Error Handling
|
|
154
191
|
|
|
155
|
-
|
|
192
|
+
Failed tasks are marked with status `FAILED`:
|
|
156
193
|
|
|
157
194
|
```python
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
# Cancel a pending task
|
|
162
|
-
task_id = pq.enqueue(my_task)
|
|
163
|
-
pq.cancel(task_id)
|
|
164
|
-
|
|
165
|
-
# Counts
|
|
166
|
-
pq.pending_count()
|
|
167
|
-
pq.periodic_count()
|
|
168
|
-
|
|
169
|
-
# List failed/completed
|
|
170
|
-
pq.list_failed(limit=10)
|
|
171
|
-
pq.list_completed(limit=10)
|
|
195
|
+
for task in pq.list_failed():
|
|
196
|
+
print(f"{task.name}: {task.error}")
|
|
172
197
|
|
|
173
|
-
#
|
|
198
|
+
# Cleanup
|
|
174
199
|
pq.clear_failed(before=datetime.now(UTC) - timedelta(days=7))
|
|
175
200
|
pq.clear_completed(before=datetime.now(UTC) - timedelta(days=1))
|
|
176
|
-
pq.clear_all()
|
|
177
201
|
```
|
|
178
202
|
|
|
179
|
-
##
|
|
203
|
+
## Architecture
|
|
204
|
+
|
|
205
|
+
### Fork Isolation
|
|
180
206
|
|
|
181
207
|
Every task runs in a forked child process:
|
|
182
208
|
|
|
@@ -190,45 +216,20 @@ Worker (parent)
|
|
|
190
216
|
```
|
|
191
217
|
|
|
192
218
|
The parent monitors via `os.wait4()` and detects:
|
|
219
|
+
|
|
193
220
|
- **Timeout** - Task exceeded `max_runtime`
|
|
194
221
|
- **OOM** - Killed by SIGKILL (OOM killer)
|
|
195
222
|
- **Signals** - Killed by other signals
|
|
196
223
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
Run multiple workers for parallel processing:
|
|
200
|
-
|
|
201
|
-
```bash
|
|
202
|
-
# Terminal 1
|
|
203
|
-
python -c "from myapp import pq; pq.run_worker()"
|
|
204
|
-
|
|
205
|
-
# Terminal 2
|
|
206
|
-
python -c "from myapp import pq; pq.run_worker()"
|
|
207
|
-
```
|
|
208
|
-
|
|
209
|
-
Tasks are claimed with `FOR UPDATE SKIP LOCKED` - each task runs exactly once.
|
|
224
|
+
### Concurrent Workers
|
|
210
225
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
Failed tasks are marked with status `FAILED`:
|
|
214
|
-
|
|
215
|
-
```python
|
|
216
|
-
for task in pq.list_failed():
|
|
217
|
-
print(f"{task.name}: {task.error}")
|
|
218
|
-
```
|
|
226
|
+
Multiple workers can run in parallel. Tasks are claimed with `FOR UPDATE SKIP LOCKED`, ensuring each task runs exactly once.
|
|
219
227
|
|
|
220
228
|
## Development
|
|
221
229
|
|
|
222
|
-
Start Postgres:
|
|
223
|
-
|
|
224
|
-
```bash
|
|
225
|
-
make dev
|
|
226
|
-
```
|
|
227
|
-
|
|
228
|
-
Run tests:
|
|
229
|
-
|
|
230
230
|
```bash
|
|
231
|
-
|
|
231
|
+
make dev # Start Postgres
|
|
232
|
+
uv run pytest # Run tests
|
|
232
233
|
```
|
|
233
234
|
|
|
234
235
|
See [CLAUDE.md](CLAUDE.md) for full development instructions.
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "python-pq"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.4"
|
|
4
4
|
description = "Postgres-backed job queue for Python"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
7
7
|
{ name = "ricwo", email = "r@cogram.com" }
|
|
8
8
|
]
|
|
9
|
-
requires-python = ">=3.
|
|
9
|
+
requires-python = ">=3.13,<3.15"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"alembic>=1.17.2",
|
|
12
12
|
"click>=8.3.1",
|
|
@@ -19,6 +19,12 @@ dependencies = [
|
|
|
19
19
|
"sqlalchemy>=2.0.45",
|
|
20
20
|
]
|
|
21
21
|
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
docs = [
|
|
24
|
+
"mkdocs-material>=9.6",
|
|
25
|
+
"mkdocstrings[python]>=0.29",
|
|
26
|
+
]
|
|
27
|
+
|
|
22
28
|
[project.scripts]
|
|
23
29
|
pq = "pq.cli:main"
|
|
24
30
|
|
|
@@ -234,7 +234,10 @@ class PQ:
|
|
|
234
234
|
Task object or None if not found.
|
|
235
235
|
"""
|
|
236
236
|
with self.session() as session:
|
|
237
|
-
|
|
237
|
+
task = session.get(Task, task_id)
|
|
238
|
+
if task:
|
|
239
|
+
session.expunge(task)
|
|
240
|
+
return task
|
|
238
241
|
|
|
239
242
|
def list_failed(self, limit: int = 100) -> list[Task]:
|
|
240
243
|
"""List failed tasks.
|
|
@@ -252,7 +255,10 @@ class PQ:
|
|
|
252
255
|
.order_by(Task.completed_at.desc())
|
|
253
256
|
.limit(limit)
|
|
254
257
|
)
|
|
255
|
-
|
|
258
|
+
tasks = list(session.execute(stmt).scalars().all())
|
|
259
|
+
for task in tasks:
|
|
260
|
+
session.expunge(task)
|
|
261
|
+
return tasks
|
|
256
262
|
|
|
257
263
|
def list_completed(self, limit: int = 100) -> list[Task]:
|
|
258
264
|
"""List completed tasks.
|
|
@@ -270,7 +276,10 @@ class PQ:
|
|
|
270
276
|
.order_by(Task.completed_at.desc())
|
|
271
277
|
.limit(limit)
|
|
272
278
|
)
|
|
273
|
-
|
|
279
|
+
tasks = list(session.execute(stmt).scalars().all())
|
|
280
|
+
for task in tasks:
|
|
281
|
+
session.expunge(task)
|
|
282
|
+
return tasks
|
|
274
283
|
|
|
275
284
|
def clear_completed(self, before: datetime | None = None) -> int:
|
|
276
285
|
"""Clear completed tasks.
|
|
@@ -13,7 +13,7 @@ import signal
|
|
|
13
13
|
import sys
|
|
14
14
|
import time
|
|
15
15
|
import traceback
|
|
16
|
-
from datetime import UTC, datetime
|
|
16
|
+
from datetime import UTC, datetime, timedelta
|
|
17
17
|
from typing import TYPE_CHECKING, Any
|
|
18
18
|
|
|
19
19
|
from croniter import croniter
|
|
@@ -33,6 +33,12 @@ if TYPE_CHECKING:
|
|
|
33
33
|
# Default max runtime: 30 minutes
|
|
34
34
|
DEFAULT_MAX_RUNTIME: float = 30 * 60
|
|
35
35
|
|
|
36
|
+
# Default retention: 7 days
|
|
37
|
+
DEFAULT_RETENTION_DAYS: int = 7
|
|
38
|
+
|
|
39
|
+
# Default cleanup interval: 1 hour
|
|
40
|
+
DEFAULT_CLEANUP_INTERVAL: float = 3600
|
|
41
|
+
|
|
36
42
|
# Exit codes for child process
|
|
37
43
|
EXIT_SUCCESS = 0
|
|
38
44
|
EXIT_FAILURE = 1
|
|
@@ -194,12 +200,45 @@ def _execute_in_fork(
|
|
|
194
200
|
raise Exception(f"Task failed with exit code {exit_code}")
|
|
195
201
|
|
|
196
202
|
|
|
203
|
+
def _maybe_run_cleanup(
|
|
204
|
+
pq: PQ,
|
|
205
|
+
retention_days: int,
|
|
206
|
+
cleanup_interval: float,
|
|
207
|
+
last_cleanup: list[float],
|
|
208
|
+
) -> None:
|
|
209
|
+
"""Run cleanup if retention is enabled and interval has passed.
|
|
210
|
+
|
|
211
|
+
Args:
|
|
212
|
+
pq: PQ client instance.
|
|
213
|
+
retention_days: Days to keep completed/failed tasks. 0 to disable.
|
|
214
|
+
cleanup_interval: Seconds between cleanup runs.
|
|
215
|
+
last_cleanup: Mutable list containing last cleanup timestamp.
|
|
216
|
+
"""
|
|
217
|
+
if retention_days <= 0:
|
|
218
|
+
return
|
|
219
|
+
|
|
220
|
+
now = time.time()
|
|
221
|
+
if now - last_cleanup[0] < cleanup_interval:
|
|
222
|
+
return
|
|
223
|
+
|
|
224
|
+
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
|
|
225
|
+
completed = pq.clear_completed(before=cutoff)
|
|
226
|
+
failed = pq.clear_failed(before=cutoff)
|
|
227
|
+
|
|
228
|
+
if completed or failed:
|
|
229
|
+
logger.info(f"Cleanup: removed {completed} completed, {failed} failed tasks")
|
|
230
|
+
|
|
231
|
+
last_cleanup[0] = now
|
|
232
|
+
|
|
233
|
+
|
|
197
234
|
def run_worker(
|
|
198
235
|
pq: PQ,
|
|
199
236
|
*,
|
|
200
237
|
poll_interval: float = 1.0,
|
|
201
238
|
max_runtime: float = DEFAULT_MAX_RUNTIME,
|
|
202
239
|
priorities: Set[Priority] | None = None,
|
|
240
|
+
retention_days: int = DEFAULT_RETENTION_DAYS,
|
|
241
|
+
cleanup_interval: float = DEFAULT_CLEANUP_INTERVAL,
|
|
203
242
|
) -> None:
|
|
204
243
|
"""Run the worker loop indefinitely.
|
|
205
244
|
|
|
@@ -211,15 +250,22 @@ def run_worker(
|
|
|
211
250
|
max_runtime: Maximum execution time per task in seconds. Default: 30 min.
|
|
212
251
|
priorities: If set, only process tasks with these priority levels.
|
|
213
252
|
Use this to dedicate workers to specific priority tiers.
|
|
253
|
+
retention_days: Days to keep completed/failed tasks. Default: 7.
|
|
254
|
+
Set to 0 to disable automatic cleanup.
|
|
255
|
+
cleanup_interval: Seconds between cleanup runs. Default: 3600 (1 hour).
|
|
214
256
|
"""
|
|
215
257
|
if priorities:
|
|
216
258
|
priority_names = ", ".join(p.name for p in sorted(priorities, reverse=True))
|
|
217
259
|
logger.info(f"Starting PQ worker (priorities: {priority_names})...")
|
|
218
260
|
else:
|
|
219
261
|
logger.info("Starting PQ worker (fork isolation enabled)...")
|
|
262
|
+
|
|
263
|
+
last_cleanup: list[float] = [0.0] # Mutable container for tracking
|
|
264
|
+
|
|
220
265
|
try:
|
|
221
266
|
while True:
|
|
222
267
|
if not run_worker_once(pq, max_runtime=max_runtime, priorities=priorities):
|
|
268
|
+
_maybe_run_cleanup(pq, retention_days, cleanup_interval, last_cleanup)
|
|
223
269
|
time.sleep(poll_interval)
|
|
224
270
|
except KeyboardInterrupt:
|
|
225
271
|
logger.info("Worker stopped.")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|