python-pq 0.1.1__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: python-pq
3
- Version: 0.1.1
3
+ Version: 0.1.4
4
4
  Summary: Postgres-backed job queue for Python
5
5
  Author: ricwo
6
6
  Author-email: ricwo <r@cogram.com>
@@ -13,21 +13,26 @@ Requires-Dist: psycopg2-binary>=2.9.11
13
13
  Requires-Dist: pydantic>=2.12.5
14
14
  Requires-Dist: pydantic-settings>=2.12.0
15
15
  Requires-Dist: sqlalchemy>=2.0.45
16
- Requires-Python: >=3.14
16
+ Requires-Dist: mkdocs-material>=9.6 ; extra == 'docs'
17
+ Requires-Dist: mkdocstrings[python]>=0.29 ; extra == 'docs'
18
+ Requires-Python: >=3.13, <3.15
19
+ Provides-Extra: docs
17
20
  Description-Content-Type: text/markdown
18
21
 
19
22
  # pq
20
23
 
21
24
  Postgres-backed job queue for Python with fork-based worker isolation.
22
25
 
26
+ **[Documentation](https://ricwo.github.io/pq/)**
27
+
23
28
  ## Features
24
29
 
25
30
  - **Fork isolation** - Each task runs in a forked process. OOM or crashes don't affect the worker.
26
- - **Natural Python API** - Pass `*args, **kwargs` directly. Pydantic models and custom objects work.
31
+ - **Natural Python API** - Pass functions directly with `*args, **kwargs`.
27
32
  - **Periodic tasks** - Schedule with intervals or cron expressions.
28
- - **Priority queues** - Five priority levels, higher priority tasks run first.
33
+ - **Priority queues** - Five levels, higher priority tasks run first.
29
34
  - **Async support** - Async handlers work seamlessly.
30
- - **Concurrent workers** - Multiple workers with `FOR UPDATE SKIP LOCKED` prevents duplicate processing.
35
+ - **Concurrent workers** - `FOR UPDATE SKIP LOCKED` prevents duplicate processing.
31
36
 
32
37
  ## Installation
33
38
 
@@ -35,7 +40,7 @@ Postgres-backed job queue for Python with fork-based worker isolation.
35
40
  uv add pq
36
41
  ```
37
42
 
38
- Requires PostgreSQL and Python 3.14+.
43
+ Requires PostgreSQL and Python 3.13+.
39
44
 
40
45
  ## Quick Start
41
46
 
@@ -45,14 +50,16 @@ from pq import PQ
45
50
  pq = PQ("postgresql://localhost/mydb")
46
51
  pq.create_tables()
47
52
 
48
- def send_email(to: str, subject: str, body: str) -> None:
49
- print(f"Sending email to {to}: {subject}")
53
+ def send_email(to: str, subject: str) -> None:
54
+ print(f"Sending to {to}: {subject}")
50
55
 
51
- pq.enqueue(send_email, to="user@example.com", subject="Hello", body="...")
56
+ pq.enqueue(send_email, to="user@example.com", subject="Hello")
52
57
  pq.run_worker()
53
58
  ```
54
59
 
55
- ## Enqueueing Tasks
60
+ ## Tasks
61
+
62
+ ### Enqueueing
56
63
 
57
64
  ```python
58
65
  def greet(name: str) -> None:
@@ -60,45 +67,108 @@ def greet(name: str) -> None:
60
67
 
61
68
  pq.enqueue(greet, name="World")
62
69
  pq.enqueue(greet, "World") # Positional args work too
70
+ ```
71
+
72
+ ### Delayed Execution
63
73
 
64
- # Delayed execution
74
+ ```python
65
75
  from datetime import datetime, timedelta, UTC
76
+
66
77
  pq.enqueue(greet, "World", run_at=datetime.now(UTC) + timedelta(hours=1))
78
+ ```
67
79
 
68
- # Priority
80
+ ### Priority
81
+
82
+ ```python
69
83
  from pq import Priority
70
- pq.enqueue(greet, "World", priority=Priority.CRITICAL) # 100
71
- pq.enqueue(greet, "World", priority=Priority.HIGH) # 75
72
- pq.enqueue(greet, "World", priority=Priority.NORMAL) # 50 (default)
73
- pq.enqueue(greet, "World", priority=Priority.LOW) # 25
74
- pq.enqueue(greet, "World", priority=Priority.BATCH) # 0
84
+
85
+ pq.enqueue(task, priority=Priority.CRITICAL) # 100 - highest
86
+ pq.enqueue(task, priority=Priority.HIGH) # 75
87
+ pq.enqueue(task, priority=Priority.NORMAL) # 50 (default)
88
+ pq.enqueue(task, priority=Priority.LOW) # 25
89
+ pq.enqueue(task, priority=Priority.BATCH) # 0 - lowest
90
+ ```
91
+
92
+ ### Cancellation
93
+
94
+ ```python
95
+ task_id = pq.enqueue(my_task)
96
+ pq.cancel(task_id) # Returns True if found and cancelled
75
97
  ```
76
98
 
77
99
  ## Periodic Tasks
78
100
 
101
+ ### Intervals
102
+
79
103
  ```python
80
104
  from datetime import timedelta
81
105
 
82
106
  def heartbeat() -> None:
83
107
  print("alive")
84
108
 
109
+ pq.schedule(heartbeat, run_every=timedelta(minutes=5))
110
+ ```
111
+
112
+ ### Cron Expressions
113
+
114
+ ```python
85
115
  def weekly_report() -> None:
86
116
  print("generating report...")
87
117
 
88
- # Fixed interval
89
- pq.schedule(heartbeat, run_every=timedelta(minutes=5))
90
-
91
- # Cron expression (Monday 9am)
92
- pq.schedule(weekly_report, cron="0 9 * * 1")
118
+ pq.schedule(weekly_report, cron="0 9 * * 1") # Monday 9am
119
+ ```
93
120
 
94
- # With arguments
95
- def report(report_type: str) -> None:
96
- print(f"generating {report_type} report...")
121
+ ### With Arguments
97
122
 
123
+ ```python
98
124
  pq.schedule(report, run_every=timedelta(hours=1), report_type="hourly")
125
+ ```
126
+
127
+ ### Unscheduling
128
+
129
+ ```python
130
+ pq.unschedule(heartbeat) # Returns True if found
131
+ ```
132
+
133
+ ## Workers
134
+
135
+ ### Running
136
+
137
+ ```python
138
+ # Run forever, poll every second when idle
139
+ pq.run_worker(poll_interval=1.0)
140
+
141
+ # Process single task (for testing)
142
+ processed = pq.run_worker_once()
143
+ ```
144
+
145
+ ### Timeout
146
+
147
+ Kill tasks that run too long:
148
+
149
+ ```python
150
+ pq.run_worker(max_runtime=300) # 5 minute timeout
151
+ ```
152
+
153
+ ### Priority-Dedicated Workers
154
+
155
+ Reserve workers for high-priority tasks:
156
+
157
+ ```python
158
+ from pq import Priority
159
+
160
+ # This worker only processes CRITICAL and HIGH
161
+ pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
162
+ ```
163
+
164
+ Run multiple workers in separate terminals:
165
+
166
+ ```bash
167
+ # Terminal 1: High-priority only
168
+ python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
99
169
 
100
- # Remove schedule
101
- pq.unschedule(heartbeat)
170
+ # Terminal 2-3: All priorities
171
+ python -c "from myapp import pq; pq.run_worker()"
102
172
  ```
103
173
 
104
174
  ## Serialization
@@ -109,20 +179,20 @@ Arguments are serialized automatically:
109
179
  |------|--------|
110
180
  | JSON-serializable (str, int, list, dict) | JSON |
111
181
  | Pydantic models | `model_dump()` → JSON |
112
- | Custom objects, functions | dill (pickle) |
182
+ | Custom objects, lambdas | dill (pickle) |
113
183
 
114
184
  ```python
185
+ from typing import Callable
115
186
  from pydantic import BaseModel
116
187
 
117
188
  class User(BaseModel):
118
189
  id: int
119
190
  email: str
120
191
 
121
- def process(user: dict, transform: callable) -> None:
122
- print(transform(user))
192
+ def process(user: User, transform: Callable[[int], int]) -> None:
193
+ print(transform(user.id))
123
194
 
124
- # Pydantic model dict, function pickled
125
- pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x["id"] * 2)
195
+ pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x * 2)
126
196
  ```
127
197
 
128
198
  ## Async Tasks
@@ -138,63 +208,22 @@ async def fetch(url: str) -> None:
138
208
  pq.enqueue(fetch, "https://example.com")
139
209
  ```
140
210
 
141
- ## Worker
142
-
143
- ```python
144
- # Run forever (poll every second when idle)
145
- pq.run_worker(poll_interval=1.0)
146
-
147
- # Process single task
148
- if pq.run_worker_once():
149
- print("Processed a task")
150
-
151
- # Timeout (kill tasks running longer than 5 minutes)
152
- pq.run_worker(max_runtime=300)
153
-
154
- # Dedicated worker for specific priorities
155
- from pq import Priority
156
- pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
157
- ```
158
-
159
- ## Dedicated Priority Workers
160
-
161
- Run separate workers for different priority tiers to ensure high-priority tasks aren't blocked:
162
-
163
- ```bash
164
- # Terminal 1: High-priority worker (CRITICAL + HIGH only)
165
- python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
166
-
167
- # Terminal 2-3: General workers (all priorities)
168
- python -c "from myapp import pq; pq.run_worker()"
169
- ```
170
-
171
- This ensures critical tasks get processed immediately even when the queue is busy.
211
+ ## Error Handling
172
212
 
173
- ## Task Management
213
+ Failed tasks are marked with status `FAILED`:
174
214
 
175
215
  ```python
176
- def my_task() -> None:
177
- pass
178
-
179
- # Cancel a pending task
180
- task_id = pq.enqueue(my_task)
181
- pq.cancel(task_id)
182
-
183
- # Counts
184
- pq.pending_count()
185
- pq.periodic_count()
186
-
187
- # List failed/completed
188
- pq.list_failed(limit=10)
189
- pq.list_completed(limit=10)
216
+ for task in pq.list_failed():
217
+ print(f"{task.name}: {task.error}")
190
218
 
191
- # Clear old tasks
219
+ # Cleanup
192
220
  pq.clear_failed(before=datetime.now(UTC) - timedelta(days=7))
193
221
  pq.clear_completed(before=datetime.now(UTC) - timedelta(days=1))
194
- pq.clear_all()
195
222
  ```
196
223
 
197
- ## Fork Isolation
224
+ ## Architecture
225
+
226
+ ### Fork Isolation
198
227
 
199
228
  Every task runs in a forked child process:
200
229
 
@@ -208,45 +237,20 @@ Worker (parent)
208
237
  ```
209
238
 
210
239
  The parent monitors via `os.wait4()` and detects:
240
+
211
241
  - **Timeout** - Task exceeded `max_runtime`
212
242
  - **OOM** - Killed by SIGKILL (OOM killer)
213
243
  - **Signals** - Killed by other signals
214
244
 
215
- ## Multiple Workers
216
-
217
- Run multiple workers for parallel processing:
218
-
219
- ```bash
220
- # Terminal 1
221
- python -c "from myapp import pq; pq.run_worker()"
222
-
223
- # Terminal 2
224
- python -c "from myapp import pq; pq.run_worker()"
225
- ```
226
-
227
- Tasks are claimed with `FOR UPDATE SKIP LOCKED` - each task runs exactly once.
245
+ ### Concurrent Workers
228
246
 
229
- ## Error Handling
230
-
231
- Failed tasks are marked with status `FAILED`:
232
-
233
- ```python
234
- for task in pq.list_failed():
235
- print(f"{task.name}: {task.error}")
236
- ```
247
+ Multiple workers can run in parallel. Tasks are claimed with `FOR UPDATE SKIP LOCKED`, ensuring each task runs exactly once.
237
248
 
238
249
  ## Development
239
250
 
240
- Start Postgres:
241
-
242
- ```bash
243
- make dev
244
- ```
245
-
246
- Run tests:
247
-
248
251
  ```bash
249
- uv run pytest
252
+ make dev # Start Postgres
253
+ uv run pytest # Run tests
250
254
  ```
251
255
 
252
256
  See [CLAUDE.md](CLAUDE.md) for full development instructions.
@@ -2,14 +2,16 @@
2
2
 
3
3
  Postgres-backed job queue for Python with fork-based worker isolation.
4
4
 
5
+ **[Documentation](https://ricwo.github.io/pq/)**
6
+
5
7
  ## Features
6
8
 
7
9
  - **Fork isolation** - Each task runs in a forked process. OOM or crashes don't affect the worker.
8
- - **Natural Python API** - Pass `*args, **kwargs` directly. Pydantic models and custom objects work.
10
+ - **Natural Python API** - Pass functions directly with `*args, **kwargs`.
9
11
  - **Periodic tasks** - Schedule with intervals or cron expressions.
10
- - **Priority queues** - Five priority levels, higher priority tasks run first.
12
+ - **Priority queues** - Five levels, higher priority tasks run first.
11
13
  - **Async support** - Async handlers work seamlessly.
12
- - **Concurrent workers** - Multiple workers with `FOR UPDATE SKIP LOCKED` prevents duplicate processing.
14
+ - **Concurrent workers** - `FOR UPDATE SKIP LOCKED` prevents duplicate processing.
13
15
 
14
16
  ## Installation
15
17
 
@@ -17,7 +19,7 @@ Postgres-backed job queue for Python with fork-based worker isolation.
17
19
  uv add pq
18
20
  ```
19
21
 
20
- Requires PostgreSQL and Python 3.14+.
22
+ Requires PostgreSQL and Python 3.13+.
21
23
 
22
24
  ## Quick Start
23
25
 
@@ -27,14 +29,16 @@ from pq import PQ
27
29
  pq = PQ("postgresql://localhost/mydb")
28
30
  pq.create_tables()
29
31
 
30
- def send_email(to: str, subject: str, body: str) -> None:
31
- print(f"Sending email to {to}: {subject}")
32
+ def send_email(to: str, subject: str) -> None:
33
+ print(f"Sending to {to}: {subject}")
32
34
 
33
- pq.enqueue(send_email, to="user@example.com", subject="Hello", body="...")
35
+ pq.enqueue(send_email, to="user@example.com", subject="Hello")
34
36
  pq.run_worker()
35
37
  ```
36
38
 
37
- ## Enqueueing Tasks
39
+ ## Tasks
40
+
41
+ ### Enqueueing
38
42
 
39
43
  ```python
40
44
  def greet(name: str) -> None:
@@ -42,45 +46,108 @@ def greet(name: str) -> None:
42
46
 
43
47
  pq.enqueue(greet, name="World")
44
48
  pq.enqueue(greet, "World") # Positional args work too
49
+ ```
50
+
51
+ ### Delayed Execution
45
52
 
46
- # Delayed execution
53
+ ```python
47
54
  from datetime import datetime, timedelta, UTC
55
+
48
56
  pq.enqueue(greet, "World", run_at=datetime.now(UTC) + timedelta(hours=1))
57
+ ```
49
58
 
50
- # Priority
59
+ ### Priority
60
+
61
+ ```python
51
62
  from pq import Priority
52
- pq.enqueue(greet, "World", priority=Priority.CRITICAL) # 100
53
- pq.enqueue(greet, "World", priority=Priority.HIGH) # 75
54
- pq.enqueue(greet, "World", priority=Priority.NORMAL) # 50 (default)
55
- pq.enqueue(greet, "World", priority=Priority.LOW) # 25
56
- pq.enqueue(greet, "World", priority=Priority.BATCH) # 0
63
+
64
+ pq.enqueue(task, priority=Priority.CRITICAL) # 100 - highest
65
+ pq.enqueue(task, priority=Priority.HIGH) # 75
66
+ pq.enqueue(task, priority=Priority.NORMAL) # 50 (default)
67
+ pq.enqueue(task, priority=Priority.LOW) # 25
68
+ pq.enqueue(task, priority=Priority.BATCH) # 0 - lowest
69
+ ```
70
+
71
+ ### Cancellation
72
+
73
+ ```python
74
+ task_id = pq.enqueue(my_task)
75
+ pq.cancel(task_id) # Returns True if found and cancelled
57
76
  ```
58
77
 
59
78
  ## Periodic Tasks
60
79
 
80
+ ### Intervals
81
+
61
82
  ```python
62
83
  from datetime import timedelta
63
84
 
64
85
  def heartbeat() -> None:
65
86
  print("alive")
66
87
 
88
+ pq.schedule(heartbeat, run_every=timedelta(minutes=5))
89
+ ```
90
+
91
+ ### Cron Expressions
92
+
93
+ ```python
67
94
  def weekly_report() -> None:
68
95
  print("generating report...")
69
96
 
70
- # Fixed interval
71
- pq.schedule(heartbeat, run_every=timedelta(minutes=5))
72
-
73
- # Cron expression (Monday 9am)
74
- pq.schedule(weekly_report, cron="0 9 * * 1")
97
+ pq.schedule(weekly_report, cron="0 9 * * 1") # Monday 9am
98
+ ```
75
99
 
76
- # With arguments
77
- def report(report_type: str) -> None:
78
- print(f"generating {report_type} report...")
100
+ ### With Arguments
79
101
 
102
+ ```python
80
103
  pq.schedule(report, run_every=timedelta(hours=1), report_type="hourly")
104
+ ```
105
+
106
+ ### Unscheduling
107
+
108
+ ```python
109
+ pq.unschedule(heartbeat) # Returns True if found
110
+ ```
111
+
112
+ ## Workers
113
+
114
+ ### Running
115
+
116
+ ```python
117
+ # Run forever, poll every second when idle
118
+ pq.run_worker(poll_interval=1.0)
119
+
120
+ # Process single task (for testing)
121
+ processed = pq.run_worker_once()
122
+ ```
123
+
124
+ ### Timeout
125
+
126
+ Kill tasks that run too long:
127
+
128
+ ```python
129
+ pq.run_worker(max_runtime=300) # 5 minute timeout
130
+ ```
131
+
132
+ ### Priority-Dedicated Workers
133
+
134
+ Reserve workers for high-priority tasks:
135
+
136
+ ```python
137
+ from pq import Priority
138
+
139
+ # This worker only processes CRITICAL and HIGH
140
+ pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
141
+ ```
142
+
143
+ Run multiple workers in separate terminals:
144
+
145
+ ```bash
146
+ # Terminal 1: High-priority only
147
+ python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
81
148
 
82
- # Remove schedule
83
- pq.unschedule(heartbeat)
149
+ # Terminal 2-3: All priorities
150
+ python -c "from myapp import pq; pq.run_worker()"
84
151
  ```
85
152
 
86
153
  ## Serialization
@@ -91,20 +158,20 @@ Arguments are serialized automatically:
91
158
  |------|--------|
92
159
  | JSON-serializable (str, int, list, dict) | JSON |
93
160
  | Pydantic models | `model_dump()` → JSON |
94
- | Custom objects, functions | dill (pickle) |
161
+ | Custom objects, lambdas | dill (pickle) |
95
162
 
96
163
  ```python
164
+ from typing import Callable
97
165
  from pydantic import BaseModel
98
166
 
99
167
  class User(BaseModel):
100
168
  id: int
101
169
  email: str
102
170
 
103
- def process(user: dict, transform: callable) -> None:
104
- print(transform(user))
171
+ def process(user: User, transform: Callable[[int], int]) -> None:
172
+ print(transform(user.id))
105
173
 
106
- # Pydantic model dict, function pickled
107
- pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x["id"] * 2)
174
+ pq.enqueue(process, User(id=1, email="a@b.com"), transform=lambda x: x * 2)
108
175
  ```
109
176
 
110
177
  ## Async Tasks
@@ -120,63 +187,22 @@ async def fetch(url: str) -> None:
120
187
  pq.enqueue(fetch, "https://example.com")
121
188
  ```
122
189
 
123
- ## Worker
124
-
125
- ```python
126
- # Run forever (poll every second when idle)
127
- pq.run_worker(poll_interval=1.0)
128
-
129
- # Process single task
130
- if pq.run_worker_once():
131
- print("Processed a task")
132
-
133
- # Timeout (kill tasks running longer than 5 minutes)
134
- pq.run_worker(max_runtime=300)
135
-
136
- # Dedicated worker for specific priorities
137
- from pq import Priority
138
- pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})
139
- ```
140
-
141
- ## Dedicated Priority Workers
142
-
143
- Run separate workers for different priority tiers to ensure high-priority tasks aren't blocked:
144
-
145
- ```bash
146
- # Terminal 1: High-priority worker (CRITICAL + HIGH only)
147
- python -c "from myapp import pq; from pq import Priority; pq.run_worker(priorities={Priority.CRITICAL, Priority.HIGH})"
148
-
149
- # Terminal 2-3: General workers (all priorities)
150
- python -c "from myapp import pq; pq.run_worker()"
151
- ```
152
-
153
- This ensures critical tasks get processed immediately even when the queue is busy.
190
+ ## Error Handling
154
191
 
155
- ## Task Management
192
+ Failed tasks are marked with status `FAILED`:
156
193
 
157
194
  ```python
158
- def my_task() -> None:
159
- pass
160
-
161
- # Cancel a pending task
162
- task_id = pq.enqueue(my_task)
163
- pq.cancel(task_id)
164
-
165
- # Counts
166
- pq.pending_count()
167
- pq.periodic_count()
168
-
169
- # List failed/completed
170
- pq.list_failed(limit=10)
171
- pq.list_completed(limit=10)
195
+ for task in pq.list_failed():
196
+ print(f"{task.name}: {task.error}")
172
197
 
173
- # Clear old tasks
198
+ # Cleanup
174
199
  pq.clear_failed(before=datetime.now(UTC) - timedelta(days=7))
175
200
  pq.clear_completed(before=datetime.now(UTC) - timedelta(days=1))
176
- pq.clear_all()
177
201
  ```
178
202
 
179
- ## Fork Isolation
203
+ ## Architecture
204
+
205
+ ### Fork Isolation
180
206
 
181
207
  Every task runs in a forked child process:
182
208
 
@@ -190,45 +216,20 @@ Worker (parent)
190
216
  ```
191
217
 
192
218
  The parent monitors via `os.wait4()` and detects:
219
+
193
220
  - **Timeout** - Task exceeded `max_runtime`
194
221
  - **OOM** - Killed by SIGKILL (OOM killer)
195
222
  - **Signals** - Killed by other signals
196
223
 
197
- ## Multiple Workers
198
-
199
- Run multiple workers for parallel processing:
200
-
201
- ```bash
202
- # Terminal 1
203
- python -c "from myapp import pq; pq.run_worker()"
204
-
205
- # Terminal 2
206
- python -c "from myapp import pq; pq.run_worker()"
207
- ```
208
-
209
- Tasks are claimed with `FOR UPDATE SKIP LOCKED` - each task runs exactly once.
224
+ ### Concurrent Workers
210
225
 
211
- ## Error Handling
212
-
213
- Failed tasks are marked with status `FAILED`:
214
-
215
- ```python
216
- for task in pq.list_failed():
217
- print(f"{task.name}: {task.error}")
218
- ```
226
+ Multiple workers can run in parallel. Tasks are claimed with `FOR UPDATE SKIP LOCKED`, ensuring each task runs exactly once.
219
227
 
220
228
  ## Development
221
229
 
222
- Start Postgres:
223
-
224
- ```bash
225
- make dev
226
- ```
227
-
228
- Run tests:
229
-
230
230
  ```bash
231
- uv run pytest
231
+ make dev # Start Postgres
232
+ uv run pytest # Run tests
232
233
  ```
233
234
 
234
235
  See [CLAUDE.md](CLAUDE.md) for full development instructions.
@@ -1,12 +1,12 @@
1
1
  [project]
2
2
  name = "python-pq"
3
- version = "0.1.1"
3
+ version = "0.1.4"
4
4
  description = "Postgres-backed job queue for Python"
5
5
  readme = "README.md"
6
6
  authors = [
7
7
  { name = "ricwo", email = "r@cogram.com" }
8
8
  ]
9
- requires-python = ">=3.14"
9
+ requires-python = ">=3.13,<3.15"
10
10
  dependencies = [
11
11
  "alembic>=1.17.2",
12
12
  "click>=8.3.1",
@@ -19,6 +19,12 @@ dependencies = [
19
19
  "sqlalchemy>=2.0.45",
20
20
  ]
21
21
 
22
+ [project.optional-dependencies]
23
+ docs = [
24
+ "mkdocs-material>=9.6",
25
+ "mkdocstrings[python]>=0.29",
26
+ ]
27
+
22
28
  [project.scripts]
23
29
  pq = "pq.cli:main"
24
30
 
@@ -234,7 +234,10 @@ class PQ:
234
234
  Task object or None if not found.
235
235
  """
236
236
  with self.session() as session:
237
- return session.get(Task, task_id)
237
+ task = session.get(Task, task_id)
238
+ if task:
239
+ session.expunge(task)
240
+ return task
238
241
 
239
242
  def list_failed(self, limit: int = 100) -> list[Task]:
240
243
  """List failed tasks.
@@ -252,7 +255,10 @@ class PQ:
252
255
  .order_by(Task.completed_at.desc())
253
256
  .limit(limit)
254
257
  )
255
- return list(session.execute(stmt).scalars().all())
258
+ tasks = list(session.execute(stmt).scalars().all())
259
+ for task in tasks:
260
+ session.expunge(task)
261
+ return tasks
256
262
 
257
263
  def list_completed(self, limit: int = 100) -> list[Task]:
258
264
  """List completed tasks.
@@ -270,7 +276,10 @@ class PQ:
270
276
  .order_by(Task.completed_at.desc())
271
277
  .limit(limit)
272
278
  )
273
- return list(session.execute(stmt).scalars().all())
279
+ tasks = list(session.execute(stmt).scalars().all())
280
+ for task in tasks:
281
+ session.expunge(task)
282
+ return tasks
274
283
 
275
284
  def clear_completed(self, before: datetime | None = None) -> int:
276
285
  """Clear completed tasks.
@@ -13,7 +13,7 @@ import signal
13
13
  import sys
14
14
  import time
15
15
  import traceback
16
- from datetime import UTC, datetime
16
+ from datetime import UTC, datetime, timedelta
17
17
  from typing import TYPE_CHECKING, Any
18
18
 
19
19
  from croniter import croniter
@@ -33,6 +33,12 @@ if TYPE_CHECKING:
33
33
  # Default max runtime: 30 minutes
34
34
  DEFAULT_MAX_RUNTIME: float = 30 * 60
35
35
 
36
+ # Default retention: 7 days
37
+ DEFAULT_RETENTION_DAYS: int = 7
38
+
39
+ # Default cleanup interval: 1 hour
40
+ DEFAULT_CLEANUP_INTERVAL: float = 3600
41
+
36
42
  # Exit codes for child process
37
43
  EXIT_SUCCESS = 0
38
44
  EXIT_FAILURE = 1
@@ -194,12 +200,45 @@ def _execute_in_fork(
194
200
  raise Exception(f"Task failed with exit code {exit_code}")
195
201
 
196
202
 
203
+ def _maybe_run_cleanup(
204
+ pq: PQ,
205
+ retention_days: int,
206
+ cleanup_interval: float,
207
+ last_cleanup: list[float],
208
+ ) -> None:
209
+ """Run cleanup if retention is enabled and interval has passed.
210
+
211
+ Args:
212
+ pq: PQ client instance.
213
+ retention_days: Days to keep completed/failed tasks. 0 to disable.
214
+ cleanup_interval: Seconds between cleanup runs.
215
+ last_cleanup: Mutable list containing last cleanup timestamp.
216
+ """
217
+ if retention_days <= 0:
218
+ return
219
+
220
+ now = time.time()
221
+ if now - last_cleanup[0] < cleanup_interval:
222
+ return
223
+
224
+ cutoff = datetime.now(UTC) - timedelta(days=retention_days)
225
+ completed = pq.clear_completed(before=cutoff)
226
+ failed = pq.clear_failed(before=cutoff)
227
+
228
+ if completed or failed:
229
+ logger.info(f"Cleanup: removed {completed} completed, {failed} failed tasks")
230
+
231
+ last_cleanup[0] = now
232
+
233
+
197
234
  def run_worker(
198
235
  pq: PQ,
199
236
  *,
200
237
  poll_interval: float = 1.0,
201
238
  max_runtime: float = DEFAULT_MAX_RUNTIME,
202
239
  priorities: Set[Priority] | None = None,
240
+ retention_days: int = DEFAULT_RETENTION_DAYS,
241
+ cleanup_interval: float = DEFAULT_CLEANUP_INTERVAL,
203
242
  ) -> None:
204
243
  """Run the worker loop indefinitely.
205
244
 
@@ -211,15 +250,22 @@ def run_worker(
211
250
  max_runtime: Maximum execution time per task in seconds. Default: 30 min.
212
251
  priorities: If set, only process tasks with these priority levels.
213
252
  Use this to dedicate workers to specific priority tiers.
253
+ retention_days: Days to keep completed/failed tasks. Default: 7.
254
+ Set to 0 to disable automatic cleanup.
255
+ cleanup_interval: Seconds between cleanup runs. Default: 3600 (1 hour).
214
256
  """
215
257
  if priorities:
216
258
  priority_names = ", ".join(p.name for p in sorted(priorities, reverse=True))
217
259
  logger.info(f"Starting PQ worker (priorities: {priority_names})...")
218
260
  else:
219
261
  logger.info("Starting PQ worker (fork isolation enabled)...")
262
+
263
+ last_cleanup: list[float] = [0.0] # Mutable container for tracking
264
+
220
265
  try:
221
266
  while True:
222
267
  if not run_worker_once(pq, max_runtime=max_runtime, priorities=priorities):
268
+ _maybe_run_cleanup(pq, retention_days, cleanup_interval, last_cleanup)
223
269
  time.sleep(poll_interval)
224
270
  except KeyboardInterrupt:
225
271
  logger.info("Worker stopped.")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes