rrq 0.3.7__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rrq-0.5.0/.claude/settings.local.json +17 -0
- rrq-0.5.0/CLAUDE.md +115 -0
- {rrq-0.3.7 → rrq-0.5.0}/PKG-INFO +125 -7
- {rrq-0.3.7 → rrq-0.5.0}/README.md +123 -5
- {rrq-0.3.7 → rrq-0.5.0}/example/example_rrq_settings.py +25 -5
- {rrq-0.3.7 → rrq-0.5.0}/example/rrq_example.py +55 -2
- {rrq-0.3.7 → rrq-0.5.0}/pyproject.toml +7 -9
- rrq-0.5.0/rrq/__init__.py +14 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/cli.py +303 -29
- rrq-0.5.0/rrq/cron.py +154 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/settings.py +5 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/store.py +122 -2
- {rrq-0.3.7 → rrq-0.5.0}/rrq/worker.py +238 -133
- {rrq-0.3.7 → rrq-0.5.0}/tests/test_cli.py +303 -10
- {rrq-0.3.7 → rrq-0.5.0}/tests/test_client.py +3 -0
- rrq-0.5.0/tests/test_cron.py +252 -0
- {rrq-0.3.7 → rrq-0.5.0}/tests/test_store.py +264 -2
- {rrq-0.3.7 → rrq-0.5.0}/tests/test_worker.py +504 -58
- rrq-0.5.0/uv.lock +570 -0
- rrq-0.3.7/FUTURE.md +0 -33
- rrq-0.3.7/tests/__init__.py +0 -0
- rrq-0.3.7/uv.lock +0 -518
- {rrq-0.3.7 → rrq-0.5.0}/.coverage +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/.gitignore +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/LICENSE +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/MANIFEST.in +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/client.py +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/constants.py +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/exc.py +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/job.py +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/rrq/registry.py +0 -0
- {rrq-0.3.7/rrq → rrq-0.5.0/tests}/__init__.py +0 -0
- {rrq-0.3.7 → rrq-0.5.0}/tests/test_registry.py +0 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"permissions": {
|
|
3
|
+
"allow": [
|
|
4
|
+
"Bash(find:*)",
|
|
5
|
+
"Bash(grep:*)",
|
|
6
|
+
"Bash(uv run pytest:*)",
|
|
7
|
+
"Bash(mkdir:*)",
|
|
8
|
+
"Bash(mv:*)",
|
|
9
|
+
"Bash(python -m pytest tests/test_worker.py::test_worker_handles_job_failure_max_retries_dlq -xvs)",
|
|
10
|
+
"Bash(rg:*)",
|
|
11
|
+
"Bash(uv run:*)",
|
|
12
|
+
"Bash(ruff format:*)",
|
|
13
|
+
"Bash(ruff check:*)"
|
|
14
|
+
],
|
|
15
|
+
"deny": []
|
|
16
|
+
}
|
|
17
|
+
}
|
rrq-0.5.0/CLAUDE.md
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: RRQ Development Guidelines
|
|
3
|
+
globs: ["**/*", "!tests/**/*"]
|
|
4
|
+
alwaysApply: true
|
|
5
|
+
---
|
|
6
|
+
# RRQ Development Guidelines
|
|
7
|
+
|
|
8
|
+
## Project Overview
|
|
9
|
+
|
|
10
|
+
**Important**: Also refer to:
|
|
11
|
+
- `tests/CLAUDE.md` - Testing guidelines
|
|
12
|
+
- `README.md` - Project setup and overview
|
|
13
|
+
|
|
14
|
+
## Quick Start Commands
|
|
15
|
+
```bash
|
|
16
|
+
# Backend
|
|
17
|
+
uv run pytest # Running tests
|
|
18
|
+
ruff check # Lint backend code
|
|
19
|
+
ruff format # Format backend code
|
|
20
|
+
|
|
21
|
+
# Package Management
|
|
22
|
+
uv add package_name # Add dependency
|
|
23
|
+
uv sync --extra dev # Sync dependencies
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Code References Format
|
|
27
|
+
**CRITICAL**: Always use VS Code clickable format for code references:
|
|
28
|
+
- `app/api/users.py:45` - Single line
|
|
29
|
+
- `app/models/job.py:120-135` - Line range
|
|
30
|
+
- Always use paths relative to project root
|
|
31
|
+
|
|
32
|
+
### Python Code Style
|
|
33
|
+
- Python 3.10+ with comprehensive type hints
|
|
34
|
+
- Double quotes for strings
|
|
35
|
+
- Max line length: 88 characters
|
|
36
|
+
- PEP 8 naming: `snake_case` functions, `PascalCase` classes
|
|
37
|
+
- Pydantic V2 for data validation
|
|
38
|
+
- Import order: stdlib → third-party → local
|
|
39
|
+
- Docstrings for public interfaces
|
|
40
|
+
- Type annotations for all function signatures
|
|
41
|
+
|
|
42
|
+
### Code Quality Practices
|
|
43
|
+
- Early returns over nested conditions
|
|
44
|
+
- Small, focused functions (single responsibility)
|
|
45
|
+
- Descriptive variable names
|
|
46
|
+
- Comprehensive error handling with specific exceptions
|
|
47
|
+
- Consistent async/await patterns
|
|
48
|
+
- Use `match/case` for complex conditionals
|
|
49
|
+
|
|
50
|
+
## Self-Review Checklist for Large Changes
|
|
51
|
+
|
|
52
|
+
Before submitting significant backend changes:
|
|
53
|
+
|
|
54
|
+
### Code Quality
|
|
55
|
+
- [ ] All functions have type hints
|
|
56
|
+
- [ ] Complex logic is well-commented
|
|
57
|
+
- [ ] No debug prints or commented code
|
|
58
|
+
- [ ] Follows existing patterns in codebase
|
|
59
|
+
- [ ] Proper error handling throughout
|
|
60
|
+
- [ ] Idiomatic code using Python, Asyncio, and Pydantic best practices
|
|
61
|
+
|
|
62
|
+
### Testing
|
|
63
|
+
- [ ] Unit tests for new functionality
|
|
64
|
+
- [ ] Integration tests for API changes
|
|
65
|
+
- [ ] All tests pass with `--warnings-as-errors`
|
|
66
|
+
- [ ] Edge cases covered
|
|
67
|
+
- [ ] Mocked external dependencies
|
|
68
|
+
|
|
69
|
+
### Performance
|
|
70
|
+
- [ ] Database queries are optimized (N+1 prevention)
|
|
71
|
+
- [ ] Async operations are properly awaited
|
|
72
|
+
- [ ] No blocking I/O in async contexts
|
|
73
|
+
- [ ] Background jobs for heavy operations
|
|
74
|
+
|
|
75
|
+
### Security
|
|
76
|
+
- [ ] Input validation on all endpoints
|
|
77
|
+
- [ ] No sensitive data in logs
|
|
78
|
+
- [ ] SQL injection prevention
|
|
79
|
+
|
|
80
|
+
### Documentation
|
|
81
|
+
- [ ] API endpoints documented
|
|
82
|
+
- [ ] Complex functions have docstrings
|
|
83
|
+
- [ ] Schema changes documented
|
|
84
|
+
- [ ] Migration files created if needed
|
|
85
|
+
|
|
86
|
+
## Linting and Pre-commit
|
|
87
|
+
|
|
88
|
+
**Always run before committing:**
|
|
89
|
+
```bash
|
|
90
|
+
# Format and lint
|
|
91
|
+
ruff format
|
|
92
|
+
ruff check --fix
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Important Development Rules
|
|
97
|
+
|
|
98
|
+
1. **Never commit broken tests** - Fix root causes
|
|
99
|
+
2. **Ask before large changes** - Especially cross-domain
|
|
100
|
+
3. **Follow existing patterns** - Check similar code first
|
|
101
|
+
4. **Quality over speed** - Do it right the first time
|
|
102
|
+
5. **Security first** - Never expose sensitive data
|
|
103
|
+
|
|
104
|
+
## Performance Considerations
|
|
105
|
+
- Profile before optimizing
|
|
106
|
+
- Use async correctly (no sync in async)
|
|
107
|
+
- Cache expensive operations
|
|
108
|
+
- Paginate large result sets
|
|
109
|
+
- Monitor query performance
|
|
110
|
+
|
|
111
|
+
## Debugging Tips
|
|
112
|
+
- Use `uv run pytest --maxfail=1` for failing tests
|
|
113
|
+
- Use debugger with `import pdb; pdb.set_trace()`
|
|
114
|
+
|
|
115
|
+
Remember: If unsure about implementation, check existing code patterns first!
|
{rrq-0.3.7 → rrq-0.5.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rrq
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: RRQ is a Python library for creating reliable job queues using Redis and asyncio
|
|
5
5
|
Project-URL: Homepage, https://github.com/getresq/rrq
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/getresq/rrq/issues
|
|
@@ -20,7 +20,7 @@ Requires-Dist: pydantic>=2.11.4
|
|
|
20
20
|
Requires-Dist: redis[hiredis]<6,>=4.2.0
|
|
21
21
|
Requires-Dist: watchfiles>=0.19.0
|
|
22
22
|
Provides-Extra: dev
|
|
23
|
-
Requires-Dist: pytest-asyncio>=0.
|
|
23
|
+
Requires-Dist: pytest-asyncio>=1.0.0; extra == 'dev'
|
|
24
24
|
Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
|
|
25
25
|
Requires-Dist: pytest>=8.3.5; extra == 'dev'
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
@@ -40,6 +40,7 @@ RRQ is a Python library for creating reliable job queues using Redis and `asynci
|
|
|
40
40
|
* **Graceful Shutdown**: Workers listen for SIGINT/SIGTERM and attempt to finish active jobs within a grace period before exiting. Interrupted jobs are re-queued.
|
|
41
41
|
* **Worker Health Checks**: Workers periodically update a health key in Redis with a TTL, allowing monitoring systems to track active workers.
|
|
42
42
|
* **Deferred Execution**: Jobs can be scheduled to run at a future time using `_defer_by` or `_defer_until`.
|
|
43
|
+
* **Cron Jobs**: Periodic jobs can be defined in `RRQSettings.cron_jobs` using a simple cron syntax.
|
|
43
44
|
|
|
44
45
|
- Using deferral with a specific `_job_id` will effectively reschedule the job associated with that ID to the new time, overwriting its previous definition and score. It does not create multiple distinct scheduled jobs with the same ID.
|
|
45
46
|
|
|
@@ -138,6 +139,125 @@ if __name__ == "__main__":
|
|
|
138
139
|
|
|
139
140
|
You can run multiple instances of `worker_script.py` for concurrent processing.
|
|
140
141
|
|
|
142
|
+
## Cron Jobs
|
|
143
|
+
|
|
144
|
+
Add instances of `CronJob` to `RRQSettings.cron_jobs` to run periodic jobs. The
|
|
145
|
+
`schedule` string follows the typical five-field cron format `minute hour day-of-month month day-of-week`.
|
|
146
|
+
It supports the most common features from Unix cron:
|
|
147
|
+
|
|
148
|
+
- numeric values
|
|
149
|
+
- ranges (e.g. `8-11`)
|
|
150
|
+
- lists separated by commas (e.g. `mon,wed,fri`)
|
|
151
|
+
- step values using `/` (e.g. `*/15`)
|
|
152
|
+
- names for months and days (`jan-dec`, `sun-sat`)
|
|
153
|
+
|
|
154
|
+
Jobs are evaluated in the server's timezone and run with minute resolution.
|
|
155
|
+
|
|
156
|
+
### Cron Schedule Examples
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
# Every minute
|
|
160
|
+
"* * * * *"
|
|
161
|
+
|
|
162
|
+
# Every hour at minute 30
|
|
163
|
+
"30 * * * *"
|
|
164
|
+
|
|
165
|
+
# Every day at 2:30 AM
|
|
166
|
+
"30 2 * * *"
|
|
167
|
+
|
|
168
|
+
# Every Monday at 9:00 AM
|
|
169
|
+
"0 9 * * mon"
|
|
170
|
+
|
|
171
|
+
# Every 15 minutes
|
|
172
|
+
"*/15 * * * *"
|
|
173
|
+
|
|
174
|
+
# Every weekday at 6:00 PM
|
|
175
|
+
"0 18 * * mon-fri"
|
|
176
|
+
|
|
177
|
+
# First day of every month at midnight
|
|
178
|
+
"0 0 1 * *"
|
|
179
|
+
|
|
180
|
+
# Every 2 hours during business hours on weekdays
|
|
181
|
+
"0 9-17/2 * * mon-fri"
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### Defining Cron Jobs
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
from rrq.settings import RRQSettings
|
|
188
|
+
from rrq.cron import CronJob
|
|
189
|
+
|
|
190
|
+
# Define your cron jobs
|
|
191
|
+
cron_jobs = [
|
|
192
|
+
# Daily cleanup at 2 AM
|
|
193
|
+
CronJob(
|
|
194
|
+
function_name="daily_cleanup",
|
|
195
|
+
schedule="0 2 * * *",
|
|
196
|
+
args=["temp_files"],
|
|
197
|
+
kwargs={"max_age_days": 7}
|
|
198
|
+
),
|
|
199
|
+
|
|
200
|
+
# Weekly report every Monday at 9 AM
|
|
201
|
+
CronJob(
|
|
202
|
+
function_name="generate_weekly_report",
|
|
203
|
+
schedule="0 9 * * mon",
|
|
204
|
+
unique=True # Prevent duplicate reports if worker restarts
|
|
205
|
+
),
|
|
206
|
+
|
|
207
|
+
# Health check every 15 minutes on a specific queue
|
|
208
|
+
CronJob(
|
|
209
|
+
function_name="system_health_check",
|
|
210
|
+
schedule="*/15 * * * *",
|
|
211
|
+
queue_name="monitoring"
|
|
212
|
+
),
|
|
213
|
+
|
|
214
|
+
# Backup database every night at 1 AM
|
|
215
|
+
CronJob(
|
|
216
|
+
function_name="backup_database",
|
|
217
|
+
schedule="0 1 * * *",
|
|
218
|
+
kwargs={"backup_type": "incremental"}
|
|
219
|
+
),
|
|
220
|
+
]
|
|
221
|
+
|
|
222
|
+
# Add to your settings
|
|
223
|
+
rrq_settings = RRQSettings(
|
|
224
|
+
redis_dsn="redis://localhost:6379/0",
|
|
225
|
+
cron_jobs=cron_jobs
|
|
226
|
+
)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Cron Job Handlers
|
|
230
|
+
|
|
231
|
+
Your cron job handlers are regular async functions, just like other job handlers:
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
async def daily_cleanup(ctx, file_type: str, max_age_days: int = 7):
|
|
235
|
+
"""Clean up old files."""
|
|
236
|
+
job_id = ctx['job_id']
|
|
237
|
+
print(f"Job {job_id}: Cleaning up {file_type} files older than {max_age_days} days")
|
|
238
|
+
# Your cleanup logic here
|
|
239
|
+
return {"cleaned_files": 42, "status": "completed"}
|
|
240
|
+
|
|
241
|
+
async def generate_weekly_report(ctx):
|
|
242
|
+
"""Generate and send weekly report."""
|
|
243
|
+
job_id = ctx['job_id']
|
|
244
|
+
print(f"Job {job_id}: Generating weekly report")
|
|
245
|
+
# Your report generation logic here
|
|
246
|
+
return {"report_id": "weekly_2024_01", "status": "sent"}
|
|
247
|
+
|
|
248
|
+
# Register your handlers
|
|
249
|
+
from rrq.registry import JobRegistry
|
|
250
|
+
|
|
251
|
+
job_registry = JobRegistry()
|
|
252
|
+
job_registry.register("daily_cleanup", daily_cleanup)
|
|
253
|
+
job_registry.register("generate_weekly_report", generate_weekly_report)
|
|
254
|
+
|
|
255
|
+
# Add the registry to your settings
|
|
256
|
+
rrq_settings.job_registry = job_registry
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
**Note:** Cron jobs are automatically enqueued by the worker when they become due. The worker checks for due cron jobs every 30 seconds and enqueues them as regular jobs to be processed.
|
|
260
|
+
|
|
141
261
|
## Command Line Interface
|
|
142
262
|
|
|
143
263
|
RRQ provides a command-line interface (CLI) for managing workers and performing health checks:
|
|
@@ -145,7 +265,8 @@ RRQ provides a command-line interface (CLI) for managing workers and performing
|
|
|
145
265
|
- **`rrq worker run`** - Run an RRQ worker process.
|
|
146
266
|
- `--settings` (optional): Specify the Python path to your settings object (e.g., `myapp.worker_config.rrq_settings`). If not provided, it will use the `RRQ_SETTINGS` environment variable or default to a basic `RRQSettings` object.
|
|
147
267
|
- `--queue` (optional, multiple): Specify queue(s) to poll. Defaults to the `default_queue_name` in settings.
|
|
148
|
-
- `--burst` (flag): Run the worker in burst mode to process one job or batch and then exit.
|
|
268
|
+
- `--burst` (flag): Run the worker in burst mode to process one job or batch and then exit. Cannot be used with `--num-workers > 1`.
|
|
269
|
+
- `--num-workers` (optional, integer): Number of parallel worker processes to start. Defaults to the number of CPU cores available on the machine. Cannot be used with `--burst` mode.
|
|
149
270
|
- **`rrq worker watch`** - Run an RRQ worker with auto-restart on file changes.
|
|
150
271
|
- `--path` (optional): Directory path to watch for changes. Defaults to the current directory.
|
|
151
272
|
- `--settings` (optional): Same as above.
|
|
@@ -178,7 +299,4 @@ RRQ can be configured in several ways, with the following precedence:
|
|
|
178
299
|
* **`JobRegistry` (`registry.py`)**: A simple registry to map string function names (used when enqueuing) to the actual asynchronous handler functions the worker should execute.
|
|
179
300
|
* **`JobStore` (`store.py`)**: An abstraction layer handling all direct interactions with Redis. It manages job definitions (Hashes), queues (Sorted Sets), processing locks (Strings with TTL), unique job locks, and worker health checks.
|
|
180
301
|
* **`Job` (`job.py`)**: A Pydantic model representing a job, containing its ID, handler name, arguments, status, retry counts, timestamps, results, etc.
|
|
181
|
-
* **`JobStatus` (`job.py`)**: An Enum defining the possible states of a job (`PENDING`, `ACTIVE`, `COMPLETED`, `FAILED`, `
|
|
182
|
-
* **`RRQSettings` (`settings.py`)**: A Pydantic `BaseSettings` model for configuring RRQ behavior (Redis DSN, queue names, timeouts, retry policies, concurrency, etc.). Loadable from environment variables (prefix `RRQ_`).
|
|
183
|
-
* **`constants.py`**: Defines shared constants like Redis key prefixes and default configuration values.
|
|
184
|
-
* **`exc.py`**: Defines custom exceptions, notably `RetryJob` which handlers can raise to explicitly request a retry, potentially with a custom delay.
|
|
302
|
+
* **`JobStatus` (`job.py`)**: An Enum defining the possible states of a job (`PENDING`, `ACTIVE`, `COMPLETED`, `FAILED`, `
|
|
@@ -13,6 +13,7 @@ RRQ is a Python library for creating reliable job queues using Redis and `asynci
|
|
|
13
13
|
* **Graceful Shutdown**: Workers listen for SIGINT/SIGTERM and attempt to finish active jobs within a grace period before exiting. Interrupted jobs are re-queued.
|
|
14
14
|
* **Worker Health Checks**: Workers periodically update a health key in Redis with a TTL, allowing monitoring systems to track active workers.
|
|
15
15
|
* **Deferred Execution**: Jobs can be scheduled to run at a future time using `_defer_by` or `_defer_until`.
|
|
16
|
+
* **Cron Jobs**: Periodic jobs can be defined in `RRQSettings.cron_jobs` using a simple cron syntax.
|
|
16
17
|
|
|
17
18
|
- Using deferral with a specific `_job_id` will effectively reschedule the job associated with that ID to the new time, overwriting its previous definition and score. It does not create multiple distinct scheduled jobs with the same ID.
|
|
18
19
|
|
|
@@ -111,6 +112,125 @@ if __name__ == "__main__":
|
|
|
111
112
|
|
|
112
113
|
You can run multiple instances of `worker_script.py` for concurrent processing.
|
|
113
114
|
|
|
115
|
+
## Cron Jobs
|
|
116
|
+
|
|
117
|
+
Add instances of `CronJob` to `RRQSettings.cron_jobs` to run periodic jobs. The
|
|
118
|
+
`schedule` string follows the typical five-field cron format `minute hour day-of-month month day-of-week`.
|
|
119
|
+
It supports the most common features from Unix cron:
|
|
120
|
+
|
|
121
|
+
- numeric values
|
|
122
|
+
- ranges (e.g. `8-11`)
|
|
123
|
+
- lists separated by commas (e.g. `mon,wed,fri`)
|
|
124
|
+
- step values using `/` (e.g. `*/15`)
|
|
125
|
+
- names for months and days (`jan-dec`, `sun-sat`)
|
|
126
|
+
|
|
127
|
+
Jobs are evaluated in the server's timezone and run with minute resolution.
|
|
128
|
+
|
|
129
|
+
### Cron Schedule Examples
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
# Every minute
|
|
133
|
+
"* * * * *"
|
|
134
|
+
|
|
135
|
+
# Every hour at minute 30
|
|
136
|
+
"30 * * * *"
|
|
137
|
+
|
|
138
|
+
# Every day at 2:30 AM
|
|
139
|
+
"30 2 * * *"
|
|
140
|
+
|
|
141
|
+
# Every Monday at 9:00 AM
|
|
142
|
+
"0 9 * * mon"
|
|
143
|
+
|
|
144
|
+
# Every 15 minutes
|
|
145
|
+
"*/15 * * * *"
|
|
146
|
+
|
|
147
|
+
# Every weekday at 6:00 PM
|
|
148
|
+
"0 18 * * mon-fri"
|
|
149
|
+
|
|
150
|
+
# First day of every month at midnight
|
|
151
|
+
"0 0 1 * *"
|
|
152
|
+
|
|
153
|
+
# Every 2 hours during business hours on weekdays
|
|
154
|
+
"0 9-17/2 * * mon-fri"
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Defining Cron Jobs
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from rrq.settings import RRQSettings
|
|
161
|
+
from rrq.cron import CronJob
|
|
162
|
+
|
|
163
|
+
# Define your cron jobs
|
|
164
|
+
cron_jobs = [
|
|
165
|
+
# Daily cleanup at 2 AM
|
|
166
|
+
CronJob(
|
|
167
|
+
function_name="daily_cleanup",
|
|
168
|
+
schedule="0 2 * * *",
|
|
169
|
+
args=["temp_files"],
|
|
170
|
+
kwargs={"max_age_days": 7}
|
|
171
|
+
),
|
|
172
|
+
|
|
173
|
+
# Weekly report every Monday at 9 AM
|
|
174
|
+
CronJob(
|
|
175
|
+
function_name="generate_weekly_report",
|
|
176
|
+
schedule="0 9 * * mon",
|
|
177
|
+
unique=True # Prevent duplicate reports if worker restarts
|
|
178
|
+
),
|
|
179
|
+
|
|
180
|
+
# Health check every 15 minutes on a specific queue
|
|
181
|
+
CronJob(
|
|
182
|
+
function_name="system_health_check",
|
|
183
|
+
schedule="*/15 * * * *",
|
|
184
|
+
queue_name="monitoring"
|
|
185
|
+
),
|
|
186
|
+
|
|
187
|
+
# Backup database every night at 1 AM
|
|
188
|
+
CronJob(
|
|
189
|
+
function_name="backup_database",
|
|
190
|
+
schedule="0 1 * * *",
|
|
191
|
+
kwargs={"backup_type": "incremental"}
|
|
192
|
+
),
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
# Add to your settings
|
|
196
|
+
rrq_settings = RRQSettings(
|
|
197
|
+
redis_dsn="redis://localhost:6379/0",
|
|
198
|
+
cron_jobs=cron_jobs
|
|
199
|
+
)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Cron Job Handlers
|
|
203
|
+
|
|
204
|
+
Your cron job handlers are regular async functions, just like other job handlers:
|
|
205
|
+
|
|
206
|
+
```python
|
|
207
|
+
async def daily_cleanup(ctx, file_type: str, max_age_days: int = 7):
|
|
208
|
+
"""Clean up old files."""
|
|
209
|
+
job_id = ctx['job_id']
|
|
210
|
+
print(f"Job {job_id}: Cleaning up {file_type} files older than {max_age_days} days")
|
|
211
|
+
# Your cleanup logic here
|
|
212
|
+
return {"cleaned_files": 42, "status": "completed"}
|
|
213
|
+
|
|
214
|
+
async def generate_weekly_report(ctx):
|
|
215
|
+
"""Generate and send weekly report."""
|
|
216
|
+
job_id = ctx['job_id']
|
|
217
|
+
print(f"Job {job_id}: Generating weekly report")
|
|
218
|
+
# Your report generation logic here
|
|
219
|
+
return {"report_id": "weekly_2024_01", "status": "sent"}
|
|
220
|
+
|
|
221
|
+
# Register your handlers
|
|
222
|
+
from rrq.registry import JobRegistry
|
|
223
|
+
|
|
224
|
+
job_registry = JobRegistry()
|
|
225
|
+
job_registry.register("daily_cleanup", daily_cleanup)
|
|
226
|
+
job_registry.register("generate_weekly_report", generate_weekly_report)
|
|
227
|
+
|
|
228
|
+
# Add the registry to your settings
|
|
229
|
+
rrq_settings.job_registry = job_registry
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
**Note:** Cron jobs are automatically enqueued by the worker when they become due. The worker checks for due cron jobs every 30 seconds and enqueues them as regular jobs to be processed.
|
|
233
|
+
|
|
114
234
|
## Command Line Interface
|
|
115
235
|
|
|
116
236
|
RRQ provides a command-line interface (CLI) for managing workers and performing health checks:
|
|
@@ -118,7 +238,8 @@ RRQ provides a command-line interface (CLI) for managing workers and performing
|
|
|
118
238
|
- **`rrq worker run`** - Run an RRQ worker process.
|
|
119
239
|
- `--settings` (optional): Specify the Python path to your settings object (e.g., `myapp.worker_config.rrq_settings`). If not provided, it will use the `RRQ_SETTINGS` environment variable or default to a basic `RRQSettings` object.
|
|
120
240
|
- `--queue` (optional, multiple): Specify queue(s) to poll. Defaults to the `default_queue_name` in settings.
|
|
121
|
-
- `--burst` (flag): Run the worker in burst mode to process one job or batch and then exit.
|
|
241
|
+
- `--burst` (flag): Run the worker in burst mode to process one job or batch and then exit. Cannot be used with `--num-workers > 1`.
|
|
242
|
+
- `--num-workers` (optional, integer): Number of parallel worker processes to start. Defaults to the number of CPU cores available on the machine. Cannot be used with `--burst` mode.
|
|
122
243
|
- **`rrq worker watch`** - Run an RRQ worker with auto-restart on file changes.
|
|
123
244
|
- `--path` (optional): Directory path to watch for changes. Defaults to the current directory.
|
|
124
245
|
- `--settings` (optional): Same as above.
|
|
@@ -151,7 +272,4 @@ RRQ can be configured in several ways, with the following precedence:
|
|
|
151
272
|
* **`JobRegistry` (`registry.py`)**: A simple registry to map string function names (used when enqueuing) to the actual asynchronous handler functions the worker should execute.
|
|
152
273
|
* **`JobStore` (`store.py`)**: An abstraction layer handling all direct interactions with Redis. It manages job definitions (Hashes), queues (Sorted Sets), processing locks (Strings with TTL), unique job locks, and worker health checks.
|
|
153
274
|
* **`Job` (`job.py`)**: A Pydantic model representing a job, containing its ID, handler name, arguments, status, retry counts, timestamps, results, etc.
|
|
154
|
-
* **`JobStatus` (`job.py`)**: An Enum defining the possible states of a job (`PENDING`, `ACTIVE`, `COMPLETED`, `FAILED`, `
|
|
155
|
-
* **`RRQSettings` (`settings.py`)**: A Pydantic `BaseSettings` model for configuring RRQ behavior (Redis DSN, queue names, timeouts, retry policies, concurrency, etc.). Loadable from environment variables (prefix `RRQ_`).
|
|
156
|
-
* **`constants.py`**: Defines shared constants like Redis key prefixes and default configuration values.
|
|
157
|
-
* **`exc.py`**: Defines custom exceptions, notably `RetryJob` which handlers can raise to explicitly request a retry, potentially with a custom delay.
|
|
275
|
+
* **`JobStatus` (`job.py`)**: An Enum defining the possible states of a job (`PENDING`, `ACTIVE`, `COMPLETED`, `FAILED`, `
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
|
|
1
|
+
"""example_rrq_settings.py: Example RRQ Application Settings"""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import logging
|
|
5
5
|
|
|
6
|
+
from rrq.cron import CronJob
|
|
6
7
|
from rrq.settings import RRQSettings
|
|
7
8
|
|
|
8
9
|
logger = logging.getLogger("rrq")
|
|
@@ -15,24 +16,43 @@ logger.addHandler(console_handler)
|
|
|
15
16
|
redis_dsn = "redis://localhost:6379/0"
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
|
|
20
19
|
async def on_startup_hook():
|
|
21
20
|
logger.info("Executing 'on_startup_hook' (application-specific startup)...")
|
|
22
21
|
await asyncio.sleep(0.1)
|
|
23
22
|
logger.info("'on_startup_hook' complete.")
|
|
24
23
|
|
|
24
|
+
|
|
25
25
|
async def on_shutdown_hook():
|
|
26
26
|
logger.info("Executing 'on_shutdown_hook' (application-specific shutdown)...")
|
|
27
27
|
await asyncio.sleep(0.1)
|
|
28
28
|
logger.info("'on_shutdown_hook' complete.")
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
|
|
32
31
|
# RRQ Settings
|
|
33
32
|
rrq_settings = RRQSettings(
|
|
34
33
|
redis_dsn=redis_dsn,
|
|
35
34
|
on_startup=on_startup_hook,
|
|
36
35
|
on_shutdown=on_shutdown_hook,
|
|
36
|
+
# Example cron jobs - these would run periodically when a worker is running
|
|
37
|
+
cron_jobs=[
|
|
38
|
+
# Run a cleanup task every day at 2 AM
|
|
39
|
+
CronJob(
|
|
40
|
+
function_name="daily_cleanup",
|
|
41
|
+
schedule="0 2 * * *",
|
|
42
|
+
args=["cleanup_logs"],
|
|
43
|
+
kwargs={"max_age_days": 30},
|
|
44
|
+
),
|
|
45
|
+
# Send a status report every Monday at 9 AM
|
|
46
|
+
CronJob(
|
|
47
|
+
function_name="send_status_report",
|
|
48
|
+
schedule="0 9 * * mon",
|
|
49
|
+
unique=True, # Prevent duplicate reports if worker restarts
|
|
50
|
+
),
|
|
51
|
+
# Health check every 15 minutes
|
|
52
|
+
CronJob(
|
|
53
|
+
function_name="health_check",
|
|
54
|
+
schedule="*/15 * * * *",
|
|
55
|
+
queue_name="monitoring", # Use a specific queue for monitoring tasks
|
|
56
|
+
),
|
|
57
|
+
],
|
|
37
58
|
)
|
|
38
|
-
|
|
@@ -8,6 +8,7 @@ from contextlib import suppress
|
|
|
8
8
|
from datetime import timedelta
|
|
9
9
|
|
|
10
10
|
from rrq.client import RRQClient
|
|
11
|
+
from rrq.cron import CronJob
|
|
11
12
|
from rrq.exc import RetryJob
|
|
12
13
|
from rrq.registry import JobRegistry
|
|
13
14
|
from rrq.settings import RRQSettings
|
|
@@ -62,6 +63,37 @@ async def retry_task(ctx, counter_limit: int):
|
|
|
62
63
|
return {"status": "completed_after_retries", "attempts": attempt}
|
|
63
64
|
|
|
64
65
|
|
|
66
|
+
# --- Example Cron Job Handlers ---
|
|
67
|
+
async def daily_cleanup(ctx, task_type: str, max_age_days: int = 30):
|
|
68
|
+
"""Example cron job handler for daily cleanup tasks."""
|
|
69
|
+
logger.info(
|
|
70
|
+
f"DAILY_CLEANUP (Job {ctx['job_id']}): Running {task_type} cleanup, max age: {max_age_days} days"
|
|
71
|
+
)
|
|
72
|
+
await asyncio.sleep(0.5) # Simulate cleanup work
|
|
73
|
+
logger.info(f"DAILY_CLEANUP (Job {ctx['job_id']}): Cleanup completed")
|
|
74
|
+
return {"task_type": task_type, "max_age_days": max_age_days, "status": "completed"}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def send_status_report(ctx):
|
|
78
|
+
"""Example cron job handler for sending status reports."""
|
|
79
|
+
logger.info(
|
|
80
|
+
f"STATUS_REPORT (Job {ctx['job_id']}): Generating and sending status report"
|
|
81
|
+
)
|
|
82
|
+
await asyncio.sleep(0.3) # Simulate report generation
|
|
83
|
+
logger.info(f"STATUS_REPORT (Job {ctx['job_id']}): Status report sent")
|
|
84
|
+
return {"report_type": "weekly", "status": "sent"}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
async def health_check(ctx):
|
|
88
|
+
"""Example cron job handler for health checks."""
|
|
89
|
+
logger.info(f"HEALTH_CHECK (Job {ctx['job_id']}): Running system health check")
|
|
90
|
+
await asyncio.sleep(0.1) # Simulate health check
|
|
91
|
+
logger.info(
|
|
92
|
+
f"HEALTH_CHECK (Job {ctx['job_id']}): Health check completed - all systems OK"
|
|
93
|
+
)
|
|
94
|
+
return {"status": "healthy", "timestamp": ctx.get("job_start_time")}
|
|
95
|
+
|
|
96
|
+
|
|
65
97
|
# --- Main Execution ---
|
|
66
98
|
async def main():
|
|
67
99
|
logger.info("--- Starting RRQ Example ---")
|
|
@@ -72,8 +104,24 @@ async def main():
|
|
|
72
104
|
default_max_retries=3, # Lower retries for example
|
|
73
105
|
worker_health_check_interval_seconds=5, # Frequent health check
|
|
74
106
|
worker_shutdown_grace_period_seconds=5,
|
|
107
|
+
# Example cron jobs - these will run periodically when the worker is running
|
|
108
|
+
cron_jobs=[
|
|
109
|
+
# Run a health check every 2 minutes (for demo purposes)
|
|
110
|
+
CronJob(
|
|
111
|
+
function_name="health_check",
|
|
112
|
+
schedule="*/2 * * * *", # Every 2 minutes
|
|
113
|
+
queue_name="monitoring",
|
|
114
|
+
),
|
|
115
|
+
# Send a status report every 5 minutes (for demo purposes)
|
|
116
|
+
CronJob(
|
|
117
|
+
function_name="send_status_report",
|
|
118
|
+
schedule="*/5 * * * *", # Every 5 minutes
|
|
119
|
+
unique=True, # Prevent duplicate reports
|
|
120
|
+
),
|
|
121
|
+
],
|
|
75
122
|
)
|
|
76
123
|
logger.info(f"Using Redis DB: {settings.redis_dsn}")
|
|
124
|
+
logger.info(f"Configured {len(settings.cron_jobs)} cron jobs")
|
|
77
125
|
|
|
78
126
|
# Ensure Redis DB is clean before starting (optional, good for examples)
|
|
79
127
|
try:
|
|
@@ -90,6 +138,10 @@ async def main():
|
|
|
90
138
|
registry.register("handle_success", successful_task)
|
|
91
139
|
registry.register("handle_failure", failing_task)
|
|
92
140
|
registry.register("handle_retry", retry_task)
|
|
141
|
+
# Register cron job handlers
|
|
142
|
+
registry.register("daily_cleanup", daily_cleanup)
|
|
143
|
+
registry.register("send_status_report", send_status_report)
|
|
144
|
+
registry.register("health_check", health_check)
|
|
93
145
|
logger.info(f"Registered handlers: {registry.get_registered_functions()}")
|
|
94
146
|
|
|
95
147
|
# 3. Client
|
|
@@ -131,7 +183,7 @@ async def main():
|
|
|
131
183
|
worker = RRQWorker(
|
|
132
184
|
settings=settings,
|
|
133
185
|
job_registry=registry,
|
|
134
|
-
queues=[settings.default_queue_name, "high_priority"],
|
|
186
|
+
queues=[settings.default_queue_name, "high_priority", "monitoring"],
|
|
135
187
|
)
|
|
136
188
|
|
|
137
189
|
# 6. Run Worker (with graceful shutdown handling)
|
|
@@ -162,7 +214,8 @@ async def main():
|
|
|
162
214
|
|
|
163
215
|
# Wait for stop event or worker task completion (e.g., if it errors out)
|
|
164
216
|
done, pending = await asyncio.wait(
|
|
165
|
-
[worker_task, asyncio.create_task(stop_event.wait())],
|
|
217
|
+
[worker_task, asyncio.create_task(stop_event.wait())],
|
|
218
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
166
219
|
)
|
|
167
220
|
|
|
168
221
|
logger.info("Stop event triggered or worker task finished.")
|
|
@@ -4,10 +4,8 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rrq"
|
|
7
|
-
version = "0.
|
|
8
|
-
authors = [
|
|
9
|
-
{ name = "Mazdak Rezvani", email = "mazdak@me.com" },
|
|
10
|
-
]
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
authors = [{ name = "Mazdak Rezvani", email = "mazdak@me.com" }]
|
|
11
9
|
description = "RRQ is a Python library for creating reliable job queues using Redis and asyncio"
|
|
12
10
|
readme = "README.md"
|
|
13
11
|
requires-python = ">=3.11"
|
|
@@ -30,11 +28,7 @@ dependencies = [
|
|
|
30
28
|
]
|
|
31
29
|
|
|
32
30
|
[project.optional-dependencies]
|
|
33
|
-
dev = [
|
|
34
|
-
"pytest>=8.3.5",
|
|
35
|
-
"pytest-asyncio>=0.26.0",
|
|
36
|
-
"pytest-cov>=6.0.0",
|
|
37
|
-
]
|
|
31
|
+
dev = ["pytest>=8.3.5", "pytest-asyncio>=1.0.0", "pytest-cov>=6.0.0"]
|
|
38
32
|
|
|
39
33
|
[project.urls]
|
|
40
34
|
"Homepage" = "https://github.com/getresq/rrq"
|
|
@@ -44,4 +38,8 @@ dev = [
|
|
|
44
38
|
rrq = "rrq.cli:rrq"
|
|
45
39
|
|
|
46
40
|
[tool.pytest.ini_options]
|
|
41
|
+
asyncio_mode = "strict"
|
|
47
42
|
asyncio_default_fixture_loop_scope = "function"
|
|
43
|
+
|
|
44
|
+
[tool.pyrefly]
|
|
45
|
+
python_interpreter = ".venv/bin/python"
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from .cron import CronJob, CronSchedule
|
|
2
|
+
from .worker import RRQWorker
|
|
3
|
+
from .client import RRQClient
|
|
4
|
+
from .registry import JobRegistry
|
|
5
|
+
from .settings import RRQSettings
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"CronJob",
|
|
9
|
+
"CronSchedule",
|
|
10
|
+
"RRQWorker",
|
|
11
|
+
"RRQClient",
|
|
12
|
+
"JobRegistry",
|
|
13
|
+
"RRQSettings",
|
|
14
|
+
]
|