django-database-task 0.2.4__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_database_task-0.2.4/django_database_task.egg-info → django_database_task-0.3.0}/PKG-INFO +214 -13
- {django_database_task-0.2.4 → django_database_task-0.3.0}/README.md +209 -9
- django_database_task-0.3.0/django_database_task/__init__.py +44 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/executor.py +14 -0
- django_database_task-0.3.0/django_database_task/management/commands/run_database_tasks.py +197 -0
- django_database_task-0.3.0/django_database_task/shutdown.py +313 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0/django_database_task.egg-info}/PKG-INFO +214 -13
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task.egg-info/SOURCES.txt +2 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task.egg-info/requires.txt +1 -1
- {django_database_task-0.2.4 → django_database_task-0.3.0}/pyproject.toml +9 -4
- django_database_task-0.3.0/tests/test_commands.py +417 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/tests/test_executor.py +61 -0
- django_database_task-0.3.0/tests/test_shutdown.py +270 -0
- django_database_task-0.2.4/django_database_task/__init__.py +0 -29
- django_database_task-0.2.4/django_database_task/management/commands/run_database_tasks.py +0 -98
- django_database_task-0.2.4/tests/test_commands.py +0 -211
- {django_database_task-0.2.4 → django_database_task-0.3.0}/LICENSE +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/MANIFEST.in +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/admin.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/apps.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/backends.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/cloudtasks/__init__.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/cloudtasks/auth.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/cloudtasks/backend.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/cloudtasks/detection.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/locale/ja/LC_MESSAGES/django.mo +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/locale/ja/LC_MESSAGES/django.po +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/management/__init__.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/management/commands/__init__.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/management/commands/purge_completed_database_tasks.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/migrations/0001_initial.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/migrations/__init__.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/models.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/urls.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task/views.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task.egg-info/dependency_links.txt +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/django_database_task.egg-info/top_level.txt +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/setup.cfg +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/tests/test_admin.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/tests/test_backend.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/tests/test_models.py +0 -0
- {django_database_task-0.2.4 → django_database_task-0.3.0}/tests/test_views.py +0 -0
{django_database_task-0.2.4/django_database_task.egg-info → django_database_task-0.3.0}/PKG-INFO
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-database-task
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: A database-backed task queue backend for Django
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A database-backed task queue backend for Django's built-in task framework
|
|
5
5
|
Author-email: Shinya Okano <tokibito@gmail.com>
|
|
6
6
|
Maintainer-email: Shinya Okano <tokibito@gmail.com>
|
|
7
7
|
License-Expression: MIT
|
|
@@ -9,10 +9,11 @@ Project-URL: Homepage, https://github.com/tokibito/django-database-task
|
|
|
9
9
|
Project-URL: Repository, https://github.com/tokibito/django-database-task
|
|
10
10
|
Project-URL: Issues, https://github.com/tokibito/django-database-task/issues
|
|
11
11
|
Keywords: django,task,queue,background,database,async
|
|
12
|
-
Classifier: Development Status ::
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
13
|
Classifier: Environment :: Web Environment
|
|
14
14
|
Classifier: Framework :: Django
|
|
15
15
|
Classifier: Framework :: Django :: 6.0
|
|
16
|
+
Classifier: Framework :: Django :: 6.1
|
|
16
17
|
Classifier: Intended Audience :: Developers
|
|
17
18
|
Classifier: Operating System :: OS Independent
|
|
18
19
|
Classifier: Programming Language :: Python
|
|
@@ -31,7 +32,7 @@ Requires-Dist: google-auth>=2.0.0; extra == "cloudtasks"
|
|
|
31
32
|
Provides-Extra: dev
|
|
32
33
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
33
34
|
Requires-Dist: pytest-django>=4.5; extra == "dev"
|
|
34
|
-
Requires-Dist: ruff
|
|
35
|
+
Requires-Dist: ruff<0.17,>=0.16; extra == "dev"
|
|
35
36
|
Dynamic: license-file
|
|
36
37
|
|
|
37
38
|
# django-database-task
|
|
@@ -41,7 +42,7 @@ Dynamic: license-file
|
|
|
41
42
|
[](https://pypi.org/project/django-database-task/)
|
|
42
43
|
[](https://opensource.org/licenses/MIT)
|
|
43
44
|
|
|
44
|
-
A database-backed task queue backend for Django
|
|
45
|
+
A database-backed task queue backend for Django's built-in task framework.
|
|
45
46
|
|
|
46
47
|
## Features
|
|
47
48
|
|
|
@@ -51,6 +52,7 @@ A database-backed task queue backend for Django 6.0's built-in task framework.
|
|
|
51
52
|
- **Exclusive locking** - Prevents duplicate task execution with `SELECT FOR UPDATE SKIP LOCKED`
|
|
52
53
|
- **Django Admin integration** - View and manage tasks from the admin interface
|
|
53
54
|
- **Async support** - Supports async task functions
|
|
55
|
+
- **Graceful shutdown** - Workers finish the running task before exiting on `SIGTERM`
|
|
54
56
|
- **Google Cloud Tasks integration** - Optional backend for GAE/Cloud Run with auto-detection
|
|
55
57
|
|
|
56
58
|
## Architecture
|
|
@@ -94,15 +96,16 @@ sequenceDiagram
|
|
|
94
96
|
|
|
95
97
|
### Supported Databases
|
|
96
98
|
|
|
97
|
-
|
|
99
|
+
The minimum database versions are the ones Django itself requires, and Django
|
|
100
|
+
6.1 raised most of them:
|
|
98
101
|
|
|
99
|
-
| Database |
|
|
100
|
-
|
|
101
|
-
| PostgreSQL | 14+ | Recommended for production. Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
102
|
-
| MySQL | 8.0.11+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
103
|
-
| MariaDB | 10.6+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
104
|
-
| SQLite | 3.31.0+ | Works for development/testing, but no row-level locking. |
|
|
105
|
-
| Oracle | 19c+ | Supported but not tested with this package. |
|
|
102
|
+
| Database | Django 6.0 | Django 6.1 | Notes |
|
|
103
|
+
|----------|------------|------------|-------|
|
|
104
|
+
| PostgreSQL | 14+ | 15+ | Recommended for production. Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
105
|
+
| MySQL | 8.0.11+ | 8.4+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
106
|
+
| MariaDB | 10.6+ | 10.11+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
107
|
+
| SQLite | 3.31.0+ | 3.37.0+ | Works for development/testing, but no row-level locking. |
|
|
108
|
+
| Oracle | 19c+ | 19c+ | Supported but not tested with this package. |
|
|
106
109
|
|
|
107
110
|
**Note**: `SELECT FOR UPDATE SKIP LOCKED` is used to prevent duplicate task execution in multi-worker environments. SQLite does not support row-level locking, so it is only recommended for development or single-worker deployments.
|
|
108
111
|
|
|
@@ -279,6 +282,24 @@ python manage.py run_database_tasks [options]
|
|
|
279
282
|
| `--continuous` | Keep polling even when no tasks |
|
|
280
283
|
| `--interval` | Polling interval in seconds (default: 5) |
|
|
281
284
|
| `--max-tasks` | Maximum number of tasks to process (0=unlimited) |
|
|
285
|
+
| `--shutdown-timeout` | Maximum seconds to wait for the running task after `SIGTERM`/`SIGINT` before forcing exit (0=wait indefinitely, default: 0) |
|
|
286
|
+
| `--no-graceful-shutdown` | Do not install signal handlers (terminate immediately, even while a task is running) |
|
|
287
|
+
| `--verbosity` | Output level: `0` silent (errors only), `1` normal (default), `2` also print an idle heartbeat dot per poll |
|
|
288
|
+
|
|
289
|
+
See [Graceful Shutdown](#graceful-shutdown) for details.
|
|
290
|
+
|
|
291
|
+
#### Output verbosity
|
|
292
|
+
|
|
293
|
+
At the default verbosity the worker only prints the startup banner and one
|
|
294
|
+
block per task, so its output stays readable in a log aggregator. Idle polls in
|
|
295
|
+
`--continuous` mode print nothing.
|
|
296
|
+
|
|
297
|
+
Pass `-v 2` to print a `.` for every poll that found no task - useful when
|
|
298
|
+
watching a worker interactively to confirm it is alive, but it buries real log
|
|
299
|
+
output if left on in production.
|
|
300
|
+
|
|
301
|
+
Pass `-v 0` to suppress the informational output entirely; task failures and
|
|
302
|
+
errors are still reported.
|
|
282
303
|
|
|
283
304
|
### purge_completed_database_tasks
|
|
284
305
|
|
|
@@ -295,6 +316,177 @@ python manage.py purge_completed_database_tasks [options]
|
|
|
295
316
|
| `--batch-size` | Number of tasks to delete at once (default: 1000) |
|
|
296
317
|
| `--dry-run` | Show count only without deleting |
|
|
297
318
|
|
|
319
|
+
## Graceful Shutdown
|
|
320
|
+
|
|
321
|
+
When a worker is redeployed, the orchestrator (Kubernetes, Cloud Run, systemd,
|
|
322
|
+
Docker, supervisord, ...) sends `SIGTERM` and kills the process with `SIGKILL`
|
|
323
|
+
after a grace period. Without any handling, a task that happens to be running
|
|
324
|
+
at that moment is killed halfway through and stays in `RUNNING` status forever.
|
|
325
|
+
|
|
326
|
+
`run_database_tasks` installs `SIGTERM` and `SIGINT` handlers by default:
|
|
327
|
+
|
|
328
|
+
1. On the first signal the worker stops fetching new tasks.
|
|
329
|
+
2. The task currently being executed keeps running until it finishes and its
|
|
330
|
+
result is written to the database.
|
|
331
|
+
3. The worker then exits with status code 0.
|
|
332
|
+
|
|
333
|
+
While no task is running (the polling sleep in `--continuous` mode), the signal
|
|
334
|
+
is handled immediately - the worker does not wait out the remaining interval.
|
|
335
|
+
|
|
336
|
+
```console
|
|
337
|
+
$ python manage.py run_database_tasks --continuous
|
|
338
|
+
Worker ID: worker-1-3f2a9c11
|
|
339
|
+
Backend: default
|
|
340
|
+
Continuous mode: interval=5.0s
|
|
341
|
+
Graceful shutdown: enabled (timeout=unlimited)
|
|
342
|
+
|
|
343
|
+
Processing task: 1e2d... (myapp.tasks.send_report)
|
|
344
|
+
^C
|
|
345
|
+
Received SIGINT: no new tasks will be started. Waiting for the running task to finish (send the signal again to force exit).
|
|
346
|
+
Task completed successfully
|
|
347
|
+
|
|
348
|
+
Shutdown complete (no task was interrupted).
|
|
349
|
+
|
|
350
|
+
Total tasks processed: 1
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
### Shutdown timeout
|
|
354
|
+
|
|
355
|
+
By default the worker waits as long as the running task needs. Use
|
|
356
|
+
`--shutdown-timeout` to put an upper bound on it, so the process exits on its
|
|
357
|
+
own terms instead of being `SIGKILL`ed by the platform:
|
|
358
|
+
|
|
359
|
+
```bash
|
|
360
|
+
python manage.py run_database_tasks --continuous --shutdown-timeout 25
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
If the task is still running when the timeout expires, the process exits
|
|
364
|
+
immediately with status code 1 and the task stays in `RUNNING` status.
|
|
365
|
+
Set this to a value slightly below the platform's termination grace period,
|
|
366
|
+
and keep the grace period longer than your longest task whenever possible.
|
|
367
|
+
|
|
368
|
+
Sending the signal a second time (for example pressing Ctrl-C twice) also
|
|
369
|
+
forces an immediate exit.
|
|
370
|
+
|
|
371
|
+
### Cooperating from inside a task
|
|
372
|
+
|
|
373
|
+
Long running tasks can check whether a shutdown was requested and stop early,
|
|
374
|
+
so the worker does not have to wait for the whole task to complete:
|
|
375
|
+
|
|
376
|
+
```python
|
|
377
|
+
from django.tasks import task
|
|
378
|
+
|
|
379
|
+
from django_database_task import is_shutdown_requested
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
@task
|
|
383
|
+
def import_rows(row_ids):
|
|
384
|
+
processed = []
|
|
385
|
+
for row_id in row_ids:
|
|
386
|
+
if is_shutdown_requested():
|
|
387
|
+
# Requeue the remaining work and return early
|
|
388
|
+
import_rows.enqueue([i for i in row_ids if i not in processed])
|
|
389
|
+
break
|
|
390
|
+
handle(row_id)
|
|
391
|
+
processed.append(row_id)
|
|
392
|
+
return len(processed)
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
`is_shutdown_requested()` returns `False` when no worker with graceful shutdown
|
|
396
|
+
is active, so tasks using it stay safe to call from a web request, a test, or
|
|
397
|
+
the HTTP endpoints.
|
|
398
|
+
|
|
399
|
+
### Deployment examples
|
|
400
|
+
|
|
401
|
+
**Kubernetes** - set `terminationGracePeriodSeconds` longer than the worker's
|
|
402
|
+
shutdown timeout:
|
|
403
|
+
|
|
404
|
+
```yaml
|
|
405
|
+
spec:
|
|
406
|
+
terminationGracePeriodSeconds: 60
|
|
407
|
+
containers:
|
|
408
|
+
- name: worker
|
|
409
|
+
command:
|
|
410
|
+
- python
|
|
411
|
+
- manage.py
|
|
412
|
+
- run_database_tasks
|
|
413
|
+
- --continuous
|
|
414
|
+
- --shutdown-timeout=50
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
**systemd** - `TimeoutStopSec` controls how long systemd waits before
|
|
418
|
+
`SIGKILL`:
|
|
419
|
+
|
|
420
|
+
```ini
|
|
421
|
+
[Service]
|
|
422
|
+
ExecStart=/srv/app/venv/bin/python manage.py run_database_tasks --continuous --shutdown-timeout=50
|
|
423
|
+
KillSignal=SIGTERM
|
|
424
|
+
TimeoutStopSec=60
|
|
425
|
+
Restart=always
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
**Docker / Docker Compose** - `docker stop` sends `SIGTERM` and waits for
|
|
429
|
+
`--time` (10 seconds by default):
|
|
430
|
+
|
|
431
|
+
```yaml
|
|
432
|
+
services:
|
|
433
|
+
worker:
|
|
434
|
+
command: python manage.py run_database_tasks --continuous --shutdown-timeout=25
|
|
435
|
+
stop_grace_period: 30s
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
Make sure the worker is PID 1 or that the signal reaches it (use the exec form
|
|
439
|
+
of `CMD`, or an init such as `tini`, rather than wrapping the command in a
|
|
440
|
+
shell script that swallows signals).
|
|
441
|
+
|
|
442
|
+
### Tasks left in RUNNING status
|
|
443
|
+
|
|
444
|
+
If a worker is killed with `SIGKILL` (grace period exceeded, node failure,
|
|
445
|
+
`--no-graceful-shutdown`), the task it was running stays in `RUNNING` status
|
|
446
|
+
because no process is left to update it. Such tasks are not picked up again by
|
|
447
|
+
other workers. They can be found and requeued from the Django admin, or with a
|
|
448
|
+
query like:
|
|
449
|
+
|
|
450
|
+
```python
|
|
451
|
+
from datetime import timedelta
|
|
452
|
+
|
|
453
|
+
from django.tasks.base import TaskResultStatus
|
|
454
|
+
from django.utils import timezone
|
|
455
|
+
|
|
456
|
+
from django_database_task.models import DatabaseTask
|
|
457
|
+
|
|
458
|
+
stale = DatabaseTask.objects.filter(
|
|
459
|
+
status=TaskResultStatus.RUNNING,
|
|
460
|
+
last_attempted_at__lt=timezone.now() - timedelta(hours=1),
|
|
461
|
+
)
|
|
462
|
+
stale.update(status=TaskResultStatus.READY)
|
|
463
|
+
```
|
|
464
|
+
|
|
465
|
+
Only requeue tasks that are safe to run twice (idempotent).
|
|
466
|
+
|
|
467
|
+
### Using it in your own worker loop
|
|
468
|
+
|
|
469
|
+
The shutdown handling is available as a public API, for custom worker loops:
|
|
470
|
+
|
|
471
|
+
```python
|
|
472
|
+
from django_database_task import GracefulShutdown, process_tasks
|
|
473
|
+
|
|
474
|
+
with GracefulShutdown(timeout=50) as shutdown:
|
|
475
|
+
while not shutdown.is_set():
|
|
476
|
+
results = process_tasks(max_tasks=10, stop_event=shutdown)
|
|
477
|
+
if not results and shutdown.wait(5): # interruptible sleep
|
|
478
|
+
break
|
|
479
|
+
```
|
|
480
|
+
|
|
481
|
+
| API | Description |
|
|
482
|
+
|-----|-------------|
|
|
483
|
+
| `GracefulShutdown(signals=None, timeout=0, on_signal=None, force_on_repeat=True)` | Context manager that installs the signal handlers |
|
|
484
|
+
| `shutdown.is_set()` | True once a shutdown has been requested |
|
|
485
|
+
| `shutdown.wait(seconds)` | Sleep, returning early (True) when a shutdown is requested |
|
|
486
|
+
| `shutdown.set()` | Request a shutdown programmatically |
|
|
487
|
+
| `process_tasks(..., stop_event=...)` | Stop starting new tasks once the event is set |
|
|
488
|
+
| `is_shutdown_requested()` | True if the active worker was asked to shut down |
|
|
489
|
+
|
|
298
490
|
## Programmatic API
|
|
299
491
|
|
|
300
492
|
You can also process tasks programmatically without management commands:
|
|
@@ -330,8 +522,17 @@ if result:
|
|
|
330
522
|
|
|
331
523
|
# Retry a failed task
|
|
332
524
|
result = run_task_by_id("...", allow_retry=True)
|
|
525
|
+
|
|
526
|
+
# Stop starting new tasks when the process receives SIGTERM/SIGINT
|
|
527
|
+
from django_database_task import GracefulShutdown
|
|
528
|
+
|
|
529
|
+
with GracefulShutdown() as shutdown:
|
|
530
|
+
results = process_tasks(stop_event=shutdown)
|
|
333
531
|
```
|
|
334
532
|
|
|
533
|
+
See [Graceful Shutdown](#graceful-shutdown) for details on `stop_event` and
|
|
534
|
+
`GracefulShutdown`.
|
|
535
|
+
|
|
335
536
|
## HTTP Endpoints (Optional)
|
|
336
537
|
|
|
337
538
|
For environments where cron or direct command execution is not available
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://pypi.org/project/django-database-task/)
|
|
6
6
|
[](https://opensource.org/licenses/MIT)
|
|
7
7
|
|
|
8
|
-
A database-backed task queue backend for Django
|
|
8
|
+
A database-backed task queue backend for Django's built-in task framework.
|
|
9
9
|
|
|
10
10
|
## Features
|
|
11
11
|
|
|
@@ -15,6 +15,7 @@ A database-backed task queue backend for Django 6.0's built-in task framework.
|
|
|
15
15
|
- **Exclusive locking** - Prevents duplicate task execution with `SELECT FOR UPDATE SKIP LOCKED`
|
|
16
16
|
- **Django Admin integration** - View and manage tasks from the admin interface
|
|
17
17
|
- **Async support** - Supports async task functions
|
|
18
|
+
- **Graceful shutdown** - Workers finish the running task before exiting on `SIGTERM`
|
|
18
19
|
- **Google Cloud Tasks integration** - Optional backend for GAE/Cloud Run with auto-detection
|
|
19
20
|
|
|
20
21
|
## Architecture
|
|
@@ -58,15 +59,16 @@ sequenceDiagram
|
|
|
58
59
|
|
|
59
60
|
### Supported Databases
|
|
60
61
|
|
|
61
|
-
|
|
62
|
+
The minimum database versions are the ones Django itself requires, and Django
|
|
63
|
+
6.1 raised most of them:
|
|
62
64
|
|
|
63
|
-
| Database |
|
|
64
|
-
|
|
65
|
-
| PostgreSQL | 14+ | Recommended for production. Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
66
|
-
| MySQL | 8.0.11+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
67
|
-
| MariaDB | 10.6+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
68
|
-
| SQLite | 3.31.0+ | Works for development/testing, but no row-level locking. |
|
|
69
|
-
| Oracle | 19c+ | Supported but not tested with this package. |
|
|
65
|
+
| Database | Django 6.0 | Django 6.1 | Notes |
|
|
66
|
+
|----------|------------|------------|-------|
|
|
67
|
+
| PostgreSQL | 14+ | 15+ | Recommended for production. Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
68
|
+
| MySQL | 8.0.11+ | 8.4+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
69
|
+
| MariaDB | 10.6+ | 10.11+ | Full `SELECT FOR UPDATE SKIP LOCKED` support. |
|
|
70
|
+
| SQLite | 3.31.0+ | 3.37.0+ | Works for development/testing, but no row-level locking. |
|
|
71
|
+
| Oracle | 19c+ | 19c+ | Supported but not tested with this package. |
|
|
70
72
|
|
|
71
73
|
**Note**: `SELECT FOR UPDATE SKIP LOCKED` is used to prevent duplicate task execution in multi-worker environments. SQLite does not support row-level locking, so it is only recommended for development or single-worker deployments.
|
|
72
74
|
|
|
@@ -243,6 +245,24 @@ python manage.py run_database_tasks [options]
|
|
|
243
245
|
| `--continuous` | Keep polling even when no tasks |
|
|
244
246
|
| `--interval` | Polling interval in seconds (default: 5) |
|
|
245
247
|
| `--max-tasks` | Maximum number of tasks to process (0=unlimited) |
|
|
248
|
+
| `--shutdown-timeout` | Maximum seconds to wait for the running task after `SIGTERM`/`SIGINT` before forcing exit (0=wait indefinitely, default: 0) |
|
|
249
|
+
| `--no-graceful-shutdown` | Do not install signal handlers (terminate immediately, even while a task is running) |
|
|
250
|
+
| `--verbosity` | Output level: `0` silent (errors only), `1` normal (default), `2` also print an idle heartbeat dot per poll |
|
|
251
|
+
|
|
252
|
+
See [Graceful Shutdown](#graceful-shutdown) for details.
|
|
253
|
+
|
|
254
|
+
#### Output verbosity
|
|
255
|
+
|
|
256
|
+
At the default verbosity the worker only prints the startup banner and one
|
|
257
|
+
block per task, so its output stays readable in a log aggregator. Idle polls in
|
|
258
|
+
`--continuous` mode print nothing.
|
|
259
|
+
|
|
260
|
+
Pass `-v 2` to print a `.` for every poll that found no task - useful when
|
|
261
|
+
watching a worker interactively to confirm it is alive, but it buries real log
|
|
262
|
+
output if left on in production.
|
|
263
|
+
|
|
264
|
+
Pass `-v 0` to suppress the informational output entirely; task failures and
|
|
265
|
+
errors are still reported.
|
|
246
266
|
|
|
247
267
|
### purge_completed_database_tasks
|
|
248
268
|
|
|
@@ -259,6 +279,177 @@ python manage.py purge_completed_database_tasks [options]
|
|
|
259
279
|
| `--batch-size` | Number of tasks to delete at once (default: 1000) |
|
|
260
280
|
| `--dry-run` | Show count only without deleting |
|
|
261
281
|
|
|
282
|
+
## Graceful Shutdown
|
|
283
|
+
|
|
284
|
+
When a worker is redeployed, the orchestrator (Kubernetes, Cloud Run, systemd,
|
|
285
|
+
Docker, supervisord, ...) sends `SIGTERM` and kills the process with `SIGKILL`
|
|
286
|
+
after a grace period. Without any handling, a task that happens to be running
|
|
287
|
+
at that moment is killed halfway through and stays in `RUNNING` status forever.
|
|
288
|
+
|
|
289
|
+
`run_database_tasks` installs `SIGTERM` and `SIGINT` handlers by default:
|
|
290
|
+
|
|
291
|
+
1. On the first signal the worker stops fetching new tasks.
|
|
292
|
+
2. The task currently being executed keeps running until it finishes and its
|
|
293
|
+
result is written to the database.
|
|
294
|
+
3. The worker then exits with status code 0.
|
|
295
|
+
|
|
296
|
+
While no task is running (the polling sleep in `--continuous` mode), the signal
|
|
297
|
+
is handled immediately - the worker does not wait out the remaining interval.
|
|
298
|
+
|
|
299
|
+
```console
|
|
300
|
+
$ python manage.py run_database_tasks --continuous
|
|
301
|
+
Worker ID: worker-1-3f2a9c11
|
|
302
|
+
Backend: default
|
|
303
|
+
Continuous mode: interval=5.0s
|
|
304
|
+
Graceful shutdown: enabled (timeout=unlimited)
|
|
305
|
+
|
|
306
|
+
Processing task: 1e2d... (myapp.tasks.send_report)
|
|
307
|
+
^C
|
|
308
|
+
Received SIGINT: no new tasks will be started. Waiting for the running task to finish (send the signal again to force exit).
|
|
309
|
+
Task completed successfully
|
|
310
|
+
|
|
311
|
+
Shutdown complete (no task was interrupted).
|
|
312
|
+
|
|
313
|
+
Total tasks processed: 1
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
### Shutdown timeout
|
|
317
|
+
|
|
318
|
+
By default the worker waits as long as the running task needs. Use
|
|
319
|
+
`--shutdown-timeout` to put an upper bound on it, so the process exits on its
|
|
320
|
+
own terms instead of being `SIGKILL`ed by the platform:
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
python manage.py run_database_tasks --continuous --shutdown-timeout 25
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
If the task is still running when the timeout expires, the process exits
|
|
327
|
+
immediately with status code 1 and the task stays in `RUNNING` status.
|
|
328
|
+
Set this to a value slightly below the platform's termination grace period,
|
|
329
|
+
and keep the grace period longer than your longest task whenever possible.
|
|
330
|
+
|
|
331
|
+
Sending the signal a second time (for example pressing Ctrl-C twice) also
|
|
332
|
+
forces an immediate exit.
|
|
333
|
+
|
|
334
|
+
### Cooperating from inside a task
|
|
335
|
+
|
|
336
|
+
Long running tasks can check whether a shutdown was requested and stop early,
|
|
337
|
+
so the worker does not have to wait for the whole task to complete:
|
|
338
|
+
|
|
339
|
+
```python
|
|
340
|
+
from django.tasks import task
|
|
341
|
+
|
|
342
|
+
from django_database_task import is_shutdown_requested
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
@task
|
|
346
|
+
def import_rows(row_ids):
|
|
347
|
+
processed = []
|
|
348
|
+
for row_id in row_ids:
|
|
349
|
+
if is_shutdown_requested():
|
|
350
|
+
# Requeue the remaining work and return early
|
|
351
|
+
import_rows.enqueue([i for i in row_ids if i not in processed])
|
|
352
|
+
break
|
|
353
|
+
handle(row_id)
|
|
354
|
+
processed.append(row_id)
|
|
355
|
+
return len(processed)
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
`is_shutdown_requested()` returns `False` when no worker with graceful shutdown
|
|
359
|
+
is active, so tasks using it stay safe to call from a web request, a test, or
|
|
360
|
+
the HTTP endpoints.
|
|
361
|
+
|
|
362
|
+
### Deployment examples
|
|
363
|
+
|
|
364
|
+
**Kubernetes** - set `terminationGracePeriodSeconds` longer than the worker's
|
|
365
|
+
shutdown timeout:
|
|
366
|
+
|
|
367
|
+
```yaml
|
|
368
|
+
spec:
|
|
369
|
+
terminationGracePeriodSeconds: 60
|
|
370
|
+
containers:
|
|
371
|
+
- name: worker
|
|
372
|
+
command:
|
|
373
|
+
- python
|
|
374
|
+
- manage.py
|
|
375
|
+
- run_database_tasks
|
|
376
|
+
- --continuous
|
|
377
|
+
- --shutdown-timeout=50
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
**systemd** - `TimeoutStopSec` controls how long systemd waits before
|
|
381
|
+
`SIGKILL`:
|
|
382
|
+
|
|
383
|
+
```ini
|
|
384
|
+
[Service]
|
|
385
|
+
ExecStart=/srv/app/venv/bin/python manage.py run_database_tasks --continuous --shutdown-timeout=50
|
|
386
|
+
KillSignal=SIGTERM
|
|
387
|
+
TimeoutStopSec=60
|
|
388
|
+
Restart=always
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
**Docker / Docker Compose** - `docker stop` sends `SIGTERM` and waits for
|
|
392
|
+
`--time` (10 seconds by default):
|
|
393
|
+
|
|
394
|
+
```yaml
|
|
395
|
+
services:
|
|
396
|
+
worker:
|
|
397
|
+
command: python manage.py run_database_tasks --continuous --shutdown-timeout=25
|
|
398
|
+
stop_grace_period: 30s
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
Make sure the worker is PID 1 or that the signal reaches it (use the exec form
|
|
402
|
+
of `CMD`, or an init such as `tini`, rather than wrapping the command in a
|
|
403
|
+
shell script that swallows signals).
|
|
404
|
+
|
|
405
|
+
### Tasks left in RUNNING status
|
|
406
|
+
|
|
407
|
+
If a worker is killed with `SIGKILL` (grace period exceeded, node failure,
|
|
408
|
+
`--no-graceful-shutdown`), the task it was running stays in `RUNNING` status
|
|
409
|
+
because no process is left to update it. Such tasks are not picked up again by
|
|
410
|
+
other workers. They can be found and requeued from the Django admin, or with a
|
|
411
|
+
query like:
|
|
412
|
+
|
|
413
|
+
```python
|
|
414
|
+
from datetime import timedelta
|
|
415
|
+
|
|
416
|
+
from django.tasks.base import TaskResultStatus
|
|
417
|
+
from django.utils import timezone
|
|
418
|
+
|
|
419
|
+
from django_database_task.models import DatabaseTask
|
|
420
|
+
|
|
421
|
+
stale = DatabaseTask.objects.filter(
|
|
422
|
+
status=TaskResultStatus.RUNNING,
|
|
423
|
+
last_attempted_at__lt=timezone.now() - timedelta(hours=1),
|
|
424
|
+
)
|
|
425
|
+
stale.update(status=TaskResultStatus.READY)
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
Only requeue tasks that are safe to run twice (idempotent).
|
|
429
|
+
|
|
430
|
+
### Using it in your own worker loop
|
|
431
|
+
|
|
432
|
+
The shutdown handling is available as a public API, for custom worker loops:
|
|
433
|
+
|
|
434
|
+
```python
|
|
435
|
+
from django_database_task import GracefulShutdown, process_tasks
|
|
436
|
+
|
|
437
|
+
with GracefulShutdown(timeout=50) as shutdown:
|
|
438
|
+
while not shutdown.is_set():
|
|
439
|
+
results = process_tasks(max_tasks=10, stop_event=shutdown)
|
|
440
|
+
if not results and shutdown.wait(5): # interruptible sleep
|
|
441
|
+
break
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
| API | Description |
|
|
445
|
+
|-----|-------------|
|
|
446
|
+
| `GracefulShutdown(signals=None, timeout=0, on_signal=None, force_on_repeat=True)` | Context manager that installs the signal handlers |
|
|
447
|
+
| `shutdown.is_set()` | True once a shutdown has been requested |
|
|
448
|
+
| `shutdown.wait(seconds)` | Sleep, returning early (True) when a shutdown is requested |
|
|
449
|
+
| `shutdown.set()` | Request a shutdown programmatically |
|
|
450
|
+
| `process_tasks(..., stop_event=...)` | Stop starting new tasks once the event is set |
|
|
451
|
+
| `is_shutdown_requested()` | True if the active worker was asked to shut down |
|
|
452
|
+
|
|
262
453
|
## Programmatic API
|
|
263
454
|
|
|
264
455
|
You can also process tasks programmatically without management commands:
|
|
@@ -294,8 +485,17 @@ if result:
|
|
|
294
485
|
|
|
295
486
|
# Retry a failed task
|
|
296
487
|
result = run_task_by_id("...", allow_retry=True)
|
|
488
|
+
|
|
489
|
+
# Stop starting new tasks when the process receives SIGTERM/SIGINT
|
|
490
|
+
from django_database_task import GracefulShutdown
|
|
491
|
+
|
|
492
|
+
with GracefulShutdown() as shutdown:
|
|
493
|
+
results = process_tasks(stop_event=shutdown)
|
|
297
494
|
```
|
|
298
495
|
|
|
496
|
+
See [Graceful Shutdown](#graceful-shutdown) for details on `stop_event` and
|
|
497
|
+
`GracefulShutdown`.
|
|
498
|
+
|
|
299
499
|
## HTTP Endpoints (Optional)
|
|
300
500
|
|
|
301
501
|
For environments where cron or direct command execution is not available
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
django-database-task: A database-backed task queue backend for Django's task framework.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.3.0"
|
|
6
|
+
|
|
7
|
+
_EXECUTOR_EXPORTS = (
|
|
8
|
+
"fetch_task",
|
|
9
|
+
"get_pending_task_count",
|
|
10
|
+
"process_one_task",
|
|
11
|
+
"process_tasks",
|
|
12
|
+
"run_task_by_id",
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
_SHUTDOWN_EXPORTS = (
|
|
16
|
+
"GracefulShutdown",
|
|
17
|
+
"get_active_shutdown",
|
|
18
|
+
"is_shutdown_requested",
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def __getattr__(name):
|
|
23
|
+
"""Lazy import to avoid AppRegistryNotReady errors."""
|
|
24
|
+
if name in _EXECUTOR_EXPORTS:
|
|
25
|
+
from . import executor
|
|
26
|
+
|
|
27
|
+
return getattr(executor, name)
|
|
28
|
+
if name in _SHUTDOWN_EXPORTS:
|
|
29
|
+
from . import shutdown
|
|
30
|
+
|
|
31
|
+
return getattr(shutdown, name)
|
|
32
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"GracefulShutdown",
|
|
37
|
+
"fetch_task",
|
|
38
|
+
"get_active_shutdown",
|
|
39
|
+
"get_pending_task_count",
|
|
40
|
+
"is_shutdown_requested",
|
|
41
|
+
"process_one_task",
|
|
42
|
+
"process_tasks",
|
|
43
|
+
"run_task_by_id",
|
|
44
|
+
]
|
|
@@ -105,6 +105,7 @@ def process_tasks(
|
|
|
105
105
|
backend_name="default",
|
|
106
106
|
max_tasks=0,
|
|
107
107
|
worker_id=None,
|
|
108
|
+
stop_event=None,
|
|
108
109
|
):
|
|
109
110
|
"""
|
|
110
111
|
Process multiple pending tasks.
|
|
@@ -114,6 +115,11 @@ def process_tasks(
|
|
|
114
115
|
backend_name: Backend name (default: "default").
|
|
115
116
|
max_tasks: Maximum number of tasks to process (0 = unlimited).
|
|
116
117
|
worker_id: Optional worker ID. If not provided, one will be generated.
|
|
118
|
+
stop_event: Optional object with an ``is_set()`` method (for example a
|
|
119
|
+
:class:`threading.Event` or a
|
|
120
|
+
:class:`~django_database_task.GracefulShutdown`). No new task is
|
|
121
|
+
started once it is set; the task already running is not
|
|
122
|
+
interrupted.
|
|
117
123
|
|
|
118
124
|
Returns:
|
|
119
125
|
List of TaskResult objects for all processed tasks.
|
|
@@ -124,6 +130,11 @@ def process_tasks(
|
|
|
124
130
|
>>> print(f"Processed {len(results)} tasks")
|
|
125
131
|
>>> for result in results:
|
|
126
132
|
... print(f" {result.id}: {result.status}")
|
|
133
|
+
|
|
134
|
+
# Stop starting new tasks on SIGTERM/SIGINT
|
|
135
|
+
>>> from django_database_task import GracefulShutdown
|
|
136
|
+
>>> with GracefulShutdown() as shutdown:
|
|
137
|
+
... results = process_tasks(stop_event=shutdown)
|
|
127
138
|
"""
|
|
128
139
|
if worker_id is None:
|
|
129
140
|
worker_id = _generate_worker_id()
|
|
@@ -132,6 +143,9 @@ def process_tasks(
|
|
|
132
143
|
tasks_processed = 0
|
|
133
144
|
|
|
134
145
|
while True:
|
|
146
|
+
if stop_event is not None and stop_event.is_set():
|
|
147
|
+
break
|
|
148
|
+
|
|
135
149
|
result = process_one_task(
|
|
136
150
|
queue_name=queue_name,
|
|
137
151
|
backend_name=backend_name,
|