dj-queue 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dj_queue-0.5.0 → dj_queue-0.6.0}/PKG-INFO +100 -46
- {dj_queue-0.5.0 → dj_queue-0.6.0}/README.md +99 -45
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/admin.py +5 -1
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/api.py +24 -11
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/config.py +48 -1
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/asgi.py +15 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/dj_queue.py +9 -4
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/observability.py +2 -5
- {dj_queue-0.5.0 → dj_queue-0.6.0}/pyproject.toml +1 -1
- {dj_queue-0.5.0 → dj_queue-0.6.0}/LICENSE +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/apps.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/backend.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/gunicorn.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/prometheus.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/dashboard.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/db.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/exceptions.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/hooks.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/log.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/dj_queue_health.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/dj_queue_prune.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0001_initial.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0004_dashboard.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/jobs.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/recurring.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/runtime.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/cleanup.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/concurrency.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/jobs.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/recurring.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/routers.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/base.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/dispatcher.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/errors.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/interruptible.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/notify.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/pidfile.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/pool.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/procline.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/scheduler.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/supervisor.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/worker.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templatetags/__init__.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templatetags/dj_queue_admin.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/urls.py +0 -0
- {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dj-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Database-backed task queue backend for Django’s Tasks framework.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -105,6 +105,12 @@ TASKS = {
|
|
|
105
105
|
The router is optional when using the default database, but harmless to include
|
|
106
106
|
and required for [multi-database setups](#multi-database-setup).
|
|
107
107
|
|
|
108
|
+
`dj_queue` can coexist with other Django task backends in the same `TASKS`
|
|
109
|
+
setting. It only manages aliases whose `BACKEND` is
|
|
110
|
+
`"dj_queue.backend.DjQueueBackend"`. If a `TASKS` alias points at some other
|
|
111
|
+
backend, `dj_queue` ignores that alias for runtime commands, admin/dashboard
|
|
112
|
+
selection, and observability.
|
|
113
|
+
|
|
108
114
|
Run migrations:
|
|
109
115
|
|
|
110
116
|
```bash
|
|
@@ -266,15 +272,17 @@ Useful command variants:
|
|
|
266
272
|
```bash
|
|
267
273
|
python manage.py dj_queue
|
|
268
274
|
python manage.py dj_queue --mode async
|
|
275
|
+
python manage.py dj_queue --backend <alias>
|
|
269
276
|
python manage.py dj_queue --only-work
|
|
270
277
|
python manage.py dj_queue --only-dispatch
|
|
271
278
|
python manage.py dj_queue --skip-recurring
|
|
272
279
|
```
|
|
273
280
|
|
|
274
|
-
|
|
281
|
+
Notes:
|
|
275
282
|
|
|
276
283
|
- `fork` is the default standalone mode
|
|
277
284
|
- `async` is also supported as a standalone mode and runs supervised actors in threads inside one process
|
|
285
|
+
- `--backend` targets a non-default backend alias
|
|
278
286
|
- `--only-work` starts workers without dispatchers or scheduler
|
|
279
287
|
- `--only-dispatch` starts dispatchers without workers or scheduler
|
|
280
288
|
- `--skip-recurring` starts without the scheduler
|
|
@@ -303,8 +311,7 @@ across queues. Prefer one primary scheduling mechanism per worker when you can.
|
|
|
303
311
|
|
|
304
312
|
### Signals and recovery
|
|
305
313
|
|
|
306
|
-
In standalone mode, both `fork` and `async` `python manage.py dj_queue`
|
|
307
|
-
supervisors own runtime signal handling:
|
|
314
|
+
In standalone mode, both `fork` and `async` `python manage.py dj_queue` supervisors own runtime signal handling:
|
|
308
315
|
|
|
309
316
|
- `SIGTERM` and `SIGINT` request graceful shutdown
|
|
310
317
|
- `SIGQUIT` takes the immediate hard-exit path
|
|
@@ -461,7 +468,7 @@ orders.resume()
|
|
|
461
468
|
orders.clear()
|
|
462
469
|
```
|
|
463
470
|
|
|
464
|
-
|
|
471
|
+
Notes:
|
|
465
472
|
|
|
466
473
|
- pausing a queue stops future claims, not enqueueing or already-claimed work
|
|
467
474
|
- pause rows are scoped per backend alias
|
|
@@ -480,8 +487,7 @@ python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
|
|
|
480
487
|
python manage.py dj_queue_prune --task-key nightly_cleanup
|
|
481
488
|
```
|
|
482
489
|
|
|
483
|
-
The
|
|
484
|
-
non-default backend alias.
|
|
490
|
+
The health and prune commands both accept `--backend` to target a non-default backend alias.
|
|
485
491
|
|
|
486
492
|
For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
|
|
487
493
|
task import path, while `--task-key` filters recurring execution cleanup by
|
|
@@ -549,11 +555,44 @@ except EnqueueError as exc:
|
|
|
549
555
|
Task execution errors are different: they become failed jobs and stay
|
|
550
556
|
inspectable in the queue database.
|
|
551
557
|
|
|
552
|
-
##
|
|
558
|
+
## Lifecycle Hooks
|
|
559
|
+
|
|
560
|
+
Register hooks before starting the runtime, typically during Django startup.
|
|
561
|
+
Each callback receives the live supervisor or runner instance.
|
|
562
|
+
|
|
563
|
+
```python
|
|
564
|
+
from dj_queue.hooks import on_start, on_worker_start, register_hook
|
|
565
|
+
|
|
566
|
+
@on_start
|
|
567
|
+
def supervisor_started(process):
|
|
568
|
+
print(process.name)
|
|
553
569
|
|
|
554
|
-
|
|
570
|
+
@on_worker_start
|
|
571
|
+
def worker_started(process):
|
|
572
|
+
print(process.metadata)
|
|
555
573
|
|
|
556
|
-
|
|
574
|
+
@register_hook("scheduler.exit")
|
|
575
|
+
def scheduler_exited(process):
|
|
576
|
+
print(process.name)
|
|
577
|
+
```
|
|
578
|
+
|
|
579
|
+
Available hook helpers:
|
|
580
|
+
|
|
581
|
+
- supervisor: `on_start`, `on_stop`, `on_exit`
|
|
582
|
+
- worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
|
|
583
|
+
- dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
|
|
584
|
+
- scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
|
|
585
|
+
- generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
|
|
586
|
+
|
|
587
|
+
Notes:
|
|
588
|
+
|
|
589
|
+
- hooks fire in registration order
|
|
590
|
+
- hook failures do not block later hooks
|
|
591
|
+
- hook failures are isolated and routed through `on_thread_error`
|
|
592
|
+
|
|
593
|
+
## Multi-Database Setup
|
|
594
|
+
|
|
595
|
+
`dj_queue` can keep queue tables on a dedicated database alias:
|
|
557
596
|
|
|
558
597
|
```python
|
|
559
598
|
DATABASES = {
|
|
@@ -591,6 +630,38 @@ python manage.py migrate dj_queue --database queue
|
|
|
591
630
|
With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
|
|
592
631
|
database.
|
|
593
632
|
|
|
633
|
+
## Backend Coexistence
|
|
634
|
+
|
|
635
|
+
Projects can mix `dj_queue` with other Django task backends in the same `TASKS` mapping:
|
|
636
|
+
|
|
637
|
+
```python
|
|
638
|
+
TASKS = {
|
|
639
|
+
"default": {
|
|
640
|
+
"BACKEND": "dj_queue.backend.DjQueueBackend",
|
|
641
|
+
"QUEUES": [],
|
|
642
|
+
"OPTIONS": {
|
|
643
|
+
"database_alias": "queue",
|
|
644
|
+
},
|
|
645
|
+
},
|
|
646
|
+
"external": {
|
|
647
|
+
"BACKEND": "some_other_backend.Backend",
|
|
648
|
+
"QUEUES": [],
|
|
649
|
+
"OPTIONS": {},
|
|
650
|
+
},
|
|
651
|
+
}
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
In that setup:
|
|
655
|
+
|
|
656
|
+
- jobs with `backend="default"` are `dj_queue`'s responsibility
|
|
657
|
+
- jobs with `backend="external"` are the other backend's responsibility
|
|
658
|
+
- `dj_queue` admin, dashboard, `/dj_queue/stats.json`, `/dj_queue/metrics`, and `manage.py dj_queue --backend ...` only operate on `dj_queue` aliases
|
|
659
|
+
|
|
660
|
+
Notes:
|
|
661
|
+
|
|
662
|
+
- if `TASKS` is empty or unset, `dj_queue` still exposes one implicit `default` alias using built-in defaults
|
|
663
|
+
- if `TASKS` is non-empty, `dj_queue` only manages aliases whose `BACKEND` is explicitly `"dj_queue.backend.DjQueueBackend"`
|
|
664
|
+
|
|
594
665
|
## Postgres Queue Health
|
|
595
666
|
|
|
596
667
|
Operational and configuration guidance for scaling with `dj_queue` in
|
|
@@ -644,6 +715,21 @@ signal handling to the host server.
|
|
|
644
715
|
|
|
645
716
|
## Configuration
|
|
646
717
|
|
|
718
|
+
### Queues, backends, and databases
|
|
719
|
+
|
|
720
|
+
`dj_queue` has three separate routing concepts. Keep them distinct:
|
|
721
|
+
|
|
722
|
+
- `queue_name`: what kind of work this job is. Use it to route lanes inside one backend, such as `email`, `webhooks`, or `search-index`.
|
|
723
|
+
- `backend_alias`: which logical queue system owns the work. Use it when you want separate runtime config, recurring tasks, pause and process visibility, retention, or admin scoping.
|
|
724
|
+
- `database_alias`: where that backend's queue tables and runtime activity live. Use it when you want a dedicated database connection path or stronger storage isolation.
|
|
725
|
+
|
|
726
|
+
Common setup choices:
|
|
727
|
+
|
|
728
|
+
- one backend, one database: simplest and usually enough
|
|
729
|
+
- one backend, separate queue database: good when you want dedicated queue connections
|
|
730
|
+
- multiple backends, same database: good for logical and operational separation without another database
|
|
731
|
+
- multiple backends, multiple databases: use when you need stronger isolation and accept more migration and deployment complexity
|
|
732
|
+
|
|
647
733
|
### Deployment topology
|
|
648
734
|
|
|
649
735
|
Once migrations are in place, start processing jobs with `python manage.py dj_queue`
|
|
@@ -792,41 +878,6 @@ Environment overrides currently supported by `dj_queue` itself:
|
|
|
792
878
|
- `DJ_QUEUE_MODE`
|
|
793
879
|
- `DJ_QUEUE_SKIP_RECURRING`
|
|
794
880
|
|
|
795
|
-
## Lifecycle Hooks
|
|
796
|
-
|
|
797
|
-
Register hooks before starting the runtime, typically during Django startup.
|
|
798
|
-
Each callback receives the live supervisor or runner instance.
|
|
799
|
-
|
|
800
|
-
```python
|
|
801
|
-
from dj_queue.hooks import on_start, on_worker_start, register_hook
|
|
802
|
-
|
|
803
|
-
@on_start
|
|
804
|
-
def supervisor_started(process):
|
|
805
|
-
print(process.name)
|
|
806
|
-
|
|
807
|
-
@on_worker_start
|
|
808
|
-
def worker_started(process):
|
|
809
|
-
print(process.metadata)
|
|
810
|
-
|
|
811
|
-
@register_hook("scheduler.exit")
|
|
812
|
-
def scheduler_exited(process):
|
|
813
|
-
print(process.name)
|
|
814
|
-
```
|
|
815
|
-
|
|
816
|
-
Available hook helpers:
|
|
817
|
-
|
|
818
|
-
- supervisor: `on_start`, `on_stop`, `on_exit`
|
|
819
|
-
- worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
|
|
820
|
-
- dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
|
|
821
|
-
- scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
|
|
822
|
-
- generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
|
|
823
|
-
|
|
824
|
-
Hook notes:
|
|
825
|
-
|
|
826
|
-
- hooks fire in registration order
|
|
827
|
-
- hook failures do not block later hooks
|
|
828
|
-
- hook failures are isolated and routed through `on_thread_error`
|
|
829
|
-
|
|
830
881
|
### Runtime infrastructure errors
|
|
831
882
|
|
|
832
883
|
Set `on_thread_error` to a dotted callable path when you want custom handling
|
|
@@ -852,7 +903,10 @@ become failed jobs instead.
|
|
|
852
903
|
## Monitoring
|
|
853
904
|
|
|
854
905
|
Queue statistics are available in JSON via `/dj_queue/stats.json` and in
|
|
855
|
-
Prometheus text format via `/dj_queue/metrics`.
|
|
906
|
+
Prometheus text format via `/dj_queue/metrics`.
|
|
907
|
+
|
|
908
|
+
These observability endpoints only report `dj_queue`-managed backend aliases and
|
|
909
|
+
ignore aliases configured for some other task backend.
|
|
856
910
|
|
|
857
911
|
Include `dj_queue.urls` to expose them:
|
|
858
912
|
|
|
@@ -77,6 +77,12 @@ TASKS = {
|
|
|
77
77
|
The router is optional when using the default database, but harmless to include
|
|
78
78
|
and required for [multi-database setups](#multi-database-setup).
|
|
79
79
|
|
|
80
|
+
`dj_queue` can coexist with other Django task backends in the same `TASKS`
|
|
81
|
+
setting. It only manages aliases whose `BACKEND` is
|
|
82
|
+
`"dj_queue.backend.DjQueueBackend"`. If a `TASKS` alias points at some other
|
|
83
|
+
backend, `dj_queue` ignores that alias for runtime commands, admin/dashboard
|
|
84
|
+
selection, and observability.
|
|
85
|
+
|
|
80
86
|
Run migrations:
|
|
81
87
|
|
|
82
88
|
```bash
|
|
@@ -238,15 +244,17 @@ Useful command variants:
|
|
|
238
244
|
```bash
|
|
239
245
|
python manage.py dj_queue
|
|
240
246
|
python manage.py dj_queue --mode async
|
|
247
|
+
python manage.py dj_queue --backend <alias>
|
|
241
248
|
python manage.py dj_queue --only-work
|
|
242
249
|
python manage.py dj_queue --only-dispatch
|
|
243
250
|
python manage.py dj_queue --skip-recurring
|
|
244
251
|
```
|
|
245
252
|
|
|
246
|
-
|
|
253
|
+
Notes:
|
|
247
254
|
|
|
248
255
|
- `fork` is the default standalone mode
|
|
249
256
|
- `async` is also supported as a standalone mode and runs supervised actors in threads inside one process
|
|
257
|
+
- `--backend` targets a non-default backend alias
|
|
250
258
|
- `--only-work` starts workers without dispatchers or scheduler
|
|
251
259
|
- `--only-dispatch` starts dispatchers without workers or scheduler
|
|
252
260
|
- `--skip-recurring` starts without the scheduler
|
|
@@ -275,8 +283,7 @@ across queues. Prefer one primary scheduling mechanism per worker when you can.
|
|
|
275
283
|
|
|
276
284
|
### Signals and recovery
|
|
277
285
|
|
|
278
|
-
In standalone mode, both `fork` and `async` `python manage.py dj_queue`
|
|
279
|
-
supervisors own runtime signal handling:
|
|
286
|
+
In standalone mode, both `fork` and `async` `python manage.py dj_queue` supervisors own runtime signal handling:
|
|
280
287
|
|
|
281
288
|
- `SIGTERM` and `SIGINT` request graceful shutdown
|
|
282
289
|
- `SIGQUIT` takes the immediate hard-exit path
|
|
@@ -433,7 +440,7 @@ orders.resume()
|
|
|
433
440
|
orders.clear()
|
|
434
441
|
```
|
|
435
442
|
|
|
436
|
-
|
|
443
|
+
Notes:
|
|
437
444
|
|
|
438
445
|
- pausing a queue stops future claims, not enqueueing or already-claimed work
|
|
439
446
|
- pause rows are scoped per backend alias
|
|
@@ -452,8 +459,7 @@ python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
|
|
|
452
459
|
python manage.py dj_queue_prune --task-key nightly_cleanup
|
|
453
460
|
```
|
|
454
461
|
|
|
455
|
-
The
|
|
456
|
-
non-default backend alias.
|
|
462
|
+
The health and prune commands both accept `--backend` to target a non-default backend alias.
|
|
457
463
|
|
|
458
464
|
For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
|
|
459
465
|
task import path, while `--task-key` filters recurring execution cleanup by
|
|
@@ -521,11 +527,44 @@ except EnqueueError as exc:
|
|
|
521
527
|
Task execution errors are different: they become failed jobs and stay
|
|
522
528
|
inspectable in the queue database.
|
|
523
529
|
|
|
524
|
-
##
|
|
530
|
+
## Lifecycle Hooks
|
|
531
|
+
|
|
532
|
+
Register hooks before starting the runtime, typically during Django startup.
|
|
533
|
+
Each callback receives the live supervisor or runner instance.
|
|
534
|
+
|
|
535
|
+
```python
|
|
536
|
+
from dj_queue.hooks import on_start, on_worker_start, register_hook
|
|
537
|
+
|
|
538
|
+
@on_start
|
|
539
|
+
def supervisor_started(process):
|
|
540
|
+
print(process.name)
|
|
525
541
|
|
|
526
|
-
|
|
542
|
+
@on_worker_start
|
|
543
|
+
def worker_started(process):
|
|
544
|
+
print(process.metadata)
|
|
527
545
|
|
|
528
|
-
|
|
546
|
+
@register_hook("scheduler.exit")
|
|
547
|
+
def scheduler_exited(process):
|
|
548
|
+
print(process.name)
|
|
549
|
+
```
|
|
550
|
+
|
|
551
|
+
Available hook helpers:
|
|
552
|
+
|
|
553
|
+
- supervisor: `on_start`, `on_stop`, `on_exit`
|
|
554
|
+
- worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
|
|
555
|
+
- dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
|
|
556
|
+
- scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
|
|
557
|
+
- generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
|
|
558
|
+
|
|
559
|
+
Notes:
|
|
560
|
+
|
|
561
|
+
- hooks fire in registration order
|
|
562
|
+
- hook failures do not block later hooks
|
|
563
|
+
- hook failures are isolated and routed through `on_thread_error`
|
|
564
|
+
|
|
565
|
+
## Multi-Database Setup
|
|
566
|
+
|
|
567
|
+
`dj_queue` can keep queue tables on a dedicated database alias:
|
|
529
568
|
|
|
530
569
|
```python
|
|
531
570
|
DATABASES = {
|
|
@@ -563,6 +602,38 @@ python manage.py migrate dj_queue --database queue
|
|
|
563
602
|
With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
|
|
564
603
|
database.
|
|
565
604
|
|
|
605
|
+
## Backend Coexistence
|
|
606
|
+
|
|
607
|
+
Projects can mix `dj_queue` with other Django task backends in the same `TASKS` mapping:
|
|
608
|
+
|
|
609
|
+
```python
|
|
610
|
+
TASKS = {
|
|
611
|
+
"default": {
|
|
612
|
+
"BACKEND": "dj_queue.backend.DjQueueBackend",
|
|
613
|
+
"QUEUES": [],
|
|
614
|
+
"OPTIONS": {
|
|
615
|
+
"database_alias": "queue",
|
|
616
|
+
},
|
|
617
|
+
},
|
|
618
|
+
"external": {
|
|
619
|
+
"BACKEND": "some_other_backend.Backend",
|
|
620
|
+
"QUEUES": [],
|
|
621
|
+
"OPTIONS": {},
|
|
622
|
+
},
|
|
623
|
+
}
|
|
624
|
+
```
|
|
625
|
+
|
|
626
|
+
In that setup:
|
|
627
|
+
|
|
628
|
+
- jobs with `backend="default"` are `dj_queue`'s responsibility
|
|
629
|
+
- jobs with `backend="external"` are the other backend's responsibility
|
|
630
|
+
- `dj_queue` admin, dashboard, `/dj_queue/stats.json`, `/dj_queue/metrics`, and `manage.py dj_queue --backend ...` only operate on `dj_queue` aliases
|
|
631
|
+
|
|
632
|
+
Notes:
|
|
633
|
+
|
|
634
|
+
- if `TASKS` is empty or unset, `dj_queue` still exposes one implicit `default` alias using built-in defaults
|
|
635
|
+
- if `TASKS` is non-empty, `dj_queue` only manages aliases whose `BACKEND` is explicitly `"dj_queue.backend.DjQueueBackend"`
|
|
636
|
+
|
|
566
637
|
## Postgres Queue Health
|
|
567
638
|
|
|
568
639
|
Operational and configuration guidance for scaling with `dj_queue` in
|
|
@@ -616,6 +687,21 @@ signal handling to the host server.
|
|
|
616
687
|
|
|
617
688
|
## Configuration
|
|
618
689
|
|
|
690
|
+
### Queues, backends, and databases
|
|
691
|
+
|
|
692
|
+
`dj_queue` has three separate routing concepts. Keep them distinct:
|
|
693
|
+
|
|
694
|
+
- `queue_name`: what kind of work this job is. Use it to route lanes inside one backend, such as `email`, `webhooks`, or `search-index`.
|
|
695
|
+
- `backend_alias`: which logical queue system owns the work. Use it when you want separate runtime config, recurring tasks, pause and process visibility, retention, or admin scoping.
|
|
696
|
+
- `database_alias`: where that backend's queue tables and runtime activity live. Use it when you want a dedicated database connection path or stronger storage isolation.
|
|
697
|
+
|
|
698
|
+
Common setup choices:
|
|
699
|
+
|
|
700
|
+
- one backend, one database: simplest and usually enough
|
|
701
|
+
- one backend, separate queue database: good when you want dedicated queue connections
|
|
702
|
+
- multiple backends, same database: good for logical and operational separation without another database
|
|
703
|
+
- multiple backends, multiple databases: use when you need stronger isolation and accept more migration and deployment complexity
|
|
704
|
+
|
|
619
705
|
### Deployment topology
|
|
620
706
|
|
|
621
707
|
Once migrations are in place, start processing jobs with `python manage.py dj_queue`
|
|
@@ -764,41 +850,6 @@ Environment overrides currently supported by `dj_queue` itself:
|
|
|
764
850
|
- `DJ_QUEUE_MODE`
|
|
765
851
|
- `DJ_QUEUE_SKIP_RECURRING`
|
|
766
852
|
|
|
767
|
-
## Lifecycle Hooks
|
|
768
|
-
|
|
769
|
-
Register hooks before starting the runtime, typically during Django startup.
|
|
770
|
-
Each callback receives the live supervisor or runner instance.
|
|
771
|
-
|
|
772
|
-
```python
|
|
773
|
-
from dj_queue.hooks import on_start, on_worker_start, register_hook
|
|
774
|
-
|
|
775
|
-
@on_start
|
|
776
|
-
def supervisor_started(process):
|
|
777
|
-
print(process.name)
|
|
778
|
-
|
|
779
|
-
@on_worker_start
|
|
780
|
-
def worker_started(process):
|
|
781
|
-
print(process.metadata)
|
|
782
|
-
|
|
783
|
-
@register_hook("scheduler.exit")
|
|
784
|
-
def scheduler_exited(process):
|
|
785
|
-
print(process.name)
|
|
786
|
-
```
|
|
787
|
-
|
|
788
|
-
Available hook helpers:
|
|
789
|
-
|
|
790
|
-
- supervisor: `on_start`, `on_stop`, `on_exit`
|
|
791
|
-
- worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
|
|
792
|
-
- dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
|
|
793
|
-
- scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
|
|
794
|
-
- generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
|
|
795
|
-
|
|
796
|
-
Hook notes:
|
|
797
|
-
|
|
798
|
-
- hooks fire in registration order
|
|
799
|
-
- hook failures do not block later hooks
|
|
800
|
-
- hook failures are isolated and routed through `on_thread_error`
|
|
801
|
-
|
|
802
853
|
### Runtime infrastructure errors
|
|
803
854
|
|
|
804
855
|
Set `on_thread_error` to a dotted callable path when you want custom handling
|
|
@@ -824,7 +875,10 @@ become failed jobs instead.
|
|
|
824
875
|
## Monitoring
|
|
825
876
|
|
|
826
877
|
Queue statistics are available in JSON via `/dj_queue/stats.json` and in
|
|
827
|
-
Prometheus text format via `/dj_queue/metrics`.
|
|
878
|
+
Prometheus text format via `/dj_queue/metrics`.
|
|
879
|
+
|
|
880
|
+
These observability endpoints only report `dj_queue`-managed backend aliases and
|
|
881
|
+
ignore aliases configured for some other task backend.
|
|
828
882
|
|
|
829
883
|
Include `dj_queue.urls` to expose them:
|
|
830
884
|
|
|
@@ -458,8 +458,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
458
458
|
)
|
|
459
459
|
readonly_fields = (
|
|
460
460
|
"task_path",
|
|
461
|
-
"
|
|
461
|
+
"queue_name_link",
|
|
462
462
|
"priority",
|
|
463
|
+
"display_status",
|
|
463
464
|
"payload",
|
|
464
465
|
"backend_alias",
|
|
465
466
|
"scheduled_at",
|
|
@@ -475,6 +476,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
475
476
|
)
|
|
476
477
|
search_fields = ("id", "task_path", "queue_name", "concurrency_key")
|
|
477
478
|
|
|
479
|
+
def get_fields(self, request, obj=None):
|
|
480
|
+
return self.get_readonly_fields(request, obj)
|
|
481
|
+
|
|
478
482
|
def get_readonly_fields(self, request, obj=None):
|
|
479
483
|
fields = super().get_readonly_fields(request, obj)
|
|
480
484
|
if obj is None or obj.status != "failed":
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from functools import partial
|
|
2
2
|
|
|
3
|
+
from django.db.models import Case, DateTimeField, DurationField, ExpressionWrapper, F, Value, When
|
|
3
4
|
from django.db.models.functions import Coalesce
|
|
4
5
|
from django.db import transaction
|
|
5
6
|
from django.utils import timezone
|
|
@@ -66,17 +67,29 @@ class QueueInfo:
|
|
|
66
67
|
|
|
67
68
|
resumed_at = timezone.now()
|
|
68
69
|
paused_at = pause.created_at
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
.
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
70
|
+
pause_duration = Value(resumed_at - paused_at, output_field=DurationField())
|
|
71
|
+
ready_row_ids = list(self._ready_queryset().values_list("id", flat=True))
|
|
72
|
+
if ready_row_ids:
|
|
73
|
+
ReadyExecution.objects.using(alias).filter(pk__in=ready_row_ids).update(
|
|
74
|
+
latency_started_at=Case(
|
|
75
|
+
When(
|
|
76
|
+
latency_started_at__isnull=True,
|
|
77
|
+
created_at__lt=paused_at,
|
|
78
|
+
then=ExpressionWrapper(
|
|
79
|
+
F("created_at") + pause_duration, output_field=DateTimeField()
|
|
80
|
+
),
|
|
81
|
+
),
|
|
82
|
+
When(
|
|
83
|
+
latency_started_at__lt=paused_at,
|
|
84
|
+
then=ExpressionWrapper(
|
|
85
|
+
F("latency_started_at") + pause_duration,
|
|
86
|
+
output_field=DateTimeField(),
|
|
87
|
+
),
|
|
88
|
+
),
|
|
89
|
+
default=Value(resumed_at, output_field=DateTimeField()),
|
|
90
|
+
output_field=DateTimeField(),
|
|
91
|
+
),
|
|
92
|
+
)
|
|
80
93
|
pause.delete()
|
|
81
94
|
|
|
82
95
|
log_event("queue.resumed", backend_alias=self.backend_alias, queue_name=self.queue_name)
|
|
@@ -54,6 +54,8 @@ DEFAULT_OPTIONS = {
|
|
|
54
54
|
"on_thread_error": None,
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
DJ_QUEUE_BACKEND_PATH = "dj_queue.backend.DjQueueBackend"
|
|
58
|
+
|
|
57
59
|
TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"}
|
|
58
60
|
FALSY_ENV_VALUES = {"0", "false", "no", "off"}
|
|
59
61
|
CONFIG_ENV_KEYS = ("DJ_QUEUE_CONFIG", "DJ_QUEUE_MODE", "DJ_QUEUE_SKIP_RECURRING")
|
|
@@ -163,6 +165,7 @@ def _load_backend_config_cached(
|
|
|
163
165
|
cli_overrides = json.loads(cli_overrides_key)
|
|
164
166
|
env = json.loads(env_key)
|
|
165
167
|
tasks_settings = json.loads(tasks_settings_key)
|
|
168
|
+
ensure_dj_queue_backend_alias(tasks_settings, backend_alias)
|
|
166
169
|
backend_block = _backend_block(tasks_settings, backend_alias)
|
|
167
170
|
resolved_options = _resolved_options(backend_alias, backend_block, cli_overrides, env)
|
|
168
171
|
|
|
@@ -241,12 +244,56 @@ def _backend_block(
|
|
|
241
244
|
if resolved_tasks_settings is None:
|
|
242
245
|
resolved_tasks_settings = getattr(settings, "TASKS", {})
|
|
243
246
|
|
|
244
|
-
|
|
247
|
+
if not resolved_tasks_settings:
|
|
248
|
+
if backend_alias == "default":
|
|
249
|
+
return {}
|
|
250
|
+
raise ImproperlyConfigured(f"dj_queue backend alias {backend_alias!r} is not configured")
|
|
251
|
+
|
|
252
|
+
if backend_alias not in resolved_tasks_settings:
|
|
253
|
+
raise ImproperlyConfigured(f"dj_queue backend alias {backend_alias!r} is not configured")
|
|
254
|
+
|
|
255
|
+
backend_block = resolved_tasks_settings[backend_alias]
|
|
245
256
|
if not isinstance(backend_block, Mapping):
|
|
246
257
|
raise ImproperlyConfigured(f"TASKS[{backend_alias!r}] must be a mapping")
|
|
247
258
|
return backend_block
|
|
248
259
|
|
|
249
260
|
|
|
261
|
+
def configured_backend_aliases(
|
|
262
|
+
tasks_settings: Mapping[str, Any] | None = None,
|
|
263
|
+
) -> tuple[str, ...]:
|
|
264
|
+
resolved_tasks_settings = tasks_settings
|
|
265
|
+
if resolved_tasks_settings is None:
|
|
266
|
+
resolved_tasks_settings = getattr(settings, "TASKS", {})
|
|
267
|
+
|
|
268
|
+
if not resolved_tasks_settings:
|
|
269
|
+
return ("default",)
|
|
270
|
+
|
|
271
|
+
aliases = []
|
|
272
|
+
for alias, backend_block in resolved_tasks_settings.items():
|
|
273
|
+
if not isinstance(backend_block, Mapping):
|
|
274
|
+
raise ImproperlyConfigured(f"TASKS[{alias!r}] must be a mapping")
|
|
275
|
+
if is_dj_queue_backend_alias(backend_block):
|
|
276
|
+
aliases.append(alias)
|
|
277
|
+
return tuple(aliases)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def ensure_dj_queue_backend_alias(
|
|
281
|
+
tasks_settings: Mapping[str, Any] | None,
|
|
282
|
+
backend_alias: str,
|
|
283
|
+
) -> None:
|
|
284
|
+
aliases = configured_backend_aliases(tasks_settings)
|
|
285
|
+
if backend_alias in aliases:
|
|
286
|
+
return
|
|
287
|
+
raise ImproperlyConfigured(
|
|
288
|
+
f"dj_queue backend alias {backend_alias!r} is not configured for DjQueueBackend"
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def is_dj_queue_backend_alias(backend_block: Mapping[str, Any]) -> bool:
|
|
293
|
+
backend_path = backend_block.get("BACKEND")
|
|
294
|
+
return backend_path == DJ_QUEUE_BACKEND_PATH
|
|
295
|
+
|
|
296
|
+
|
|
250
297
|
def _resolved_options(
|
|
251
298
|
backend_alias: str,
|
|
252
299
|
backend_block: Mapping[str, Any],
|
|
@@ -12,6 +12,12 @@ class DjQueueLifespan:
|
|
|
12
12
|
self.app = app
|
|
13
13
|
self.backend_alias = backend_alias
|
|
14
14
|
self.supervisor = None
|
|
15
|
+
self._poll_task = None
|
|
16
|
+
|
|
17
|
+
async def _poll_supervisor(self):
|
|
18
|
+
while self.supervisor is not None:
|
|
19
|
+
await asyncio.to_thread(self.supervisor.poll_once)
|
|
20
|
+
await asyncio.sleep(self.supervisor.polling_interval)
|
|
15
21
|
|
|
16
22
|
async def __call__(self, scope, receive, send):
|
|
17
23
|
if scope["type"] != "lifespan":
|
|
@@ -23,8 +29,17 @@ class DjQueueLifespan:
|
|
|
23
29
|
if message["type"] == "lifespan.startup":
|
|
24
30
|
self.supervisor = build_supervisor(self.backend_alias)
|
|
25
31
|
await asyncio.to_thread(self.supervisor.start)
|
|
32
|
+
self._poll_task = asyncio.create_task(self._poll_supervisor())
|
|
26
33
|
await send({"type": "lifespan.startup.complete"})
|
|
27
34
|
elif message["type"] == "lifespan.shutdown":
|
|
35
|
+
poll_task = self._poll_task
|
|
36
|
+
self._poll_task = None
|
|
37
|
+
if poll_task is not None:
|
|
38
|
+
poll_task.cancel()
|
|
39
|
+
try:
|
|
40
|
+
await poll_task
|
|
41
|
+
except asyncio.CancelledError:
|
|
42
|
+
pass
|
|
28
43
|
if self.supervisor is not None:
|
|
29
44
|
await asyncio.to_thread(self.supervisor.stop)
|
|
30
45
|
self.supervisor = None
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
1
2
|
from django.core.management.base import BaseCommand
|
|
3
|
+
from django.core.management.base import CommandError
|
|
2
4
|
|
|
3
5
|
from dj_queue.config import load_backend_config
|
|
4
6
|
from dj_queue.runtime.supervisor import AsyncSupervisor, ForkSupervisor
|
|
@@ -32,8 +34,11 @@ class Command(BaseCommand):
|
|
|
32
34
|
"only_dispatch": options["only_dispatch"],
|
|
33
35
|
"skip_recurring": options["skip_recurring"],
|
|
34
36
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
try:
|
|
38
|
+
supervisor = build_supervisor(
|
|
39
|
+
backend_alias=options["backend"],
|
|
40
|
+
cli_overrides=cli_overrides,
|
|
41
|
+
)
|
|
42
|
+
except ImproperlyConfigured as exc:
|
|
43
|
+
raise CommandError(str(exc)) from exc
|
|
39
44
|
supervisor.run()
|
|
@@ -9,6 +9,7 @@ from django.db.models import Count, Max, Min
|
|
|
9
9
|
from django.db.models.functions import Coalesce
|
|
10
10
|
from django.utils import timezone
|
|
11
11
|
|
|
12
|
+
from dj_queue.config import configured_backend_aliases as configured_dj_queue_backend_aliases
|
|
12
13
|
from dj_queue.config import load_backend_config
|
|
13
14
|
from dj_queue.db import get_database_alias
|
|
14
15
|
from dj_queue.models import (
|
|
@@ -33,11 +34,7 @@ class BackendChoice:
|
|
|
33
34
|
|
|
34
35
|
|
|
35
36
|
def configured_backend_aliases():
|
|
36
|
-
|
|
37
|
-
aliases = tuple(tasks)
|
|
38
|
-
if aliases:
|
|
39
|
-
return aliases
|
|
40
|
-
return ("default",)
|
|
37
|
+
return configured_dj_queue_backend_aliases(getattr(settings, "TASKS", {}))
|
|
41
38
|
|
|
42
39
|
|
|
43
40
|
def backend_choices():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html
RENAMED
|
File without changes
|
{dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html
RENAMED
|
File without changes
|
{dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html
RENAMED
|
File without changes
|
{dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|