dj-queue 0.5.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. {dj_queue-0.5.0 → dj_queue-0.6.0}/PKG-INFO +100 -46
  2. {dj_queue-0.5.0 → dj_queue-0.6.0}/README.md +99 -45
  3. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/admin.py +5 -1
  4. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/api.py +24 -11
  5. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/config.py +48 -1
  6. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/asgi.py +15 -0
  7. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/dj_queue.py +9 -4
  8. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/observability.py +2 -5
  9. {dj_queue-0.5.0 → dj_queue-0.6.0}/pyproject.toml +1 -1
  10. {dj_queue-0.5.0 → dj_queue-0.6.0}/LICENSE +0 -0
  11. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/__init__.py +0 -0
  12. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/apps.py +0 -0
  13. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/backend.py +0 -0
  14. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/__init__.py +0 -0
  15. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/gunicorn.py +0 -0
  16. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/contrib/prometheus.py +0 -0
  17. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/dashboard.py +0 -0
  18. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/db.py +0 -0
  19. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/exceptions.py +0 -0
  20. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/hooks.py +0 -0
  21. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/log.py +0 -0
  22. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/__init__.py +0 -0
  23. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/__init__.py +0 -0
  24. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/dj_queue_health.py +0 -0
  25. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/management/commands/dj_queue_prune.py +0 -0
  26. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0001_initial.py +0 -0
  27. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
  28. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
  29. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0004_dashboard.py +0 -0
  30. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
  31. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/migrations/__init__.py +0 -0
  32. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/__init__.py +0 -0
  33. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/jobs.py +0 -0
  34. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/recurring.py +0 -0
  35. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/models/runtime.py +0 -0
  36. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/__init__.py +0 -0
  37. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/cleanup.py +0 -0
  38. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/concurrency.py +0 -0
  39. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/jobs.py +0 -0
  40. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/operations/recurring.py +0 -0
  41. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/routers.py +0 -0
  42. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/__init__.py +0 -0
  43. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/base.py +0 -0
  44. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/dispatcher.py +0 -0
  45. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/errors.py +0 -0
  46. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/interruptible.py +0 -0
  47. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/notify.py +0 -0
  48. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/pidfile.py +0 -0
  49. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/pool.py +0 -0
  50. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/procline.py +0 -0
  51. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/scheduler.py +0 -0
  52. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/supervisor.py +0 -0
  53. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/runtime/worker.py +0 -0
  54. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
  55. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
  56. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
  57. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
  58. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
  59. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
  60. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
  61. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
  62. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
  63. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
  64. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
  65. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
  66. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templatetags/__init__.py +0 -0
  67. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/templatetags/dj_queue_admin.py +0 -0
  68. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/urls.py +0 -0
  69. {dj_queue-0.5.0 → dj_queue-0.6.0}/dj_queue/views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dj-queue
3
- Version: 0.5.0
3
+ Version: 0.6.0
4
4
  Summary: Database-backed task queue backend for Django’s Tasks framework.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -105,6 +105,12 @@ TASKS = {
105
105
  The router is optional when using the default database, but harmless to include
106
106
  and required for [multi-database setups](#multi-database-setup).
107
107
 
108
+ `dj_queue` can coexist with other Django task backends in the same `TASKS`
109
+ setting. It only manages aliases whose `BACKEND` is
110
+ `"dj_queue.backend.DjQueueBackend"`. If a `TASKS` alias points at some other
111
+ backend, `dj_queue` ignores that alias for runtime commands, admin/dashboard
112
+ selection, and observability.
113
+
108
114
  Run migrations:
109
115
 
110
116
  ```bash
@@ -266,15 +272,17 @@ Useful command variants:
266
272
  ```bash
267
273
  python manage.py dj_queue
268
274
  python manage.py dj_queue --mode async
275
+ python manage.py dj_queue --backend <alias>
269
276
  python manage.py dj_queue --only-work
270
277
  python manage.py dj_queue --only-dispatch
271
278
  python manage.py dj_queue --skip-recurring
272
279
  ```
273
280
 
274
- Mode and topology notes:
281
+ Notes:
275
282
 
276
283
  - `fork` is the default standalone mode
277
284
  - `async` is also supported as a standalone mode and runs supervised actors in threads inside one process
285
+ - `--backend` targets a non-default backend alias
278
286
  - `--only-work` starts workers without dispatchers or scheduler
279
287
  - `--only-dispatch` starts dispatchers without workers or scheduler
280
288
  - `--skip-recurring` starts without the scheduler
@@ -303,8 +311,7 @@ across queues. Prefer one primary scheduling mechanism per worker when you can.
303
311
 
304
312
  ### Signals and recovery
305
313
 
306
- In standalone mode, both `fork` and `async` `python manage.py dj_queue`
307
- supervisors own runtime signal handling:
314
+ In standalone mode, both `fork` and `async` `python manage.py dj_queue` supervisors own runtime signal handling:
308
315
 
309
316
  - `SIGTERM` and `SIGINT` request graceful shutdown
310
317
  - `SIGQUIT` takes the immediate hard-exit path
@@ -461,7 +468,7 @@ orders.resume()
461
468
  orders.clear()
462
469
  ```
463
470
 
464
- Queue control notes:
471
+ Notes:
465
472
 
466
473
  - pausing a queue stops future claims, not enqueueing or already-claimed work
467
474
  - pause rows are scoped per backend alias
@@ -480,8 +487,7 @@ python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
480
487
  python manage.py dj_queue_prune --task-key nightly_cleanup
481
488
  ```
482
489
 
483
- The runtime, health, and prune commands all accept `--backend` to target a
484
- non-default backend alias.
490
+ The health and prune commands both accept `--backend` to target a non-default backend alias.
485
491
 
486
492
  For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
487
493
  task import path, while `--task-key` filters recurring execution cleanup by
@@ -549,11 +555,44 @@ except EnqueueError as exc:
549
555
  Task execution errors are different: they become failed jobs and stay
550
556
  inspectable in the queue database.
551
557
 
552
- ## Multi-Database Setup
558
+ ## Lifecycle Hooks
559
+
560
+ Register hooks before starting the runtime, typically during Django startup.
561
+ Each callback receives the live supervisor or runner instance.
562
+
563
+ ```python
564
+ from dj_queue.hooks import on_start, on_worker_start, register_hook
565
+
566
+ @on_start
567
+ def supervisor_started(process):
568
+ print(process.name)
553
569
 
554
- `dj_queue` can keep queue tables on a dedicated database alias.
570
+ @on_worker_start
571
+ def worker_started(process):
572
+ print(process.metadata)
555
573
 
556
- Example configuration:
574
+ @register_hook("scheduler.exit")
575
+ def scheduler_exited(process):
576
+ print(process.name)
577
+ ```
578
+
579
+ Available hook helpers:
580
+
581
+ - supervisor: `on_start`, `on_stop`, `on_exit`
582
+ - worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
583
+ - dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
584
+ - scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
585
+ - generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
586
+
587
+ Notes:
588
+
589
+ - hooks fire in registration order
590
+ - hook failures do not block later hooks
591
+ - hook failures are isolated and routed through `on_thread_error`
592
+
593
+ ## Multi-Database Setup
594
+
595
+ `dj_queue` can keep queue tables on a dedicated database alias:
557
596
 
558
597
  ```python
559
598
  DATABASES = {
@@ -591,6 +630,38 @@ python manage.py migrate dj_queue --database queue
591
630
  With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
592
631
  database.
593
632
 
633
+ ## Backend Coexistence
634
+
635
+ Projects can mix `dj_queue` with other Django task backends in the same `TASKS` mapping:
636
+
637
+ ```python
638
+ TASKS = {
639
+ "default": {
640
+ "BACKEND": "dj_queue.backend.DjQueueBackend",
641
+ "QUEUES": [],
642
+ "OPTIONS": {
643
+ "database_alias": "queue",
644
+ },
645
+ },
646
+ "external": {
647
+ "BACKEND": "some_other_backend.Backend",
648
+ "QUEUES": [],
649
+ "OPTIONS": {},
650
+ },
651
+ }
652
+ ```
653
+
654
+ In that setup:
655
+
656
+ - jobs with `backend="default"` are `dj_queue`'s responsibility
657
+ - jobs with `backend="external"` are the other backend's responsibility
658
+ - `dj_queue` admin, dashboard, `/dj_queue/stats.json`, `/dj_queue/metrics`, and `manage.py dj_queue --backend ...` only operate on `dj_queue` aliases
659
+
660
+ Notes:
661
+
662
+ - if `TASKS` is empty or unset, `dj_queue` still exposes one implicit `default` alias using built-in defaults
663
+ - if `TASKS` is non-empty, `dj_queue` only manages aliases whose `BACKEND` is explicitly `"dj_queue.backend.DjQueueBackend"`
664
+
594
665
  ## Postgres Queue Health
595
666
 
596
667
  Operational and configuration guidance for scaling with `dj_queue` in
@@ -644,6 +715,21 @@ signal handling to the host server.
644
715
 
645
716
  ## Configuration
646
717
 
718
+ ### Queues, backends, and databases
719
+
720
+ `dj_queue` has three separate routing concepts. Keep them distinct:
721
+
722
+ - `queue_name`: what kind of work this job is. Use it to route lanes inside one backend, such as `email`, `webhooks`, or `search-index`.
723
+ - `backend_alias`: which logical queue system owns the work. Use it when you want separate runtime config, recurring tasks, pause and process visibility, retention, or admin scoping.
724
+ - `database_alias`: where that backend's queue tables and runtime activity live. Use it when you want a dedicated database connection path or stronger storage isolation.
725
+
726
+ Common setup choices:
727
+
728
+ - one backend, one database: simplest and usually enough
729
+ - one backend, separate queue database: good when you want dedicated queue connections
730
+ - multiple backends, same database: good for logical and operational separation without another database
731
+ - multiple backends, multiple databases: use when you need stronger isolation and accept more migration and deployment complexity
732
+
647
733
  ### Deployment topology
648
734
 
649
735
  Once migrations are in place, start processing jobs with `python manage.py dj_queue`
@@ -792,41 +878,6 @@ Environment overrides currently supported by `dj_queue` itself:
792
878
  - `DJ_QUEUE_MODE`
793
879
  - `DJ_QUEUE_SKIP_RECURRING`
794
880
 
795
- ## Lifecycle Hooks
796
-
797
- Register hooks before starting the runtime, typically during Django startup.
798
- Each callback receives the live supervisor or runner instance.
799
-
800
- ```python
801
- from dj_queue.hooks import on_start, on_worker_start, register_hook
802
-
803
- @on_start
804
- def supervisor_started(process):
805
- print(process.name)
806
-
807
- @on_worker_start
808
- def worker_started(process):
809
- print(process.metadata)
810
-
811
- @register_hook("scheduler.exit")
812
- def scheduler_exited(process):
813
- print(process.name)
814
- ```
815
-
816
- Available hook helpers:
817
-
818
- - supervisor: `on_start`, `on_stop`, `on_exit`
819
- - worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
820
- - dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
821
- - scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
822
- - generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
823
-
824
- Hook notes:
825
-
826
- - hooks fire in registration order
827
- - hook failures do not block later hooks
828
- - hook failures are isolated and routed through `on_thread_error`
829
-
830
881
  ### Runtime infrastructure errors
831
882
 
832
883
  Set `on_thread_error` to a dotted callable path when you want custom handling
@@ -852,7 +903,10 @@ become failed jobs instead.
852
903
  ## Monitoring
853
904
 
854
905
  Queue statistics are available in JSON via `/dj_queue/stats.json` and in
855
- Prometheus text format via `/dj_queue/metrics`.
906
+ Prometheus text format via `/dj_queue/metrics`.
907
+
908
+ These observability endpoints only report `dj_queue`-managed backend aliases and
909
+ ignore aliases configured for some other task backend.
856
910
 
857
911
  Include `dj_queue.urls` to expose them:
858
912
 
@@ -77,6 +77,12 @@ TASKS = {
77
77
  The router is optional when using the default database, but harmless to include
78
78
  and required for [multi-database setups](#multi-database-setup).
79
79
 
80
+ `dj_queue` can coexist with other Django task backends in the same `TASKS`
81
+ setting. It only manages aliases whose `BACKEND` is
82
+ `"dj_queue.backend.DjQueueBackend"`. If a `TASKS` alias points at some other
83
+ backend, `dj_queue` ignores that alias for runtime commands, admin/dashboard
84
+ selection, and observability.
85
+
80
86
  Run migrations:
81
87
 
82
88
  ```bash
@@ -238,15 +244,17 @@ Useful command variants:
238
244
  ```bash
239
245
  python manage.py dj_queue
240
246
  python manage.py dj_queue --mode async
247
+ python manage.py dj_queue --backend <alias>
241
248
  python manage.py dj_queue --only-work
242
249
  python manage.py dj_queue --only-dispatch
243
250
  python manage.py dj_queue --skip-recurring
244
251
  ```
245
252
 
246
- Mode and topology notes:
253
+ Notes:
247
254
 
248
255
  - `fork` is the default standalone mode
249
256
  - `async` is also supported as a standalone mode and runs supervised actors in threads inside one process
257
+ - `--backend` targets a non-default backend alias
250
258
  - `--only-work` starts workers without dispatchers or scheduler
251
259
  - `--only-dispatch` starts dispatchers without workers or scheduler
252
260
  - `--skip-recurring` starts without the scheduler
@@ -275,8 +283,7 @@ across queues. Prefer one primary scheduling mechanism per worker when you can.
275
283
 
276
284
  ### Signals and recovery
277
285
 
278
- In standalone mode, both `fork` and `async` `python manage.py dj_queue`
279
- supervisors own runtime signal handling:
286
+ In standalone mode, both `fork` and `async` `python manage.py dj_queue` supervisors own runtime signal handling:
280
287
 
281
288
  - `SIGTERM` and `SIGINT` request graceful shutdown
282
289
  - `SIGQUIT` takes the immediate hard-exit path
@@ -433,7 +440,7 @@ orders.resume()
433
440
  orders.clear()
434
441
  ```
435
442
 
436
- Queue control notes:
443
+ Notes:
437
444
 
438
445
  - pausing a queue stops future claims, not enqueueing or already-claimed work
439
446
  - pause rows are scoped per backend alias
@@ -452,8 +459,7 @@ python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
452
459
  python manage.py dj_queue_prune --task-key nightly_cleanup
453
460
  ```
454
461
 
455
- The runtime, health, and prune commands all accept `--backend` to target a
456
- non-default backend alias.
462
+ The health and prune commands both accept `--backend` to target a non-default backend alias.
457
463
 
458
464
  For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
459
465
  task import path, while `--task-key` filters recurring execution cleanup by
@@ -521,11 +527,44 @@ except EnqueueError as exc:
521
527
  Task execution errors are different: they become failed jobs and stay
522
528
  inspectable in the queue database.
523
529
 
524
- ## Multi-Database Setup
530
+ ## Lifecycle Hooks
531
+
532
+ Register hooks before starting the runtime, typically during Django startup.
533
+ Each callback receives the live supervisor or runner instance.
534
+
535
+ ```python
536
+ from dj_queue.hooks import on_start, on_worker_start, register_hook
537
+
538
+ @on_start
539
+ def supervisor_started(process):
540
+ print(process.name)
525
541
 
526
- `dj_queue` can keep queue tables on a dedicated database alias.
542
+ @on_worker_start
543
+ def worker_started(process):
544
+ print(process.metadata)
527
545
 
528
- Example configuration:
546
+ @register_hook("scheduler.exit")
547
+ def scheduler_exited(process):
548
+ print(process.name)
549
+ ```
550
+
551
+ Available hook helpers:
552
+
553
+ - supervisor: `on_start`, `on_stop`, `on_exit`
554
+ - worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
555
+ - dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
556
+ - scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
557
+ - generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
558
+
559
+ Notes:
560
+
561
+ - hooks fire in registration order
562
+ - hook failures do not block later hooks
563
+ - hook failures are isolated and routed through `on_thread_error`
564
+
565
+ ## Multi-Database Setup
566
+
567
+ `dj_queue` can keep queue tables on a dedicated database alias:
529
568
 
530
569
  ```python
531
570
  DATABASES = {
@@ -563,6 +602,38 @@ python manage.py migrate dj_queue --database queue
563
602
  With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
564
603
  database.
565
604
 
605
+ ## Backend Coexistence
606
+
607
+ Projects can mix `dj_queue` with other Django task backends in the same `TASKS` mapping:
608
+
609
+ ```python
610
+ TASKS = {
611
+ "default": {
612
+ "BACKEND": "dj_queue.backend.DjQueueBackend",
613
+ "QUEUES": [],
614
+ "OPTIONS": {
615
+ "database_alias": "queue",
616
+ },
617
+ },
618
+ "external": {
619
+ "BACKEND": "some_other_backend.Backend",
620
+ "QUEUES": [],
621
+ "OPTIONS": {},
622
+ },
623
+ }
624
+ ```
625
+
626
+ In that setup:
627
+
628
+ - jobs with `backend="default"` are `dj_queue`'s responsibility
629
+ - jobs with `backend="external"` are the other backend's responsibility
630
+ - `dj_queue` admin, dashboard, `/dj_queue/stats.json`, `/dj_queue/metrics`, and `manage.py dj_queue --backend ...` only operate on `dj_queue` aliases
631
+
632
+ Notes:
633
+
634
+ - if `TASKS` is empty or unset, `dj_queue` still exposes one implicit `default` alias using built-in defaults
635
+ - if `TASKS` is non-empty, `dj_queue` only manages aliases whose `BACKEND` is explicitly `"dj_queue.backend.DjQueueBackend"`
636
+
566
637
  ## Postgres Queue Health
567
638
 
568
639
  Operational and configuration guidance for scaling with `dj_queue` in
@@ -616,6 +687,21 @@ signal handling to the host server.
616
687
 
617
688
  ## Configuration
618
689
 
690
+ ### Queues, backends, and databases
691
+
692
+ `dj_queue` has three separate routing concepts. Keep them distinct:
693
+
694
+ - `queue_name`: what kind of work this job is. Use it to route lanes inside one backend, such as `email`, `webhooks`, or `search-index`.
695
+ - `backend_alias`: which logical queue system owns the work. Use it when you want separate runtime config, recurring tasks, pause and process visibility, retention, or admin scoping.
696
+ - `database_alias`: where that backend's queue tables and runtime activity live. Use it when you want a dedicated database connection path or stronger storage isolation.
697
+
698
+ Common setup choices:
699
+
700
+ - one backend, one database: simplest and usually enough
701
+ - one backend, separate queue database: good when you want dedicated queue connections
702
+ - multiple backends, same database: good for logical and operational separation without another database
703
+ - multiple backends, multiple databases: use when you need stronger isolation and accept more migration and deployment complexity
704
+
619
705
  ### Deployment topology
620
706
 
621
707
  Once migrations are in place, start processing jobs with `python manage.py dj_queue`
@@ -764,41 +850,6 @@ Environment overrides currently supported by `dj_queue` itself:
764
850
  - `DJ_QUEUE_MODE`
765
851
  - `DJ_QUEUE_SKIP_RECURRING`
766
852
 
767
- ## Lifecycle Hooks
768
-
769
- Register hooks before starting the runtime, typically during Django startup.
770
- Each callback receives the live supervisor or runner instance.
771
-
772
- ```python
773
- from dj_queue.hooks import on_start, on_worker_start, register_hook
774
-
775
- @on_start
776
- def supervisor_started(process):
777
- print(process.name)
778
-
779
- @on_worker_start
780
- def worker_started(process):
781
- print(process.metadata)
782
-
783
- @register_hook("scheduler.exit")
784
- def scheduler_exited(process):
785
- print(process.name)
786
- ```
787
-
788
- Available hook helpers:
789
-
790
- - supervisor: `on_start`, `on_stop`, `on_exit`
791
- - worker: `on_worker_start`, `on_worker_stop`, `on_worker_exit`
792
- - dispatcher: `on_dispatcher_start`, `on_dispatcher_stop`, `on_dispatcher_exit`
793
- - scheduler: `on_scheduler_start`, `on_scheduler_stop`, `on_scheduler_exit`
794
- - generic events: `register_hook("worker.start")`, `register_hook("dispatcher.stop")`, and so on
795
-
796
- Hook notes:
797
-
798
- - hooks fire in registration order
799
- - hook failures do not block later hooks
800
- - hook failures are isolated and routed through `on_thread_error`
801
-
802
853
  ### Runtime infrastructure errors
803
854
 
804
855
  Set `on_thread_error` to a dotted callable path when you want custom handling
@@ -824,7 +875,10 @@ become failed jobs instead.
824
875
  ## Monitoring
825
876
 
826
877
  Queue statistics are available in JSON via `/dj_queue/stats.json` and in
827
- Prometheus text format via `/dj_queue/metrics`.
878
+ Prometheus text format via `/dj_queue/metrics`.
879
+
880
+ These observability endpoints only report `dj_queue`-managed backend aliases and
881
+ ignore aliases configured for some other task backend.
828
882
 
829
883
  Include `dj_queue.urls` to expose them:
830
884
 
@@ -458,8 +458,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
458
458
  )
459
459
  readonly_fields = (
460
460
  "task_path",
461
- "queue_name",
461
+ "queue_name_link",
462
462
  "priority",
463
+ "display_status",
463
464
  "payload",
464
465
  "backend_alias",
465
466
  "scheduled_at",
@@ -475,6 +476,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
475
476
  )
476
477
  search_fields = ("id", "task_path", "queue_name", "concurrency_key")
477
478
 
479
+ def get_fields(self, request, obj=None):
480
+ return self.get_readonly_fields(request, obj)
481
+
478
482
  def get_readonly_fields(self, request, obj=None):
479
483
  fields = super().get_readonly_fields(request, obj)
480
484
  if obj is None or obj.status != "failed":
@@ -1,5 +1,6 @@
1
1
  from functools import partial
2
2
 
3
+ from django.db.models import Case, DateTimeField, DurationField, ExpressionWrapper, F, Value, When
3
4
  from django.db.models.functions import Coalesce
4
5
  from django.db import transaction
5
6
  from django.utils import timezone
@@ -66,17 +67,29 @@ class QueueInfo:
66
67
 
67
68
  resumed_at = timezone.now()
68
69
  paused_at = pause.created_at
69
- ready_rows = list(
70
- ReadyExecution.objects.using(alias)
71
- .filter(queue_name=self.queue_name)
72
- .only("id", "created_at", "latency_started_at")
73
- )
74
- for ready_row in ready_rows:
75
- started_at = ready_row.latency_started_at or ready_row.created_at
76
- overlap_started_at = max(started_at, paused_at)
77
- ready_row.latency_started_at = started_at + (resumed_at - overlap_started_at)
78
- if ready_rows:
79
- ReadyExecution.objects.using(alias).bulk_update(ready_rows, ["latency_started_at"])
70
+ pause_duration = Value(resumed_at - paused_at, output_field=DurationField())
71
+ ready_row_ids = list(self._ready_queryset().values_list("id", flat=True))
72
+ if ready_row_ids:
73
+ ReadyExecution.objects.using(alias).filter(pk__in=ready_row_ids).update(
74
+ latency_started_at=Case(
75
+ When(
76
+ latency_started_at__isnull=True,
77
+ created_at__lt=paused_at,
78
+ then=ExpressionWrapper(
79
+ F("created_at") + pause_duration, output_field=DateTimeField()
80
+ ),
81
+ ),
82
+ When(
83
+ latency_started_at__lt=paused_at,
84
+ then=ExpressionWrapper(
85
+ F("latency_started_at") + pause_duration,
86
+ output_field=DateTimeField(),
87
+ ),
88
+ ),
89
+ default=Value(resumed_at, output_field=DateTimeField()),
90
+ output_field=DateTimeField(),
91
+ ),
92
+ )
80
93
  pause.delete()
81
94
 
82
95
  log_event("queue.resumed", backend_alias=self.backend_alias, queue_name=self.queue_name)
@@ -54,6 +54,8 @@ DEFAULT_OPTIONS = {
54
54
  "on_thread_error": None,
55
55
  }
56
56
 
57
+ DJ_QUEUE_BACKEND_PATH = "dj_queue.backend.DjQueueBackend"
58
+
57
59
  TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"}
58
60
  FALSY_ENV_VALUES = {"0", "false", "no", "off"}
59
61
  CONFIG_ENV_KEYS = ("DJ_QUEUE_CONFIG", "DJ_QUEUE_MODE", "DJ_QUEUE_SKIP_RECURRING")
@@ -163,6 +165,7 @@ def _load_backend_config_cached(
163
165
  cli_overrides = json.loads(cli_overrides_key)
164
166
  env = json.loads(env_key)
165
167
  tasks_settings = json.loads(tasks_settings_key)
168
+ ensure_dj_queue_backend_alias(tasks_settings, backend_alias)
166
169
  backend_block = _backend_block(tasks_settings, backend_alias)
167
170
  resolved_options = _resolved_options(backend_alias, backend_block, cli_overrides, env)
168
171
 
@@ -241,12 +244,56 @@ def _backend_block(
241
244
  if resolved_tasks_settings is None:
242
245
  resolved_tasks_settings = getattr(settings, "TASKS", {})
243
246
 
244
- backend_block = resolved_tasks_settings.get(backend_alias, {})
247
+ if not resolved_tasks_settings:
248
+ if backend_alias == "default":
249
+ return {}
250
+ raise ImproperlyConfigured(f"dj_queue backend alias {backend_alias!r} is not configured")
251
+
252
+ if backend_alias not in resolved_tasks_settings:
253
+ raise ImproperlyConfigured(f"dj_queue backend alias {backend_alias!r} is not configured")
254
+
255
+ backend_block = resolved_tasks_settings[backend_alias]
245
256
  if not isinstance(backend_block, Mapping):
246
257
  raise ImproperlyConfigured(f"TASKS[{backend_alias!r}] must be a mapping")
247
258
  return backend_block
248
259
 
249
260
 
261
+ def configured_backend_aliases(
262
+ tasks_settings: Mapping[str, Any] | None = None,
263
+ ) -> tuple[str, ...]:
264
+ resolved_tasks_settings = tasks_settings
265
+ if resolved_tasks_settings is None:
266
+ resolved_tasks_settings = getattr(settings, "TASKS", {})
267
+
268
+ if not resolved_tasks_settings:
269
+ return ("default",)
270
+
271
+ aliases = []
272
+ for alias, backend_block in resolved_tasks_settings.items():
273
+ if not isinstance(backend_block, Mapping):
274
+ raise ImproperlyConfigured(f"TASKS[{alias!r}] must be a mapping")
275
+ if is_dj_queue_backend_alias(backend_block):
276
+ aliases.append(alias)
277
+ return tuple(aliases)
278
+
279
+
280
+ def ensure_dj_queue_backend_alias(
281
+ tasks_settings: Mapping[str, Any] | None,
282
+ backend_alias: str,
283
+ ) -> None:
284
+ aliases = configured_backend_aliases(tasks_settings)
285
+ if backend_alias in aliases:
286
+ return
287
+ raise ImproperlyConfigured(
288
+ f"dj_queue backend alias {backend_alias!r} is not configured for DjQueueBackend"
289
+ )
290
+
291
+
292
+ def is_dj_queue_backend_alias(backend_block: Mapping[str, Any]) -> bool:
293
+ backend_path = backend_block.get("BACKEND")
294
+ return backend_path == DJ_QUEUE_BACKEND_PATH
295
+
296
+
250
297
  def _resolved_options(
251
298
  backend_alias: str,
252
299
  backend_block: Mapping[str, Any],
@@ -12,6 +12,12 @@ class DjQueueLifespan:
12
12
  self.app = app
13
13
  self.backend_alias = backend_alias
14
14
  self.supervisor = None
15
+ self._poll_task = None
16
+
17
+ async def _poll_supervisor(self):
18
+ while self.supervisor is not None:
19
+ await asyncio.to_thread(self.supervisor.poll_once)
20
+ await asyncio.sleep(self.supervisor.polling_interval)
15
21
 
16
22
  async def __call__(self, scope, receive, send):
17
23
  if scope["type"] != "lifespan":
@@ -23,8 +29,17 @@ class DjQueueLifespan:
23
29
  if message["type"] == "lifespan.startup":
24
30
  self.supervisor = build_supervisor(self.backend_alias)
25
31
  await asyncio.to_thread(self.supervisor.start)
32
+ self._poll_task = asyncio.create_task(self._poll_supervisor())
26
33
  await send({"type": "lifespan.startup.complete"})
27
34
  elif message["type"] == "lifespan.shutdown":
35
+ poll_task = self._poll_task
36
+ self._poll_task = None
37
+ if poll_task is not None:
38
+ poll_task.cancel()
39
+ try:
40
+ await poll_task
41
+ except asyncio.CancelledError:
42
+ pass
28
43
  if self.supervisor is not None:
29
44
  await asyncio.to_thread(self.supervisor.stop)
30
45
  self.supervisor = None
@@ -1,4 +1,6 @@
1
+ from django.core.exceptions import ImproperlyConfigured
1
2
  from django.core.management.base import BaseCommand
3
+ from django.core.management.base import CommandError
2
4
 
3
5
  from dj_queue.config import load_backend_config
4
6
  from dj_queue.runtime.supervisor import AsyncSupervisor, ForkSupervisor
@@ -32,8 +34,11 @@ class Command(BaseCommand):
32
34
  "only_dispatch": options["only_dispatch"],
33
35
  "skip_recurring": options["skip_recurring"],
34
36
  }
35
- supervisor = build_supervisor(
36
- backend_alias=options["backend"],
37
- cli_overrides=cli_overrides,
38
- )
37
+ try:
38
+ supervisor = build_supervisor(
39
+ backend_alias=options["backend"],
40
+ cli_overrides=cli_overrides,
41
+ )
42
+ except ImproperlyConfigured as exc:
43
+ raise CommandError(str(exc)) from exc
39
44
  supervisor.run()
@@ -9,6 +9,7 @@ from django.db.models import Count, Max, Min
9
9
  from django.db.models.functions import Coalesce
10
10
  from django.utils import timezone
11
11
 
12
+ from dj_queue.config import configured_backend_aliases as configured_dj_queue_backend_aliases
12
13
  from dj_queue.config import load_backend_config
13
14
  from dj_queue.db import get_database_alias
14
15
  from dj_queue.models import (
@@ -33,11 +34,7 @@ class BackendChoice:
33
34
 
34
35
 
35
36
  def configured_backend_aliases():
36
- tasks = getattr(settings, "TASKS", {})
37
- aliases = tuple(tasks)
38
- if aliases:
39
- return aliases
40
- return ("default",)
37
+ return configured_dj_queue_backend_aliases(getattr(settings, "TASKS", {}))
41
38
 
42
39
 
43
40
  def backend_choices():
@@ -4,7 +4,7 @@ build-backend = "uv_build"
4
4
 
5
5
  [project]
6
6
  name = "dj-queue"
7
- version = "0.5.0"
7
+ version = "0.6.0"
8
8
  description = "Database-backed task queue backend for Django’s Tasks framework."
9
9
  readme = "README.md"
10
10
  license = "MIT"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes