dj-queue 0.9.0__tar.gz → 0.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {dj_queue-0.9.0 → dj_queue-0.9.2}/PKG-INFO +65 -21
  2. {dj_queue-0.9.0 → dj_queue-0.9.2}/README.md +64 -20
  3. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/admin.py +16 -35
  4. dj_queue-0.9.2/dj_queue/backend.py +91 -0
  5. dj_queue-0.9.2/dj_queue/contrib/prometheus.py +26 -0
  6. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/dashboard.py +33 -69
  7. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/log.py +12 -1
  8. dj_queue-0.9.2/dj_queue/metrics.py +163 -0
  9. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/observability.py +15 -38
  10. dj_queue-0.9.2/dj_queue/operations/_helpers.py +197 -0
  11. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/concurrency.py +28 -20
  12. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/jobs.py +252 -165
  13. dj_queue-0.9.2/dj_queue/queue_selectors.py +62 -0
  14. dj_queue-0.9.2/dj_queue/queue_state.py +138 -0
  15. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/base.py +34 -14
  16. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/notify.py +4 -18
  17. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/supervisor.py +52 -182
  18. dj_queue-0.9.2/dj_queue/runtime/topology.py +51 -0
  19. dj_queue-0.9.2/dj_queue/task_results.py +107 -0
  20. {dj_queue-0.9.0 → dj_queue-0.9.2}/pyproject.toml +1 -1
  21. dj_queue-0.9.0/dj_queue/backend.py +0 -170
  22. dj_queue-0.9.0/dj_queue/contrib/prometheus.py +0 -128
  23. dj_queue-0.9.0/dj_queue/operations/_helpers.py +0 -41
  24. {dj_queue-0.9.0 → dj_queue-0.9.2}/LICENSE +0 -0
  25. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/__init__.py +0 -0
  26. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/api.py +0 -0
  27. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/apps.py +0 -0
  28. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/config.py +0 -0
  29. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/contrib/__init__.py +0 -0
  30. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/contrib/asgi.py +0 -0
  31. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/contrib/gunicorn.py +0 -0
  32. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/cron.py +0 -0
  33. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/db.py +0 -0
  34. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/exceptions.py +0 -0
  35. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/hooks.py +0 -0
  36. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/management/__init__.py +0 -0
  37. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/management/commands/__init__.py +0 -0
  38. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue.py +0 -0
  39. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue_health.py +0 -0
  40. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue_prune.py +0 -0
  41. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0001_initial.py +0 -0
  42. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
  43. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
  44. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0004_dashboard.py +0 -0
  45. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
  46. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
  47. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
  48. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
  49. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/migrations/__init__.py +0 -0
  50. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/models/__init__.py +0 -0
  51. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/models/jobs.py +0 -0
  52. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/models/recurring.py +0 -0
  53. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/models/runtime.py +0 -0
  54. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/__init__.py +0 -0
  55. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/_insert.py +0 -0
  56. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/cleanup.py +0 -0
  57. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/queues.py +0 -0
  58. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/operations/recurring.py +0 -0
  59. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/routers.py +0 -0
  60. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/__init__.py +0 -0
  61. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/connection_budget.py +0 -0
  62. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/dispatcher.py +0 -0
  63. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/errors.py +0 -0
  64. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/interruptible.py +0 -0
  65. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/pidfile.py +0 -0
  66. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/pool.py +0 -0
  67. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/procline.py +0 -0
  68. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/scheduler.py +0 -0
  69. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/runtime/worker.py +0 -0
  70. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
  71. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
  72. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
  73. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
  74. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
  75. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
  76. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
  77. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
  78. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
  79. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
  80. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
  81. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
  82. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templatetags/__init__.py +0 -0
  83. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/templatetags/dj_queue_admin.py +0 -0
  84. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/urls.py +0 -0
  85. {dj_queue-0.9.0 → dj_queue-0.9.2}/dj_queue/views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dj-queue
3
- Version: 0.9.0
3
+ Version: 0.9.2
4
4
  Summary: Database-backed task queue backend for Django’s Tasks framework.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -780,26 +780,70 @@ concurrency-maintenance throughput.
780
780
 
781
781
  The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
782
782
 
783
- Start with these options:
784
-
785
- - `mode`: `"fork"` or `"async"`
786
- - `workers`: queue selectors, thread counts, and process counts
787
- - `dispatchers`: scheduled promotion and concurrency maintenance settings
788
- - `scheduler`: dynamic recurring polling settings
789
- - `database_alias`: database alias for queue tables and runtime activity
790
- - `preserve_finished_jobs` and `clear_finished_jobs_after`: successful result retention and cleanup
791
- - `clear_failed_jobs_after`: optional failed-job retention window
792
- - `clear_recurring_executions_after`: optional recurring reservation retention window
793
-
794
- Additional operational tuning is available when needed:
795
-
796
- - `use_skip_locked`: use `SKIP LOCKED` when the active backend supports it
797
- - `listen_notify`: PostgreSQL-only worker wakeup optimization layered on top of polling
798
- - `silence_polling`: suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger
799
- - `process_heartbeat_interval` and `process_alive_threshold`: process liveness reporting and stale-runner detection
800
- - `shutdown_timeout`: graceful drain window before standalone shutdown gives up on waiting
801
- - `supervisor_pidfile`: optional pidfile guard for standalone supervisors
802
- - `on_thread_error`: dotted callback path for runtime infrastructure exceptions
783
+ No top-level `OPTIONS` key is required. Omit a key to use its default. Static
784
+ `recurring` entries are the exception: each named recurring task requires
785
+ `task_path` and `schedule`.
786
+
787
+ Global options:
788
+
789
+ | Option | Default | Meaning |
790
+ |---|---|---|
791
+ | `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
792
+ | `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
793
+ | `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
794
+ | `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
795
+ | `recurring` | `{}` | static recurring task definitions keyed by name |
796
+ | `database_alias` | `"default"` | database alias for queue tables and runtime activity |
797
+ | `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
798
+ | `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
799
+ | `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
800
+ | `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
801
+ | `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
802
+ | `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
803
+ | `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
804
+ | `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
805
+ | `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
806
+ | `process_alive_threshold` | `300` | seconds before a process row is stale |
807
+ | `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
808
+ | `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
809
+ | `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
810
+
811
+ Worker entry options:
812
+
813
+ | Option | Default | Meaning |
814
+ |---|---|---|
815
+ | `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
816
+ | `threads` | `3` | worker threads per worker process |
817
+ | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
818
+ | `polling_interval` | `0.1` | seconds between worker polls |
819
+
820
+ Dispatcher entry options:
821
+
822
+ | Option | Default | Meaning |
823
+ |---|---|---|
824
+ | `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
825
+ | `polling_interval` | `1` | seconds between dispatcher polls |
826
+ | `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
827
+ | `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
828
+
829
+ Scheduler entry options:
830
+
831
+ | Option | Default | Meaning |
832
+ |---|---|---|
833
+ | `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
834
+ | `polling_interval` | `5` | seconds between scheduler polls |
835
+
836
+ Recurring entry options:
837
+
838
+ | Option | Default | Meaning |
839
+ |---|---|---|
840
+ | `task_path` | none | required dotted import path for the task to enqueue |
841
+ | `schedule` | none | required cron or supported Fugit-style cronish schedule |
842
+ | `args` | `[]` | positional arguments for the task |
843
+ | `kwargs` | `{}` | keyword arguments for the task |
844
+ | `queue_name` | `"default"` | queue used for jobs created from this recurring task |
845
+ | `priority` | `0` | priority used for jobs created from this recurring task |
846
+ | `description` | `""` | operator-facing description |
803
847
 
804
848
  On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
805
849
  configuration as the main database connection. Install a compatible driver in
@@ -754,26 +754,70 @@ concurrency-maintenance throughput.
754
754
 
755
755
  The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
756
756
 
757
- Start with these options:
758
-
759
- - `mode`: `"fork"` or `"async"`
760
- - `workers`: queue selectors, thread counts, and process counts
761
- - `dispatchers`: scheduled promotion and concurrency maintenance settings
762
- - `scheduler`: dynamic recurring polling settings
763
- - `database_alias`: database alias for queue tables and runtime activity
764
- - `preserve_finished_jobs` and `clear_finished_jobs_after`: successful result retention and cleanup
765
- - `clear_failed_jobs_after`: optional failed-job retention window
766
- - `clear_recurring_executions_after`: optional recurring reservation retention window
767
-
768
- Additional operational tuning is available when needed:
769
-
770
- - `use_skip_locked`: use `SKIP LOCKED` when the active backend supports it
771
- - `listen_notify`: PostgreSQL-only worker wakeup optimization layered on top of polling
772
- - `silence_polling`: suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger
773
- - `process_heartbeat_interval` and `process_alive_threshold`: process liveness reporting and stale-runner detection
774
- - `shutdown_timeout`: graceful drain window before standalone shutdown gives up on waiting
775
- - `supervisor_pidfile`: optional pidfile guard for standalone supervisors
776
- - `on_thread_error`: dotted callback path for runtime infrastructure exceptions
757
+ No top-level `OPTIONS` key is required. Omit a key to use its default. Static
758
+ `recurring` entries are the exception: each named recurring task requires
759
+ `task_path` and `schedule`.
760
+
761
+ Global options:
762
+
763
+ | Option | Default | Meaning |
764
+ |---|---|---|
765
+ | `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
766
+ | `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
767
+ | `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
768
+ | `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
769
+ | `recurring` | `{}` | static recurring task definitions keyed by name |
770
+ | `database_alias` | `"default"` | database alias for queue tables and runtime activity |
771
+ | `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
772
+ | `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
773
+ | `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
774
+ | `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
775
+ | `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
776
+ | `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
777
+ | `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
778
+ | `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
779
+ | `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
780
+ | `process_alive_threshold` | `300` | seconds before a process row is stale |
781
+ | `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
782
+ | `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
783
+ | `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
784
+
785
+ Worker entry options:
786
+
787
+ | Option | Default | Meaning |
788
+ |---|---|---|
789
+ | `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
790
+ | `threads` | `3` | worker threads per worker process |
791
+ | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
792
+ | `polling_interval` | `0.1` | seconds between worker polls |
793
+
794
+ Dispatcher entry options:
795
+
796
+ | Option | Default | Meaning |
797
+ |---|---|---|
798
+ | `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
799
+ | `polling_interval` | `1` | seconds between dispatcher polls |
800
+ | `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
801
+ | `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
802
+
803
+ Scheduler entry options:
804
+
805
+ | Option | Default | Meaning |
806
+ |---|---|---|
807
+ | `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
808
+ | `polling_interval` | `5` | seconds between scheduler polls |
809
+
810
+ Recurring entry options:
811
+
812
+ | Option | Default | Meaning |
813
+ |---|---|---|
814
+ | `task_path` | none | required dotted import path for the task to enqueue |
815
+ | `schedule` | none | required cron or supported Fugit-style cronish schedule |
816
+ | `args` | `[]` | positional arguments for the task |
817
+ | `kwargs` | `{}` | keyword arguments for the task |
818
+ | `queue_name` | `"default"` | queue used for jobs created from this recurring task |
819
+ | `priority` | `0` | priority used for jobs created from this recurring task |
820
+ | `description` | `""` | operator-facing description |
777
821
 
778
822
  On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
779
823
  configuration as the main database connection. Install a compatible driver in
@@ -29,12 +29,19 @@ from dj_queue.models import (
29
29
  Semaphore,
30
30
  )
31
31
  from dj_queue.operations.jobs import (
32
+ DispatchOutcome,
32
33
  discard_failed_job,
33
34
  dispatch_scheduled_job_now,
34
35
  enqueue_job_again,
35
36
  retry_failed_job,
36
37
  retry_failed_jobs,
37
38
  )
39
+ from dj_queue.queue_state import (
40
+ QUEUE_STATES,
41
+ filter_queue_state,
42
+ is_queue_state,
43
+ status_rank_expression,
44
+ )
38
45
 
39
46
 
40
47
  class DjQueueFirstAdminSite(admin.AdminSite):
@@ -327,29 +334,12 @@ class JobStatusListFilter(admin.SimpleListFilter):
327
334
  parameter_name = "status"
328
335
 
329
336
  def lookups(self, request, model_admin):
330
- return (
331
- ("ready", "ready"),
332
- ("scheduled", "scheduled"),
333
- ("claimed", "claimed"),
334
- ("blocked", "blocked"),
335
- ("failed", "failed"),
336
- ("finished", "finished"),
337
- )
337
+ return QUEUE_STATES
338
338
 
339
339
  def queryset(self, request, queryset):
340
340
  value = self.value()
341
- if value == "ready":
342
- return queryset.ready()
343
- if value == "scheduled":
344
- return queryset.scheduled()
345
- if value == "claimed":
346
- return queryset.claimed()
347
- if value == "blocked":
348
- return queryset.blocked()
349
- if value == "failed":
350
- return queryset.failed()
351
- if value == "finished":
352
- return queryset.finished()
341
+ if is_queue_state(value):
342
+ return filter_queue_state(queryset, value)
353
343
  return queryset
354
344
 
355
345
 
@@ -503,18 +493,7 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
503
493
 
504
494
  def get_queryset(self, request):
505
495
  queryset = super().get_queryset(request)
506
- return queryset.annotate(
507
- status_rank=Case(
508
- When(ready_execution__isnull=False, then=Value(0)),
509
- When(scheduled_execution__isnull=False, then=Value(1)),
510
- When(claimed_execution__isnull=False, then=Value(2)),
511
- When(blocked_execution__isnull=False, then=Value(3)),
512
- When(failed_execution__isnull=False, then=Value(4)),
513
- When(finished_at__isnull=False, then=Value(5)),
514
- default=Value(99),
515
- output_field=IntegerField(),
516
- )
517
- )
496
+ return queryset.annotate(status_rank=status_rank_expression())
518
497
 
519
498
  @admin.display(description="status", ordering="status_rank")
520
499
  def display_status(self, obj):
@@ -600,15 +579,17 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
600
579
  def handle_change_action(self, request, obj, action):
601
580
  if action == "run_now":
602
581
  try:
603
- _job, dispatched_as = dispatch_scheduled_job_now(obj.pk, backend_alias=obj.backend_alias)
582
+ _job, dispatch_outcome = dispatch_scheduled_job_now(
583
+ obj.pk, backend_alias=obj.backend_alias
584
+ )
604
585
  except (EnqueueError, ImportError, AttributeError) as exc:
605
586
  self.message_user(request, f"Could not dispatch job now: {exc}", level=messages.ERROR)
606
587
  return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
607
588
 
608
589
  message = "Dispatched scheduled job for immediate execution"
609
- if dispatched_as == "blocked":
590
+ if dispatch_outcome is DispatchOutcome.BLOCKED:
610
591
  message = "Dispatched scheduled job immediately and it is now blocked"
611
- if dispatched_as == "discarded":
592
+ if dispatch_outcome is DispatchOutcome.DISCARDED:
612
593
  message = "Dispatched scheduled job immediately and it was discarded"
613
594
  self.message_user(request, message, level=messages.SUCCESS)
614
595
  return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
@@ -0,0 +1,91 @@
1
+ from asgiref.sync import sync_to_async
2
+ from django.db import close_old_connections, connections
3
+ from django.tasks.backends.base import BaseTaskBackend
4
+ from django.tasks.exceptions import TaskResultDoesNotExist
5
+
6
+ from dj_queue.db import get_database_alias
7
+ from dj_queue.models import Job
8
+ from dj_queue.operations.jobs import (
9
+ DispatchOutcome,
10
+ enqueue_job_with_dispatch,
11
+ enqueue_jobs_bulk,
12
+ validate_queue_allowed,
13
+ )
14
+ from dj_queue.task_results import task_result_from_enqueued_job, task_result_from_job
15
+
16
+
17
+ class DjQueueBackend(BaseTaskBackend):
18
+ supports_async_task = True
19
+ supports_defer = True
20
+ supports_get_result = True
21
+ supports_priority = True
22
+
23
+ def validate_task(self, task):
24
+ validate_queue_allowed(task.queue_name, backend_alias=self.alias)
25
+ return super().validate_task(task)
26
+
27
+ def enqueue(self, task, args, kwargs):
28
+ self.validate_task(task)
29
+ job, dispatch_outcome = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
30
+ return task_result_from_enqueued_job(
31
+ job,
32
+ task,
33
+ successful=dispatch_outcome is DispatchOutcome.DISCARDED,
34
+ )
35
+
36
+ async def aenqueue(self, task, args, kwargs):
37
+ return await sync_to_async(_async_backend_call, thread_sensitive=True)(
38
+ self.enqueue,
39
+ task=task,
40
+ args=args,
41
+ kwargs=kwargs,
42
+ )
43
+
44
+ def enqueue_all(self, task_calls):
45
+ jobs = []
46
+ for task, args, kwargs in task_calls:
47
+ self.validate_task(task)
48
+ jobs.append((task, args, kwargs))
49
+
50
+ created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias)
51
+ return [
52
+ task_result_from_enqueued_job(
53
+ job,
54
+ task,
55
+ successful=dispatch_outcome is DispatchOutcome.DISCARDED,
56
+ )
57
+ for job, task, dispatch_outcome in created_jobs
58
+ ]
59
+
60
+ def get_result(self, result_id):
61
+ alias = get_database_alias(self.alias)
62
+ try:
63
+ job = (
64
+ Job.objects.using(alias)
65
+ .select_related(
66
+ "ready_execution",
67
+ "scheduled_execution",
68
+ "claimed_execution__process",
69
+ "blocked_execution",
70
+ "failed_execution",
71
+ )
72
+ .get(pk=result_id, backend_alias=self.alias)
73
+ )
74
+ except Job.DoesNotExist as exc:
75
+ raise TaskResultDoesNotExist(str(result_id)) from exc
76
+
77
+ return task_result_from_job(job)
78
+
79
+ async def aget_result(self, result_id):
80
+ return await sync_to_async(_async_backend_call, thread_sensitive=True)(
81
+ self.get_result,
82
+ result_id=result_id,
83
+ )
84
+
85
+
86
+ def _async_backend_call(method, /, **kwargs):
87
+ close_old_connections()
88
+ try:
89
+ return method(**kwargs)
90
+ finally:
91
+ connections.close_all()
@@ -0,0 +1,26 @@
1
+ try:
2
+ from prometheus_client import CollectorRegistry, generate_latest
3
+ from prometheus_client.core import GaugeMetricFamily
4
+ except ImportError:
5
+ DjQueueCollector = None
6
+ registry = None
7
+ generate_latest = None
8
+ else:
9
+ from dj_queue.metrics import metric_families
10
+
11
+ class DjQueueCollector:
12
+ """Prometheus collector that exposes dj_queue metrics from the shared observability snapshot."""
13
+
14
+ def collect(self):
15
+ for family in metric_families():
16
+ gauge = GaugeMetricFamily(
17
+ family.name,
18
+ family.help_text,
19
+ labels=list(family.labels),
20
+ )
21
+ for sample in family.samples:
22
+ gauge.add_metric(list(sample.labels), sample.value)
23
+ yield gauge
24
+
25
+ registry = CollectorRegistry(auto_describe=False)
26
+ registry.register(DjQueueCollector())
@@ -22,51 +22,26 @@ from dj_queue.operations.jobs import (
22
22
  enqueue_job_again,
23
23
  retry_failed_jobs,
24
24
  )
25
+ from dj_queue.queue_state import (
26
+ QUEUE_STATE_DEFINITIONS,
27
+ QUEUE_STATE_LABELS,
28
+ QUEUE_STATES,
29
+ queue_state_count_key,
30
+ queue_state_queryset,
31
+ )
25
32
 
26
- QUEUE_STATE_CONFIG = {
27
- "ready": {
28
- "label": "ready",
29
- "job_actions": ({"name": "discard", "label": "discard selected"},),
30
- "query_filter": {"ready_execution__isnull": False},
31
- "query_order": ("-priority", "ready_execution__id"),
32
- },
33
- "claimed": {
34
- "label": "claimed",
35
- "job_actions": (),
36
- "query_filter": {"claimed_execution__isnull": False},
37
- "query_order": ("claimed_execution__created_at", "id"),
38
- },
39
- "scheduled": {
40
- "label": "scheduled",
41
- "job_actions": ({"name": "discard", "label": "discard selected"},),
42
- "query_filter": {"scheduled_execution__isnull": False},
43
- "query_order": ("scheduled_execution__scheduled_at", "-priority", "scheduled_execution__id"),
44
- },
45
- "blocked": {
46
- "label": "blocked",
47
- "job_actions": ({"name": "discard", "label": "discard selected"},),
48
- "query_filter": {"blocked_execution__isnull": False},
49
- "query_order": ("blocked_execution__expires_at", "-priority", "blocked_execution__id"),
50
- },
51
- "failed": {
52
- "label": "failed",
53
- "job_actions": (
54
- {"name": "retry", "label": "retry selected"},
55
- {"name": "discard", "label": "discard selected"},
56
- ),
57
- "query_filter": {"failed_execution__isnull": False},
58
- "query_order": ("-failed_execution__created_at", "id"),
59
- },
60
- "finished": {
61
- "label": "finished",
62
- "job_actions": ({"name": "enqueue", "label": "enqueue selected again"},),
63
- "query_filter": {"finished_at__isnull": False},
64
- "query_order": ("-finished_at", "id"),
65
- },
66
- }
67
33
 
68
- QUEUE_STATES = tuple((state, config["label"]) for state, config in QUEUE_STATE_CONFIG.items())
69
- QUEUE_STATE_LABELS = {state: config["label"] for state, config in QUEUE_STATE_CONFIG.items()}
34
+ QUEUE_JOB_ACTIONS = {
35
+ "ready": ({"name": "discard", "label": "discard selected"},),
36
+ "claimed": (),
37
+ "scheduled": ({"name": "discard", "label": "discard selected"},),
38
+ "blocked": ({"name": "discard", "label": "discard selected"},),
39
+ "failed": (
40
+ {"name": "retry", "label": "retry selected"},
41
+ {"name": "discard", "label": "discard selected"},
42
+ ),
43
+ "finished": ({"name": "enqueue", "label": "enqueue selected again"},),
44
+ }
70
45
  PAGE_SIZE = 100
71
46
  OVERVIEW_PAGE_SIZES = {
72
47
  "queues": 18,
@@ -87,12 +62,14 @@ OVERVIEW_SORTS = {
87
62
  "default": "name",
88
63
  "fields": {
89
64
  "name": {"label": "name", "key": "name", "default_desc": False, "css_class": "djq-col-name"},
90
- "ready": {"label": "ready", "key": "ready_count", "default_desc": True},
91
- "claimed": {"label": "claimed", "key": "claimed_count", "default_desc": True},
92
- "scheduled": {"label": "scheduled", "key": "scheduled_count", "default_desc": True},
93
- "blocked": {"label": "blocked", "key": "blocked_count", "default_desc": True},
94
- "failed": {"label": "failed", "key": "failed_count", "default_desc": True},
95
- "finished": {"label": "finished", "key": "finished_count", "default_desc": True},
65
+ **{
66
+ definition.name: {
67
+ "label": definition.label,
68
+ "key": definition.count_key,
69
+ "default_desc": True,
70
+ }
71
+ for definition in QUEUE_STATE_DEFINITIONS
72
+ },
96
73
  "paused": {"label": "paused", "key": "paused", "default_desc": True},
97
74
  "latency": {"label": "latency", "key": "latency_seconds", "default_desc": True},
98
75
  "workers": {"label": "workers", "key": "live_worker_count", "default_desc": True},
@@ -399,7 +376,7 @@ def queue_page_context(*, backend_alias, queue_name, state, page_number, query_p
399
376
  )
400
377
  queue_info = QueueInfo(queue_name, backend_alias=backend_alias)
401
378
  state_counts = {
402
- state_name: queue_row[f"{state_name}_count"] for state_name, _label in QUEUE_STATES
379
+ definition.name: queue_row[definition.count_key] for definition in QUEUE_STATE_DEFINITIONS
403
380
  }
404
381
  state_tabs = [
405
382
  {
@@ -555,15 +532,15 @@ def apply_job_action(*, backend_alias, queue_name, state, action, job_ids):
555
532
 
556
533
 
557
534
  def job_actions_for_state(state):
558
- return QUEUE_STATE_CONFIG[state]["job_actions"]
535
+ return QUEUE_JOB_ACTIONS[state]
559
536
 
560
537
 
561
538
  def _summary_cards(*, backend_alias, queue_rows, process_rows, recurring_rows, semaphore_rows):
562
539
  paused_count = sum(1 for row in queue_rows if row["paused"])
563
- ready_count = sum(row["ready_count"] for row in queue_rows)
564
- scheduled_count = sum(row["scheduled_count"] for row in queue_rows)
565
- failed_count = sum(row["failed_count"] for row in queue_rows)
566
- blocked_count = sum(row["blocked_count"] for row in queue_rows)
540
+ ready_count = sum(row[queue_state_count_key("ready")] for row in queue_rows)
541
+ scheduled_count = sum(row[queue_state_count_key("scheduled")] for row in queue_rows)
542
+ failed_count = sum(row[queue_state_count_key("failed")] for row in queue_rows)
543
+ blocked_count = sum(row[queue_state_count_key("blocked")] for row in queue_rows)
567
544
  live_processes = sum(1 for row in process_rows if row["is_live"])
568
545
  stale_processes = len(process_rows) - live_processes
569
546
 
@@ -1181,20 +1158,7 @@ def _failed_execution_changelist_url(backend_alias, **filters):
1181
1158
 
1182
1159
 
1183
1160
  def _jobs_for_queue_state(*, backend_alias, queue_name, state):
1184
- alias = get_database_alias(backend_alias)
1185
- queryset = (
1186
- Job.objects.using(alias)
1187
- .filter(backend_alias=backend_alias, queue_name=queue_name)
1188
- .select_related(
1189
- "ready_execution",
1190
- "scheduled_execution",
1191
- "claimed_execution__process",
1192
- "blocked_execution",
1193
- "failed_execution",
1194
- )
1195
- )
1196
- state_config = QUEUE_STATE_CONFIG[state]
1197
- return queryset.filter(**state_config["query_filter"]).order_by(*state_config["query_order"])
1161
+ return queue_state_queryset(backend_alias=backend_alias, queue_name=queue_name, state=state)
1198
1162
 
1199
1163
 
1200
1164
  def _next_run_at(schedule, now):
@@ -6,6 +6,17 @@ from dj_queue.config import load_backend_config
6
6
  logger = logging.getLogger("dj_queue")
7
7
 
8
8
 
9
+ def event_logging_enabled(
10
+ level: int = logging.INFO,
11
+ *,
12
+ backend_alias: str = "default",
13
+ polling: bool = False,
14
+ ):
15
+ if polling and load_backend_config(backend_alias).silence_polling:
16
+ return False
17
+ return logger.isEnabledFor(level)
18
+
19
+
9
20
  def log_event(
10
21
  event: str,
11
22
  *,
@@ -14,7 +25,7 @@ def log_event(
14
25
  polling: bool = False,
15
26
  **fields: Any,
16
27
  ):
17
- if polling and load_backend_config(backend_alias).silence_polling:
28
+ if not event_logging_enabled(level, backend_alias=backend_alias, polling=polling):
18
29
  return
19
30
 
20
31
  logger.log(