dj-queue 0.9.1__tar.gz → 0.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {dj_queue-0.9.1 → dj_queue-0.9.2}/PKG-INFO +65 -21
  2. {dj_queue-0.9.1 → dj_queue-0.9.2}/README.md +64 -20
  3. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/admin.py +15 -24
  4. dj_queue-0.9.2/dj_queue/backend.py +91 -0
  5. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/dashboard.py +20 -12
  6. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/log.py +12 -1
  7. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/metrics.py +4 -3
  8. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/observability.py +15 -23
  9. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/_helpers.py +7 -0
  10. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/concurrency.py +17 -11
  11. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/jobs.py +134 -134
  12. dj_queue-0.9.2/dj_queue/queue_selectors.py +62 -0
  13. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/queue_state.py +26 -6
  14. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/notify.py +4 -18
  15. dj_queue-0.9.2/dj_queue/task_results.py +107 -0
  16. {dj_queue-0.9.1 → dj_queue-0.9.2}/pyproject.toml +1 -1
  17. dj_queue-0.9.1/dj_queue/backend.py +0 -170
  18. {dj_queue-0.9.1 → dj_queue-0.9.2}/LICENSE +0 -0
  19. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/__init__.py +0 -0
  20. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/api.py +0 -0
  21. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/apps.py +0 -0
  22. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/config.py +0 -0
  23. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/__init__.py +0 -0
  24. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/asgi.py +0 -0
  25. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/gunicorn.py +0 -0
  26. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/prometheus.py +0 -0
  27. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/cron.py +0 -0
  28. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/db.py +0 -0
  29. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/exceptions.py +0 -0
  30. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/hooks.py +0 -0
  31. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/__init__.py +0 -0
  32. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/__init__.py +0 -0
  33. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue.py +0 -0
  34. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue_health.py +0 -0
  35. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue_prune.py +0 -0
  36. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0001_initial.py +0 -0
  37. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
  38. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
  39. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0004_dashboard.py +0 -0
  40. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
  41. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
  42. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
  43. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
  44. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/__init__.py +0 -0
  45. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/__init__.py +0 -0
  46. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/jobs.py +0 -0
  47. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/recurring.py +0 -0
  48. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/runtime.py +0 -0
  49. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/__init__.py +0 -0
  50. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/_insert.py +0 -0
  51. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/cleanup.py +0 -0
  52. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/queues.py +0 -0
  53. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/recurring.py +0 -0
  54. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/routers.py +0 -0
  55. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/__init__.py +0 -0
  56. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/base.py +0 -0
  57. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/connection_budget.py +0 -0
  58. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/dispatcher.py +0 -0
  59. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/errors.py +0 -0
  60. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/interruptible.py +0 -0
  61. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/pidfile.py +0 -0
  62. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/pool.py +0 -0
  63. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/procline.py +0 -0
  64. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/scheduler.py +0 -0
  65. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/supervisor.py +0 -0
  66. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/topology.py +0 -0
  67. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/worker.py +0 -0
  68. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
  69. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
  70. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
  71. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
  72. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
  73. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
  74. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
  75. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
  76. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
  77. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
  78. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
  79. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
  80. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templatetags/__init__.py +0 -0
  81. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templatetags/dj_queue_admin.py +0 -0
  82. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/urls.py +0 -0
  83. {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dj-queue
3
- Version: 0.9.1
3
+ Version: 0.9.2
4
4
  Summary: Database-backed task queue backend for Django’s Tasks framework.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -780,26 +780,70 @@ concurrency-maintenance throughput.
780
780
 
781
781
  The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
782
782
 
783
- Start with these options:
784
-
785
- - `mode`: `"fork"` or `"async"`
786
- - `workers`: queue selectors, thread counts, and process counts
787
- - `dispatchers`: scheduled promotion and concurrency maintenance settings
788
- - `scheduler`: dynamic recurring polling settings
789
- - `database_alias`: database alias for queue tables and runtime activity
790
- - `preserve_finished_jobs` and `clear_finished_jobs_after`: successful result retention and cleanup
791
- - `clear_failed_jobs_after`: optional failed-job retention window
792
- - `clear_recurring_executions_after`: optional recurring reservation retention window
793
-
794
- Additional operational tuning is available when needed:
795
-
796
- - `use_skip_locked`: use `SKIP LOCKED` when the active backend supports it
797
- - `listen_notify`: PostgreSQL-only worker wakeup optimization layered on top of polling
798
- - `silence_polling`: suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger
799
- - `process_heartbeat_interval` and `process_alive_threshold`: process liveness reporting and stale-runner detection
800
- - `shutdown_timeout`: graceful drain window before standalone shutdown gives up on waiting
801
- - `supervisor_pidfile`: optional pidfile guard for standalone supervisors
802
- - `on_thread_error`: dotted callback path for runtime infrastructure exceptions
783
+ No top-level `OPTIONS` key is required. Omit a key to use its default. Static
784
+ `recurring` entries are the exception: each named recurring task requires
785
+ `task_path` and `schedule`.
786
+
787
+ Global options:
788
+
789
+ | Option | Default | Meaning |
790
+ |---|---|---|
791
+ | `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
792
+ | `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
793
+ | `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
794
+ | `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
795
+ | `recurring` | `{}` | static recurring task definitions keyed by name |
796
+ | `database_alias` | `"default"` | database alias for queue tables and runtime activity |
797
+ | `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
798
+ | `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
799
+ | `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
800
+ | `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
801
+ | `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
802
+ | `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
803
+ | `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
804
+ | `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
805
+ | `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
806
+ | `process_alive_threshold` | `300` | seconds before a process row is stale |
807
+ | `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
808
+ | `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
809
+ | `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
810
+
811
+ Worker entry options:
812
+
813
+ | Option | Default | Meaning |
814
+ |---|---|---|
815
+ | `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
816
+ | `threads` | `3` | worker threads per worker process |
817
+ | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
818
+ | `polling_interval` | `0.1` | seconds between worker polls |
819
+
820
+ Dispatcher entry options:
821
+
822
+ | Option | Default | Meaning |
823
+ |---|---|---|
824
+ | `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
825
+ | `polling_interval` | `1` | seconds between dispatcher polls |
826
+ | `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
827
+ | `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
828
+
829
+ Scheduler entry options:
830
+
831
+ | Option | Default | Meaning |
832
+ |---|---|---|
833
+ | `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
834
+ | `polling_interval` | `5` | seconds between scheduler polls |
835
+
836
+ Recurring entry options:
837
+
838
+ | Option | Default | Meaning |
839
+ |---|---|---|
840
+ | `task_path` | none | required dotted import path for the task to enqueue |
841
+ | `schedule` | none | required cron or supported Fugit-style cronish schedule |
842
+ | `args` | `[]` | positional arguments for the task |
843
+ | `kwargs` | `{}` | keyword arguments for the task |
844
+ | `queue_name` | `"default"` | queue used for jobs created from this recurring task |
845
+ | `priority` | `0` | priority used for jobs created from this recurring task |
846
+ | `description` | `""` | operator-facing description |
803
847
 
804
848
  On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
805
849
  configuration as the main database connection. Install a compatible driver in
@@ -754,26 +754,70 @@ concurrency-maintenance throughput.
754
754
 
755
755
  The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
756
756
 
757
- Start with these options:
758
-
759
- - `mode`: `"fork"` or `"async"`
760
- - `workers`: queue selectors, thread counts, and process counts
761
- - `dispatchers`: scheduled promotion and concurrency maintenance settings
762
- - `scheduler`: dynamic recurring polling settings
763
- - `database_alias`: database alias for queue tables and runtime activity
764
- - `preserve_finished_jobs` and `clear_finished_jobs_after`: successful result retention and cleanup
765
- - `clear_failed_jobs_after`: optional failed-job retention window
766
- - `clear_recurring_executions_after`: optional recurring reservation retention window
767
-
768
- Additional operational tuning is available when needed:
769
-
770
- - `use_skip_locked`: use `SKIP LOCKED` when the active backend supports it
771
- - `listen_notify`: PostgreSQL-only worker wakeup optimization layered on top of polling
772
- - `silence_polling`: suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger
773
- - `process_heartbeat_interval` and `process_alive_threshold`: process liveness reporting and stale-runner detection
774
- - `shutdown_timeout`: graceful drain window before standalone shutdown gives up on waiting
775
- - `supervisor_pidfile`: optional pidfile guard for standalone supervisors
776
- - `on_thread_error`: dotted callback path for runtime infrastructure exceptions
757
+ No top-level `OPTIONS` key is required. Omit a key to use its default. Static
758
+ `recurring` entries are the exception: each named recurring task requires
759
+ `task_path` and `schedule`.
760
+
761
+ Global options:
762
+
763
+ | Option | Default | Meaning |
764
+ |---|---|---|
765
+ | `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
766
+ | `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
767
+ | `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
768
+ | `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
769
+ | `recurring` | `{}` | static recurring task definitions keyed by name |
770
+ | `database_alias` | `"default"` | database alias for queue tables and runtime activity |
771
+ | `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
772
+ | `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
773
+ | `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
774
+ | `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
775
+ | `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
776
+ | `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
777
+ | `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
778
+ | `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
779
+ | `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
780
+ | `process_alive_threshold` | `300` | seconds before a process row is stale |
781
+ | `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
782
+ | `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
783
+ | `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
784
+
785
+ Worker entry options:
786
+
787
+ | Option | Default | Meaning |
788
+ |---|---|---|
789
+ | `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
790
+ | `threads` | `3` | worker threads per worker process |
791
+ | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
792
+ | `polling_interval` | `0.1` | seconds between worker polls |
793
+
794
+ Dispatcher entry options:
795
+
796
+ | Option | Default | Meaning |
797
+ |---|---|---|
798
+ | `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
799
+ | `polling_interval` | `1` | seconds between dispatcher polls |
800
+ | `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
801
+ | `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
802
+
803
+ Scheduler entry options:
804
+
805
+ | Option | Default | Meaning |
806
+ |---|---|---|
807
+ | `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
808
+ | `polling_interval` | `5` | seconds between scheduler polls |
809
+
810
+ Recurring entry options:
811
+
812
+ | Option | Default | Meaning |
813
+ |---|---|---|
814
+ | `task_path` | none | required dotted import path for the task to enqueue |
815
+ | `schedule` | none | required cron or supported Fugit-style cronish schedule |
816
+ | `args` | `[]` | positional arguments for the task |
817
+ | `kwargs` | `{}` | keyword arguments for the task |
818
+ | `queue_name` | `"default"` | queue used for jobs created from this recurring task |
819
+ | `priority` | `0` | priority used for jobs created from this recurring task |
820
+ | `description` | `""` | operator-facing description |
777
821
 
778
822
  On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
779
823
  configuration as the main database connection. Install a compatible driver in
@@ -29,13 +29,19 @@ from dj_queue.models import (
29
29
  Semaphore,
30
30
  )
31
31
  from dj_queue.operations.jobs import (
32
+ DispatchOutcome,
32
33
  discard_failed_job,
33
34
  dispatch_scheduled_job_now,
34
35
  enqueue_job_again,
35
36
  retry_failed_job,
36
37
  retry_failed_jobs,
37
38
  )
38
- from dj_queue.queue_state import status_rank_expression
39
+ from dj_queue.queue_state import (
40
+ QUEUE_STATES,
41
+ filter_queue_state,
42
+ is_queue_state,
43
+ status_rank_expression,
44
+ )
39
45
 
40
46
 
41
47
  class DjQueueFirstAdminSite(admin.AdminSite):
@@ -328,29 +334,12 @@ class JobStatusListFilter(admin.SimpleListFilter):
328
334
  parameter_name = "status"
329
335
 
330
336
  def lookups(self, request, model_admin):
331
- return (
332
- ("ready", "ready"),
333
- ("scheduled", "scheduled"),
334
- ("claimed", "claimed"),
335
- ("blocked", "blocked"),
336
- ("failed", "failed"),
337
- ("finished", "finished"),
338
- )
337
+ return QUEUE_STATES
339
338
 
340
339
  def queryset(self, request, queryset):
341
340
  value = self.value()
342
- if value == "ready":
343
- return queryset.ready()
344
- if value == "scheduled":
345
- return queryset.scheduled()
346
- if value == "claimed":
347
- return queryset.claimed()
348
- if value == "blocked":
349
- return queryset.blocked()
350
- if value == "failed":
351
- return queryset.failed()
352
- if value == "finished":
353
- return queryset.finished()
341
+ if is_queue_state(value):
342
+ return filter_queue_state(queryset, value)
354
343
  return queryset
355
344
 
356
345
 
@@ -590,15 +579,17 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
590
579
  def handle_change_action(self, request, obj, action):
591
580
  if action == "run_now":
592
581
  try:
593
- _job, dispatched_as = dispatch_scheduled_job_now(obj.pk, backend_alias=obj.backend_alias)
582
+ _job, dispatch_outcome = dispatch_scheduled_job_now(
583
+ obj.pk, backend_alias=obj.backend_alias
584
+ )
594
585
  except (EnqueueError, ImportError, AttributeError) as exc:
595
586
  self.message_user(request, f"Could not dispatch job now: {exc}", level=messages.ERROR)
596
587
  return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
597
588
 
598
589
  message = "Dispatched scheduled job for immediate execution"
599
- if dispatched_as == "blocked":
590
+ if dispatch_outcome is DispatchOutcome.BLOCKED:
600
591
  message = "Dispatched scheduled job immediately and it is now blocked"
601
- if dispatched_as == "discarded":
592
+ if dispatch_outcome is DispatchOutcome.DISCARDED:
602
593
  message = "Dispatched scheduled job immediately and it was discarded"
603
594
  self.message_user(request, message, level=messages.SUCCESS)
604
595
  return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
@@ -0,0 +1,91 @@
1
+ from asgiref.sync import sync_to_async
2
+ from django.db import close_old_connections, connections
3
+ from django.tasks.backends.base import BaseTaskBackend
4
+ from django.tasks.exceptions import TaskResultDoesNotExist
5
+
6
+ from dj_queue.db import get_database_alias
7
+ from dj_queue.models import Job
8
+ from dj_queue.operations.jobs import (
9
+ DispatchOutcome,
10
+ enqueue_job_with_dispatch,
11
+ enqueue_jobs_bulk,
12
+ validate_queue_allowed,
13
+ )
14
+ from dj_queue.task_results import task_result_from_enqueued_job, task_result_from_job
15
+
16
+
17
+ class DjQueueBackend(BaseTaskBackend):
18
+ supports_async_task = True
19
+ supports_defer = True
20
+ supports_get_result = True
21
+ supports_priority = True
22
+
23
+ def validate_task(self, task):
24
+ validate_queue_allowed(task.queue_name, backend_alias=self.alias)
25
+ return super().validate_task(task)
26
+
27
+ def enqueue(self, task, args, kwargs):
28
+ self.validate_task(task)
29
+ job, dispatch_outcome = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
30
+ return task_result_from_enqueued_job(
31
+ job,
32
+ task,
33
+ successful=dispatch_outcome is DispatchOutcome.DISCARDED,
34
+ )
35
+
36
+ async def aenqueue(self, task, args, kwargs):
37
+ return await sync_to_async(_async_backend_call, thread_sensitive=True)(
38
+ self.enqueue,
39
+ task=task,
40
+ args=args,
41
+ kwargs=kwargs,
42
+ )
43
+
44
+ def enqueue_all(self, task_calls):
45
+ jobs = []
46
+ for task, args, kwargs in task_calls:
47
+ self.validate_task(task)
48
+ jobs.append((task, args, kwargs))
49
+
50
+ created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias)
51
+ return [
52
+ task_result_from_enqueued_job(
53
+ job,
54
+ task,
55
+ successful=dispatch_outcome is DispatchOutcome.DISCARDED,
56
+ )
57
+ for job, task, dispatch_outcome in created_jobs
58
+ ]
59
+
60
+ def get_result(self, result_id):
61
+ alias = get_database_alias(self.alias)
62
+ try:
63
+ job = (
64
+ Job.objects.using(alias)
65
+ .select_related(
66
+ "ready_execution",
67
+ "scheduled_execution",
68
+ "claimed_execution__process",
69
+ "blocked_execution",
70
+ "failed_execution",
71
+ )
72
+ .get(pk=result_id, backend_alias=self.alias)
73
+ )
74
+ except Job.DoesNotExist as exc:
75
+ raise TaskResultDoesNotExist(str(result_id)) from exc
76
+
77
+ return task_result_from_job(job)
78
+
79
+ async def aget_result(self, result_id):
80
+ return await sync_to_async(_async_backend_call, thread_sensitive=True)(
81
+ self.get_result,
82
+ result_id=result_id,
83
+ )
84
+
85
+
86
+ def _async_backend_call(method, /, **kwargs):
87
+ close_old_connections()
88
+ try:
89
+ return method(**kwargs)
90
+ finally:
91
+ connections.close_all()
@@ -22,7 +22,13 @@ from dj_queue.operations.jobs import (
22
22
  enqueue_job_again,
23
23
  retry_failed_jobs,
24
24
  )
25
- from dj_queue.queue_state import QUEUE_STATE_LABELS, QUEUE_STATES, queue_state_queryset
25
+ from dj_queue.queue_state import (
26
+ QUEUE_STATE_DEFINITIONS,
27
+ QUEUE_STATE_LABELS,
28
+ QUEUE_STATES,
29
+ queue_state_count_key,
30
+ queue_state_queryset,
31
+ )
26
32
 
27
33
 
28
34
  QUEUE_JOB_ACTIONS = {
@@ -56,12 +62,14 @@ OVERVIEW_SORTS = {
56
62
  "default": "name",
57
63
  "fields": {
58
64
  "name": {"label": "name", "key": "name", "default_desc": False, "css_class": "djq-col-name"},
59
- "ready": {"label": "ready", "key": "ready_count", "default_desc": True},
60
- "claimed": {"label": "claimed", "key": "claimed_count", "default_desc": True},
61
- "scheduled": {"label": "scheduled", "key": "scheduled_count", "default_desc": True},
62
- "blocked": {"label": "blocked", "key": "blocked_count", "default_desc": True},
63
- "failed": {"label": "failed", "key": "failed_count", "default_desc": True},
64
- "finished": {"label": "finished", "key": "finished_count", "default_desc": True},
65
+ **{
66
+ definition.name: {
67
+ "label": definition.label,
68
+ "key": definition.count_key,
69
+ "default_desc": True,
70
+ }
71
+ for definition in QUEUE_STATE_DEFINITIONS
72
+ },
65
73
  "paused": {"label": "paused", "key": "paused", "default_desc": True},
66
74
  "latency": {"label": "latency", "key": "latency_seconds", "default_desc": True},
67
75
  "workers": {"label": "workers", "key": "live_worker_count", "default_desc": True},
@@ -368,7 +376,7 @@ def queue_page_context(*, backend_alias, queue_name, state, page_number, query_p
368
376
  )
369
377
  queue_info = QueueInfo(queue_name, backend_alias=backend_alias)
370
378
  state_counts = {
371
- state_name: queue_row[f"{state_name}_count"] for state_name, _label in QUEUE_STATES
379
+ definition.name: queue_row[definition.count_key] for definition in QUEUE_STATE_DEFINITIONS
372
380
  }
373
381
  state_tabs = [
374
382
  {
@@ -529,10 +537,10 @@ def job_actions_for_state(state):
529
537
 
530
538
  def _summary_cards(*, backend_alias, queue_rows, process_rows, recurring_rows, semaphore_rows):
531
539
  paused_count = sum(1 for row in queue_rows if row["paused"])
532
- ready_count = sum(row["ready_count"] for row in queue_rows)
533
- scheduled_count = sum(row["scheduled_count"] for row in queue_rows)
534
- failed_count = sum(row["failed_count"] for row in queue_rows)
535
- blocked_count = sum(row["blocked_count"] for row in queue_rows)
540
+ ready_count = sum(row[queue_state_count_key("ready")] for row in queue_rows)
541
+ scheduled_count = sum(row[queue_state_count_key("scheduled")] for row in queue_rows)
542
+ failed_count = sum(row[queue_state_count_key("failed")] for row in queue_rows)
543
+ blocked_count = sum(row[queue_state_count_key("blocked")] for row in queue_rows)
536
544
  live_processes = sum(1 for row in process_rows if row["is_live"])
537
545
  stale_processes = len(process_rows) - live_processes
538
546
 
@@ -6,6 +6,17 @@ from dj_queue.config import load_backend_config
6
6
  logger = logging.getLogger("dj_queue")
7
7
 
8
8
 
9
+ def event_logging_enabled(
10
+ level: int = logging.INFO,
11
+ *,
12
+ backend_alias: str = "default",
13
+ polling: bool = False,
14
+ ):
15
+ if polling and load_backend_config(backend_alias).silence_polling:
16
+ return False
17
+ return logger.isEnabledFor(level)
18
+
19
+
9
20
  def log_event(
10
21
  event: str,
11
22
  *,
@@ -14,7 +25,7 @@ def log_event(
14
25
  polling: bool = False,
15
26
  **fields: Any,
16
27
  ):
17
- if polling and load_backend_config(backend_alias).silence_polling:
28
+ if not event_logging_enabled(level, backend_alias=backend_alias, polling=polling):
18
29
  return
19
30
 
20
31
  logger.log(
@@ -1,6 +1,7 @@
1
1
  from dataclasses import dataclass
2
2
 
3
3
  from dj_queue import observability
4
+ from dj_queue.queue_state import QUEUE_STATE_DEFINITIONS
4
5
 
5
6
 
6
7
  @dataclass(frozen=True, slots=True)
@@ -38,11 +39,11 @@ def metric_families(*, snapshots=None):
38
39
  runner_metrics = snapshot["runner_metrics"]
39
40
 
40
41
  for queue in snapshot["queue_rows"]:
41
- for state in ("ready", "claimed", "scheduled", "blocked", "failed", "finished"):
42
+ for definition in QUEUE_STATE_DEFINITIONS:
42
43
  queue_jobs.append(
43
44
  MetricSample(
44
- labels=(backend_alias, queue["name"], state),
45
- value=queue[f"{state}_count"],
45
+ labels=(backend_alias, queue["name"], definition.name),
46
+ value=queue[definition.count_key],
46
47
  )
47
48
  )
48
49
  queue_paused.append(
@@ -25,7 +25,8 @@ from dj_queue.models import (
25
25
  ScheduledExecution,
26
26
  Semaphore,
27
27
  )
28
- from dj_queue.queue_state import queue_state_counts
28
+ from dj_queue.queue_selectors import queue_matches_selectors
29
+ from dj_queue.queue_state import queue_state_count_fields, queue_state_counts
29
30
 
30
31
 
31
32
  @dataclass(frozen=True, slots=True)
@@ -294,14 +295,20 @@ def queue_snapshot(
294
295
  if oldest_ready_at is not None and paused is False:
295
296
  latency_seconds = max((now - oldest_ready_at).total_seconds(), 0.0)
296
297
 
298
+ state_count_fields = queue_state_count_fields(
299
+ {
300
+ "ready": ready_count,
301
+ "claimed": claimed_count,
302
+ "scheduled": scheduled_count,
303
+ "blocked": blocked_count,
304
+ "failed": failed_count,
305
+ "finished": finished_count,
306
+ }
307
+ )
308
+
297
309
  return {
298
310
  "name": queue_name,
299
- "ready_count": ready_count,
300
- "claimed_count": claimed_count,
301
- "scheduled_count": scheduled_count,
302
- "blocked_count": blocked_count,
303
- "failed_count": failed_count,
304
- "finished_count": finished_count,
311
+ **state_count_fields,
305
312
  "paused": paused,
306
313
  "latency_seconds": latency_seconds,
307
314
  "oldest_scheduled_at": oldest_scheduled_at,
@@ -309,7 +316,7 @@ def queue_snapshot(
309
316
  "live_worker_count": sum(
310
317
  1
311
318
  for worker in live_workers
312
- if queue_matches_selectors(queue_name, worker.metadata.get("queues", []))
319
+ if queue_matches_selectors(queue_name, worker.metadata.get("queues") or ("*",))
313
320
  ),
314
321
  }
315
322
 
@@ -419,21 +426,6 @@ def next_run_at(schedule, now):
419
426
  return next_cron_run(schedule, now)
420
427
 
421
428
 
422
- def queue_matches_selectors(queue_name, selectors):
423
- normalized = tuple(selectors or ())
424
- if normalized in ((), ("*",)):
425
- return True
426
-
427
- for selector in normalized:
428
- if selector == "*":
429
- return True
430
- if selector.endswith("*") and queue_name.startswith(selector[:-1]):
431
- return True
432
- if selector == queue_name:
433
- return True
434
- return False
435
-
436
-
437
429
  def _live_processes_for_backend(*, alias, backend_alias, kind, process_cutoff):
438
430
  return [
439
431
  process
@@ -28,6 +28,13 @@ def _lock_active_pauses(alias, backend_alias, queue_names=None):
28
28
  return set(queryset.values_list("queue_name", flat=True))
29
29
 
30
30
 
31
+ def _exclude_active_pauses(queryset, alias, backend_alias):
32
+ paused_queue_names = (
33
+ Pause.objects.using(alias).filter(backend_alias=backend_alias).values("queue_name")
34
+ )
35
+ return queryset.exclude(queue_name__in=paused_queue_names)
36
+
37
+
31
38
  def _ready_execution_row(
32
39
  job,
33
40
  *,
@@ -1,7 +1,7 @@
1
1
  from datetime import timedelta
2
2
 
3
3
  from django.db import connections, transaction
4
- from django.db.models import F
4
+ from django.db.models import Case, F, IntegerField, When
5
5
  from django.utils import timezone
6
6
  from django.utils.module_loading import import_string
7
7
 
@@ -104,17 +104,23 @@ def _mysql_family_semaphore_acquire(alias, key, *, limit, expires_at, now):
104
104
 
105
105
  def semaphore_release(key, *, duration_seconds, backend_alias="default"):
106
106
  alias = get_database_alias(backend_alias)
107
- expires_at = timezone.now() + timedelta(seconds=duration_seconds)
107
+ now = timezone.now()
108
+ expires_at = now + timedelta(seconds=duration_seconds)
108
109
 
109
- with transaction.atomic(using=alias):
110
- semaphore = Semaphore.objects.using(alias).select_for_update().filter(key=key).first()
111
- if semaphore is None:
112
- return False
113
-
114
- semaphore.value = min(semaphore.limit, semaphore.value + 1)
115
- semaphore.expires_at = expires_at
116
- semaphore.save(using=alias, update_fields=["value", "expires_at", "updated_at"])
117
- return True
110
+ updated = (
111
+ Semaphore.objects.using(alias)
112
+ .filter(key=key)
113
+ .update(
114
+ value=Case(
115
+ When(value__gte=F("limit"), then=F("limit")),
116
+ default=F("value") + 1,
117
+ output_field=IntegerField(),
118
+ ),
119
+ expires_at=expires_at,
120
+ updated_at=now,
121
+ )
122
+ )
123
+ return updated > 0
118
124
 
119
125
 
120
126
  def concurrency_settings(task, *, backend_alias):