dj-queue 0.9.1__tar.gz → 0.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dj_queue-0.9.1 → dj_queue-0.9.2}/PKG-INFO +65 -21
- {dj_queue-0.9.1 → dj_queue-0.9.2}/README.md +64 -20
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/admin.py +15 -24
- dj_queue-0.9.2/dj_queue/backend.py +91 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/dashboard.py +20 -12
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/log.py +12 -1
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/metrics.py +4 -3
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/observability.py +15 -23
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/_helpers.py +7 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/concurrency.py +17 -11
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/jobs.py +134 -134
- dj_queue-0.9.2/dj_queue/queue_selectors.py +62 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/queue_state.py +26 -6
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/notify.py +4 -18
- dj_queue-0.9.2/dj_queue/task_results.py +107 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/pyproject.toml +1 -1
- dj_queue-0.9.1/dj_queue/backend.py +0 -170
- {dj_queue-0.9.1 → dj_queue-0.9.2}/LICENSE +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/api.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/apps.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/config.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/asgi.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/gunicorn.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/contrib/prometheus.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/cron.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/db.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/exceptions.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/hooks.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue_health.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/management/commands/dj_queue_prune.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0001_initial.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0004_dashboard.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/migrations/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/jobs.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/recurring.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/models/runtime.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/_insert.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/cleanup.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/queues.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/operations/recurring.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/routers.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/base.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/connection_budget.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/dispatcher.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/errors.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/interruptible.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/pidfile.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/pool.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/procline.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/scheduler.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/supervisor.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/topology.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/runtime/worker.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templatetags/__init__.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/templatetags/dj_queue_admin.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/urls.py +0 -0
- {dj_queue-0.9.1 → dj_queue-0.9.2}/dj_queue/views.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dj-queue
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Database-backed task queue backend for Django’s Tasks framework.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -780,26 +780,70 @@ concurrency-maintenance throughput.
|
|
|
780
780
|
|
|
781
781
|
The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
|
|
782
782
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
783
|
+
No top-level `OPTIONS` key is required. Omit a key to use its default. Static
|
|
784
|
+
`recurring` entries are the exception: each named recurring task requires
|
|
785
|
+
`task_path` and `schedule`.
|
|
786
|
+
|
|
787
|
+
Global options:
|
|
788
|
+
|
|
789
|
+
| Option | Default | Meaning |
|
|
790
|
+
|---|---|---|
|
|
791
|
+
| `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
|
|
792
|
+
| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
|
|
793
|
+
| `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
|
|
794
|
+
| `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
|
|
795
|
+
| `recurring` | `{}` | static recurring task definitions keyed by name |
|
|
796
|
+
| `database_alias` | `"default"` | database alias for queue tables and runtime activity |
|
|
797
|
+
| `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
|
|
798
|
+
| `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
|
|
799
|
+
| `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
|
|
800
|
+
| `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
|
|
801
|
+
| `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
|
|
802
|
+
| `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
|
|
803
|
+
| `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
|
|
804
|
+
| `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
|
|
805
|
+
| `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
|
|
806
|
+
| `process_alive_threshold` | `300` | seconds before a process row is stale |
|
|
807
|
+
| `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
|
|
808
|
+
| `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
|
|
809
|
+
| `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
|
|
810
|
+
|
|
811
|
+
Worker entry options:
|
|
812
|
+
|
|
813
|
+
| Option | Default | Meaning |
|
|
814
|
+
|---|---|---|
|
|
815
|
+
| `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
|
|
816
|
+
| `threads` | `3` | worker threads per worker process |
|
|
817
|
+
| `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
|
|
818
|
+
| `polling_interval` | `0.1` | seconds between worker polls |
|
|
819
|
+
|
|
820
|
+
Dispatcher entry options:
|
|
821
|
+
|
|
822
|
+
| Option | Default | Meaning |
|
|
823
|
+
|---|---|---|
|
|
824
|
+
| `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
|
|
825
|
+
| `polling_interval` | `1` | seconds between dispatcher polls |
|
|
826
|
+
| `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
|
|
827
|
+
| `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
|
|
828
|
+
|
|
829
|
+
Scheduler entry options:
|
|
830
|
+
|
|
831
|
+
| Option | Default | Meaning |
|
|
832
|
+
|---|---|---|
|
|
833
|
+
| `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
|
|
834
|
+
| `polling_interval` | `5` | seconds between scheduler polls |
|
|
835
|
+
|
|
836
|
+
Recurring entry options:
|
|
837
|
+
|
|
838
|
+
| Option | Default | Meaning |
|
|
839
|
+
|---|---|---|
|
|
840
|
+
| `task_path` | none | required dotted import path for the task to enqueue |
|
|
841
|
+
| `schedule` | none | required cron or supported Fugit-style cronish schedule |
|
|
842
|
+
| `args` | `[]` | positional arguments for the task |
|
|
843
|
+
| `kwargs` | `{}` | keyword arguments for the task |
|
|
844
|
+
| `queue_name` | `"default"` | queue used for jobs created from this recurring task |
|
|
845
|
+
| `priority` | `0` | priority used for jobs created from this recurring task |
|
|
846
|
+
| `description` | `""` | operator-facing description |
|
|
803
847
|
|
|
804
848
|
On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
|
|
805
849
|
configuration as the main database connection. Install a compatible driver in
|
|
@@ -754,26 +754,70 @@ concurrency-maintenance throughput.
|
|
|
754
754
|
|
|
755
755
|
The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
|
|
756
756
|
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
757
|
+
No top-level `OPTIONS` key is required. Omit a key to use its default. Static
|
|
758
|
+
`recurring` entries are the exception: each named recurring task requires
|
|
759
|
+
`task_path` and `schedule`.
|
|
760
|
+
|
|
761
|
+
Global options:
|
|
762
|
+
|
|
763
|
+
| Option | Default | Meaning |
|
|
764
|
+
|---|---|---|
|
|
765
|
+
| `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
|
|
766
|
+
| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
|
|
767
|
+
| `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
|
|
768
|
+
| `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
|
|
769
|
+
| `recurring` | `{}` | static recurring task definitions keyed by name |
|
|
770
|
+
| `database_alias` | `"default"` | database alias for queue tables and runtime activity |
|
|
771
|
+
| `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
|
|
772
|
+
| `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
|
|
773
|
+
| `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
|
|
774
|
+
| `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
|
|
775
|
+
| `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
|
|
776
|
+
| `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
|
|
777
|
+
| `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
|
|
778
|
+
| `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
|
|
779
|
+
| `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
|
|
780
|
+
| `process_alive_threshold` | `300` | seconds before a process row is stale |
|
|
781
|
+
| `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
|
|
782
|
+
| `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
|
|
783
|
+
| `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
|
|
784
|
+
|
|
785
|
+
Worker entry options:
|
|
786
|
+
|
|
787
|
+
| Option | Default | Meaning |
|
|
788
|
+
|---|---|---|
|
|
789
|
+
| `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
|
|
790
|
+
| `threads` | `3` | worker threads per worker process |
|
|
791
|
+
| `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
|
|
792
|
+
| `polling_interval` | `0.1` | seconds between worker polls |
|
|
793
|
+
|
|
794
|
+
Dispatcher entry options:
|
|
795
|
+
|
|
796
|
+
| Option | Default | Meaning |
|
|
797
|
+
|---|---|---|
|
|
798
|
+
| `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
|
|
799
|
+
| `polling_interval` | `1` | seconds between dispatcher polls |
|
|
800
|
+
| `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
|
|
801
|
+
| `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
|
|
802
|
+
|
|
803
|
+
Scheduler entry options:
|
|
804
|
+
|
|
805
|
+
| Option | Default | Meaning |
|
|
806
|
+
|---|---|---|
|
|
807
|
+
| `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
|
|
808
|
+
| `polling_interval` | `5` | seconds between scheduler polls |
|
|
809
|
+
|
|
810
|
+
Recurring entry options:
|
|
811
|
+
|
|
812
|
+
| Option | Default | Meaning |
|
|
813
|
+
|---|---|---|
|
|
814
|
+
| `task_path` | none | required dotted import path for the task to enqueue |
|
|
815
|
+
| `schedule` | none | required cron or supported Fugit-style cronish schedule |
|
|
816
|
+
| `args` | `[]` | positional arguments for the task |
|
|
817
|
+
| `kwargs` | `{}` | keyword arguments for the task |
|
|
818
|
+
| `queue_name` | `"default"` | queue used for jobs created from this recurring task |
|
|
819
|
+
| `priority` | `0` | priority used for jobs created from this recurring task |
|
|
820
|
+
| `description` | `""` | operator-facing description |
|
|
777
821
|
|
|
778
822
|
On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
|
|
779
823
|
configuration as the main database connection. Install a compatible driver in
|
|
@@ -29,13 +29,19 @@ from dj_queue.models import (
|
|
|
29
29
|
Semaphore,
|
|
30
30
|
)
|
|
31
31
|
from dj_queue.operations.jobs import (
|
|
32
|
+
DispatchOutcome,
|
|
32
33
|
discard_failed_job,
|
|
33
34
|
dispatch_scheduled_job_now,
|
|
34
35
|
enqueue_job_again,
|
|
35
36
|
retry_failed_job,
|
|
36
37
|
retry_failed_jobs,
|
|
37
38
|
)
|
|
38
|
-
from dj_queue.queue_state import
|
|
39
|
+
from dj_queue.queue_state import (
|
|
40
|
+
QUEUE_STATES,
|
|
41
|
+
filter_queue_state,
|
|
42
|
+
is_queue_state,
|
|
43
|
+
status_rank_expression,
|
|
44
|
+
)
|
|
39
45
|
|
|
40
46
|
|
|
41
47
|
class DjQueueFirstAdminSite(admin.AdminSite):
|
|
@@ -328,29 +334,12 @@ class JobStatusListFilter(admin.SimpleListFilter):
|
|
|
328
334
|
parameter_name = "status"
|
|
329
335
|
|
|
330
336
|
def lookups(self, request, model_admin):
|
|
331
|
-
return
|
|
332
|
-
("ready", "ready"),
|
|
333
|
-
("scheduled", "scheduled"),
|
|
334
|
-
("claimed", "claimed"),
|
|
335
|
-
("blocked", "blocked"),
|
|
336
|
-
("failed", "failed"),
|
|
337
|
-
("finished", "finished"),
|
|
338
|
-
)
|
|
337
|
+
return QUEUE_STATES
|
|
339
338
|
|
|
340
339
|
def queryset(self, request, queryset):
|
|
341
340
|
value = self.value()
|
|
342
|
-
if value
|
|
343
|
-
return queryset
|
|
344
|
-
if value == "scheduled":
|
|
345
|
-
return queryset.scheduled()
|
|
346
|
-
if value == "claimed":
|
|
347
|
-
return queryset.claimed()
|
|
348
|
-
if value == "blocked":
|
|
349
|
-
return queryset.blocked()
|
|
350
|
-
if value == "failed":
|
|
351
|
-
return queryset.failed()
|
|
352
|
-
if value == "finished":
|
|
353
|
-
return queryset.finished()
|
|
341
|
+
if is_queue_state(value):
|
|
342
|
+
return filter_queue_state(queryset, value)
|
|
354
343
|
return queryset
|
|
355
344
|
|
|
356
345
|
|
|
@@ -590,15 +579,17 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
590
579
|
def handle_change_action(self, request, obj, action):
|
|
591
580
|
if action == "run_now":
|
|
592
581
|
try:
|
|
593
|
-
_job,
|
|
582
|
+
_job, dispatch_outcome = dispatch_scheduled_job_now(
|
|
583
|
+
obj.pk, backend_alias=obj.backend_alias
|
|
584
|
+
)
|
|
594
585
|
except (EnqueueError, ImportError, AttributeError) as exc:
|
|
595
586
|
self.message_user(request, f"Could not dispatch job now: {exc}", level=messages.ERROR)
|
|
596
587
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
597
588
|
|
|
598
589
|
message = "Dispatched scheduled job for immediate execution"
|
|
599
|
-
if
|
|
590
|
+
if dispatch_outcome is DispatchOutcome.BLOCKED:
|
|
600
591
|
message = "Dispatched scheduled job immediately and it is now blocked"
|
|
601
|
-
if
|
|
592
|
+
if dispatch_outcome is DispatchOutcome.DISCARDED:
|
|
602
593
|
message = "Dispatched scheduled job immediately and it was discarded"
|
|
603
594
|
self.message_user(request, message, level=messages.SUCCESS)
|
|
604
595
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from asgiref.sync import sync_to_async
|
|
2
|
+
from django.db import close_old_connections, connections
|
|
3
|
+
from django.tasks.backends.base import BaseTaskBackend
|
|
4
|
+
from django.tasks.exceptions import TaskResultDoesNotExist
|
|
5
|
+
|
|
6
|
+
from dj_queue.db import get_database_alias
|
|
7
|
+
from dj_queue.models import Job
|
|
8
|
+
from dj_queue.operations.jobs import (
|
|
9
|
+
DispatchOutcome,
|
|
10
|
+
enqueue_job_with_dispatch,
|
|
11
|
+
enqueue_jobs_bulk,
|
|
12
|
+
validate_queue_allowed,
|
|
13
|
+
)
|
|
14
|
+
from dj_queue.task_results import task_result_from_enqueued_job, task_result_from_job
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DjQueueBackend(BaseTaskBackend):
|
|
18
|
+
supports_async_task = True
|
|
19
|
+
supports_defer = True
|
|
20
|
+
supports_get_result = True
|
|
21
|
+
supports_priority = True
|
|
22
|
+
|
|
23
|
+
def validate_task(self, task):
|
|
24
|
+
validate_queue_allowed(task.queue_name, backend_alias=self.alias)
|
|
25
|
+
return super().validate_task(task)
|
|
26
|
+
|
|
27
|
+
def enqueue(self, task, args, kwargs):
|
|
28
|
+
self.validate_task(task)
|
|
29
|
+
job, dispatch_outcome = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
|
|
30
|
+
return task_result_from_enqueued_job(
|
|
31
|
+
job,
|
|
32
|
+
task,
|
|
33
|
+
successful=dispatch_outcome is DispatchOutcome.DISCARDED,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
async def aenqueue(self, task, args, kwargs):
|
|
37
|
+
return await sync_to_async(_async_backend_call, thread_sensitive=True)(
|
|
38
|
+
self.enqueue,
|
|
39
|
+
task=task,
|
|
40
|
+
args=args,
|
|
41
|
+
kwargs=kwargs,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def enqueue_all(self, task_calls):
|
|
45
|
+
jobs = []
|
|
46
|
+
for task, args, kwargs in task_calls:
|
|
47
|
+
self.validate_task(task)
|
|
48
|
+
jobs.append((task, args, kwargs))
|
|
49
|
+
|
|
50
|
+
created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias)
|
|
51
|
+
return [
|
|
52
|
+
task_result_from_enqueued_job(
|
|
53
|
+
job,
|
|
54
|
+
task,
|
|
55
|
+
successful=dispatch_outcome is DispatchOutcome.DISCARDED,
|
|
56
|
+
)
|
|
57
|
+
for job, task, dispatch_outcome in created_jobs
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
def get_result(self, result_id):
|
|
61
|
+
alias = get_database_alias(self.alias)
|
|
62
|
+
try:
|
|
63
|
+
job = (
|
|
64
|
+
Job.objects.using(alias)
|
|
65
|
+
.select_related(
|
|
66
|
+
"ready_execution",
|
|
67
|
+
"scheduled_execution",
|
|
68
|
+
"claimed_execution__process",
|
|
69
|
+
"blocked_execution",
|
|
70
|
+
"failed_execution",
|
|
71
|
+
)
|
|
72
|
+
.get(pk=result_id, backend_alias=self.alias)
|
|
73
|
+
)
|
|
74
|
+
except Job.DoesNotExist as exc:
|
|
75
|
+
raise TaskResultDoesNotExist(str(result_id)) from exc
|
|
76
|
+
|
|
77
|
+
return task_result_from_job(job)
|
|
78
|
+
|
|
79
|
+
async def aget_result(self, result_id):
|
|
80
|
+
return await sync_to_async(_async_backend_call, thread_sensitive=True)(
|
|
81
|
+
self.get_result,
|
|
82
|
+
result_id=result_id,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _async_backend_call(method, /, **kwargs):
|
|
87
|
+
close_old_connections()
|
|
88
|
+
try:
|
|
89
|
+
return method(**kwargs)
|
|
90
|
+
finally:
|
|
91
|
+
connections.close_all()
|
|
@@ -22,7 +22,13 @@ from dj_queue.operations.jobs import (
|
|
|
22
22
|
enqueue_job_again,
|
|
23
23
|
retry_failed_jobs,
|
|
24
24
|
)
|
|
25
|
-
from dj_queue.queue_state import
|
|
25
|
+
from dj_queue.queue_state import (
|
|
26
|
+
QUEUE_STATE_DEFINITIONS,
|
|
27
|
+
QUEUE_STATE_LABELS,
|
|
28
|
+
QUEUE_STATES,
|
|
29
|
+
queue_state_count_key,
|
|
30
|
+
queue_state_queryset,
|
|
31
|
+
)
|
|
26
32
|
|
|
27
33
|
|
|
28
34
|
QUEUE_JOB_ACTIONS = {
|
|
@@ -56,12 +62,14 @@ OVERVIEW_SORTS = {
|
|
|
56
62
|
"default": "name",
|
|
57
63
|
"fields": {
|
|
58
64
|
"name": {"label": "name", "key": "name", "default_desc": False, "css_class": "djq-col-name"},
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
+
**{
|
|
66
|
+
definition.name: {
|
|
67
|
+
"label": definition.label,
|
|
68
|
+
"key": definition.count_key,
|
|
69
|
+
"default_desc": True,
|
|
70
|
+
}
|
|
71
|
+
for definition in QUEUE_STATE_DEFINITIONS
|
|
72
|
+
},
|
|
65
73
|
"paused": {"label": "paused", "key": "paused", "default_desc": True},
|
|
66
74
|
"latency": {"label": "latency", "key": "latency_seconds", "default_desc": True},
|
|
67
75
|
"workers": {"label": "workers", "key": "live_worker_count", "default_desc": True},
|
|
@@ -368,7 +376,7 @@ def queue_page_context(*, backend_alias, queue_name, state, page_number, query_p
|
|
|
368
376
|
)
|
|
369
377
|
queue_info = QueueInfo(queue_name, backend_alias=backend_alias)
|
|
370
378
|
state_counts = {
|
|
371
|
-
|
|
379
|
+
definition.name: queue_row[definition.count_key] for definition in QUEUE_STATE_DEFINITIONS
|
|
372
380
|
}
|
|
373
381
|
state_tabs = [
|
|
374
382
|
{
|
|
@@ -529,10 +537,10 @@ def job_actions_for_state(state):
|
|
|
529
537
|
|
|
530
538
|
def _summary_cards(*, backend_alias, queue_rows, process_rows, recurring_rows, semaphore_rows):
|
|
531
539
|
paused_count = sum(1 for row in queue_rows if row["paused"])
|
|
532
|
-
ready_count = sum(row["
|
|
533
|
-
scheduled_count = sum(row["
|
|
534
|
-
failed_count = sum(row["
|
|
535
|
-
blocked_count = sum(row["
|
|
540
|
+
ready_count = sum(row[queue_state_count_key("ready")] for row in queue_rows)
|
|
541
|
+
scheduled_count = sum(row[queue_state_count_key("scheduled")] for row in queue_rows)
|
|
542
|
+
failed_count = sum(row[queue_state_count_key("failed")] for row in queue_rows)
|
|
543
|
+
blocked_count = sum(row[queue_state_count_key("blocked")] for row in queue_rows)
|
|
536
544
|
live_processes = sum(1 for row in process_rows if row["is_live"])
|
|
537
545
|
stale_processes = len(process_rows) - live_processes
|
|
538
546
|
|
|
@@ -6,6 +6,17 @@ from dj_queue.config import load_backend_config
|
|
|
6
6
|
logger = logging.getLogger("dj_queue")
|
|
7
7
|
|
|
8
8
|
|
|
9
|
+
def event_logging_enabled(
|
|
10
|
+
level: int = logging.INFO,
|
|
11
|
+
*,
|
|
12
|
+
backend_alias: str = "default",
|
|
13
|
+
polling: bool = False,
|
|
14
|
+
):
|
|
15
|
+
if polling and load_backend_config(backend_alias).silence_polling:
|
|
16
|
+
return False
|
|
17
|
+
return logger.isEnabledFor(level)
|
|
18
|
+
|
|
19
|
+
|
|
9
20
|
def log_event(
|
|
10
21
|
event: str,
|
|
11
22
|
*,
|
|
@@ -14,7 +25,7 @@ def log_event(
|
|
|
14
25
|
polling: bool = False,
|
|
15
26
|
**fields: Any,
|
|
16
27
|
):
|
|
17
|
-
if
|
|
28
|
+
if not event_logging_enabled(level, backend_alias=backend_alias, polling=polling):
|
|
18
29
|
return
|
|
19
30
|
|
|
20
31
|
logger.log(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
|
|
3
3
|
from dj_queue import observability
|
|
4
|
+
from dj_queue.queue_state import QUEUE_STATE_DEFINITIONS
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
@dataclass(frozen=True, slots=True)
|
|
@@ -38,11 +39,11 @@ def metric_families(*, snapshots=None):
|
|
|
38
39
|
runner_metrics = snapshot["runner_metrics"]
|
|
39
40
|
|
|
40
41
|
for queue in snapshot["queue_rows"]:
|
|
41
|
-
for
|
|
42
|
+
for definition in QUEUE_STATE_DEFINITIONS:
|
|
42
43
|
queue_jobs.append(
|
|
43
44
|
MetricSample(
|
|
44
|
-
labels=(backend_alias, queue["name"],
|
|
45
|
-
value=queue[
|
|
45
|
+
labels=(backend_alias, queue["name"], definition.name),
|
|
46
|
+
value=queue[definition.count_key],
|
|
46
47
|
)
|
|
47
48
|
)
|
|
48
49
|
queue_paused.append(
|
|
@@ -25,7 +25,8 @@ from dj_queue.models import (
|
|
|
25
25
|
ScheduledExecution,
|
|
26
26
|
Semaphore,
|
|
27
27
|
)
|
|
28
|
-
from dj_queue.
|
|
28
|
+
from dj_queue.queue_selectors import queue_matches_selectors
|
|
29
|
+
from dj_queue.queue_state import queue_state_count_fields, queue_state_counts
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
@dataclass(frozen=True, slots=True)
|
|
@@ -294,14 +295,20 @@ def queue_snapshot(
|
|
|
294
295
|
if oldest_ready_at is not None and paused is False:
|
|
295
296
|
latency_seconds = max((now - oldest_ready_at).total_seconds(), 0.0)
|
|
296
297
|
|
|
298
|
+
state_count_fields = queue_state_count_fields(
|
|
299
|
+
{
|
|
300
|
+
"ready": ready_count,
|
|
301
|
+
"claimed": claimed_count,
|
|
302
|
+
"scheduled": scheduled_count,
|
|
303
|
+
"blocked": blocked_count,
|
|
304
|
+
"failed": failed_count,
|
|
305
|
+
"finished": finished_count,
|
|
306
|
+
}
|
|
307
|
+
)
|
|
308
|
+
|
|
297
309
|
return {
|
|
298
310
|
"name": queue_name,
|
|
299
|
-
|
|
300
|
-
"claimed_count": claimed_count,
|
|
301
|
-
"scheduled_count": scheduled_count,
|
|
302
|
-
"blocked_count": blocked_count,
|
|
303
|
-
"failed_count": failed_count,
|
|
304
|
-
"finished_count": finished_count,
|
|
311
|
+
**state_count_fields,
|
|
305
312
|
"paused": paused,
|
|
306
313
|
"latency_seconds": latency_seconds,
|
|
307
314
|
"oldest_scheduled_at": oldest_scheduled_at,
|
|
@@ -309,7 +316,7 @@ def queue_snapshot(
|
|
|
309
316
|
"live_worker_count": sum(
|
|
310
317
|
1
|
|
311
318
|
for worker in live_workers
|
|
312
|
-
if queue_matches_selectors(queue_name, worker.metadata.get("queues",
|
|
319
|
+
if queue_matches_selectors(queue_name, worker.metadata.get("queues") or ("*",))
|
|
313
320
|
),
|
|
314
321
|
}
|
|
315
322
|
|
|
@@ -419,21 +426,6 @@ def next_run_at(schedule, now):
|
|
|
419
426
|
return next_cron_run(schedule, now)
|
|
420
427
|
|
|
421
428
|
|
|
422
|
-
def queue_matches_selectors(queue_name, selectors):
|
|
423
|
-
normalized = tuple(selectors or ())
|
|
424
|
-
if normalized in ((), ("*",)):
|
|
425
|
-
return True
|
|
426
|
-
|
|
427
|
-
for selector in normalized:
|
|
428
|
-
if selector == "*":
|
|
429
|
-
return True
|
|
430
|
-
if selector.endswith("*") and queue_name.startswith(selector[:-1]):
|
|
431
|
-
return True
|
|
432
|
-
if selector == queue_name:
|
|
433
|
-
return True
|
|
434
|
-
return False
|
|
435
|
-
|
|
436
|
-
|
|
437
429
|
def _live_processes_for_backend(*, alias, backend_alias, kind, process_cutoff):
|
|
438
430
|
return [
|
|
439
431
|
process
|
|
@@ -28,6 +28,13 @@ def _lock_active_pauses(alias, backend_alias, queue_names=None):
|
|
|
28
28
|
return set(queryset.values_list("queue_name", flat=True))
|
|
29
29
|
|
|
30
30
|
|
|
31
|
+
def _exclude_active_pauses(queryset, alias, backend_alias):
|
|
32
|
+
paused_queue_names = (
|
|
33
|
+
Pause.objects.using(alias).filter(backend_alias=backend_alias).values("queue_name")
|
|
34
|
+
)
|
|
35
|
+
return queryset.exclude(queue_name__in=paused_queue_names)
|
|
36
|
+
|
|
37
|
+
|
|
31
38
|
def _ready_execution_row(
|
|
32
39
|
job,
|
|
33
40
|
*,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from datetime import timedelta
|
|
2
2
|
|
|
3
3
|
from django.db import connections, transaction
|
|
4
|
-
from django.db.models import F
|
|
4
|
+
from django.db.models import Case, F, IntegerField, When
|
|
5
5
|
from django.utils import timezone
|
|
6
6
|
from django.utils.module_loading import import_string
|
|
7
7
|
|
|
@@ -104,17 +104,23 @@ def _mysql_family_semaphore_acquire(alias, key, *, limit, expires_at, now):
|
|
|
104
104
|
|
|
105
105
|
def semaphore_release(key, *, duration_seconds, backend_alias="default"):
|
|
106
106
|
alias = get_database_alias(backend_alias)
|
|
107
|
-
|
|
107
|
+
now = timezone.now()
|
|
108
|
+
expires_at = now + timedelta(seconds=duration_seconds)
|
|
108
109
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
110
|
+
updated = (
|
|
111
|
+
Semaphore.objects.using(alias)
|
|
112
|
+
.filter(key=key)
|
|
113
|
+
.update(
|
|
114
|
+
value=Case(
|
|
115
|
+
When(value__gte=F("limit"), then=F("limit")),
|
|
116
|
+
default=F("value") + 1,
|
|
117
|
+
output_field=IntegerField(),
|
|
118
|
+
),
|
|
119
|
+
expires_at=expires_at,
|
|
120
|
+
updated_at=now,
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
return updated > 0
|
|
118
124
|
|
|
119
125
|
|
|
120
126
|
def concurrency_settings(task, *, backend_alias):
|