dj-queue 0.10.5__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dj_queue-0.10.5 → dj_queue-0.11.0}/PKG-INFO +10 -2
- {dj_queue-0.10.5 → dj_queue-0.11.0}/README.md +9 -1
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/admin.py +142 -90
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/api.py +27 -28
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/config.py +28 -8
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/contrib/asgi.py +6 -1
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/contrib/gunicorn.py +43 -9
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/cron.py +52 -29
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/dashboard.py +74 -173
- dj_queue-0.11.0/dj_queue/dashboard_actions.py +141 -0
- dj_queue-0.11.0/dj_queue/management/commands/dj_queue_health.py +28 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/metrics.py +7 -7
- dj_queue-0.11.0/dj_queue/migrations/0011_remove_blockedexecution_djq_bl_b_conc_idx_and_more.py +32 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/models/jobs.py +78 -26
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/observability.py +230 -136
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/_helpers.py +92 -12
- dj_queue-0.11.0/dj_queue/operations/_insert.py +43 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/cleanup.py +31 -18
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/concurrency.py +319 -48
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/jobs.py +158 -135
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/recurring.py +46 -4
- dj_queue-0.11.0/dj_queue/queue_state.py +257 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/base.py +17 -7
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/errors.py +0 -1
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/notify.py +59 -16
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/pool.py +10 -6
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/scheduler.py +22 -32
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/supervisor.py +159 -27
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/worker.py +24 -1
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/task_results.py +16 -1
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +7 -1
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/dashboard.html +1 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/pyproject.toml +1 -1
- dj_queue-0.10.5/dj_queue/management/commands/dj_queue_health.py +0 -39
- dj_queue-0.10.5/dj_queue/operations/_insert.py +0 -24
- dj_queue-0.10.5/dj_queue/queue_state.py +0 -138
- {dj_queue-0.10.5 → dj_queue-0.11.0}/LICENSE +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/apps.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/backend.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/contrib/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/contrib/prometheus.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/db.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/exceptions.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/hooks.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/log.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/management/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/management/commands/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/management/commands/dj_queue.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/management/commands/dj_queue_prune.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0001_initial.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0004_dashboard.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0009_remove_process_dj_queue_processes_name_supervisor_unique_and_more.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/0010_remove_process_djq_pr_name_parent_uniq_and_more.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/migrations/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/models/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/models/recurring.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/models/runtime.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/operations/queues.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/queue_selectors.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/routers.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/connection_budget.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/dispatcher.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/interruptible.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/pidfile.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/procline.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/runtime/topology.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templatetags/__init__.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/templatetags/dj_queue_admin.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/urls.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/views.py +0 -0
- {dj_queue-0.10.5 → dj_queue-0.11.0}/dj_queue/wakeup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dj-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Database-backed task queue backend for Django’s Tasks framework.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -323,7 +323,7 @@ across queues. Prefer one primary scheduling mechanism per worker when you can.
|
|
|
323
323
|
In standalone mode, both `fork` and `async` `python manage.py dj_queue` supervisors own runtime signal handling:
|
|
324
324
|
|
|
325
325
|
- `SIGTERM` and `SIGINT` request graceful shutdown
|
|
326
|
-
- `SIGQUIT`
|
|
326
|
+
- `SIGQUIT` requests immediate process exit through normal Python cleanup
|
|
327
327
|
- `shutdown_timeout` controls how long the runtime waits for in-flight work to drain
|
|
328
328
|
- `supervisor_pidfile` can prevent duplicate standalone supervisors on one host
|
|
329
329
|
|
|
@@ -397,6 +397,9 @@ TASKS = {
|
|
|
397
397
|
}
|
|
398
398
|
```
|
|
399
399
|
|
|
400
|
+
Static recurring definitions are synced when the scheduler runner starts, then
|
|
401
|
+
normal polling uses the persisted recurring rows and their `next_run_at` cursor.
|
|
402
|
+
|
|
400
403
|
### Dynamic recurring tasks
|
|
401
404
|
|
|
402
405
|
Create, update, and remove recurring tasks at runtime:
|
|
@@ -428,6 +431,7 @@ Notes:
|
|
|
428
431
|
- cron supports five fields or an optional leading seconds field, presets like `@daily`, optional timezone suffixes, wraparound ranges, `L`/`last` and negative monthdays, weekday `#` and `%` extensions, and normal day-of-month/day-of-week OR semantics
|
|
429
432
|
- add `&` to either day field to require day-of-month and day-of-week to both match
|
|
430
433
|
- natural-language schedules support `every`, `at`, `on`, and `from` forms such as `every day at noon`, `every weekday at five`, `every 5 minutes`, `every month on day 2 at 10:00`, `from monday to friday at 9`, and `at minute 5`
|
|
434
|
+
- natural interval counts must fit inside the cron field they map to; misleading forms such as `every 90 seconds`, `every 90 minutes`, and `every 24 hours` are rejected
|
|
431
435
|
- natural-language schedules that expand to multiple cron expressions, such as `every day at 16:15 and 18:30`, are treated as the union of those schedules
|
|
432
436
|
- recurring task keys are scoped per backend alias
|
|
433
437
|
- only dynamic tasks can be unscheduled at runtime; unscheduling a static task returns `0`
|
|
@@ -490,6 +494,10 @@ if claimed_jobs:
|
|
|
490
494
|
execute_claimed_job(claimed_jobs[0])
|
|
491
495
|
```
|
|
492
496
|
|
|
497
|
+
`QueueInfo.all()` discovers queues through the same read model as the dashboard
|
|
498
|
+
and reuses that snapshot for `size`, `latency`, and `paused` until a queue
|
|
499
|
+
mutation invalidates it.
|
|
500
|
+
|
|
493
501
|
Notes:
|
|
494
502
|
|
|
495
503
|
- pausing a queue stops future claims, not enqueueing or already-claimed work
|
|
@@ -297,7 +297,7 @@ across queues. Prefer one primary scheduling mechanism per worker when you can.
|
|
|
297
297
|
In standalone mode, both `fork` and `async` `python manage.py dj_queue` supervisors own runtime signal handling:
|
|
298
298
|
|
|
299
299
|
- `SIGTERM` and `SIGINT` request graceful shutdown
|
|
300
|
-
- `SIGQUIT`
|
|
300
|
+
- `SIGQUIT` requests immediate process exit through normal Python cleanup
|
|
301
301
|
- `shutdown_timeout` controls how long the runtime waits for in-flight work to drain
|
|
302
302
|
- `supervisor_pidfile` can prevent duplicate standalone supervisors on one host
|
|
303
303
|
|
|
@@ -371,6 +371,9 @@ TASKS = {
|
|
|
371
371
|
}
|
|
372
372
|
```
|
|
373
373
|
|
|
374
|
+
Static recurring definitions are synced when the scheduler runner starts, then
|
|
375
|
+
normal polling uses the persisted recurring rows and their `next_run_at` cursor.
|
|
376
|
+
|
|
374
377
|
### Dynamic recurring tasks
|
|
375
378
|
|
|
376
379
|
Create, update, and remove recurring tasks at runtime:
|
|
@@ -402,6 +405,7 @@ Notes:
|
|
|
402
405
|
- cron supports five fields or an optional leading seconds field, presets like `@daily`, optional timezone suffixes, wraparound ranges, `L`/`last` and negative monthdays, weekday `#` and `%` extensions, and normal day-of-month/day-of-week OR semantics
|
|
403
406
|
- add `&` to either day field to require day-of-month and day-of-week to both match
|
|
404
407
|
- natural-language schedules support `every`, `at`, `on`, and `from` forms such as `every day at noon`, `every weekday at five`, `every 5 minutes`, `every month on day 2 at 10:00`, `from monday to friday at 9`, and `at minute 5`
|
|
408
|
+
- natural interval counts must fit inside the cron field they map to; misleading forms such as `every 90 seconds`, `every 90 minutes`, and `every 24 hours` are rejected
|
|
405
409
|
- natural-language schedules that expand to multiple cron expressions, such as `every day at 16:15 and 18:30`, are treated as the union of those schedules
|
|
406
410
|
- recurring task keys are scoped per backend alias
|
|
407
411
|
- only dynamic tasks can be unscheduled at runtime; unscheduling a static task returns `0`
|
|
@@ -464,6 +468,10 @@ if claimed_jobs:
|
|
|
464
468
|
execute_claimed_job(claimed_jobs[0])
|
|
465
469
|
```
|
|
466
470
|
|
|
471
|
+
`QueueInfo.all()` discovers queues through the same read model as the dashboard
|
|
472
|
+
and reuses that snapshot for `size`, `latency`, and `paused` until a queue
|
|
473
|
+
mutation invalidates it.
|
|
474
|
+
|
|
467
475
|
Notes:
|
|
468
476
|
|
|
469
477
|
- pausing a queue stops future claims, not enqueueing or already-claimed work
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import json
|
|
2
|
-
from datetime import timedelta
|
|
3
2
|
from functools import wraps
|
|
4
3
|
from urllib.parse import parse_qsl, urlencode
|
|
5
4
|
|
|
6
5
|
from django.contrib import admin, messages
|
|
7
|
-
from django.db.models import Case, Count, IntegerField, OuterRef, Subquery, Value, When
|
|
8
|
-
from django.db.models.functions import Coalesce
|
|
9
6
|
from django.http import HttpResponseNotAllowed, HttpResponseRedirect
|
|
10
7
|
from django.template.response import TemplateResponse
|
|
11
8
|
from django.urls import path, reverse
|
|
@@ -13,13 +10,13 @@ from django.utils.html import format_html
|
|
|
13
10
|
from django.utils.http import url_has_allowed_host_and_scheme
|
|
14
11
|
from django.utils import timezone
|
|
15
12
|
|
|
16
|
-
from dj_queue.config import load_backend_config
|
|
17
13
|
from dj_queue import dashboard
|
|
14
|
+
from dj_queue import dashboard_actions
|
|
15
|
+
from dj_queue import observability
|
|
18
16
|
from dj_queue.api import QueueInfo, unschedule_recurring_task
|
|
19
17
|
from dj_queue.db import get_database_alias
|
|
20
18
|
from dj_queue.exceptions import EnqueueError
|
|
21
19
|
from dj_queue.models import (
|
|
22
|
-
BlockedExecution,
|
|
23
20
|
Dashboard,
|
|
24
21
|
FailedExecution,
|
|
25
22
|
Job,
|
|
@@ -43,37 +40,46 @@ from dj_queue.queue_state import (
|
|
|
43
40
|
status_rank_expression,
|
|
44
41
|
)
|
|
45
42
|
|
|
43
|
+
ADMIN_ACTION_ERRORS = (
|
|
44
|
+
EnqueueError,
|
|
45
|
+
ImportError,
|
|
46
|
+
AttributeError,
|
|
47
|
+
Job.DoesNotExist,
|
|
48
|
+
FailedExecution.DoesNotExist,
|
|
49
|
+
RecurringTask.DoesNotExist,
|
|
50
|
+
)
|
|
51
|
+
ADMIN_POST_ACTION_ERRORS = (ValueError, *ADMIN_ACTION_ERRORS)
|
|
52
|
+
|
|
46
53
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
return
|
|
54
|
+
def _install_dj_queue_admin_site(site):
|
|
55
|
+
if getattr(site, "_dj_queue_dashboard_installed", False):
|
|
56
|
+
return
|
|
50
57
|
|
|
51
|
-
|
|
52
|
-
|
|
58
|
+
original_get_app_list = site.get_app_list
|
|
59
|
+
original_app_index = site.app_index
|
|
60
|
+
|
|
61
|
+
def dashboard_app_url():
|
|
62
|
+
return reverse("admin:dj_queue_dashboard_changelist", current_app=site.name)
|
|
63
|
+
|
|
64
|
+
def get_app_list(request, app_label=None):
|
|
65
|
+
app_list = original_get_app_list(request, app_label=app_label)
|
|
53
66
|
for app in app_list:
|
|
54
67
|
if app["app_label"] == "dj_queue":
|
|
55
|
-
app["app_url"] =
|
|
68
|
+
app["app_url"] = dashboard_app_url()
|
|
56
69
|
return sorted(app_list, key=lambda app: app["app_label"] != "dj_queue")
|
|
57
70
|
|
|
58
|
-
def app_index(
|
|
71
|
+
def app_index(request, app_label, extra_context=None):
|
|
59
72
|
if app_label == "dj_queue":
|
|
60
|
-
url =
|
|
73
|
+
url = dashboard_app_url()
|
|
61
74
|
query = request.GET.urlencode()
|
|
62
75
|
if query:
|
|
63
76
|
url = f"{url}?{query}"
|
|
64
77
|
return HttpResponseRedirect(url)
|
|
65
|
-
return
|
|
78
|
+
return original_app_index(request, app_label, extra_context=extra_context)
|
|
66
79
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
return
|
|
71
|
-
|
|
72
|
-
site.__class__ = type(
|
|
73
|
-
f"DjQueue{site.__class__.__name__}",
|
|
74
|
-
(DjQueueAdminSiteMixin, site.__class__),
|
|
75
|
-
{"__module__": __name__},
|
|
76
|
-
)
|
|
80
|
+
site.get_app_list = get_app_list
|
|
81
|
+
site.app_index = app_index
|
|
82
|
+
site._dj_queue_dashboard_installed = True
|
|
77
83
|
|
|
78
84
|
|
|
79
85
|
_install_dj_queue_admin_site(admin.site)
|
|
@@ -128,16 +134,20 @@ class DashboardAdmin(admin.ModelAdmin):
|
|
|
128
134
|
return wrapper
|
|
129
135
|
|
|
130
136
|
return [
|
|
131
|
-
path("queue/<str:queue_name>/", wrap(self.queue_view), name="dj_queue_dashboard_queue"),
|
|
132
137
|
path(
|
|
133
|
-
"queue/<
|
|
138
|
+
"queue/<path:queue_name>/jobs/action/",
|
|
139
|
+
wrap(self.job_action_view),
|
|
140
|
+
name="dj_queue_dashboard_job_action",
|
|
141
|
+
),
|
|
142
|
+
path(
|
|
143
|
+
"queue/<path:queue_name>/action/",
|
|
134
144
|
wrap(self.queue_action_view),
|
|
135
145
|
name="dj_queue_dashboard_queue_action",
|
|
136
146
|
),
|
|
137
147
|
path(
|
|
138
|
-
"queue/<
|
|
139
|
-
wrap(self.
|
|
140
|
-
name="
|
|
148
|
+
"queue/<path:queue_name>/",
|
|
149
|
+
wrap(self.queue_view),
|
|
150
|
+
name="dj_queue_dashboard_queue",
|
|
141
151
|
),
|
|
142
152
|
] + super().get_urls()
|
|
143
153
|
|
|
@@ -155,7 +165,7 @@ class DashboardAdmin(admin.ModelAdmin):
|
|
|
155
165
|
context = {
|
|
156
166
|
**self.admin_site.each_context(request),
|
|
157
167
|
**queue_context,
|
|
158
|
-
"job_actions":
|
|
168
|
+
"job_actions": dashboard_actions.job_actions_for_state(state),
|
|
159
169
|
"title": "dj_queue",
|
|
160
170
|
"subtitle": queue_name,
|
|
161
171
|
}
|
|
@@ -165,7 +175,7 @@ class DashboardAdmin(admin.ModelAdmin):
|
|
|
165
175
|
backend_alias = dashboard.resolve_backend_alias(request.POST.get("backend"))
|
|
166
176
|
return self._post_action_response(
|
|
167
177
|
request,
|
|
168
|
-
operation=lambda:
|
|
178
|
+
operation=lambda: dashboard_actions.apply_queue_action(
|
|
169
179
|
backend_alias=backend_alias,
|
|
170
180
|
queue_name=queue_name,
|
|
171
181
|
action=request.POST.get("action"),
|
|
@@ -178,7 +188,7 @@ class DashboardAdmin(admin.ModelAdmin):
|
|
|
178
188
|
state = request.POST.get("state", "ready")
|
|
179
189
|
return self._post_action_response(
|
|
180
190
|
request,
|
|
181
|
-
operation=lambda:
|
|
191
|
+
operation=lambda: dashboard_actions.apply_job_action(
|
|
182
192
|
backend_alias=backend_alias,
|
|
183
193
|
queue_name=queue_name,
|
|
184
194
|
state=state,
|
|
@@ -195,7 +205,7 @@ class DashboardAdmin(admin.ModelAdmin):
|
|
|
195
205
|
return HttpResponseNotAllowed(["POST"])
|
|
196
206
|
try:
|
|
197
207
|
message = operation()
|
|
198
|
-
except
|
|
208
|
+
except ADMIN_POST_ACTION_ERRORS as exc:
|
|
199
209
|
self.message_user(request, str(exc), level=messages.ERROR)
|
|
200
210
|
else:
|
|
201
211
|
self.message_user(request, message, level=messages.SUCCESS)
|
|
@@ -322,6 +332,26 @@ class HiddenSidebarAdminMixin:
|
|
|
322
332
|
def handle_change_action(self, request, obj, action):
|
|
323
333
|
return HttpResponseRedirect(request.get_full_path())
|
|
324
334
|
|
|
335
|
+
def _run_change_operation(
|
|
336
|
+
self,
|
|
337
|
+
request,
|
|
338
|
+
*,
|
|
339
|
+
operation,
|
|
340
|
+
success_message,
|
|
341
|
+
error_message,
|
|
342
|
+
success_redirect,
|
|
343
|
+
error_redirect,
|
|
344
|
+
):
|
|
345
|
+
try:
|
|
346
|
+
result = operation()
|
|
347
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
348
|
+
self.message_user(request, f"{error_message}: {exc}", level=messages.ERROR)
|
|
349
|
+
return error_redirect()
|
|
350
|
+
|
|
351
|
+
message = success_message(result) if callable(success_message) else success_message
|
|
352
|
+
self.message_user(request, message, level=messages.SUCCESS)
|
|
353
|
+
return success_redirect(result)
|
|
354
|
+
|
|
325
355
|
def _change_redirect(self, *, object_id, backend_alias):
|
|
326
356
|
return HttpResponseRedirect(self._change_url(object_id=object_id, backend_alias=backend_alias))
|
|
327
357
|
|
|
@@ -428,16 +458,10 @@ class ProcessStatusListFilter(admin.SimpleListFilter):
|
|
|
428
458
|
value = self.value()
|
|
429
459
|
if not value:
|
|
430
460
|
return queryset
|
|
431
|
-
cutoff =
|
|
432
|
-
|
|
433
|
-
dashboard.resolve_backend_alias(request.GET.get("backend"))
|
|
434
|
-
).process_alive_threshold
|
|
461
|
+
cutoff = observability.process_cutoff_for_backend(
|
|
462
|
+
dashboard.resolve_backend_alias(request.GET.get("backend"))
|
|
435
463
|
)
|
|
436
|
-
|
|
437
|
-
return queryset.filter(last_heartbeat_at__gte=cutoff)
|
|
438
|
-
if value == "stale":
|
|
439
|
-
return queryset.filter(last_heartbeat_at__lt=cutoff)
|
|
440
|
-
return queryset
|
|
464
|
+
return observability.filter_process_status(queryset, value, process_cutoff=cutoff)
|
|
441
465
|
|
|
442
466
|
|
|
443
467
|
@admin.register(Job)
|
|
@@ -515,8 +539,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
515
539
|
@admin.display(description="queue name", ordering="queue_name")
|
|
516
540
|
def queue_name_link(self, obj):
|
|
517
541
|
query = {"backend": obj.backend_alias}
|
|
518
|
-
|
|
519
|
-
|
|
542
|
+
status = obj.status
|
|
543
|
+
if is_queue_state(status):
|
|
544
|
+
query["state"] = status
|
|
520
545
|
url = f"{reverse('admin:dj_queue_dashboard_queue', args=[obj.queue_name])}?{urlencode(query)}"
|
|
521
546
|
return format_html('<a href="{}">{}</a>', url, obj.queue_name)
|
|
522
547
|
|
|
@@ -595,7 +620,7 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
595
620
|
_job, dispatch_outcome = dispatch_scheduled_job_now(
|
|
596
621
|
obj.pk, backend_alias=obj.backend_alias
|
|
597
622
|
)
|
|
598
|
-
except
|
|
623
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
599
624
|
self.message_user(request, f"Could not dispatch job now: {exc}", level=messages.ERROR)
|
|
600
625
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
601
626
|
|
|
@@ -610,7 +635,7 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
610
635
|
if action == "enqueue_copy_now":
|
|
611
636
|
try:
|
|
612
637
|
new_job = enqueue_job_again(obj.pk, backend_alias=obj.backend_alias, run_after=None)
|
|
613
|
-
except
|
|
638
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
614
639
|
self.message_user(request, f"Could not enqueue job: {exc}", level=messages.ERROR)
|
|
615
640
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
616
641
|
|
|
@@ -628,7 +653,7 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
628
653
|
if action == "enqueue":
|
|
629
654
|
try:
|
|
630
655
|
new_job = enqueue_job_again(obj.pk, backend_alias=obj.backend_alias)
|
|
631
|
-
except
|
|
656
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
632
657
|
self.message_user(request, f"Could not enqueue job: {exc}", level=messages.ERROR)
|
|
633
658
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
634
659
|
|
|
@@ -648,14 +673,32 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
648
673
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
649
674
|
|
|
650
675
|
if action == "retry":
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
676
|
+
return self._run_change_operation(
|
|
677
|
+
request,
|
|
678
|
+
operation=lambda: retry_failed_job(
|
|
679
|
+
obj.failed_execution.job_id, backend_alias=obj.backend_alias
|
|
680
|
+
),
|
|
681
|
+
success_message="Retried failed job",
|
|
682
|
+
error_message="Could not retry failed job",
|
|
683
|
+
success_redirect=lambda _result: self._current_object_redirect(
|
|
684
|
+
obj, backend_alias=obj.backend_alias
|
|
685
|
+
),
|
|
686
|
+
error_redirect=lambda: self._current_object_redirect(obj, backend_alias=obj.backend_alias),
|
|
687
|
+
)
|
|
654
688
|
|
|
655
689
|
if action == "discard":
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
690
|
+
return self._run_change_operation(
|
|
691
|
+
request,
|
|
692
|
+
operation=lambda: discard_failed_job(
|
|
693
|
+
obj.failed_execution.job_id, backend_alias=obj.backend_alias
|
|
694
|
+
),
|
|
695
|
+
success_message="Discarded failed job",
|
|
696
|
+
error_message="Could not discard failed job",
|
|
697
|
+
success_redirect=lambda _result: HttpResponseRedirect(
|
|
698
|
+
self._changelist_url(backend_alias=obj.backend_alias)
|
|
699
|
+
),
|
|
700
|
+
error_redirect=lambda: self._current_object_redirect(obj, backend_alias=obj.backend_alias),
|
|
701
|
+
)
|
|
659
702
|
|
|
660
703
|
return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
|
|
661
704
|
|
|
@@ -676,18 +719,28 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
676
719
|
|
|
677
720
|
@admin.action(description="Retry selected failed jobs")
|
|
678
721
|
def retry_jobs(self, request, queryset):
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
722
|
+
try:
|
|
723
|
+
retried = retry_failed_jobs(
|
|
724
|
+
job_ids=list(queryset.values_list("job_id", flat=True)),
|
|
725
|
+
batch_size=queryset.count() or 1,
|
|
726
|
+
backend_alias=self._backend_alias(request),
|
|
727
|
+
)
|
|
728
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
729
|
+
self.message_user(request, f"Could not retry failed jobs: {exc}", level=messages.ERROR)
|
|
730
|
+
return
|
|
684
731
|
self.message_user(request, f"Retried {retried} failed jobs", level=messages.SUCCESS)
|
|
685
732
|
|
|
686
733
|
@admin.action(description="Discard selected failed jobs")
|
|
687
734
|
def discard_jobs(self, request, queryset):
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
735
|
+
try:
|
|
736
|
+
discarded = 0
|
|
737
|
+
for execution in queryset.select_related("job"):
|
|
738
|
+
discarded += discard_failed_job(
|
|
739
|
+
execution.job_id, backend_alias=execution.job.backend_alias
|
|
740
|
+
)
|
|
741
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
742
|
+
self.message_user(request, f"Could not discard failed jobs: {exc}", level=messages.ERROR)
|
|
743
|
+
return
|
|
691
744
|
self.message_user(request, f"Discarded {discarded} failed jobs", level=messages.SUCCESS)
|
|
692
745
|
|
|
693
746
|
@admin.display(description="created at", ordering="created_at")
|
|
@@ -711,15 +764,28 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
711
764
|
|
|
712
765
|
if action == "retry":
|
|
713
766
|
job_id = obj.job_id
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
767
|
+
return self._run_change_operation(
|
|
768
|
+
request,
|
|
769
|
+
operation=lambda: retry_failed_job(job_id, backend_alias=backend_alias),
|
|
770
|
+
success_message="Retried failed job",
|
|
771
|
+
error_message="Could not retry failed job",
|
|
772
|
+
success_redirect=lambda _result: HttpResponseRedirect(
|
|
773
|
+
f"{reverse('admin:dj_queue_job_change', args=[job_id])}?{urlencode({'backend': backend_alias})}"
|
|
774
|
+
),
|
|
775
|
+
error_redirect=lambda: self._current_object_redirect(obj, backend_alias=backend_alias),
|
|
776
|
+
)
|
|
718
777
|
|
|
719
778
|
if action == "discard":
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
779
|
+
return self._run_change_operation(
|
|
780
|
+
request,
|
|
781
|
+
operation=lambda: discard_failed_job(obj.job_id, backend_alias=backend_alias),
|
|
782
|
+
success_message="Discarded failed job",
|
|
783
|
+
error_message="Could not discard failed job",
|
|
784
|
+
success_redirect=lambda _result: HttpResponseRedirect(
|
|
785
|
+
self._changelist_url(backend_alias=backend_alias)
|
|
786
|
+
),
|
|
787
|
+
error_redirect=lambda: self._current_object_redirect(obj, backend_alias=backend_alias),
|
|
788
|
+
)
|
|
723
789
|
|
|
724
790
|
return self._current_object_redirect(obj, backend_alias=backend_alias)
|
|
725
791
|
|
|
@@ -752,16 +818,8 @@ class ProcessAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
752
818
|
|
|
753
819
|
def get_queryset(self, request):
|
|
754
820
|
queryset = super().get_queryset(request)
|
|
755
|
-
cutoff =
|
|
756
|
-
|
|
757
|
-
)
|
|
758
|
-
return queryset.annotate(
|
|
759
|
-
live_rank=Case(
|
|
760
|
-
When(last_heartbeat_at__gte=cutoff, then=Value(0)),
|
|
761
|
-
default=Value(1),
|
|
762
|
-
output_field=IntegerField(),
|
|
763
|
-
)
|
|
764
|
-
)
|
|
821
|
+
cutoff = observability.process_cutoff_for_backend(self._backend_alias(request))
|
|
822
|
+
return queryset.annotate(live_rank=observability.process_live_rank_expression(cutoff))
|
|
765
823
|
|
|
766
824
|
@admin.display(description="status", ordering="live_rank")
|
|
767
825
|
def display_status(self, obj):
|
|
@@ -872,7 +930,11 @@ class PauseAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
872
930
|
def handle_change_action(self, request, obj, action):
|
|
873
931
|
backend_alias = self._backend_alias(request)
|
|
874
932
|
if action == "resume":
|
|
875
|
-
|
|
933
|
+
try:
|
|
934
|
+
QueueInfo(obj.queue_name, backend_alias=backend_alias).resume()
|
|
935
|
+
except ADMIN_ACTION_ERRORS as exc:
|
|
936
|
+
self.message_user(request, f"Could not resume queue: {exc}", level=messages.ERROR)
|
|
937
|
+
return self._current_object_redirect(obj, backend_alias=backend_alias)
|
|
876
938
|
self.message_user(
|
|
877
939
|
request,
|
|
878
940
|
format_html(
|
|
@@ -896,18 +958,8 @@ class SemaphoreAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
896
958
|
def get_queryset(self, request):
|
|
897
959
|
queryset = super().get_queryset(request)
|
|
898
960
|
alias = self._backend_database_alias(request)
|
|
899
|
-
blocked_waiters = (
|
|
900
|
-
BlockedExecution.objects.using(alias)
|
|
901
|
-
.filter(concurrency_key=OuterRef("key"))
|
|
902
|
-
.values("concurrency_key")
|
|
903
|
-
.annotate(total=Count("id"))
|
|
904
|
-
.values("total")[:1]
|
|
905
|
-
)
|
|
906
961
|
return queryset.annotate(
|
|
907
|
-
blocked_waiter_count=
|
|
908
|
-
Subquery(blocked_waiters, output_field=IntegerField()),
|
|
909
|
-
Value(0),
|
|
910
|
-
)
|
|
962
|
+
blocked_waiter_count=observability.semaphore_blocked_waiter_count_expression(alias)
|
|
911
963
|
)
|
|
912
964
|
|
|
913
965
|
@admin.display(description="blocked waiters", ordering="blocked_waiter_count")
|
|
@@ -1,19 +1,15 @@
|
|
|
1
|
-
from datetime import timedelta
|
|
2
1
|
from functools import partial
|
|
3
2
|
|
|
4
3
|
from django.db import transaction
|
|
5
|
-
from django.utils import timezone
|
|
6
4
|
|
|
7
5
|
from dj_queue import observability
|
|
8
|
-
from dj_queue.config import load_backend_config
|
|
9
|
-
from dj_queue.db import get_database_alias
|
|
10
|
-
from dj_queue.models import ReadyExecution
|
|
11
6
|
from dj_queue.operations.jobs import (
|
|
12
7
|
ClaimedJob,
|
|
13
8
|
claim_ready_jobs,
|
|
14
9
|
discard_blocked_jobs,
|
|
15
10
|
discard_failed_job,
|
|
16
11
|
discard_failed_jobs,
|
|
12
|
+
discard_ready_jobs_for_queue,
|
|
17
13
|
discard_ready_jobs,
|
|
18
14
|
discard_scheduled_jobs,
|
|
19
15
|
execute_claimed_job,
|
|
@@ -42,16 +38,28 @@ __all__ = [
|
|
|
42
38
|
|
|
43
39
|
|
|
44
40
|
class QueueInfo:
|
|
45
|
-
def __init__(self, queue_name, *, backend_alias="default"):
|
|
41
|
+
def __init__(self, queue_name, *, backend_alias="default", snapshot=None):
|
|
46
42
|
self.queue_name = queue_name
|
|
47
43
|
self.backend_alias = backend_alias
|
|
44
|
+
self._snapshot = snapshot
|
|
48
45
|
|
|
49
46
|
@property
|
|
50
47
|
def size(self):
|
|
51
|
-
|
|
48
|
+
if self._snapshot is not None:
|
|
49
|
+
return self._snapshot["ready_count"]
|
|
50
|
+
return observability.queue_ready_count(
|
|
51
|
+
backend_alias=self.backend_alias,
|
|
52
|
+
queue_name=self.queue_name,
|
|
53
|
+
)
|
|
52
54
|
|
|
53
55
|
@property
|
|
54
56
|
def latency(self):
|
|
57
|
+
if self._snapshot is not None:
|
|
58
|
+
latency = self._snapshot["latency_seconds"]
|
|
59
|
+
if latency is None and not self._snapshot["paused"]:
|
|
60
|
+
return 0.0
|
|
61
|
+
return latency
|
|
62
|
+
|
|
55
63
|
paused = observability.queue_is_paused(
|
|
56
64
|
backend_alias=self.backend_alias,
|
|
57
65
|
queue_name=self.queue_name,
|
|
@@ -68,6 +76,8 @@ class QueueInfo:
|
|
|
68
76
|
|
|
69
77
|
@property
|
|
70
78
|
def paused(self):
|
|
79
|
+
if self._snapshot is not None:
|
|
80
|
+
return self._snapshot["paused"]
|
|
71
81
|
return observability.queue_is_paused(
|
|
72
82
|
backend_alias=self.backend_alias,
|
|
73
83
|
queue_name=self.queue_name,
|
|
@@ -75,40 +85,29 @@ class QueueInfo:
|
|
|
75
85
|
|
|
76
86
|
def pause(self):
|
|
77
87
|
pause_queue(self.queue_name, backend_alias=self.backend_alias)
|
|
88
|
+
self._snapshot = None
|
|
78
89
|
|
|
79
90
|
def resume(self):
|
|
80
91
|
resume_queue(self.queue_name, backend_alias=self.backend_alias)
|
|
92
|
+
self._snapshot = None
|
|
81
93
|
|
|
82
94
|
def clear(self, *, batch_size=500):
|
|
83
95
|
deleted = 0
|
|
84
96
|
while True:
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
return deleted
|
|
88
|
-
deleted += discard_ready_jobs(
|
|
89
|
-
job_ids=job_ids,
|
|
97
|
+
batch_deleted = discard_ready_jobs_for_queue(
|
|
98
|
+
self.queue_name,
|
|
90
99
|
batch_size=batch_size,
|
|
91
100
|
backend_alias=self.backend_alias,
|
|
92
101
|
)
|
|
102
|
+
if not batch_deleted:
|
|
103
|
+
self._snapshot = None
|
|
104
|
+
return deleted
|
|
105
|
+
deleted += batch_deleted
|
|
93
106
|
|
|
94
107
|
@classmethod
|
|
95
108
|
def all(cls, *, backend_alias="default"):
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
process_cutoff = now - timedelta(seconds=config.process_alive_threshold)
|
|
99
|
-
queue_rows = observability.queue_rows(
|
|
100
|
-
backend_alias=backend_alias,
|
|
101
|
-
now=now,
|
|
102
|
-
process_cutoff=process_cutoff,
|
|
103
|
-
)
|
|
104
|
-
return [cls(row["name"], backend_alias=backend_alias) for row in queue_rows]
|
|
105
|
-
|
|
106
|
-
def _ready_queryset(self):
|
|
107
|
-
alias = get_database_alias(self.backend_alias)
|
|
108
|
-
return ReadyExecution.objects.using(alias).filter(
|
|
109
|
-
backend_alias=self.backend_alias,
|
|
110
|
-
queue_name=self.queue_name,
|
|
111
|
-
)
|
|
109
|
+
queue_rows = observability.queue_rows_for_backend(backend_alias=backend_alias)
|
|
110
|
+
return [cls(row["name"], backend_alias=backend_alias, snapshot=row) for row in queue_rows]
|
|
112
111
|
|
|
113
112
|
|
|
114
113
|
def enqueue_on_commit(task, *args, using=None, **kwargs):
|