dj-queue 0.6.4__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dj_queue-0.6.4 → dj_queue-0.7.1}/PKG-INFO +25 -1
- {dj_queue-0.6.4 → dj_queue-0.7.1}/README.md +24 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/api.py +19 -75
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/backend.py +9 -1
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/config.py +82 -21
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/contrib/prometheus.py +5 -6
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/db.py +7 -1
- dj_queue-0.7.1/dj_queue/migrations/0007_recurringtask_next_run_at.py +23 -0
- dj_queue-0.7.1/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +161 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/models/jobs.py +46 -11
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/models/recurring.py +8 -1
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/observability.py +10 -10
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/operations/concurrency.py +85 -10
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/operations/jobs.py +205 -93
- dj_queue-0.7.1/dj_queue/operations/queues.py +69 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/operations/recurring.py +93 -1
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/base.py +8 -4
- dj_queue-0.7.1/dj_queue/runtime/connection_budget.py +98 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/notify.py +38 -5
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/pool.py +40 -2
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/scheduler.py +5 -1
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/supervisor.py +5 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/worker.py +5 -4
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/urls.py +1 -6
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/views.py +4 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/pyproject.toml +1 -1
- {dj_queue-0.6.4 → dj_queue-0.7.1}/LICENSE +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/admin.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/apps.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/contrib/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/contrib/asgi.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/contrib/gunicorn.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/dashboard.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/exceptions.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/hooks.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/log.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/management/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/management/commands/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/management/commands/dj_queue.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/management/commands/dj_queue_health.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/management/commands/dj_queue_prune.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/0001_initial.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/0004_dashboard.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/migrations/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/models/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/models/runtime.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/operations/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/operations/_insert.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/operations/cleanup.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/routers.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/dispatcher.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/errors.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/interruptible.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/pidfile.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/runtime/procline.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templatetags/__init__.py +0 -0
- {dj_queue-0.6.4 → dj_queue-0.7.1}/dj_queue/templatetags/dj_queue_admin.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dj-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Database-backed task queue backend for Django’s Tasks framework.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -349,6 +349,8 @@ If you need to pass model instances, files, or custom objects, store them elsewh
|
|
|
349
349
|
|
|
350
350
|
For MySQL or MariaDB, install and configure a Django-compatible driver following Django's database docs.
|
|
351
351
|
|
|
352
|
+
Other Django database vendors are rejected explicitly.
|
|
353
|
+
|
|
352
354
|
Polling is the portability path everywhere. Backend-specific features improve latency and throughput but are not correctness requirements.
|
|
353
355
|
|
|
354
356
|
For production PostgreSQL operational guidance, see [Postgres Queue Health](#postgres-queue-health).
|
|
@@ -358,6 +360,10 @@ For production PostgreSQL operational guidance, see [Postgres Queue Health](#pos
|
|
|
358
360
|
`dj_queue` supports both static recurring tasks from settings and dynamic
|
|
359
361
|
recurring tasks managed at runtime.
|
|
360
362
|
|
|
363
|
+
Recurring tasks persist their next due cursor after the first scheduler poll so
|
|
364
|
+
large recurring sets do not need cron parsing on every tick for rows that are not
|
|
365
|
+
due yet.
|
|
366
|
+
|
|
361
367
|
### Static recurring tasks
|
|
362
368
|
|
|
363
369
|
Define recurring tasks in `TASKS[...]["OPTIONS"]["recurring"]`:
|
|
@@ -538,7 +544,9 @@ backend-side validation failures instead of silently dropping work.
|
|
|
538
544
|
Common reasons include:
|
|
539
545
|
|
|
540
546
|
- args or kwargs are not JSON round-trippable
|
|
547
|
+
- the task targets a queue outside a non-empty `QUEUES` allow-list
|
|
541
548
|
- `concurrency_key` is set without `concurrency_limit`
|
|
549
|
+
- `concurrency_limit` or `concurrency_duration` is not a positive integer
|
|
542
550
|
- `concurrency_key` cannot be resolved from the enqueue arguments
|
|
543
551
|
- `concurrency_key` does not resolve to a non-empty string up to 255 chars
|
|
544
552
|
- `on_conflict` is not `"block"` or `"discard"`
|
|
@@ -670,6 +678,12 @@ policy, and autovacuum tuning.
|
|
|
670
678
|
|
|
671
679
|
- Use a dedicated queue database via `database_alias`. Keep reporting and
|
|
672
680
|
long-running transactions off the queue database.
|
|
681
|
+
- Consider a positive Django `CONN_MAX_AGE` for the queue database connection
|
|
682
|
+
path. It can materially improve worker throughput by reusing worker-thread
|
|
683
|
+
connections, but size PostgreSQL `max_connections` or your connection pool
|
|
684
|
+
for the total web, runner, and worker-thread footprint first. `dj_queue`
|
|
685
|
+
logs `connection_budget.warning` on startup when the local worker footprint
|
|
686
|
+
appears close to PostgreSQL connection capacity.
|
|
673
687
|
- Keep retention short. Set `preserve_finished_jobs = False` if you do not need
|
|
674
688
|
successful results. Otherwise use bounded `clear_finished_jobs_after`,
|
|
675
689
|
`clear_failed_jobs_after`, and `clear_recurring_executions_after` values.
|
|
@@ -730,6 +744,9 @@ Common setup choices:
|
|
|
730
744
|
- multiple backends, same database: good for logical and operational separation without another database
|
|
731
745
|
- multiple backends, multiple databases: use when you need stronger isolation and accept more migration and deployment complexity
|
|
732
746
|
|
|
747
|
+
`TASKS[backend_alias]["QUEUES"]` is an enqueue allow-list. Leave it as `[]` to
|
|
748
|
+
allow any queue name, or set an exact list to reject work outside those lanes.
|
|
749
|
+
|
|
733
750
|
### Deployment topology
|
|
734
751
|
|
|
735
752
|
Once migrations are in place, start processing jobs with `python manage.py dj_queue`
|
|
@@ -920,6 +937,8 @@ The `/dj_queue/metrics` endpoint requires the `prometheus` extra:
|
|
|
920
937
|
pip install "dj-queue[prometheus]"
|
|
921
938
|
```
|
|
922
939
|
|
|
940
|
+
If the extra is missing, the endpoint returns `503` with an installation hint.
|
|
941
|
+
|
|
923
942
|
Exported metric families:
|
|
924
943
|
|
|
925
944
|
- `dj_queue_queue_jobs{backend,queue,state}`
|
|
@@ -937,6 +956,11 @@ Both endpoints support bearer token authentication. Set
|
|
|
937
956
|
`Authorization: Bearer <token>`. Leave it unset if you protect these URLs at
|
|
938
957
|
the network or proxy layer.
|
|
939
958
|
|
|
959
|
+
## Benchmarks
|
|
960
|
+
|
|
961
|
+
The repository includes a standalone benchmark harness for enqueue, promotion, recurring, worker-drain, and concurrency-contention scenarios.
|
|
962
|
+
See [`docs/benchmarks/`](docs/benchmarks/) for the latest published reports.
|
|
963
|
+
|
|
940
964
|
## License
|
|
941
965
|
|
|
942
966
|
MIT
|
|
@@ -321,6 +321,8 @@ If you need to pass model instances, files, or custom objects, store them elsewh
|
|
|
321
321
|
|
|
322
322
|
For MySQL or MariaDB, install and configure a Django-compatible driver following Django's database docs.
|
|
323
323
|
|
|
324
|
+
Other Django database vendors are rejected explicitly.
|
|
325
|
+
|
|
324
326
|
Polling is the portability path everywhere. Backend-specific features improve latency and throughput but are not correctness requirements.
|
|
325
327
|
|
|
326
328
|
For production PostgreSQL operational guidance, see [Postgres Queue Health](#postgres-queue-health).
|
|
@@ -330,6 +332,10 @@ For production PostgreSQL operational guidance, see [Postgres Queue Health](#pos
|
|
|
330
332
|
`dj_queue` supports both static recurring tasks from settings and dynamic
|
|
331
333
|
recurring tasks managed at runtime.
|
|
332
334
|
|
|
335
|
+
Recurring tasks persist their next due cursor after the first scheduler poll so
|
|
336
|
+
large recurring sets do not need cron parsing on every tick for rows that are not
|
|
337
|
+
due yet.
|
|
338
|
+
|
|
333
339
|
### Static recurring tasks
|
|
334
340
|
|
|
335
341
|
Define recurring tasks in `TASKS[...]["OPTIONS"]["recurring"]`:
|
|
@@ -510,7 +516,9 @@ backend-side validation failures instead of silently dropping work.
|
|
|
510
516
|
Common reasons include:
|
|
511
517
|
|
|
512
518
|
- args or kwargs are not JSON round-trippable
|
|
519
|
+
- the task targets a queue outside a non-empty `QUEUES` allow-list
|
|
513
520
|
- `concurrency_key` is set without `concurrency_limit`
|
|
521
|
+
- `concurrency_limit` or `concurrency_duration` is not a positive integer
|
|
514
522
|
- `concurrency_key` cannot be resolved from the enqueue arguments
|
|
515
523
|
- `concurrency_key` does not resolve to a non-empty string up to 255 chars
|
|
516
524
|
- `on_conflict` is not `"block"` or `"discard"`
|
|
@@ -642,6 +650,12 @@ policy, and autovacuum tuning.
|
|
|
642
650
|
|
|
643
651
|
- Use a dedicated queue database via `database_alias`. Keep reporting and
|
|
644
652
|
long-running transactions off the queue database.
|
|
653
|
+
- Consider a positive Django `CONN_MAX_AGE` for the queue database connection
|
|
654
|
+
path. It can materially improve worker throughput by reusing worker-thread
|
|
655
|
+
connections, but size PostgreSQL `max_connections` or your connection pool
|
|
656
|
+
for the total web, runner, and worker-thread footprint first. `dj_queue`
|
|
657
|
+
logs `connection_budget.warning` on startup when the local worker footprint
|
|
658
|
+
appears close to PostgreSQL connection capacity.
|
|
645
659
|
- Keep retention short. Set `preserve_finished_jobs = False` if you do not need
|
|
646
660
|
successful results. Otherwise use bounded `clear_finished_jobs_after`,
|
|
647
661
|
`clear_failed_jobs_after`, and `clear_recurring_executions_after` values.
|
|
@@ -702,6 +716,9 @@ Common setup choices:
|
|
|
702
716
|
- multiple backends, same database: good for logical and operational separation without another database
|
|
703
717
|
- multiple backends, multiple databases: use when you need stronger isolation and accept more migration and deployment complexity
|
|
704
718
|
|
|
719
|
+
`TASKS[backend_alias]["QUEUES"]` is an enqueue allow-list. Leave it as `[]` to
|
|
720
|
+
allow any queue name, or set an exact list to reject work outside those lanes.
|
|
721
|
+
|
|
705
722
|
### Deployment topology
|
|
706
723
|
|
|
707
724
|
Once migrations are in place, start processing jobs with `python manage.py dj_queue`
|
|
@@ -892,6 +909,8 @@ The `/dj_queue/metrics` endpoint requires the `prometheus` extra:
|
|
|
892
909
|
pip install "dj-queue[prometheus]"
|
|
893
910
|
```
|
|
894
911
|
|
|
912
|
+
If the extra is missing, the endpoint returns `503` with an installation hint.
|
|
913
|
+
|
|
895
914
|
Exported metric families:
|
|
896
915
|
|
|
897
916
|
- `dj_queue_queue_jobs{backend,queue,state}`
|
|
@@ -909,6 +928,11 @@ Both endpoints support bearer token authentication. Set
|
|
|
909
928
|
`Authorization: Bearer <token>`. Leave it unset if you protect these URLs at
|
|
910
929
|
the network or proxy layer.
|
|
911
930
|
|
|
931
|
+
## Benchmarks
|
|
932
|
+
|
|
933
|
+
The repository includes a standalone benchmark harness for enqueue, promotion, recurring, worker-drain, and concurrency-contention scenarios.
|
|
934
|
+
See [`docs/benchmarks/`](docs/benchmarks/) for the latest published reports.
|
|
935
|
+
|
|
912
936
|
## License
|
|
913
937
|
|
|
914
938
|
MIT
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
from functools import partial
|
|
2
2
|
|
|
3
|
-
from django.db.models import Case, DateTimeField, DurationField, ExpressionWrapper, F, Value, When
|
|
4
3
|
from django.db.models.functions import Coalesce
|
|
5
4
|
from django.db import transaction
|
|
6
5
|
from django.utils import timezone
|
|
7
6
|
from django.utils.module_loading import import_string
|
|
8
7
|
|
|
9
8
|
from dj_queue.db import get_database_alias
|
|
10
|
-
from dj_queue.
|
|
11
|
-
from dj_queue.models import Pause, ReadyExecution, RecurringTask
|
|
9
|
+
from dj_queue.models import Pause, ReadyExecution
|
|
12
10
|
|
|
13
11
|
|
|
14
12
|
class QueueInfo:
|
|
@@ -46,53 +44,12 @@ class QueueInfo:
|
|
|
46
44
|
)
|
|
47
45
|
|
|
48
46
|
def pause(self):
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
backend_alias=self.backend_alias,
|
|
52
|
-
queue_name=self.queue_name,
|
|
53
|
-
)
|
|
54
|
-
log_event("queue.paused", backend_alias=self.backend_alias, queue_name=self.queue_name)
|
|
47
|
+
pause_queue = import_string("dj_queue.operations.queues.pause_queue")
|
|
48
|
+
pause_queue(self.queue_name, backend_alias=self.backend_alias)
|
|
55
49
|
|
|
56
50
|
def resume(self):
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
pause = (
|
|
60
|
-
Pause.objects.using(alias)
|
|
61
|
-
.select_for_update()
|
|
62
|
-
.filter(backend_alias=self.backend_alias, queue_name=self.queue_name)
|
|
63
|
-
.first()
|
|
64
|
-
)
|
|
65
|
-
if pause is None:
|
|
66
|
-
return
|
|
67
|
-
|
|
68
|
-
resumed_at = timezone.now()
|
|
69
|
-
paused_at = pause.created_at
|
|
70
|
-
pause_duration = Value(resumed_at - paused_at, output_field=DurationField())
|
|
71
|
-
ready_row_ids = list(self._ready_queryset().values_list("id", flat=True))
|
|
72
|
-
if ready_row_ids:
|
|
73
|
-
ReadyExecution.objects.using(alias).filter(pk__in=ready_row_ids).update(
|
|
74
|
-
latency_started_at=Case(
|
|
75
|
-
When(
|
|
76
|
-
latency_started_at__isnull=True,
|
|
77
|
-
created_at__lt=paused_at,
|
|
78
|
-
then=ExpressionWrapper(
|
|
79
|
-
F("created_at") + pause_duration, output_field=DateTimeField()
|
|
80
|
-
),
|
|
81
|
-
),
|
|
82
|
-
When(
|
|
83
|
-
latency_started_at__lt=paused_at,
|
|
84
|
-
then=ExpressionWrapper(
|
|
85
|
-
F("latency_started_at") + pause_duration,
|
|
86
|
-
output_field=DateTimeField(),
|
|
87
|
-
),
|
|
88
|
-
),
|
|
89
|
-
default=Value(resumed_at, output_field=DateTimeField()),
|
|
90
|
-
output_field=DateTimeField(),
|
|
91
|
-
),
|
|
92
|
-
)
|
|
93
|
-
pause.delete()
|
|
94
|
-
|
|
95
|
-
log_event("queue.resumed", backend_alias=self.backend_alias, queue_name=self.queue_name)
|
|
51
|
+
resume_queue = import_string("dj_queue.operations.queues.resume_queue")
|
|
52
|
+
resume_queue(self.queue_name, backend_alias=self.backend_alias)
|
|
96
53
|
|
|
97
54
|
def clear(self, *, batch_size=500):
|
|
98
55
|
deleted = 0
|
|
@@ -111,7 +68,7 @@ class QueueInfo:
|
|
|
111
68
|
alias = get_database_alias(backend_alias)
|
|
112
69
|
queue_names = (
|
|
113
70
|
ReadyExecution.objects.using(alias)
|
|
114
|
-
.filter(
|
|
71
|
+
.filter(backend_alias=backend_alias)
|
|
115
72
|
.order_by("queue_name")
|
|
116
73
|
.values_list(
|
|
117
74
|
"queue_name",
|
|
@@ -124,8 +81,8 @@ class QueueInfo:
|
|
|
124
81
|
def _ready_queryset(self):
|
|
125
82
|
alias = get_database_alias(self.backend_alias)
|
|
126
83
|
return ReadyExecution.objects.using(alias).filter(
|
|
84
|
+
backend_alias=self.backend_alias,
|
|
127
85
|
queue_name=self.queue_name,
|
|
128
|
-
job__backend_alias=self.backend_alias,
|
|
129
86
|
)
|
|
130
87
|
|
|
131
88
|
|
|
@@ -150,33 +107,20 @@ def schedule_recurring_task(
|
|
|
150
107
|
description="",
|
|
151
108
|
backend_alias="default",
|
|
152
109
|
):
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
kwargs = {}
|
|
156
|
-
|
|
157
|
-
recurring_task, _ = RecurringTask.objects.using(alias).update_or_create(
|
|
158
|
-
backend_alias=backend_alias,
|
|
110
|
+
operation = import_string("dj_queue.operations.recurring.schedule_recurring_task")
|
|
111
|
+
return operation(
|
|
159
112
|
key=key,
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
},
|
|
113
|
+
task_path=task_path,
|
|
114
|
+
schedule=schedule,
|
|
115
|
+
args=args,
|
|
116
|
+
kwargs=kwargs,
|
|
117
|
+
queue_name=queue_name,
|
|
118
|
+
priority=priority,
|
|
119
|
+
description=description,
|
|
120
|
+
backend_alias=backend_alias,
|
|
169
121
|
)
|
|
170
|
-
return recurring_task
|
|
171
122
|
|
|
172
123
|
|
|
173
124
|
def unschedule_recurring_task(key, *, backend_alias="default"):
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
backend_alias=backend_alias,
|
|
177
|
-
key=key,
|
|
178
|
-
static=False,
|
|
179
|
-
)
|
|
180
|
-
deleted = queryset.count()
|
|
181
|
-
queryset.delete()
|
|
182
|
-
return deleted
|
|
125
|
+
operation = import_string("dj_queue.operations.recurring.unschedule_recurring_task")
|
|
126
|
+
return operation(key, backend_alias=backend_alias)
|
|
@@ -8,7 +8,11 @@ from django.utils.module_loading import import_string
|
|
|
8
8
|
|
|
9
9
|
from dj_queue.db import get_database_alias
|
|
10
10
|
from dj_queue.models import Job
|
|
11
|
-
from dj_queue.operations.jobs import
|
|
11
|
+
from dj_queue.operations.jobs import (
|
|
12
|
+
enqueue_job_with_dispatch,
|
|
13
|
+
enqueue_jobs_bulk,
|
|
14
|
+
validate_queue_allowed,
|
|
15
|
+
)
|
|
12
16
|
|
|
13
17
|
|
|
14
18
|
class DjQueueBackend(BaseTaskBackend):
|
|
@@ -17,6 +21,10 @@ class DjQueueBackend(BaseTaskBackend):
|
|
|
17
21
|
supports_get_result = True
|
|
18
22
|
supports_priority = True
|
|
19
23
|
|
|
24
|
+
def validate_task(self, task):
|
|
25
|
+
validate_queue_allowed(task.queue_name, backend_alias=self.alias)
|
|
26
|
+
return super().validate_task(task)
|
|
27
|
+
|
|
20
28
|
def enqueue(self, task, args, kwargs):
|
|
21
29
|
self.validate_task(task)
|
|
22
30
|
job, dispatched_as = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
|
|
@@ -174,12 +174,15 @@ def _load_backend_config_cached(
|
|
|
174
174
|
if mode not in {"fork", "async"}:
|
|
175
175
|
raise ImproperlyConfigured(f"dj_queue mode must be 'fork' or 'async', got {mode!r}")
|
|
176
176
|
|
|
177
|
-
only_work =
|
|
178
|
-
only_dispatch =
|
|
177
|
+
only_work = _bool_option(cli_overrides.get("only_work", False), "--only-work")
|
|
178
|
+
only_dispatch = _bool_option(cli_overrides.get("only_dispatch", False), "--only-dispatch")
|
|
179
179
|
if only_work and only_dispatch:
|
|
180
180
|
raise ImproperlyConfigured("--only-work and --only-dispatch cannot be combined")
|
|
181
181
|
|
|
182
182
|
skip_recurring = _resolve_skip_recurring(cli_overrides, env)
|
|
183
|
+
preserve_finished_jobs = _bool_option(
|
|
184
|
+
resolved_options["preserve_finished_jobs"], "preserve_finished_jobs"
|
|
185
|
+
)
|
|
183
186
|
on_thread_error = _validated_callback_path(resolved_options.get("on_thread_error"))
|
|
184
187
|
recurring = _build_recurring_config(resolved_options.get("recurring", {}))
|
|
185
188
|
scheduler = _build_scheduler_config(resolved_options.get("scheduler", DEFAULT_SCHEDULER))
|
|
@@ -195,7 +198,7 @@ def _load_backend_config_cached(
|
|
|
195
198
|
elif skip_recurring or not _scheduler_has_work(
|
|
196
199
|
scheduler,
|
|
197
200
|
recurring,
|
|
198
|
-
preserve_finished_jobs=
|
|
201
|
+
preserve_finished_jobs=preserve_finished_jobs,
|
|
199
202
|
clear_finished_jobs_after=resolved_options["clear_finished_jobs_after"],
|
|
200
203
|
clear_failed_jobs_after=resolved_options["clear_failed_jobs_after"],
|
|
201
204
|
clear_recurring_executions_after=resolved_options["clear_recurring_executions_after"],
|
|
@@ -215,21 +218,28 @@ def _load_backend_config_cached(
|
|
|
215
218
|
dispatchers=dispatchers,
|
|
216
219
|
scheduler=scheduler,
|
|
217
220
|
recurring=recurring,
|
|
218
|
-
process_heartbeat_interval=
|
|
219
|
-
|
|
220
|
-
|
|
221
|
+
process_heartbeat_interval=_nonnegative_float(
|
|
222
|
+
resolved_options["process_heartbeat_interval"], "process_heartbeat_interval"
|
|
223
|
+
),
|
|
224
|
+
process_alive_threshold=_positive_float(
|
|
225
|
+
resolved_options["process_alive_threshold"], "process_alive_threshold"
|
|
226
|
+
),
|
|
227
|
+
shutdown_timeout=_nonnegative_float(resolved_options["shutdown_timeout"], "shutdown_timeout"),
|
|
221
228
|
supervisor_pidfile=resolved_options["supervisor_pidfile"],
|
|
222
|
-
preserve_finished_jobs=
|
|
229
|
+
preserve_finished_jobs=preserve_finished_jobs,
|
|
223
230
|
clear_finished_jobs_after=_optional_int(resolved_options["clear_finished_jobs_after"]),
|
|
224
231
|
clear_failed_jobs_after=_optional_int(resolved_options["clear_failed_jobs_after"]),
|
|
225
232
|
clear_recurring_executions_after=_optional_int(
|
|
226
233
|
resolved_options["clear_recurring_executions_after"]
|
|
227
234
|
),
|
|
228
|
-
default_concurrency_duration=
|
|
235
|
+
default_concurrency_duration=_positive_int(
|
|
236
|
+
resolved_options["default_concurrency_duration"],
|
|
237
|
+
"default_concurrency_duration",
|
|
238
|
+
),
|
|
229
239
|
database_alias=str(resolved_options["database_alias"]),
|
|
230
|
-
use_skip_locked=
|
|
231
|
-
listen_notify=
|
|
232
|
-
silence_polling=
|
|
240
|
+
use_skip_locked=_bool_option(resolved_options["use_skip_locked"], "use_skip_locked"),
|
|
241
|
+
listen_notify=_bool_option(resolved_options["listen_notify"], "listen_notify"),
|
|
242
|
+
silence_polling=_bool_option(resolved_options["silence_polling"], "silence_polling"),
|
|
233
243
|
on_thread_error=on_thread_error,
|
|
234
244
|
skip_recurring=skip_recurring,
|
|
235
245
|
only_work=only_work,
|
|
@@ -356,7 +366,7 @@ def _resolve_skip_recurring(
|
|
|
356
366
|
env: Mapping[str, str],
|
|
357
367
|
) -> bool:
|
|
358
368
|
if "skip_recurring" in cli_overrides:
|
|
359
|
-
return
|
|
369
|
+
return _bool_option(cli_overrides["skip_recurring"], "skip_recurring")
|
|
360
370
|
|
|
361
371
|
value = env.get("DJ_QUEUE_SKIP_RECURRING")
|
|
362
372
|
if value is None:
|
|
@@ -375,6 +385,16 @@ def _parse_bool(value: str, setting_name: str) -> bool:
|
|
|
375
385
|
)
|
|
376
386
|
|
|
377
387
|
|
|
388
|
+
def _bool_option(value: Any, setting_name: str) -> bool:
|
|
389
|
+
if isinstance(value, bool):
|
|
390
|
+
return value
|
|
391
|
+
if isinstance(value, str):
|
|
392
|
+
return _parse_bool(value, setting_name)
|
|
393
|
+
if isinstance(value, int) and value in (0, 1):
|
|
394
|
+
return bool(value)
|
|
395
|
+
raise ImproperlyConfigured(f"dj_queue {setting_name} must be a boolean")
|
|
396
|
+
|
|
397
|
+
|
|
378
398
|
def _validated_callback_path(callback_path: Any) -> str | None:
|
|
379
399
|
if callback_path in (None, ""):
|
|
380
400
|
return None
|
|
@@ -400,8 +420,13 @@ def _build_worker_configs(raw_workers: Any, mode: str) -> tuple[WorkerConfig, ..
|
|
|
400
420
|
|
|
401
421
|
worker = WorkerConfig(
|
|
402
422
|
queues=_as_queue_selectors(raw_worker.get("queues", DEFAULT_WORKER["queues"])),
|
|
403
|
-
threads=
|
|
404
|
-
|
|
423
|
+
threads=_positive_int(
|
|
424
|
+
raw_worker.get("threads", DEFAULT_WORKER["threads"]), f"workers[{index}].threads"
|
|
425
|
+
),
|
|
426
|
+
processes=_positive_int(
|
|
427
|
+
raw_worker.get("processes", DEFAULT_WORKER["processes"]),
|
|
428
|
+
f"workers[{index}].processes",
|
|
429
|
+
),
|
|
405
430
|
polling_interval=_positive_float(
|
|
406
431
|
raw_worker.get("polling_interval", DEFAULT_WORKER["polling_interval"]),
|
|
407
432
|
f"workers[{index}].polling_interval",
|
|
@@ -431,22 +456,27 @@ def _build_dispatcher_configs(raw_dispatchers: Any) -> tuple[DispatcherConfig, .
|
|
|
431
456
|
|
|
432
457
|
dispatchers.append(
|
|
433
458
|
DispatcherConfig(
|
|
434
|
-
batch_size=
|
|
459
|
+
batch_size=_positive_int(
|
|
460
|
+
raw_dispatcher.get("batch_size", DEFAULT_DISPATCHER["batch_size"]),
|
|
461
|
+
f"dispatchers[{index}].batch_size",
|
|
462
|
+
),
|
|
435
463
|
polling_interval=_positive_float(
|
|
436
464
|
raw_dispatcher.get("polling_interval", DEFAULT_DISPATCHER["polling_interval"]),
|
|
437
465
|
f"dispatchers[{index}].polling_interval",
|
|
438
466
|
),
|
|
439
|
-
concurrency_maintenance=
|
|
467
|
+
concurrency_maintenance=_bool_option(
|
|
440
468
|
raw_dispatcher.get(
|
|
441
469
|
"concurrency_maintenance",
|
|
442
470
|
DEFAULT_DISPATCHER["concurrency_maintenance"],
|
|
443
|
-
)
|
|
471
|
+
),
|
|
472
|
+
f"dispatchers[{index}].concurrency_maintenance",
|
|
444
473
|
),
|
|
445
|
-
concurrency_maintenance_interval=
|
|
474
|
+
concurrency_maintenance_interval=_nonnegative_float(
|
|
446
475
|
raw_dispatcher.get(
|
|
447
476
|
"concurrency_maintenance_interval",
|
|
448
477
|
DEFAULT_DISPATCHER["concurrency_maintenance_interval"],
|
|
449
|
-
)
|
|
478
|
+
),
|
|
479
|
+
f"dispatchers[{index}].concurrency_maintenance_interval",
|
|
450
480
|
),
|
|
451
481
|
)
|
|
452
482
|
)
|
|
@@ -460,11 +490,12 @@ def _build_scheduler_config(raw_scheduler: Any) -> SchedulerConfig:
|
|
|
460
490
|
raise ImproperlyConfigured("scheduler config must be a mapping")
|
|
461
491
|
|
|
462
492
|
return SchedulerConfig(
|
|
463
|
-
dynamic_tasks_enabled=
|
|
493
|
+
dynamic_tasks_enabled=_bool_option(
|
|
464
494
|
raw_scheduler.get(
|
|
465
495
|
"dynamic_tasks_enabled",
|
|
466
496
|
DEFAULT_SCHEDULER["dynamic_tasks_enabled"],
|
|
467
|
-
)
|
|
497
|
+
),
|
|
498
|
+
"scheduler.dynamic_tasks_enabled",
|
|
468
499
|
),
|
|
469
500
|
polling_interval=_positive_float(
|
|
470
501
|
raw_scheduler.get("polling_interval", DEFAULT_SCHEDULER["polling_interval"]),
|
|
@@ -556,5 +587,35 @@ def _positive_float(value: Any, setting_name: str) -> float:
|
|
|
556
587
|
return number
|
|
557
588
|
|
|
558
589
|
|
|
590
|
+
def _nonnegative_float(value: Any, setting_name: str) -> float:
|
|
591
|
+
try:
|
|
592
|
+
number = float(value)
|
|
593
|
+
except (TypeError, ValueError) as exc:
|
|
594
|
+
raise ImproperlyConfigured(
|
|
595
|
+
f"dj_queue {setting_name} must be a non-negative number, got {value!r}"
|
|
596
|
+
) from exc
|
|
597
|
+
|
|
598
|
+
if not math.isfinite(number) or number < 0:
|
|
599
|
+
raise ImproperlyConfigured(
|
|
600
|
+
f"dj_queue {setting_name} must be a non-negative number, got {value!r}"
|
|
601
|
+
)
|
|
602
|
+
return number
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _positive_int(value: Any, setting_name: str) -> int:
|
|
606
|
+
try:
|
|
607
|
+
number = int(value)
|
|
608
|
+
except (TypeError, ValueError, OverflowError) as exc:
|
|
609
|
+
raise ImproperlyConfigured(
|
|
610
|
+
f"dj_queue {setting_name} must be a positive integer, got {value!r}"
|
|
611
|
+
) from exc
|
|
612
|
+
|
|
613
|
+
if number <= 0:
|
|
614
|
+
raise ImproperlyConfigured(
|
|
615
|
+
f"dj_queue {setting_name} must be a positive integer, got {value!r}"
|
|
616
|
+
)
|
|
617
|
+
return number
|
|
618
|
+
|
|
619
|
+
|
|
559
620
|
def _cache_key(value: Any) -> str:
|
|
560
621
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
try:
|
|
2
2
|
from prometheus_client import CollectorRegistry, generate_latest
|
|
3
3
|
from prometheus_client.core import GaugeMetricFamily
|
|
4
|
-
|
|
4
|
+
except ImportError:
|
|
5
|
+
DjQueueCollector = None
|
|
6
|
+
registry = None
|
|
7
|
+
generate_latest = None
|
|
8
|
+
else:
|
|
5
9
|
from dj_queue import observability
|
|
6
10
|
|
|
7
11
|
class DjQueueCollector:
|
|
@@ -122,8 +126,3 @@ try:
|
|
|
122
126
|
|
|
123
127
|
registry = CollectorRegistry(auto_describe=False)
|
|
124
128
|
registry.register(DjQueueCollector())
|
|
125
|
-
|
|
126
|
-
except ImportError:
|
|
127
|
-
DjQueueCollector = None
|
|
128
|
-
registry = None
|
|
129
|
-
generate_latest = None
|
|
@@ -3,6 +3,7 @@ from contextlib import contextmanager
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from typing import Literal
|
|
5
5
|
|
|
6
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
6
7
|
from django.db import DEFAULT_DB_ALIAS, connections
|
|
7
8
|
|
|
8
9
|
from dj_queue.config import load_backend_config
|
|
@@ -65,4 +66,9 @@ def _backend_family(connection) -> Literal["postgresql", "mysql", "mariadb", "sq
|
|
|
65
66
|
return "sqlite"
|
|
66
67
|
if connection.vendor == "mysql" and getattr(connection, "mysql_is_mariadb", False):
|
|
67
68
|
return "mariadb"
|
|
68
|
-
|
|
69
|
+
if connection.vendor == "mysql":
|
|
70
|
+
return "mysql"
|
|
71
|
+
raise ImproperlyConfigured(
|
|
72
|
+
f"dj_queue unsupported database vendor {connection.vendor!r}; "
|
|
73
|
+
"supported vendors are 'postgresql', 'mysql', and 'sqlite'"
|
|
74
|
+
)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Generated by Django 6.0.4 on 2026-05-06 06:30
|
|
2
|
+
|
|
3
|
+
from django.db import migrations, models
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Migration(migrations.Migration):
|
|
7
|
+
dependencies = [
|
|
8
|
+
("dj_queue", "0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more"),
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
operations = [
|
|
12
|
+
migrations.AddField(
|
|
13
|
+
model_name="recurringtask",
|
|
14
|
+
name="next_run_at",
|
|
15
|
+
field=models.DateTimeField(blank=True, null=True),
|
|
16
|
+
),
|
|
17
|
+
migrations.AddIndex(
|
|
18
|
+
model_name="recurringtask",
|
|
19
|
+
index=models.Index(
|
|
20
|
+
fields=["backend_alias", "next_run_at", "key"], name="dj_queue_rt_next_run_idx"
|
|
21
|
+
),
|
|
22
|
+
),
|
|
23
|
+
]
|