dj-queue 0.13.0__tar.gz → 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dj_queue-0.13.0 → dj_queue-0.14.0}/PKG-INFO +118 -15
- {dj_queue-0.13.0 → dj_queue-0.14.0}/README.md +115 -13
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/admin.py +31 -6
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/api.py +47 -12
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/apps.py +1 -2
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/config.py +34 -2
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/asgi.py +1 -1
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/gunicorn.py +6 -5
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/cron.py +1 -4
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/dashboard.py +12 -9
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/dashboard_actions.py +0 -1
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/db.py +74 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/exceptions.py +4 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/commands/dj_queue.py +33 -9
- dj_queue-0.14.0/dj_queue/management/commands/dj_queue_health.py +59 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/commands/dj_queue_postgres_autovacuum.py +11 -4
- dj_queue-0.14.0/dj_queue/management/commands/dj_queue_prune.py +86 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0001_initial.py +2 -1
- dj_queue-0.14.0/dj_queue/migrations/0014_semaphore_active_count.py +18 -0
- dj_queue-0.14.0/dj_queue/migrations/0015_job_concurrency_duration.py +18 -0
- dj_queue-0.14.0/dj_queue/migrations/0016_job_concurrency_limit.py +18 -0
- dj_queue-0.14.0/dj_queue/migrations/0017_job_concurrency_on_conflict.py +18 -0
- dj_queue-0.14.0/dj_queue/migrations/0018_recurringexecution_intended_job_id.py +18 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/jobs.py +4 -1
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/recurring.py +2 -1
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/runtime.py +9 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/observability.py +72 -10
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/_helpers.py +8 -2
- dj_queue-0.14.0/dj_queue/operations/claiming.py +297 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/concurrency.py +157 -63
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/jobs.py +514 -620
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/recurring.py +174 -68
- dj_queue-0.14.0/dj_queue/py.typed +1 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/base.py +20 -14
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/notify.py +7 -8
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/pool.py +3 -3
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/scheduler.py +4 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/supervisor.py +15 -9
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/worker.py +4 -3
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/mysql.py +77 -2
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/postgres.py +23 -2
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +1 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/urls.py +0 -1
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/wakeup.py +3 -3
- {dj_queue-0.13.0 → dj_queue-0.14.0}/pyproject.toml +35 -3
- dj_queue-0.13.0/dj_queue/management/commands/dj_queue_health.py +0 -28
- dj_queue-0.13.0/dj_queue/management/commands/dj_queue_prune.py +0 -44
- {dj_queue-0.13.0 → dj_queue-0.14.0}/LICENSE +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/backend.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/prometheus.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/hooks.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/log.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/commands/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/metrics.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0004_dashboard.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0009_remove_process_dj_queue_processes_name_supervisor_unique_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0010_remove_process_djq_pr_name_parent_uniq_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0011_remove_blockedexecution_djq_bl_b_conc_idx_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0012_job_djq_jobs_b_queue_id_idx_job_djq_jobs_b_conc_idx_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0013_failedexecution_retry_at_and_more.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/_insert.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/cleanup.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/queues.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/queue_selectors.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/queue_state.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/routers.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/connection_budget.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/dispatcher.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/errors.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/interruptible.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/pidfile.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/procline.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/topology.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/common.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/sqlite.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/state.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/task_results.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templatetags/__init__.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templatetags/dj_queue_admin.py +0 -0
- {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/views.py +0 -0
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dj-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.0
|
|
4
4
|
Summary: Database-backed task queue backend for Django’s Tasks framework.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
7
7
|
Classifier: Development Status :: 4 - Beta
|
|
8
8
|
Classifier: Framework :: Django
|
|
9
9
|
Classifier: Framework :: Django :: 6.0
|
|
10
|
+
Classifier: Framework :: Django :: 6.1
|
|
10
11
|
Classifier: Intended Audience :: Developers
|
|
11
12
|
Classifier: Programming Language :: Python :: 3
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
15
16
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
|
-
Requires-Dist: django>=6.0.0
|
|
17
|
+
Requires-Dist: django>=6.0.0,<6.2
|
|
17
18
|
Requires-Dist: psycopg>=3.3.3 ; extra == 'postgres'
|
|
18
19
|
Requires-Dist: prometheus-client>=0.4.0 ; extra == 'prometheus'
|
|
19
20
|
Requires-Python: >=3.12
|
|
@@ -39,7 +40,7 @@ It keeps the queue, live execution state, runtime metadata, and task results in
|
|
|
39
40
|
|
|
40
41
|
- no Redis, RabbitMQ, or separate result store
|
|
41
42
|
- PostgreSQL is the first-class production backend
|
|
42
|
-
- MySQL
|
|
43
|
+
- MySQL, MariaDB, and SQLite are supported at Django's version floors
|
|
43
44
|
- immediate, scheduled, recurring, and concurrency-limited work
|
|
44
45
|
|
|
45
46
|
`dj_queue` is inspired by Rails' [Solid Queue](https://github.com/rails/solid_queue),
|
|
@@ -66,7 +67,7 @@ see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
|
|
|
66
67
|
|
|
67
68
|
## Installation
|
|
68
69
|
|
|
69
|
-
`dj_queue` requires Python 3.12+ and Django 6.0
|
|
70
|
+
`dj_queue` requires Python 3.12+ and supports Django 6.0 and 6.1.
|
|
70
71
|
|
|
71
72
|
Install the package:
|
|
72
73
|
|
|
@@ -74,6 +75,9 @@ Install the package:
|
|
|
74
75
|
pip install dj-queue
|
|
75
76
|
```
|
|
76
77
|
|
|
78
|
+
The package includes inline public API annotations and a `py.typed` marker for
|
|
79
|
+
PEP 561-compatible type checkers.
|
|
80
|
+
|
|
77
81
|
Optional extras:
|
|
78
82
|
|
|
79
83
|
```bash
|
|
@@ -131,6 +135,7 @@ Define a task with Django's `@task` decorator:
|
|
|
131
135
|
# myapp/tasks.py
|
|
132
136
|
from django.tasks import task
|
|
133
137
|
|
|
138
|
+
|
|
134
139
|
@task
|
|
135
140
|
def add(a, b):
|
|
136
141
|
return a + b
|
|
@@ -352,10 +357,21 @@ If you need to pass model instances, files, or custom objects, store them elsewh
|
|
|
352
357
|
| Backend | Support level | Notes |
|
|
353
358
|
|---|---|---|
|
|
354
359
|
| PostgreSQL | first-class | polling, `SKIP LOCKED`, and optional `LISTEN/NOTIFY` |
|
|
355
|
-
| MySQL
|
|
356
|
-
| MariaDB
|
|
360
|
+
| MySQL | supported | polling plus `SKIP LOCKED` |
|
|
361
|
+
| MariaDB | supported | polling plus `SKIP LOCKED` |
|
|
357
362
|
| SQLite | supported with limits | polling only, serialized writes, no `SKIP LOCKED`, no `LISTEN/NOTIFY`; practical for development, CI, and smaller deployments |
|
|
358
363
|
|
|
364
|
+
The database floor follows the selected Django release:
|
|
365
|
+
|
|
366
|
+
| Django | PostgreSQL | MySQL | MariaDB | SQLite |
|
|
367
|
+
|---|---|---|---|---|
|
|
368
|
+
| 6.0 | 14+ | 8.0.11+ | 10.6+ | 3.31+ |
|
|
369
|
+
| 6.1 | 15+ | 8.4+ | 10.11+ | 3.37+ |
|
|
370
|
+
|
|
371
|
+
CI keeps the floors on Django 6.0 and tests newer database lines through
|
|
372
|
+
PostgreSQL 18, MySQL 9.7 and 26.7, and MariaDB 12.3. Django 6.1 only runs with
|
|
373
|
+
database versions that it supports.
|
|
374
|
+
|
|
359
375
|
For MySQL or MariaDB, install and configure a Django-compatible driver following Django's database docs.
|
|
360
376
|
|
|
361
377
|
Other Django database vendors are rejected explicitly.
|
|
@@ -364,6 +380,70 @@ Polling is the portability path everywhere. Backend-specific features improve la
|
|
|
364
380
|
|
|
365
381
|
For production PostgreSQL operational guidance, see [Postgres Queue Health](#postgres-queue-health).
|
|
366
382
|
|
|
383
|
+
## Rolling Upgrades
|
|
384
|
+
|
|
385
|
+
`dj_queue` supports live queue writes during a one-generation N/N−1 rolling
|
|
386
|
+
upgrade. Schema changes use an expand, bridge, activate, and contract sequence:
|
|
387
|
+
|
|
388
|
+
- release N adds nullable fields and writes both old and new representations
|
|
389
|
+
- N and N−1 workers and producers can use the expanded schema together
|
|
390
|
+
- release N+1 can activate the new representation after N−1 has left the fleet
|
|
391
|
+
- a later release removes compatibility fields only after its oldest supported writer no longer needs them
|
|
392
|
+
|
|
393
|
+
Each N release must name its minimum rollout-compatible N−1 patch. That patch
|
|
394
|
+
must already retry transient database conflicts because release N cannot change
|
|
395
|
+
the behavior of a process that is still running N−1.
|
|
396
|
+
|
|
397
|
+
Use this deployment order:
|
|
398
|
+
|
|
399
|
+
1. Upgrade all application and queue processes to the minimum compatible N−1 patch.
|
|
400
|
+
2. Run `python manage.py dj_queue_health --deep` for queue processes and
|
|
401
|
+
invariants. Use your deployment platform to confirm that no older application
|
|
402
|
+
producer remains.
|
|
403
|
+
3. Apply release N migrations once.
|
|
404
|
+
4. Roll application producers and queue processes from N−1 to N.
|
|
405
|
+
5. Run `python manage.py dj_queue_health --deep --require-version <N version>` for
|
|
406
|
+
queue processes. Use your deployment platform to confirm that no N−1
|
|
407
|
+
application producer remains.
|
|
408
|
+
|
|
409
|
+
Each registered queue process stores its exact `dj_queue` version and rollout
|
|
410
|
+
protocol generation. Deep health rejects a missing or incompatible protocol.
|
|
411
|
+
`--require-version` also rejects a live queue process on another package
|
|
412
|
+
version. Application producers do not register process rows, so check those in
|
|
413
|
+
your deployment platform.
|
|
414
|
+
|
|
415
|
+
A code rollback from N to N−1 keeps the expanded schema in place. Do not reverse
|
|
416
|
+
the database migration during a live rollback. N−2 processes and skipped bridge
|
|
417
|
+
releases are not supported.
|
|
418
|
+
|
|
419
|
+
Expansion migrations keep one DDL operation per hot table migration so lock
|
|
420
|
+
conflicts can retry safely. Constraints that a supported database cannot add
|
|
421
|
+
online stay deferred until a later activation or contract release.
|
|
422
|
+
|
|
423
|
+
This contract covers `dj_queue`'s schema and runtime. Jobs can outlive an
|
|
424
|
+
application deployment, so keep your task import paths and accepted argument
|
|
425
|
+
shapes compatible. Use a new task name when a payload change cannot be backward
|
|
426
|
+
compatible.
|
|
427
|
+
|
|
428
|
+
Maintainers can prove a candidate rollout against two immutable revisions:
|
|
429
|
+
|
|
430
|
+
```bash
|
|
431
|
+
bin/prerelease.py \
|
|
432
|
+
--from-ref <release-n-1-ref> \
|
|
433
|
+
--to-ref <release-n-ref> \
|
|
434
|
+
--backend postgres \
|
|
435
|
+
--django '>=6.0,<6.1'
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
The command calibrates release N−1, applies release N migrations under live
|
|
439
|
+
writes, runs both versions together, drains with release N, and verifies queue
|
|
440
|
+
state and side effects. It rejects revisions that do not publish one shared
|
|
441
|
+
rollout protocol. It writes wheel hashes, one-second metrics, logs, and a result
|
|
442
|
+
manifest under `benchmark-results/`. The **Pre-release migration load** workflow
|
|
443
|
+
runs the database floor and current-version lanes. `--smoke` and custom workload
|
|
444
|
+
profiles keep all validity and stability gates and record performance. Only the
|
|
445
|
+
default ten-minute release profile enforces the X-to-Y performance ratios.
|
|
446
|
+
|
|
367
447
|
## Recurring Tasks
|
|
368
448
|
|
|
369
449
|
`dj_queue` supports both static recurring tasks from settings and dynamic
|
|
@@ -445,27 +525,33 @@ Notes:
|
|
|
445
525
|
|
|
446
526
|
Tasks can opt into database-backed concurrency limits.
|
|
447
527
|
|
|
448
|
-
`
|
|
449
|
-
`@task` decorator, so `dj_queue` reads them as attributes on the wrapped function:
|
|
528
|
+
Use `dj_queue.api.concurrency` with Django's `@task` decorator:
|
|
450
529
|
|
|
451
530
|
```python
|
|
452
531
|
from django.tasks import task
|
|
532
|
+
from dj_queue.api import concurrency
|
|
533
|
+
|
|
453
534
|
|
|
454
535
|
@task
|
|
536
|
+
@concurrency(
|
|
537
|
+
key="account:{account_id}",
|
|
538
|
+
limit=1,
|
|
539
|
+
duration=60,
|
|
540
|
+
on_conflict="block",
|
|
541
|
+
)
|
|
455
542
|
def sync_account(account_id, action):
|
|
456
543
|
return f"{account_id}:{action}"
|
|
457
|
-
|
|
458
|
-
sync_account.func.concurrency_key = "account:{account_id}"
|
|
459
|
-
sync_account.func.concurrency_limit = 1
|
|
460
|
-
sync_account.func.concurrency_duration = 60
|
|
461
|
-
sync_account.func.on_conflict = "block"
|
|
462
544
|
```
|
|
463
545
|
|
|
464
546
|
With this configuration:
|
|
465
547
|
|
|
466
548
|
- the first matching job can run immediately
|
|
467
549
|
- later jobs for the same key can block until capacity is released
|
|
468
|
-
- `on_conflict
|
|
550
|
+
- `on_conflict="discard"` turns the same pattern into singleton-style work
|
|
551
|
+
|
|
552
|
+
The concurrency key can be a format string or a callable. Both decorator orders
|
|
553
|
+
are supported, but placing `@task` above `@concurrency` keeps backend-specific
|
|
554
|
+
options next to the task function.
|
|
469
555
|
|
|
470
556
|
Semaphore rows remain shared on the queue database. If you want per-backend
|
|
471
557
|
isolation for a limit, express that in the `concurrency_key` itself rather than
|
|
@@ -511,10 +597,13 @@ Operational commands:
|
|
|
511
597
|
|
|
512
598
|
```bash
|
|
513
599
|
python manage.py dj_queue_health
|
|
600
|
+
python manage.py dj_queue_health --deep
|
|
514
601
|
python manage.py dj_queue_health --max-age 120
|
|
602
|
+
python manage.py dj_queue_health --deep --require-version 0.13.0
|
|
515
603
|
python manage.py dj_queue_prune --older-than 86400
|
|
516
604
|
python manage.py dj_queue_prune --failed-older-than 604800
|
|
517
605
|
python manage.py dj_queue_prune --recurring-older-than 2592000
|
|
606
|
+
python manage.py dj_queue_prune --batch-size 500
|
|
518
607
|
python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
|
|
519
608
|
python manage.py dj_queue_prune --task-key nightly_cleanup
|
|
520
609
|
python manage.py dj_queue_postgres_autovacuum
|
|
@@ -522,9 +611,16 @@ python manage.py dj_queue_postgres_autovacuum
|
|
|
522
611
|
|
|
523
612
|
The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
|
|
524
613
|
|
|
614
|
+
`dj_queue_health` exits with status `1` and prints a reason when no process is
|
|
615
|
+
live. Add `--deep` to check persisted queue invariants and live process rollout
|
|
616
|
+
protocols. Add `--require-version` to require one exact package version across
|
|
617
|
+
all live queue processes.
|
|
618
|
+
|
|
525
619
|
For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
|
|
526
620
|
task import path, while `--task-key` filters recurring execution cleanup by
|
|
527
|
-
recurring task key.
|
|
621
|
+
recurring task key. One invocation deletes at most `--batch-size` rows from
|
|
622
|
+
each retained row type. Run it again when it reports that the batch limit was
|
|
623
|
+
reached.
|
|
528
624
|
|
|
529
625
|
## Failed Jobs
|
|
530
626
|
|
|
@@ -596,14 +692,17 @@ Each callback receives the live supervisor or runner instance.
|
|
|
596
692
|
```python
|
|
597
693
|
from dj_queue.hooks import on_start, on_worker_start, register_hook
|
|
598
694
|
|
|
695
|
+
|
|
599
696
|
@on_start
|
|
600
697
|
def supervisor_started(process):
|
|
601
698
|
print(process.name)
|
|
602
699
|
|
|
700
|
+
|
|
603
701
|
@on_worker_start
|
|
604
702
|
def worker_started(process):
|
|
605
703
|
print(process.metadata)
|
|
606
704
|
|
|
705
|
+
|
|
607
706
|
@register_hook("scheduler.exit")
|
|
608
707
|
def scheduler_exited(process):
|
|
609
708
|
print(process.name)
|
|
@@ -890,6 +989,10 @@ Configuration precedence is explicit:
|
|
|
890
989
|
- TOML file pointed to by `DJ_QUEUE_CONFIG`
|
|
891
990
|
- Django `TASKS` settings
|
|
892
991
|
|
|
992
|
+
Unknown option names are rejected in Django settings and in the selected TOML
|
|
993
|
+
overlay, including nested worker, dispatcher, scheduler, and recurring task
|
|
994
|
+
options. The error identifies the exact configuration path and option name.
|
|
995
|
+
|
|
893
996
|
### TOML file config
|
|
894
997
|
|
|
895
998
|
```bash
|
|
@@ -13,7 +13,7 @@ It keeps the queue, live execution state, runtime metadata, and task results in
|
|
|
13
13
|
|
|
14
14
|
- no Redis, RabbitMQ, or separate result store
|
|
15
15
|
- PostgreSQL is the first-class production backend
|
|
16
|
-
- MySQL
|
|
16
|
+
- MySQL, MariaDB, and SQLite are supported at Django's version floors
|
|
17
17
|
- immediate, scheduled, recurring, and concurrency-limited work
|
|
18
18
|
|
|
19
19
|
`dj_queue` is inspired by Rails' [Solid Queue](https://github.com/rails/solid_queue),
|
|
@@ -40,7 +40,7 @@ see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
|
|
|
40
40
|
|
|
41
41
|
## Installation
|
|
42
42
|
|
|
43
|
-
`dj_queue` requires Python 3.12+ and Django 6.0
|
|
43
|
+
`dj_queue` requires Python 3.12+ and supports Django 6.0 and 6.1.
|
|
44
44
|
|
|
45
45
|
Install the package:
|
|
46
46
|
|
|
@@ -48,6 +48,9 @@ Install the package:
|
|
|
48
48
|
pip install dj-queue
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
+
The package includes inline public API annotations and a `py.typed` marker for
|
|
52
|
+
PEP 561-compatible type checkers.
|
|
53
|
+
|
|
51
54
|
Optional extras:
|
|
52
55
|
|
|
53
56
|
```bash
|
|
@@ -105,6 +108,7 @@ Define a task with Django's `@task` decorator:
|
|
|
105
108
|
# myapp/tasks.py
|
|
106
109
|
from django.tasks import task
|
|
107
110
|
|
|
111
|
+
|
|
108
112
|
@task
|
|
109
113
|
def add(a, b):
|
|
110
114
|
return a + b
|
|
@@ -326,10 +330,21 @@ If you need to pass model instances, files, or custom objects, store them elsewh
|
|
|
326
330
|
| Backend | Support level | Notes |
|
|
327
331
|
|---|---|---|
|
|
328
332
|
| PostgreSQL | first-class | polling, `SKIP LOCKED`, and optional `LISTEN/NOTIFY` |
|
|
329
|
-
| MySQL
|
|
330
|
-
| MariaDB
|
|
333
|
+
| MySQL | supported | polling plus `SKIP LOCKED` |
|
|
334
|
+
| MariaDB | supported | polling plus `SKIP LOCKED` |
|
|
331
335
|
| SQLite | supported with limits | polling only, serialized writes, no `SKIP LOCKED`, no `LISTEN/NOTIFY`; practical for development, CI, and smaller deployments |
|
|
332
336
|
|
|
337
|
+
The database floor follows the selected Django release:
|
|
338
|
+
|
|
339
|
+
| Django | PostgreSQL | MySQL | MariaDB | SQLite |
|
|
340
|
+
|---|---|---|---|---|
|
|
341
|
+
| 6.0 | 14+ | 8.0.11+ | 10.6+ | 3.31+ |
|
|
342
|
+
| 6.1 | 15+ | 8.4+ | 10.11+ | 3.37+ |
|
|
343
|
+
|
|
344
|
+
CI keeps the floors on Django 6.0 and tests newer database lines through
|
|
345
|
+
PostgreSQL 18, MySQL 9.7 and 26.7, and MariaDB 12.3. Django 6.1 only runs with
|
|
346
|
+
database versions that it supports.
|
|
347
|
+
|
|
333
348
|
For MySQL or MariaDB, install and configure a Django-compatible driver following Django's database docs.
|
|
334
349
|
|
|
335
350
|
Other Django database vendors are rejected explicitly.
|
|
@@ -338,6 +353,70 @@ Polling is the portability path everywhere. Backend-specific features improve la
|
|
|
338
353
|
|
|
339
354
|
For production PostgreSQL operational guidance, see [Postgres Queue Health](#postgres-queue-health).
|
|
340
355
|
|
|
356
|
+
## Rolling Upgrades
|
|
357
|
+
|
|
358
|
+
`dj_queue` supports live queue writes during a one-generation N/N−1 rolling
|
|
359
|
+
upgrade. Schema changes use an expand, bridge, activate, and contract sequence:
|
|
360
|
+
|
|
361
|
+
- release N adds nullable fields and writes both old and new representations
|
|
362
|
+
- N and N−1 workers and producers can use the expanded schema together
|
|
363
|
+
- release N+1 can activate the new representation after N−1 has left the fleet
|
|
364
|
+
- a later release removes compatibility fields only after its oldest supported writer no longer needs them
|
|
365
|
+
|
|
366
|
+
Each N release must name its minimum rollout-compatible N−1 patch. That patch
|
|
367
|
+
must already retry transient database conflicts because release N cannot change
|
|
368
|
+
the behavior of a process that is still running N−1.
|
|
369
|
+
|
|
370
|
+
Use this deployment order:
|
|
371
|
+
|
|
372
|
+
1. Upgrade all application and queue processes to the minimum compatible N−1 patch.
|
|
373
|
+
2. Run `python manage.py dj_queue_health --deep` for queue processes and
|
|
374
|
+
invariants. Use your deployment platform to confirm that no older application
|
|
375
|
+
producer remains.
|
|
376
|
+
3. Apply release N migrations once.
|
|
377
|
+
4. Roll application producers and queue processes from N−1 to N.
|
|
378
|
+
5. Run `python manage.py dj_queue_health --deep --require-version <N version>` for
|
|
379
|
+
queue processes. Use your deployment platform to confirm that no N−1
|
|
380
|
+
application producer remains.
|
|
381
|
+
|
|
382
|
+
Each registered queue process stores its exact `dj_queue` version and rollout
|
|
383
|
+
protocol generation. Deep health rejects a missing or incompatible protocol.
|
|
384
|
+
`--require-version` also rejects a live queue process on another package
|
|
385
|
+
version. Application producers do not register process rows, so check those in
|
|
386
|
+
your deployment platform.
|
|
387
|
+
|
|
388
|
+
A code rollback from N to N−1 keeps the expanded schema in place. Do not reverse
|
|
389
|
+
the database migration during a live rollback. N−2 processes and skipped bridge
|
|
390
|
+
releases are not supported.
|
|
391
|
+
|
|
392
|
+
Expansion migrations keep one DDL operation per hot table migration so lock
|
|
393
|
+
conflicts can retry safely. Constraints that a supported database cannot add
|
|
394
|
+
online stay deferred until a later activation or contract release.
|
|
395
|
+
|
|
396
|
+
This contract covers `dj_queue`'s schema and runtime. Jobs can outlive an
|
|
397
|
+
application deployment, so keep your task import paths and accepted argument
|
|
398
|
+
shapes compatible. Use a new task name when a payload change cannot be backward
|
|
399
|
+
compatible.
|
|
400
|
+
|
|
401
|
+
Maintainers can prove a candidate rollout against two immutable revisions:
|
|
402
|
+
|
|
403
|
+
```bash
|
|
404
|
+
bin/prerelease.py \
|
|
405
|
+
--from-ref <release-n-1-ref> \
|
|
406
|
+
--to-ref <release-n-ref> \
|
|
407
|
+
--backend postgres \
|
|
408
|
+
--django '>=6.0,<6.1'
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
The command calibrates release N−1, applies release N migrations under live
|
|
412
|
+
writes, runs both versions together, drains with release N, and verifies queue
|
|
413
|
+
state and side effects. It rejects revisions that do not publish one shared
|
|
414
|
+
rollout protocol. It writes wheel hashes, one-second metrics, logs, and a result
|
|
415
|
+
manifest under `benchmark-results/`. The **Pre-release migration load** workflow
|
|
416
|
+
runs the database floor and current-version lanes. `--smoke` and custom workload
|
|
417
|
+
profiles keep all validity and stability gates and record performance. Only the
|
|
418
|
+
default ten-minute release profile enforces the X-to-Y performance ratios.
|
|
419
|
+
|
|
341
420
|
## Recurring Tasks
|
|
342
421
|
|
|
343
422
|
`dj_queue` supports both static recurring tasks from settings and dynamic
|
|
@@ -419,27 +498,33 @@ Notes:
|
|
|
419
498
|
|
|
420
499
|
Tasks can opt into database-backed concurrency limits.
|
|
421
500
|
|
|
422
|
-
`
|
|
423
|
-
`@task` decorator, so `dj_queue` reads them as attributes on the wrapped function:
|
|
501
|
+
Use `dj_queue.api.concurrency` with Django's `@task` decorator:
|
|
424
502
|
|
|
425
503
|
```python
|
|
426
504
|
from django.tasks import task
|
|
505
|
+
from dj_queue.api import concurrency
|
|
506
|
+
|
|
427
507
|
|
|
428
508
|
@task
|
|
509
|
+
@concurrency(
|
|
510
|
+
key="account:{account_id}",
|
|
511
|
+
limit=1,
|
|
512
|
+
duration=60,
|
|
513
|
+
on_conflict="block",
|
|
514
|
+
)
|
|
429
515
|
def sync_account(account_id, action):
|
|
430
516
|
return f"{account_id}:{action}"
|
|
431
|
-
|
|
432
|
-
sync_account.func.concurrency_key = "account:{account_id}"
|
|
433
|
-
sync_account.func.concurrency_limit = 1
|
|
434
|
-
sync_account.func.concurrency_duration = 60
|
|
435
|
-
sync_account.func.on_conflict = "block"
|
|
436
517
|
```
|
|
437
518
|
|
|
438
519
|
With this configuration:
|
|
439
520
|
|
|
440
521
|
- the first matching job can run immediately
|
|
441
522
|
- later jobs for the same key can block until capacity is released
|
|
442
|
-
- `on_conflict
|
|
523
|
+
- `on_conflict="discard"` turns the same pattern into singleton-style work
|
|
524
|
+
|
|
525
|
+
The concurrency key can be a format string or a callable. Both decorator orders
|
|
526
|
+
are supported, but placing `@task` above `@concurrency` keeps backend-specific
|
|
527
|
+
options next to the task function.
|
|
443
528
|
|
|
444
529
|
Semaphore rows remain shared on the queue database. If you want per-backend
|
|
445
530
|
isolation for a limit, express that in the `concurrency_key` itself rather than
|
|
@@ -485,10 +570,13 @@ Operational commands:
|
|
|
485
570
|
|
|
486
571
|
```bash
|
|
487
572
|
python manage.py dj_queue_health
|
|
573
|
+
python manage.py dj_queue_health --deep
|
|
488
574
|
python manage.py dj_queue_health --max-age 120
|
|
575
|
+
python manage.py dj_queue_health --deep --require-version 0.13.0
|
|
489
576
|
python manage.py dj_queue_prune --older-than 86400
|
|
490
577
|
python manage.py dj_queue_prune --failed-older-than 604800
|
|
491
578
|
python manage.py dj_queue_prune --recurring-older-than 2592000
|
|
579
|
+
python manage.py dj_queue_prune --batch-size 500
|
|
492
580
|
python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
|
|
493
581
|
python manage.py dj_queue_prune --task-key nightly_cleanup
|
|
494
582
|
python manage.py dj_queue_postgres_autovacuum
|
|
@@ -496,9 +584,16 @@ python manage.py dj_queue_postgres_autovacuum
|
|
|
496
584
|
|
|
497
585
|
The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
|
|
498
586
|
|
|
587
|
+
`dj_queue_health` exits with status `1` and prints a reason when no process is
|
|
588
|
+
live. Add `--deep` to check persisted queue invariants and live process rollout
|
|
589
|
+
protocols. Add `--require-version` to require one exact package version across
|
|
590
|
+
all live queue processes.
|
|
591
|
+
|
|
499
592
|
For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
|
|
500
593
|
task import path, while `--task-key` filters recurring execution cleanup by
|
|
501
|
-
recurring task key.
|
|
594
|
+
recurring task key. One invocation deletes at most `--batch-size` rows from
|
|
595
|
+
each retained row type. Run it again when it reports that the batch limit was
|
|
596
|
+
reached.
|
|
502
597
|
|
|
503
598
|
## Failed Jobs
|
|
504
599
|
|
|
@@ -570,14 +665,17 @@ Each callback receives the live supervisor or runner instance.
|
|
|
570
665
|
```python
|
|
571
666
|
from dj_queue.hooks import on_start, on_worker_start, register_hook
|
|
572
667
|
|
|
668
|
+
|
|
573
669
|
@on_start
|
|
574
670
|
def supervisor_started(process):
|
|
575
671
|
print(process.name)
|
|
576
672
|
|
|
673
|
+
|
|
577
674
|
@on_worker_start
|
|
578
675
|
def worker_started(process):
|
|
579
676
|
print(process.metadata)
|
|
580
677
|
|
|
678
|
+
|
|
581
679
|
@register_hook("scheduler.exit")
|
|
582
680
|
def scheduler_exited(process):
|
|
583
681
|
print(process.name)
|
|
@@ -864,6 +962,10 @@ Configuration precedence is explicit:
|
|
|
864
962
|
- TOML file pointed to by `DJ_QUEUE_CONFIG`
|
|
865
963
|
- Django `TASKS` settings
|
|
866
964
|
|
|
965
|
+
Unknown option names are rejected in Django settings and in the selected TOML
|
|
966
|
+
overlay, including nested worker, dispatcher, scheduler, and recurring task
|
|
967
|
+
options. The error identifies the exact configuration path and option name.
|
|
968
|
+
|
|
867
969
|
### TOML file config
|
|
868
970
|
|
|
869
971
|
```bash
|
|
@@ -6,13 +6,11 @@ from django.contrib import admin, messages
|
|
|
6
6
|
from django.http import HttpResponseNotAllowed, HttpResponseRedirect
|
|
7
7
|
from django.template.response import TemplateResponse
|
|
8
8
|
from django.urls import path, reverse
|
|
9
|
+
from django.utils import timezone
|
|
9
10
|
from django.utils.html import format_html
|
|
10
11
|
from django.utils.http import url_has_allowed_host_and_scheme
|
|
11
|
-
from django.utils import timezone
|
|
12
12
|
|
|
13
|
-
from dj_queue import dashboard
|
|
14
|
-
from dj_queue import dashboard_actions
|
|
15
|
-
from dj_queue import observability
|
|
13
|
+
from dj_queue import dashboard, dashboard_actions, observability
|
|
16
14
|
from dj_queue.api import QueueInfo, unschedule_recurring_task
|
|
17
15
|
from dj_queue.db import get_database_alias
|
|
18
16
|
from dj_queue.exceptions import EnqueueError
|
|
@@ -495,6 +493,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
495
493
|
"backend_alias",
|
|
496
494
|
"scheduled_at",
|
|
497
495
|
"concurrency_key",
|
|
496
|
+
"concurrency_limit",
|
|
497
|
+
"concurrency_duration",
|
|
498
|
+
"concurrency_on_conflict",
|
|
498
499
|
"finished_at",
|
|
499
500
|
"return_value",
|
|
500
501
|
"created_at",
|
|
@@ -964,8 +965,24 @@ class PauseAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
964
965
|
|
|
965
966
|
@admin.register(Semaphore)
|
|
966
967
|
class SemaphoreAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
967
|
-
list_display = (
|
|
968
|
-
|
|
968
|
+
list_display = (
|
|
969
|
+
"key",
|
|
970
|
+
"display_active_count",
|
|
971
|
+
"display_available_count",
|
|
972
|
+
"limit",
|
|
973
|
+
"display_blocked_waiters",
|
|
974
|
+
"display_expires_at",
|
|
975
|
+
)
|
|
976
|
+
readonly_fields = (
|
|
977
|
+
"key",
|
|
978
|
+
"display_active_count",
|
|
979
|
+
"display_available_count",
|
|
980
|
+
"value",
|
|
981
|
+
"limit",
|
|
982
|
+
"expires_at",
|
|
983
|
+
"created_at",
|
|
984
|
+
"updated_at",
|
|
985
|
+
)
|
|
969
986
|
search_fields = ("key",)
|
|
970
987
|
|
|
971
988
|
def get_queryset(self, request):
|
|
@@ -975,6 +992,14 @@ class SemaphoreAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
|
|
|
975
992
|
blocked_waiter_count=observability.semaphore_blocked_waiter_count_expression(alias)
|
|
976
993
|
)
|
|
977
994
|
|
|
995
|
+
@admin.display(description="active")
|
|
996
|
+
def display_active_count(self, obj):
|
|
997
|
+
return obj.occupied_count
|
|
998
|
+
|
|
999
|
+
@admin.display(description="available")
|
|
1000
|
+
def display_available_count(self, obj):
|
|
1001
|
+
return obj.available_count
|
|
1002
|
+
|
|
978
1003
|
@admin.display(description="blocked waiters", ordering="blocked_waiter_count")
|
|
979
1004
|
def display_blocked_waiters(self, obj):
|
|
980
1005
|
return obj.blocked_waiter_count
|
|
@@ -1,16 +1,18 @@
|
|
|
1
|
+
from collections.abc import Callable, Mapping
|
|
1
2
|
from functools import partial
|
|
3
|
+
from typing import Any, Literal, Self, TypeVar
|
|
2
4
|
|
|
3
5
|
from django.db import transaction
|
|
6
|
+
from django.tasks import Task
|
|
4
7
|
|
|
5
8
|
from dj_queue import observability
|
|
9
|
+
from dj_queue.operations.claiming import ClaimedJob, claim_ready_jobs
|
|
6
10
|
from dj_queue.operations.jobs import (
|
|
7
|
-
ClaimedJob,
|
|
8
|
-
claim_ready_jobs,
|
|
9
11
|
discard_blocked_jobs,
|
|
10
12
|
discard_failed_job,
|
|
11
13
|
discard_failed_jobs,
|
|
12
|
-
discard_ready_jobs_for_queue,
|
|
13
14
|
discard_ready_jobs,
|
|
15
|
+
discard_ready_jobs_for_queue,
|
|
14
16
|
discard_scheduled_jobs,
|
|
15
17
|
execute_claimed_job,
|
|
16
18
|
retry_failed_job,
|
|
@@ -24,6 +26,7 @@ __all__ = [
|
|
|
24
26
|
"ClaimedJob",
|
|
25
27
|
"QueueInfo",
|
|
26
28
|
"claim_ready_jobs",
|
|
29
|
+
"concurrency",
|
|
27
30
|
"discard_blocked_jobs",
|
|
28
31
|
"discard_failed_job",
|
|
29
32
|
"discard_failed_jobs",
|
|
@@ -38,15 +41,42 @@ __all__ = [
|
|
|
38
41
|
"unschedule_recurring_task",
|
|
39
42
|
]
|
|
40
43
|
|
|
44
|
+
_Decorated = TypeVar("_Decorated")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def concurrency(
|
|
48
|
+
*,
|
|
49
|
+
key: str | Callable[..., str],
|
|
50
|
+
limit: int,
|
|
51
|
+
duration: int | None = None,
|
|
52
|
+
on_conflict: Literal["block", "discard"] = "block",
|
|
53
|
+
) -> Callable[[_Decorated], _Decorated]:
|
|
54
|
+
def decorator(target: _Decorated) -> _Decorated:
|
|
55
|
+
func = getattr(target, "func", target)
|
|
56
|
+
func.concurrency_key = key
|
|
57
|
+
func.concurrency_limit = limit
|
|
58
|
+
if duration is not None:
|
|
59
|
+
func.concurrency_duration = duration
|
|
60
|
+
func.on_conflict = on_conflict
|
|
61
|
+
return target
|
|
62
|
+
|
|
63
|
+
return decorator
|
|
64
|
+
|
|
41
65
|
|
|
42
66
|
class QueueInfo:
|
|
43
|
-
def __init__(
|
|
67
|
+
def __init__(
|
|
68
|
+
self,
|
|
69
|
+
queue_name: str,
|
|
70
|
+
*,
|
|
71
|
+
backend_alias: str = "default",
|
|
72
|
+
snapshot: Mapping[str, Any] | None = None,
|
|
73
|
+
) -> None:
|
|
44
74
|
self.queue_name = queue_name
|
|
45
75
|
self.backend_alias = backend_alias
|
|
46
76
|
self._snapshot = snapshot
|
|
47
77
|
|
|
48
78
|
@property
|
|
49
|
-
def size(self):
|
|
79
|
+
def size(self) -> int:
|
|
50
80
|
if self._snapshot is not None:
|
|
51
81
|
return self._snapshot["ready_count"]
|
|
52
82
|
return observability.queue_ready_count(
|
|
@@ -55,7 +85,7 @@ class QueueInfo:
|
|
|
55
85
|
)
|
|
56
86
|
|
|
57
87
|
@property
|
|
58
|
-
def latency(self):
|
|
88
|
+
def latency(self) -> float | None:
|
|
59
89
|
if self._snapshot is not None:
|
|
60
90
|
latency = self._snapshot["latency_seconds"]
|
|
61
91
|
if latency is None and not self._snapshot["paused"]:
|
|
@@ -77,7 +107,7 @@ class QueueInfo:
|
|
|
77
107
|
return 0.0 if latency is None else latency
|
|
78
108
|
|
|
79
109
|
@property
|
|
80
|
-
def paused(self):
|
|
110
|
+
def paused(self) -> bool:
|
|
81
111
|
if self._snapshot is not None:
|
|
82
112
|
return self._snapshot["paused"]
|
|
83
113
|
return observability.queue_is_paused(
|
|
@@ -85,15 +115,15 @@ class QueueInfo:
|
|
|
85
115
|
queue_name=self.queue_name,
|
|
86
116
|
)
|
|
87
117
|
|
|
88
|
-
def pause(self):
|
|
118
|
+
def pause(self) -> None:
|
|
89
119
|
pause_queue(self.queue_name, backend_alias=self.backend_alias)
|
|
90
120
|
self._snapshot = None
|
|
91
121
|
|
|
92
|
-
def resume(self):
|
|
122
|
+
def resume(self) -> None:
|
|
93
123
|
resume_queue(self.queue_name, backend_alias=self.backend_alias)
|
|
94
124
|
self._snapshot = None
|
|
95
125
|
|
|
96
|
-
def clear(self, *, batch_size=500):
|
|
126
|
+
def clear(self, *, batch_size: int = 500) -> int:
|
|
97
127
|
deleted = 0
|
|
98
128
|
while True:
|
|
99
129
|
batch_deleted = discard_ready_jobs_for_queue(
|
|
@@ -107,10 +137,15 @@ class QueueInfo:
|
|
|
107
137
|
deleted += batch_deleted
|
|
108
138
|
|
|
109
139
|
@classmethod
|
|
110
|
-
def all(cls, *, backend_alias="default"):
|
|
140
|
+
def all(cls, *, backend_alias: str = "default") -> list[Self]:
|
|
111
141
|
queue_rows = observability.queue_rows_for_backend(backend_alias=backend_alias)
|
|
112
142
|
return [cls(row["name"], backend_alias=backend_alias, snapshot=row) for row in queue_rows]
|
|
113
143
|
|
|
114
144
|
|
|
115
|
-
def enqueue_on_commit(
|
|
145
|
+
def enqueue_on_commit(
|
|
146
|
+
task: Task,
|
|
147
|
+
*args: Any,
|
|
148
|
+
using: str | None = None,
|
|
149
|
+
**kwargs: Any,
|
|
150
|
+
) -> None:
|
|
116
151
|
transaction.on_commit(partial(task.enqueue, *args, **kwargs), using=using)
|