dj-queue 0.13.0__tar.gz → 0.14.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. {dj_queue-0.13.0 → dj_queue-0.14.0}/PKG-INFO +118 -15
  2. {dj_queue-0.13.0 → dj_queue-0.14.0}/README.md +115 -13
  3. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/admin.py +31 -6
  4. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/api.py +47 -12
  5. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/apps.py +1 -2
  6. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/config.py +34 -2
  7. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/asgi.py +1 -1
  8. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/gunicorn.py +6 -5
  9. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/cron.py +1 -4
  10. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/dashboard.py +12 -9
  11. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/dashboard_actions.py +0 -1
  12. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/db.py +74 -0
  13. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/exceptions.py +4 -0
  14. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/commands/dj_queue.py +33 -9
  15. dj_queue-0.14.0/dj_queue/management/commands/dj_queue_health.py +59 -0
  16. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/commands/dj_queue_postgres_autovacuum.py +11 -4
  17. dj_queue-0.14.0/dj_queue/management/commands/dj_queue_prune.py +86 -0
  18. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0001_initial.py +2 -1
  19. dj_queue-0.14.0/dj_queue/migrations/0014_semaphore_active_count.py +18 -0
  20. dj_queue-0.14.0/dj_queue/migrations/0015_job_concurrency_duration.py +18 -0
  21. dj_queue-0.14.0/dj_queue/migrations/0016_job_concurrency_limit.py +18 -0
  22. dj_queue-0.14.0/dj_queue/migrations/0017_job_concurrency_on_conflict.py +18 -0
  23. dj_queue-0.14.0/dj_queue/migrations/0018_recurringexecution_intended_job_id.py +18 -0
  24. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/jobs.py +4 -1
  25. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/recurring.py +2 -1
  26. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/runtime.py +9 -0
  27. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/observability.py +72 -10
  28. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/_helpers.py +8 -2
  29. dj_queue-0.14.0/dj_queue/operations/claiming.py +297 -0
  30. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/concurrency.py +157 -63
  31. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/jobs.py +514 -620
  32. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/recurring.py +174 -68
  33. dj_queue-0.14.0/dj_queue/py.typed +1 -0
  34. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/base.py +20 -14
  35. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/notify.py +7 -8
  36. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/pool.py +3 -3
  37. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/scheduler.py +4 -0
  38. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/supervisor.py +15 -9
  39. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/worker.py +4 -3
  40. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/mysql.py +77 -2
  41. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/postgres.py +23 -2
  42. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +1 -0
  43. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/urls.py +0 -1
  44. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/wakeup.py +3 -3
  45. {dj_queue-0.13.0 → dj_queue-0.14.0}/pyproject.toml +35 -3
  46. dj_queue-0.13.0/dj_queue/management/commands/dj_queue_health.py +0 -28
  47. dj_queue-0.13.0/dj_queue/management/commands/dj_queue_prune.py +0 -44
  48. {dj_queue-0.13.0 → dj_queue-0.14.0}/LICENSE +0 -0
  49. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/__init__.py +0 -0
  50. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/backend.py +0 -0
  51. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/__init__.py +0 -0
  52. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/contrib/prometheus.py +0 -0
  53. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/hooks.py +0 -0
  54. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/log.py +0 -0
  55. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/__init__.py +0 -0
  56. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/management/commands/__init__.py +0 -0
  57. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/metrics.py +0 -0
  58. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
  59. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
  60. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0004_dashboard.py +0 -0
  61. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
  62. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
  63. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
  64. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
  65. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0009_remove_process_dj_queue_processes_name_supervisor_unique_and_more.py +0 -0
  66. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0010_remove_process_djq_pr_name_parent_uniq_and_more.py +0 -0
  67. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0011_remove_blockedexecution_djq_bl_b_conc_idx_and_more.py +0 -0
  68. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0012_job_djq_jobs_b_queue_id_idx_job_djq_jobs_b_conc_idx_and_more.py +0 -0
  69. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/0013_failedexecution_retry_at_and_more.py +0 -0
  70. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/migrations/__init__.py +0 -0
  71. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/models/__init__.py +0 -0
  72. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/__init__.py +0 -0
  73. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/_insert.py +0 -0
  74. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/cleanup.py +0 -0
  75. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/operations/queues.py +0 -0
  76. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/queue_selectors.py +0 -0
  77. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/queue_state.py +0 -0
  78. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/routers.py +0 -0
  79. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/__init__.py +0 -0
  80. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/connection_budget.py +0 -0
  81. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/dispatcher.py +0 -0
  82. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/errors.py +0 -0
  83. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/interruptible.py +0 -0
  84. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/pidfile.py +0 -0
  85. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/procline.py +0 -0
  86. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/runtime/topology.py +0 -0
  87. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/__init__.py +0 -0
  88. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/common.py +0 -0
  89. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/sqlite.py +0 -0
  90. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/sql/state.py +0 -0
  91. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/task_results.py +0 -0
  92. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
  93. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
  94. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
  95. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
  96. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
  97. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
  98. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
  99. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
  100. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
  101. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
  102. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
  103. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templatetags/__init__.py +0 -0
  104. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/templatetags/dj_queue_admin.py +0 -0
  105. {dj_queue-0.13.0 → dj_queue-0.14.0}/dj_queue/views.py +0 -0
@@ -1,19 +1,20 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dj-queue
3
- Version: 0.13.0
3
+ Version: 0.14.0
4
4
  Summary: Database-backed task queue backend for Django’s Tasks framework.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
7
7
  Classifier: Development Status :: 4 - Beta
8
8
  Classifier: Framework :: Django
9
9
  Classifier: Framework :: Django :: 6.0
10
+ Classifier: Framework :: Django :: 6.1
10
11
  Classifier: Intended Audience :: Developers
11
12
  Classifier: Programming Language :: Python :: 3
12
13
  Classifier: Programming Language :: Python :: 3.12
13
14
  Classifier: Programming Language :: Python :: 3.13
14
15
  Classifier: Programming Language :: Python :: 3.14
15
16
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
- Requires-Dist: django>=6.0.0
17
+ Requires-Dist: django>=6.0.0,<6.2
17
18
  Requires-Dist: psycopg>=3.3.3 ; extra == 'postgres'
18
19
  Requires-Dist: prometheus-client>=0.4.0 ; extra == 'prometheus'
19
20
  Requires-Python: >=3.12
@@ -39,7 +40,7 @@ It keeps the queue, live execution state, runtime metadata, and task results in
39
40
 
40
41
  - no Redis, RabbitMQ, or separate result store
41
42
  - PostgreSQL is the first-class production backend
42
- - MySQL 8+, MariaDB 10.6+, and SQLite are supported
43
+ - MySQL, MariaDB, and SQLite are supported at Django's version floors
43
44
  - immediate, scheduled, recurring, and concurrency-limited work
44
45
 
45
46
  `dj_queue` is inspired by Rails' [Solid Queue](https://github.com/rails/solid_queue),
@@ -66,7 +67,7 @@ see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
66
67
 
67
68
  ## Installation
68
69
 
69
- `dj_queue` requires Python 3.12+ and Django 6.0+.
70
+ `dj_queue` requires Python 3.12+ and supports Django 6.0 and 6.1.
70
71
 
71
72
  Install the package:
72
73
 
@@ -74,6 +75,9 @@ Install the package:
74
75
  pip install dj-queue
75
76
  ```
76
77
 
78
+ The package includes inline public API annotations and a `py.typed` marker for
79
+ PEP 561-compatible type checkers.
80
+
77
81
  Optional extras:
78
82
 
79
83
  ```bash
@@ -131,6 +135,7 @@ Define a task with Django's `@task` decorator:
131
135
  # myapp/tasks.py
132
136
  from django.tasks import task
133
137
 
138
+
134
139
  @task
135
140
  def add(a, b):
136
141
  return a + b
@@ -352,10 +357,21 @@ If you need to pass model instances, files, or custom objects, store them elsewh
352
357
  | Backend | Support level | Notes |
353
358
  |---|---|---|
354
359
  | PostgreSQL | first-class | polling, `SKIP LOCKED`, and optional `LISTEN/NOTIFY` |
355
- | MySQL 8+ | supported | polling plus `SKIP LOCKED` |
356
- | MariaDB 10.6+ | supported | polling plus `SKIP LOCKED` |
360
+ | MySQL | supported | polling plus `SKIP LOCKED` |
361
+ | MariaDB | supported | polling plus `SKIP LOCKED` |
357
362
  | SQLite | supported with limits | polling only, serialized writes, no `SKIP LOCKED`, no `LISTEN/NOTIFY`; practical for development, CI, and smaller deployments |
358
363
 
364
+ The database floor follows the selected Django release:
365
+
366
+ | Django | PostgreSQL | MySQL | MariaDB | SQLite |
367
+ |---|---|---|---|---|
368
+ | 6.0 | 14+ | 8.0.11+ | 10.6+ | 3.31+ |
369
+ | 6.1 | 15+ | 8.4+ | 10.11+ | 3.37+ |
370
+
371
+ CI keeps the floors on Django 6.0 and tests newer database lines through
372
+ PostgreSQL 18, MySQL 9.7 and 26.7, and MariaDB 12.3. Django 6.1 only runs with
373
+ database versions that it supports.
374
+
359
375
  For MySQL or MariaDB, install and configure a Django-compatible driver following Django's database docs.
360
376
 
361
377
  Other Django database vendors are rejected explicitly.
@@ -364,6 +380,70 @@ Polling is the portability path everywhere. Backend-specific features improve la
364
380
 
365
381
  For production PostgreSQL operational guidance, see [Postgres Queue Health](#postgres-queue-health).
366
382
 
383
+ ## Rolling Upgrades
384
+
385
+ `dj_queue` supports live queue writes during a one-generation N/N−1 rolling
386
+ upgrade. Schema changes use an expand, bridge, activate, and contract sequence:
387
+
388
+ - release N adds nullable fields and writes both old and new representations
389
+ - N and N−1 workers and producers can use the expanded schema together
390
+ - release N+1 can activate the new representation after N−1 has left the fleet
391
+ - a later release removes compatibility fields only after its oldest supported writer no longer needs them
392
+
393
+ Each N release must name its minimum rollout-compatible N−1 patch. That patch
394
+ must already retry transient database conflicts because release N cannot change
395
+ the behavior of a process that is still running N−1.
396
+
397
+ Use this deployment order:
398
+
399
+ 1. Upgrade all application and queue processes to the minimum compatible N−1 patch.
400
+ 2. Run `python manage.py dj_queue_health --deep` for queue processes and
401
+ invariants. Use your deployment platform to confirm that no older application
402
+ producer remains.
403
+ 3. Apply release N migrations once.
404
+ 4. Roll application producers and queue processes from N−1 to N.
405
+ 5. Run `python manage.py dj_queue_health --deep --require-version <N version>` for
406
+ queue processes. Use your deployment platform to confirm that no N−1
407
+ application producer remains.
408
+
409
+ Each registered queue process stores its exact `dj_queue` version and rollout
410
+ protocol generation. Deep health rejects a missing or incompatible protocol.
411
+ `--require-version` also rejects a live queue process on another package
412
+ version. Application producers do not register process rows, so check those in
413
+ your deployment platform.
414
+
415
+ A code rollback from N to N−1 keeps the expanded schema in place. Do not reverse
416
+ the database migration during a live rollback. N−2 processes and skipped bridge
417
+ releases are not supported.
418
+
419
+ Expansion migrations keep one DDL operation per hot table migration so lock
420
+ conflicts can retry safely. Constraints that a supported database cannot add
421
+ online stay deferred until a later activation or contract release.
422
+
423
+ This contract covers `dj_queue`'s schema and runtime. Jobs can outlive an
424
+ application deployment, so keep your task import paths and accepted argument
425
+ shapes compatible. Use a new task name when a payload change cannot be backward
426
+ compatible.
427
+
428
+ Maintainers can prove a candidate rollout against two immutable revisions:
429
+
430
+ ```bash
431
+ bin/prerelease.py \
432
+ --from-ref <release-n-1-ref> \
433
+ --to-ref <release-n-ref> \
434
+ --backend postgres \
435
+ --django '>=6.0,<6.1'
436
+ ```
437
+
438
+ The command calibrates release N−1, applies release N migrations under live
439
+ writes, runs both versions together, drains with release N, and verifies queue
440
+ state and side effects. It rejects revisions that do not publish one shared
441
+ rollout protocol. It writes wheel hashes, one-second metrics, logs, and a result
442
+ manifest under `benchmark-results/`. The **Pre-release migration load** workflow
443
+ runs the database floor and current-version lanes. `--smoke` and custom workload
444
+ profiles keep all validity and stability gates and record performance. Only the
445
+ default ten-minute release profile enforces the X-to-Y performance ratios.
446
+
367
447
  ## Recurring Tasks
368
448
 
369
449
  `dj_queue` supports both static recurring tasks from settings and dynamic
@@ -445,27 +525,33 @@ Notes:
445
525
 
446
526
  Tasks can opt into database-backed concurrency limits.
447
527
 
448
- `django.tasks` has no standard way to pass backend-specific options through the
449
- `@task` decorator, so `dj_queue` reads them as attributes on the wrapped function:
528
+ Use `dj_queue.api.concurrency` with Django's `@task` decorator:
450
529
 
451
530
  ```python
452
531
  from django.tasks import task
532
+ from dj_queue.api import concurrency
533
+
453
534
 
454
535
  @task
536
+ @concurrency(
537
+ key="account:{account_id}",
538
+ limit=1,
539
+ duration=60,
540
+ on_conflict="block",
541
+ )
455
542
  def sync_account(account_id, action):
456
543
  return f"{account_id}:{action}"
457
-
458
- sync_account.func.concurrency_key = "account:{account_id}"
459
- sync_account.func.concurrency_limit = 1
460
- sync_account.func.concurrency_duration = 60
461
- sync_account.func.on_conflict = "block"
462
544
  ```
463
545
 
464
546
  With this configuration:
465
547
 
466
548
  - the first matching job can run immediately
467
549
  - later jobs for the same key can block until capacity is released
468
- - `on_conflict = "discard"` turns the same pattern into singleton-style work
550
+ - `on_conflict="discard"` turns the same pattern into singleton-style work
551
+
552
+ The concurrency key can be a format string or a callable. Both decorator orders
553
+ are supported, but placing `@task` above `@concurrency` keeps backend-specific
554
+ options next to the task function.
469
555
 
470
556
  Semaphore rows remain shared on the queue database. If you want per-backend
471
557
  isolation for a limit, express that in the `concurrency_key` itself rather than
@@ -511,10 +597,13 @@ Operational commands:
511
597
 
512
598
  ```bash
513
599
  python manage.py dj_queue_health
600
+ python manage.py dj_queue_health --deep
514
601
  python manage.py dj_queue_health --max-age 120
602
+ python manage.py dj_queue_health --deep --require-version 0.13.0
515
603
  python manage.py dj_queue_prune --older-than 86400
516
604
  python manage.py dj_queue_prune --failed-older-than 604800
517
605
  python manage.py dj_queue_prune --recurring-older-than 2592000
606
+ python manage.py dj_queue_prune --batch-size 500
518
607
  python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
519
608
  python manage.py dj_queue_prune --task-key nightly_cleanup
520
609
  python manage.py dj_queue_postgres_autovacuum
@@ -522,9 +611,16 @@ python manage.py dj_queue_postgres_autovacuum
522
611
 
523
612
  The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
524
613
 
614
+ `dj_queue_health` exits with status `1` and prints a reason when no process is
615
+ live. Add `--deep` to check persisted queue invariants and live process rollout
616
+ protocols. Add `--require-version` to require one exact package version across
617
+ all live queue processes.
618
+
525
619
  For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
526
620
  task import path, while `--task-key` filters recurring execution cleanup by
527
- recurring task key.
621
+ recurring task key. One invocation deletes at most `--batch-size` rows from
622
+ each retained row type. Run it again when it reports that the batch limit was
623
+ reached.
528
624
 
529
625
  ## Failed Jobs
530
626
 
@@ -596,14 +692,17 @@ Each callback receives the live supervisor or runner instance.
596
692
  ```python
597
693
  from dj_queue.hooks import on_start, on_worker_start, register_hook
598
694
 
695
+
599
696
  @on_start
600
697
  def supervisor_started(process):
601
698
  print(process.name)
602
699
 
700
+
603
701
  @on_worker_start
604
702
  def worker_started(process):
605
703
  print(process.metadata)
606
704
 
705
+
607
706
  @register_hook("scheduler.exit")
608
707
  def scheduler_exited(process):
609
708
  print(process.name)
@@ -890,6 +989,10 @@ Configuration precedence is explicit:
890
989
  - TOML file pointed to by `DJ_QUEUE_CONFIG`
891
990
  - Django `TASKS` settings
892
991
 
992
+ Unknown option names are rejected in Django settings and in the selected TOML
993
+ overlay, including nested worker, dispatcher, scheduler, and recurring task
994
+ options. The error identifies the exact configuration path and option name.
995
+
893
996
  ### TOML file config
894
997
 
895
998
  ```bash
@@ -13,7 +13,7 @@ It keeps the queue, live execution state, runtime metadata, and task results in
13
13
 
14
14
  - no Redis, RabbitMQ, or separate result store
15
15
  - PostgreSQL is the first-class production backend
16
- - MySQL 8+, MariaDB 10.6+, and SQLite are supported
16
+ - MySQL, MariaDB, and SQLite are supported at Django's version floors
17
17
  - immediate, scheduled, recurring, and concurrency-limited work
18
18
 
19
19
  `dj_queue` is inspired by Rails' [Solid Queue](https://github.com/rails/solid_queue),
@@ -40,7 +40,7 @@ see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
40
40
 
41
41
  ## Installation
42
42
 
43
- `dj_queue` requires Python 3.12+ and Django 6.0+.
43
+ `dj_queue` requires Python 3.12+ and supports Django 6.0 and 6.1.
44
44
 
45
45
  Install the package:
46
46
 
@@ -48,6 +48,9 @@ Install the package:
48
48
  pip install dj-queue
49
49
  ```
50
50
 
51
+ The package includes inline public API annotations and a `py.typed` marker for
52
+ PEP 561-compatible type checkers.
53
+
51
54
  Optional extras:
52
55
 
53
56
  ```bash
@@ -105,6 +108,7 @@ Define a task with Django's `@task` decorator:
105
108
  # myapp/tasks.py
106
109
  from django.tasks import task
107
110
 
111
+
108
112
  @task
109
113
  def add(a, b):
110
114
  return a + b
@@ -326,10 +330,21 @@ If you need to pass model instances, files, or custom objects, store them elsewh
326
330
  | Backend | Support level | Notes |
327
331
  |---|---|---|
328
332
  | PostgreSQL | first-class | polling, `SKIP LOCKED`, and optional `LISTEN/NOTIFY` |
329
- | MySQL 8+ | supported | polling plus `SKIP LOCKED` |
330
- | MariaDB 10.6+ | supported | polling plus `SKIP LOCKED` |
333
+ | MySQL | supported | polling plus `SKIP LOCKED` |
334
+ | MariaDB | supported | polling plus `SKIP LOCKED` |
331
335
  | SQLite | supported with limits | polling only, serialized writes, no `SKIP LOCKED`, no `LISTEN/NOTIFY`; practical for development, CI, and smaller deployments |
332
336
 
337
+ The database floor follows the selected Django release:
338
+
339
+ | Django | PostgreSQL | MySQL | MariaDB | SQLite |
340
+ |---|---|---|---|---|
341
+ | 6.0 | 14+ | 8.0.11+ | 10.6+ | 3.31+ |
342
+ | 6.1 | 15+ | 8.4+ | 10.11+ | 3.37+ |
343
+
344
+ CI keeps the floors on Django 6.0 and tests newer database lines through
345
+ PostgreSQL 18, MySQL 9.7 and 26.7, and MariaDB 12.3. Django 6.1 only runs with
346
+ database versions that it supports.
347
+
333
348
  For MySQL or MariaDB, install and configure a Django-compatible driver following Django's database docs.
334
349
 
335
350
  Other Django database vendors are rejected explicitly.
@@ -338,6 +353,70 @@ Polling is the portability path everywhere. Backend-specific features improve la
338
353
 
339
354
  For production PostgreSQL operational guidance, see [Postgres Queue Health](#postgres-queue-health).
340
355
 
356
+ ## Rolling Upgrades
357
+
358
+ `dj_queue` supports live queue writes during a one-generation N/N−1 rolling
359
+ upgrade. Schema changes use an expand, bridge, activate, and contract sequence:
360
+
361
+ - release N adds nullable fields and writes both old and new representations
362
+ - N and N−1 workers and producers can use the expanded schema together
363
+ - release N+1 can activate the new representation after N−1 has left the fleet
364
+ - a later release removes compatibility fields only after its oldest supported writer no longer needs them
365
+
366
+ Each N release must name its minimum rollout-compatible N−1 patch. That patch
367
+ must already retry transient database conflicts because release N cannot change
368
+ the behavior of a process that is still running N−1.
369
+
370
+ Use this deployment order:
371
+
372
+ 1. Upgrade all application and queue processes to the minimum compatible N−1 patch.
373
+ 2. Run `python manage.py dj_queue_health --deep` for queue processes and
374
+ invariants. Use your deployment platform to confirm that no older application
375
+ producer remains.
376
+ 3. Apply release N migrations once.
377
+ 4. Roll application producers and queue processes from N−1 to N.
378
+ 5. Run `python manage.py dj_queue_health --deep --require-version <N version>` for
379
+ queue processes. Use your deployment platform to confirm that no N−1
380
+ application producer remains.
381
+
382
+ Each registered queue process stores its exact `dj_queue` version and rollout
383
+ protocol generation. Deep health rejects a missing or incompatible protocol.
384
+ `--require-version` also rejects a live queue process on another package
385
+ version. Application producers do not register process rows, so check those in
386
+ your deployment platform.
387
+
388
+ A code rollback from N to N−1 keeps the expanded schema in place. Do not reverse
389
+ the database migration during a live rollback. N−2 processes and skipped bridge
390
+ releases are not supported.
391
+
392
+ Expansion migrations keep one DDL operation per hot table migration so lock
393
+ conflicts can retry safely. Constraints that a supported database cannot add
394
+ online stay deferred until a later activation or contract release.
395
+
396
+ This contract covers `dj_queue`'s schema and runtime. Jobs can outlive an
397
+ application deployment, so keep your task import paths and accepted argument
398
+ shapes compatible. Use a new task name when a payload change cannot be backward
399
+ compatible.
400
+
401
+ Maintainers can prove a candidate rollout against two immutable revisions:
402
+
403
+ ```bash
404
+ bin/prerelease.py \
405
+ --from-ref <release-n-1-ref> \
406
+ --to-ref <release-n-ref> \
407
+ --backend postgres \
408
+ --django '>=6.0,<6.1'
409
+ ```
410
+
411
+ The command calibrates release N−1, applies release N migrations under live
412
+ writes, runs both versions together, drains with release N, and verifies queue
413
+ state and side effects. It rejects revisions that do not publish one shared
414
+ rollout protocol. It writes wheel hashes, one-second metrics, logs, and a result
415
+ manifest under `benchmark-results/`. The **Pre-release migration load** workflow
416
+ runs the database floor and current-version lanes. `--smoke` and custom workload
417
+ profiles keep all validity and stability gates and record performance. Only the
418
+ default ten-minute release profile enforces the X-to-Y performance ratios.
419
+
341
420
  ## Recurring Tasks
342
421
 
343
422
  `dj_queue` supports both static recurring tasks from settings and dynamic
@@ -419,27 +498,33 @@ Notes:
419
498
 
420
499
  Tasks can opt into database-backed concurrency limits.
421
500
 
422
- `django.tasks` has no standard way to pass backend-specific options through the
423
- `@task` decorator, so `dj_queue` reads them as attributes on the wrapped function:
501
+ Use `dj_queue.api.concurrency` with Django's `@task` decorator:
424
502
 
425
503
  ```python
426
504
  from django.tasks import task
505
+ from dj_queue.api import concurrency
506
+
427
507
 
428
508
  @task
509
+ @concurrency(
510
+ key="account:{account_id}",
511
+ limit=1,
512
+ duration=60,
513
+ on_conflict="block",
514
+ )
429
515
  def sync_account(account_id, action):
430
516
  return f"{account_id}:{action}"
431
-
432
- sync_account.func.concurrency_key = "account:{account_id}"
433
- sync_account.func.concurrency_limit = 1
434
- sync_account.func.concurrency_duration = 60
435
- sync_account.func.on_conflict = "block"
436
517
  ```
437
518
 
438
519
  With this configuration:
439
520
 
440
521
  - the first matching job can run immediately
441
522
  - later jobs for the same key can block until capacity is released
442
- - `on_conflict = "discard"` turns the same pattern into singleton-style work
523
+ - `on_conflict="discard"` turns the same pattern into singleton-style work
524
+
525
+ The concurrency key can be a format string or a callable. Both decorator orders
526
+ are supported, but placing `@task` above `@concurrency` keeps backend-specific
527
+ options next to the task function.
443
528
 
444
529
  Semaphore rows remain shared on the queue database. If you want per-backend
445
530
  isolation for a limit, express that in the `concurrency_key` itself rather than
@@ -485,10 +570,13 @@ Operational commands:
485
570
 
486
571
  ```bash
487
572
  python manage.py dj_queue_health
573
+ python manage.py dj_queue_health --deep
488
574
  python manage.py dj_queue_health --max-age 120
575
+ python manage.py dj_queue_health --deep --require-version 0.13.0
489
576
  python manage.py dj_queue_prune --older-than 86400
490
577
  python manage.py dj_queue_prune --failed-older-than 604800
491
578
  python manage.py dj_queue_prune --recurring-older-than 2592000
579
+ python manage.py dj_queue_prune --batch-size 500
492
580
  python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
493
581
  python manage.py dj_queue_prune --task-key nightly_cleanup
494
582
  python manage.py dj_queue_postgres_autovacuum
@@ -496,9 +584,16 @@ python manage.py dj_queue_postgres_autovacuum
496
584
 
497
585
  The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
498
586
 
587
+ `dj_queue_health` exits with status `1` and prints a reason when no process is
588
+ live. Add `--deep` to check persisted queue invariants and live process rollout
589
+ protocols. Add `--require-version` to require one exact package version across
590
+ all live queue processes.
591
+
499
592
  For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
500
593
  task import path, while `--task-key` filters recurring execution cleanup by
501
- recurring task key.
594
+ recurring task key. One invocation deletes at most `--batch-size` rows from
595
+ each retained row type. Run it again when it reports that the batch limit was
596
+ reached.
502
597
 
503
598
  ## Failed Jobs
504
599
 
@@ -570,14 +665,17 @@ Each callback receives the live supervisor or runner instance.
570
665
  ```python
571
666
  from dj_queue.hooks import on_start, on_worker_start, register_hook
572
667
 
668
+
573
669
  @on_start
574
670
  def supervisor_started(process):
575
671
  print(process.name)
576
672
 
673
+
577
674
  @on_worker_start
578
675
  def worker_started(process):
579
676
  print(process.metadata)
580
677
 
678
+
581
679
  @register_hook("scheduler.exit")
582
680
  def scheduler_exited(process):
583
681
  print(process.name)
@@ -864,6 +962,10 @@ Configuration precedence is explicit:
864
962
  - TOML file pointed to by `DJ_QUEUE_CONFIG`
865
963
  - Django `TASKS` settings
866
964
 
965
+ Unknown option names are rejected in Django settings and in the selected TOML
966
+ overlay, including nested worker, dispatcher, scheduler, and recurring task
967
+ options. The error identifies the exact configuration path and option name.
968
+
867
969
  ### TOML file config
868
970
 
869
971
  ```bash
@@ -6,13 +6,11 @@ from django.contrib import admin, messages
6
6
  from django.http import HttpResponseNotAllowed, HttpResponseRedirect
7
7
  from django.template.response import TemplateResponse
8
8
  from django.urls import path, reverse
9
+ from django.utils import timezone
9
10
  from django.utils.html import format_html
10
11
  from django.utils.http import url_has_allowed_host_and_scheme
11
- from django.utils import timezone
12
12
 
13
- from dj_queue import dashboard
14
- from dj_queue import dashboard_actions
15
- from dj_queue import observability
13
+ from dj_queue import dashboard, dashboard_actions, observability
16
14
  from dj_queue.api import QueueInfo, unschedule_recurring_task
17
15
  from dj_queue.db import get_database_alias
18
16
  from dj_queue.exceptions import EnqueueError
@@ -495,6 +493,9 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
495
493
  "backend_alias",
496
494
  "scheduled_at",
497
495
  "concurrency_key",
496
+ "concurrency_limit",
497
+ "concurrency_duration",
498
+ "concurrency_on_conflict",
498
499
  "finished_at",
499
500
  "return_value",
500
501
  "created_at",
@@ -964,8 +965,24 @@ class PauseAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
964
965
 
965
966
  @admin.register(Semaphore)
966
967
  class SemaphoreAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
967
- list_display = ("key", "value", "limit", "display_blocked_waiters", "display_expires_at")
968
- readonly_fields = ("key", "value", "limit", "expires_at", "created_at", "updated_at")
968
+ list_display = (
969
+ "key",
970
+ "display_active_count",
971
+ "display_available_count",
972
+ "limit",
973
+ "display_blocked_waiters",
974
+ "display_expires_at",
975
+ )
976
+ readonly_fields = (
977
+ "key",
978
+ "display_active_count",
979
+ "display_available_count",
980
+ "value",
981
+ "limit",
982
+ "expires_at",
983
+ "created_at",
984
+ "updated_at",
985
+ )
969
986
  search_fields = ("key",)
970
987
 
971
988
  def get_queryset(self, request):
@@ -975,6 +992,14 @@ class SemaphoreAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
975
992
  blocked_waiter_count=observability.semaphore_blocked_waiter_count_expression(alias)
976
993
  )
977
994
 
995
+ @admin.display(description="active")
996
+ def display_active_count(self, obj):
997
+ return obj.occupied_count
998
+
999
+ @admin.display(description="available")
1000
+ def display_available_count(self, obj):
1001
+ return obj.available_count
1002
+
978
1003
  @admin.display(description="blocked waiters", ordering="blocked_waiter_count")
979
1004
  def display_blocked_waiters(self, obj):
980
1005
  return obj.blocked_waiter_count
@@ -1,16 +1,18 @@
1
+ from collections.abc import Callable, Mapping
1
2
  from functools import partial
3
+ from typing import Any, Literal, Self, TypeVar
2
4
 
3
5
  from django.db import transaction
6
+ from django.tasks import Task
4
7
 
5
8
  from dj_queue import observability
9
+ from dj_queue.operations.claiming import ClaimedJob, claim_ready_jobs
6
10
  from dj_queue.operations.jobs import (
7
- ClaimedJob,
8
- claim_ready_jobs,
9
11
  discard_blocked_jobs,
10
12
  discard_failed_job,
11
13
  discard_failed_jobs,
12
- discard_ready_jobs_for_queue,
13
14
  discard_ready_jobs,
15
+ discard_ready_jobs_for_queue,
14
16
  discard_scheduled_jobs,
15
17
  execute_claimed_job,
16
18
  retry_failed_job,
@@ -24,6 +26,7 @@ __all__ = [
24
26
  "ClaimedJob",
25
27
  "QueueInfo",
26
28
  "claim_ready_jobs",
29
+ "concurrency",
27
30
  "discard_blocked_jobs",
28
31
  "discard_failed_job",
29
32
  "discard_failed_jobs",
@@ -38,15 +41,42 @@ __all__ = [
38
41
  "unschedule_recurring_task",
39
42
  ]
40
43
 
44
+ _Decorated = TypeVar("_Decorated")
45
+
46
+
47
+ def concurrency(
48
+ *,
49
+ key: str | Callable[..., str],
50
+ limit: int,
51
+ duration: int | None = None,
52
+ on_conflict: Literal["block", "discard"] = "block",
53
+ ) -> Callable[[_Decorated], _Decorated]:
54
+ def decorator(target: _Decorated) -> _Decorated:
55
+ func = getattr(target, "func", target)
56
+ func.concurrency_key = key
57
+ func.concurrency_limit = limit
58
+ if duration is not None:
59
+ func.concurrency_duration = duration
60
+ func.on_conflict = on_conflict
61
+ return target
62
+
63
+ return decorator
64
+
41
65
 
42
66
  class QueueInfo:
43
- def __init__(self, queue_name, *, backend_alias="default", snapshot=None):
67
+ def __init__(
68
+ self,
69
+ queue_name: str,
70
+ *,
71
+ backend_alias: str = "default",
72
+ snapshot: Mapping[str, Any] | None = None,
73
+ ) -> None:
44
74
  self.queue_name = queue_name
45
75
  self.backend_alias = backend_alias
46
76
  self._snapshot = snapshot
47
77
 
48
78
  @property
49
- def size(self):
79
+ def size(self) -> int:
50
80
  if self._snapshot is not None:
51
81
  return self._snapshot["ready_count"]
52
82
  return observability.queue_ready_count(
@@ -55,7 +85,7 @@ class QueueInfo:
55
85
  )
56
86
 
57
87
  @property
58
- def latency(self):
88
+ def latency(self) -> float | None:
59
89
  if self._snapshot is not None:
60
90
  latency = self._snapshot["latency_seconds"]
61
91
  if latency is None and not self._snapshot["paused"]:
@@ -77,7 +107,7 @@ class QueueInfo:
77
107
  return 0.0 if latency is None else latency
78
108
 
79
109
  @property
80
- def paused(self):
110
+ def paused(self) -> bool:
81
111
  if self._snapshot is not None:
82
112
  return self._snapshot["paused"]
83
113
  return observability.queue_is_paused(
@@ -85,15 +115,15 @@ class QueueInfo:
85
115
  queue_name=self.queue_name,
86
116
  )
87
117
 
88
- def pause(self):
118
+ def pause(self) -> None:
89
119
  pause_queue(self.queue_name, backend_alias=self.backend_alias)
90
120
  self._snapshot = None
91
121
 
92
- def resume(self):
122
+ def resume(self) -> None:
93
123
  resume_queue(self.queue_name, backend_alias=self.backend_alias)
94
124
  self._snapshot = None
95
125
 
96
- def clear(self, *, batch_size=500):
126
+ def clear(self, *, batch_size: int = 500) -> int:
97
127
  deleted = 0
98
128
  while True:
99
129
  batch_deleted = discard_ready_jobs_for_queue(
@@ -107,10 +137,15 @@ class QueueInfo:
107
137
  deleted += batch_deleted
108
138
 
109
139
  @classmethod
110
- def all(cls, *, backend_alias="default"):
140
+ def all(cls, *, backend_alias: str = "default") -> list[Self]:
111
141
  queue_rows = observability.queue_rows_for_backend(backend_alias=backend_alias)
112
142
  return [cls(row["name"], backend_alias=backend_alias, snapshot=row) for row in queue_rows]
113
143
 
114
144
 
115
- def enqueue_on_commit(task, *args, using=None, **kwargs):
145
+ def enqueue_on_commit(
146
+ task: Task,
147
+ *args: Any,
148
+ using: str | None = None,
149
+ **kwargs: Any,
150
+ ) -> None:
116
151
  transaction.on_commit(partial(task.enqueue, *args, **kwargs), using=using)
@@ -1,5 +1,4 @@
1
- from django.apps import AppConfig
2
- from django.apps import apps
1
+ from django.apps import AppConfig, apps
3
2
 
4
3
 
5
4
  class DjQueueConfig(AppConfig):