dj-queue 0.12.0__tar.gz → 0.13.0__tar.gz

{dj_queue-0.12.0 → dj_queue-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.12.0
+Version: 0.13.0
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE
@@ -517,9 +517,10 @@ python manage.py dj_queue_prune --failed-older-than 604800
 python manage.py dj_queue_prune --recurring-older-than 2592000
 python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
 python manage.py dj_queue_prune --task-key nightly_cleanup
+python manage.py dj_queue_postgres_autovacuum
 ```
 
-The health and prune commands both accept `--backend` to target a non-default backend alias.
+The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
 
 For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
 task import path, while `--task-key` filters recurring execution cleanup by
@@ -543,9 +544,11 @@ from dj_queue.api import (
   discard_scheduled_jobs,
   retry_failed_job,
   retry_failed_jobs,
+  schedule_failed_job_retry,
 )
 
 retry_failed_job(job_id)
+schedule_failed_job_retry(job_id, retry_at=retry_at)
 discard_failed_job(job_id)
 retry_failed_jobs(job_ids=[job_id_a, job_id_b], batch_size=2)
 discard_ready_jobs(job_ids=[ready_job_id], batch_size=1)
@@ -554,7 +557,9 @@ discard_scheduled_jobs(job_ids=[scheduled_job_id], batch_size=1)
 discard_blocked_jobs(job_ids=[blocked_job_id], batch_size=1)
 ```
 
-Failures stay inspectable until you act on them.
+Failures stay inspectable until you act on them. Scheduled failed retries stay
+failed until a dispatcher promotes rows whose `retry_at` is due; `dj_queue` does
+not add an automatic retry policy or backoff engine.
 
 ## Errors When Enqueuing
 
@@ -714,6 +719,9 @@ policy, and autovacuum tuning.
 - Tune autovacuum for `dj_queue_jobs` and the high-churn
   `dj_queue_*_executions` tables, often default OLTP settings are too
   conservative for queue workloads.
+- Use `python manage.py dj_queue_postgres_autovacuum` to print reviewable
+  `ALTER TABLE ... SET (...)` SQL for queue-table autovacuum storage
+  parameters. `dj_queue` does not apply these settings in migrations.
 - Keep transactions short across workers and the rest of your app. Long-lived
   transactions pin dead tuples and delay vacuum.
 - Monitor dead tuples, autovacuum frequency, and long-running queries before
@@ -839,7 +847,7 @@ Worker entry options:
 | `threads` | `3` | worker threads per worker process |
 | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
 | `polling_interval` | `0.1` | seconds between worker polls |
-| `prefetch_multiplier` | `2` | max claimed running/queued jobs per worker thread |
+| `prefetch_multiplier` | `2` | upper bound on running plus queued claimed work; workers only poll when at least one thread is idle |
 
 Dispatcher entry options:
 
@@ -1020,6 +1028,10 @@ Exported metric families:
 - `dj_queue_recurring_tasks{backend}`
 - `dj_queue_semaphores{queue_database}`
 - `dj_queue_process_rows{backend}`
+- `dj_queue_failed_jobs{backend}`
+- `dj_queue_failed_job_oldest_age_seconds{backend}`
+- `dj_queue_failed_job_retention_seconds{backend}`
+- `dj_queue_failed_jobs_over_retention{backend}`
 
 Both endpoints support bearer token authentication. Set
 `DJ_QUEUE_OBSERVABILITY_TOKEN` in `settings.py` and include it as

{dj_queue-0.12.0 → dj_queue-0.13.0}/README.md

@@ -491,9 +491,10 @@ python manage.py dj_queue_prune --failed-older-than 604800
 python manage.py dj_queue_prune --recurring-older-than 2592000
 python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
 python manage.py dj_queue_prune --task-key nightly_cleanup
+python manage.py dj_queue_postgres_autovacuum
 ```
 
-The health and prune commands both accept `--backend` to target a non-default backend alias.
+The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
 
 For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
 task import path, while `--task-key` filters recurring execution cleanup by
@@ -517,9 +518,11 @@ from dj_queue.api import (
   discard_scheduled_jobs,
   retry_failed_job,
   retry_failed_jobs,
+  schedule_failed_job_retry,
 )
 
 retry_failed_job(job_id)
+schedule_failed_job_retry(job_id, retry_at=retry_at)
 discard_failed_job(job_id)
 retry_failed_jobs(job_ids=[job_id_a, job_id_b], batch_size=2)
 discard_ready_jobs(job_ids=[ready_job_id], batch_size=1)
@@ -528,7 +531,9 @@ discard_scheduled_jobs(job_ids=[scheduled_job_id], batch_size=1)
 discard_blocked_jobs(job_ids=[blocked_job_id], batch_size=1)
 ```
 
-Failures stay inspectable until you act on them.
+Failures stay inspectable until you act on them. Scheduled failed retries stay
+failed until a dispatcher promotes rows whose `retry_at` is due; `dj_queue` does
+not add an automatic retry policy or backoff engine.
 
 ## Errors When Enqueuing
 
@@ -688,6 +693,9 @@ policy, and autovacuum tuning.
 - Tune autovacuum for `dj_queue_jobs` and the high-churn
   `dj_queue_*_executions` tables, often default OLTP settings are too
   conservative for queue workloads.
+- Use `python manage.py dj_queue_postgres_autovacuum` to print reviewable
+  `ALTER TABLE ... SET (...)` SQL for queue-table autovacuum storage
+  parameters. `dj_queue` does not apply these settings in migrations.
 - Keep transactions short across workers and the rest of your app. Long-lived
   transactions pin dead tuples and delay vacuum.
 - Monitor dead tuples, autovacuum frequency, and long-running queries before
@@ -813,7 +821,7 @@ Worker entry options:
 | `threads` | `3` | worker threads per worker process |
 | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
 | `polling_interval` | `0.1` | seconds between worker polls |
-| `prefetch_multiplier` | `2` | max claimed running/queued jobs per worker thread |
+| `prefetch_multiplier` | `2` | upper bound on running plus queued claimed work; workers only poll when at least one thread is idle |
 
 Dispatcher entry options:
 
@@ -994,6 +1002,10 @@ Exported metric families:
 - `dj_queue_recurring_tasks{backend}`
 - `dj_queue_semaphores{queue_database}`
 - `dj_queue_process_rows{backend}`
+- `dj_queue_failed_jobs{backend}`
+- `dj_queue_failed_job_oldest_age_seconds{backend}`
+- `dj_queue_failed_job_retention_seconds{backend}`
+- `dj_queue_failed_jobs_over_retention{backend}`
 
 Both endpoints support bearer token authentication. Set
 `DJ_QUEUE_OBSERVABILITY_TOKEN` in `settings.py` and include it as

{dj_queue-0.12.0 → dj_queue-0.13.0}/dj_queue/admin.py

@@ -716,14 +716,14 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
 class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
   backend_filter_field = "job__backend_alias"
   ignored_filter_params = ("job__backend_alias",)
-  list_display = ("job", "exception_class", "message", "display_created_at")
+  list_display = ("job", "exception_class", "message", "display_retry_at", "display_created_at")
   list_filter = (
     ("job__queue_name", admin.AllValuesFieldListFilter),
     "exception_class",
   )
   list_select_related = ("job",)
   actions = ("retry_jobs", "discard_jobs")
-  readonly_fields = ("job", "exception_class", "message", "traceback", "created_at")
+  readonly_fields = ("job", "exception_class", "message", "traceback", "retry_at", "created_at")
   search_fields = ("job__id", "job__task_path", "message", "exception_class")
 
   @admin.action(description="Retry selected failed jobs")
@@ -756,6 +756,10 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
   def display_created_at(self, obj):
     return _format_admin_datetime(obj.created_at)
 
+  @admin.display(description="retry at", ordering="retry_at")
+  def display_retry_at(self, obj):
+    return _format_admin_datetime(obj.retry_at)
+
   def get_change_actions(self, request, obj):
     if obj is None:
       return ()

{dj_queue-0.12.0 → dj_queue-0.13.0}/dj_queue/api.py

@@ -15,6 +15,7 @@ from dj_queue.operations.jobs import (
   execute_claimed_job,
   retry_failed_job,
   retry_failed_jobs,
+  schedule_failed_job_retry,
 )
 from dj_queue.operations.queues import pause_queue, resume_queue
 from dj_queue.operations.recurring import schedule_recurring_task, unschedule_recurring_task
@@ -32,6 +33,7 @@ __all__ = [
   "execute_claimed_job",
   "retry_failed_job",
   "retry_failed_jobs",
+  "schedule_failed_job_retry",
   "schedule_recurring_task",
   "unschedule_recurring_task",
 ]

{dj_queue-0.12.0 → dj_queue-0.13.0}/dj_queue/config.py

@@ -157,21 +157,26 @@ def load_backend_config(
 
   ensure_dj_queue_backend_alias(tasks_settings, backend_alias)
   backend_block = _backend_block(tasks_settings, backend_alias)
-  env_values = {key: env.get(key) for key in CONFIG_ENV_KEYS if env.get(key) is not None}
+  env_key = tuple(env.get(key) for key in CONFIG_ENV_KEYS)
   cache_key = (
     backend_alias,
-    _cache_key(cli_overrides),
-    _cache_key(env_values),
+    _EMPTY_CLI_CACHE_KEY if not cli_overrides else _cache_key(cli_overrides),
+    env_key,
     _cache_key(backend_block),
   )
-  if cache_key not in _BACKEND_CONFIG_CACHE:
-    _BACKEND_CONFIG_CACHE[cache_key] = _load_backend_config_uncached(
-      backend_alias,
-      cli_overrides,
-      env_values,
-      tasks_settings,
-    )
-  return _BACKEND_CONFIG_CACHE[cache_key]
+  cached = _BACKEND_CONFIG_CACHE.get(cache_key)
+  if cached is not None:
+    return cached
+
+  env_values = {key: value for key, value in zip(CONFIG_ENV_KEYS, env_key) if value is not None}
+  config = _load_backend_config_uncached(
+    backend_alias,
+    cli_overrides,
+    env_values,
+    tasks_settings,
+  )
+  _BACKEND_CONFIG_CACHE[cache_key] = config
+  return config
 
 
 def load_allowed_queues(
@@ -776,3 +781,6 @@ def _cache_key(value: Any) -> str:
     return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
   except (TypeError, ValueError) as exc:
     raise ImproperlyConfigured("dj_queue config values must be JSON-serializable") from exc
+
+
+_EMPTY_CLI_CACHE_KEY = _cache_key({})

{dj_queue-0.12.0 → dj_queue-0.13.0}/dj_queue/db.py

@@ -32,17 +32,26 @@ def locked_queryset(qs, use_skip_locked: bool = True):
   return qs.select_for_update(**select_for_update_kwargs)
 
 
+_CAPABILITIES_CACHE: dict[str, DatabaseCapabilities] = {}
+
+
 def database_capabilities(alias: str) -> DatabaseCapabilities:
+  cached = _CAPABILITIES_CACHE.get(alias)
+  if cached is not None:
+    return cached
+
   connection = connections[alias]
   backend_family = _backend_family(connection)
   supports_skip_locked_flag = bool(connection.features.has_select_for_update_skip_locked)
 
-  return DatabaseCapabilities(
+  capabilities = DatabaseCapabilities(
     backend_family=backend_family,
     supports_skip_locked=supports_skip_locked_flag,
     supports_listen_notify=backend_family == "postgresql",
     uses_serialized_writes=backend_family == "sqlite",
   )
+  _CAPABILITIES_CACHE[alias] = capabilities
+  return capabilities
 
 
 def supports_skip_locked(alias: str) -> bool:

dj_queue-0.13.0/dj_queue/management/commands/dj_queue_postgres_autovacuum.py

@@ -0,0 +1,26 @@
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+from django.db import connections
+
+from dj_queue import observability
+from dj_queue.config import load_backend_config
+from dj_queue.db import database_capabilities
+
+
+class Command(BaseCommand):
+  help = "Print PostgreSQL autovacuum storage-parameter SQL for dj_queue tables"
+
+  def add_arguments(self, parser):
+    parser.add_argument("--backend", default="default")
+
+  def handle(self, *args, **options):
+    backend_alias = options["backend"]
+    database_alias = load_backend_config(backend_alias).database_alias
+    if database_capabilities(database_alias).backend_family != "postgresql":
+      raise CommandError("dj_queue_postgres_autovacuum requires PostgreSQL")
+
+    connection = connections[database_alias]
+    self.stdout.write("-- dj_queue PostgreSQL autovacuum guidance")
+    self.stdout.write("-- review before applying; dj_queue does not apply these in migrations")
+    for statement in observability.postgres_autovacuum_sql(connection):
+      self.stdout.write(statement)

{dj_queue-0.12.0 → dj_queue-0.13.0}/dj_queue/metrics.py

@@ -32,6 +32,10 @@ def metric_families(*, snapshots=None):
   recurring_tasks = []
   semaphores = []
   process_rows = []
+  failed_jobs = []
+  failed_oldest_age = []
+  failed_retention = []
+  failed_over_retention = []
   seen_queue_databases = set()
 
   for snapshot in snapshots:
@@ -95,6 +99,33 @@ def metric_families(*, snapshots=None):
         value=len(snapshot.process_rows),
       )
     )
+    if snapshot.failed_metrics is not None:
+      failed_jobs.append(
+        MetricSample(
+          labels=(backend_alias,),
+          value=snapshot.failed_metrics["count"],
+        )
+      )
+      if snapshot.failed_metrics["oldest_age_seconds"] is not None:
+        failed_oldest_age.append(
+          MetricSample(
+            labels=(backend_alias,),
+            value=snapshot.failed_metrics["oldest_age_seconds"],
+          )
+        )
+      if snapshot.failed_metrics["retention_seconds"] is not None:
+        failed_retention.append(
+          MetricSample(
+            labels=(backend_alias,),
+            value=snapshot.failed_metrics["retention_seconds"],
+          )
+        )
+        failed_over_retention.append(
+          MetricSample(
+            labels=(backend_alias,),
+            value=snapshot.failed_metrics["over_retention_count"],
+          )
+        )
 
     if queue_database_alias in seen_queue_databases:
       continue
@@ -170,4 +201,32 @@ def metric_families(*, snapshots=None):
       labels=("backend",),
       samples=tuple(process_rows),
     ),
+    MetricFamily(
+      name="dj_queue_failed_jobs",
+      help_text="Current failed job count by backend",
+      metric_type="gauge",
+      labels=("backend",),
+      samples=tuple(failed_jobs),
+    ),
+    MetricFamily(
+      name="dj_queue_failed_job_oldest_age_seconds",
+      help_text="Age of the oldest failed job by backend",
+      metric_type="gauge",
+      labels=("backend",),
+      samples=tuple(failed_oldest_age),
+    ),
+    MetricFamily(
+      name="dj_queue_failed_job_retention_seconds",
+      help_text="Configured failed-job retention window by backend",
+      metric_type="gauge",
+      labels=("backend",),
+      samples=tuple(failed_retention),
+    ),
+    MetricFamily(
+      name="dj_queue_failed_jobs_over_retention",
+      help_text="Current failed job count older than configured retention by backend",
+      metric_type="gauge",
+      labels=("backend",),
+      samples=tuple(failed_over_retention),
+    ),
   )

dj_queue-0.13.0/dj_queue/migrations/0013_failedexecution_retry_at_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 6.0.5 on 2026-06-06 09:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+  dependencies = [
+    ("dj_queue", "0012_job_djq_jobs_b_queue_id_idx_job_djq_jobs_b_conc_idx_and_more"),
+  ]
+
+  operations = [
+    migrations.AddField(
+      model_name="failedexecution",
+      name="retry_at",
+      field=models.DateTimeField(blank=True, null=True),
+    ),
+    migrations.AddIndex(
+      model_name="failedexecution",
+      index=models.Index(fields=["retry_at", "job"], name="dj_queue_fa_retry_a_2ff769_idx"),
+    ),
+  ]

{dj_queue-0.12.0 → dj_queue-0.13.0}/dj_queue/models/jobs.py

@@ -281,8 +281,12 @@ class FailedExecution(models.Model):
   exception_class = models.CharField(max_length=255)
   message = models.TextField(default="")
   traceback = models.TextField(default="")
+  retry_at = models.DateTimeField(null=True, blank=True)
   created_at = models.DateTimeField(auto_now_add=True)
 
   class Meta:
     db_table = "dj_queue_failed_executions"
-    indexes = [models.Index(fields=["created_at", "job"])]
+    indexes = [
+      models.Index(fields=["created_at", "job"]),
+      models.Index(fields=["retry_at", "job"]),
+    ]