dj-queue 0.11.0__tar.gz → 0.13.0__tar.gz

{dj_queue-0.11.0 → dj_queue-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.11.0
+Version: 0.13.0
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE
@@ -517,9 +517,10 @@ python manage.py dj_queue_prune --failed-older-than 604800
 python manage.py dj_queue_prune --recurring-older-than 2592000
 python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
 python manage.py dj_queue_prune --task-key nightly_cleanup
+python manage.py dj_queue_postgres_autovacuum
 ```
 
-The health and prune commands both accept `--backend` to target a non-default backend alias.
+The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
 
 For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
 task import path, while `--task-key` filters recurring execution cleanup by
@@ -543,9 +544,11 @@ from dj_queue.api import (
   discard_scheduled_jobs,
   retry_failed_job,
   retry_failed_jobs,
+  schedule_failed_job_retry,
 )
 
 retry_failed_job(job_id)
+schedule_failed_job_retry(job_id, retry_at=retry_at)
 discard_failed_job(job_id)
 retry_failed_jobs(job_ids=[job_id_a, job_id_b], batch_size=2)
 discard_ready_jobs(job_ids=[ready_job_id], batch_size=1)
@@ -554,7 +557,9 @@ discard_scheduled_jobs(job_ids=[scheduled_job_id], batch_size=1)
 discard_blocked_jobs(job_ids=[blocked_job_id], batch_size=1)
 ```
 
-Failures stay inspectable until you act on them.
+Failures stay inspectable until you act on them. Scheduled failed retries stay
+failed until a dispatcher promotes rows whose `retry_at` is due; `dj_queue` does
+not add an automatic retry policy or backoff engine.
 
 ## Errors When Enqueuing
 
@@ -714,6 +719,9 @@ policy, and autovacuum tuning.
 - Tune autovacuum for `dj_queue_jobs` and the high-churn
   `dj_queue_*_executions` tables, often default OLTP settings are too
   conservative for queue workloads.
+- Use `python manage.py dj_queue_postgres_autovacuum` to print reviewable
+  `ALTER TABLE ... SET (...)` SQL for queue-table autovacuum storage
+  parameters. `dj_queue` does not apply these settings in migrations.
 - Keep transactions short across workers and the rest of your app. Long-lived
   transactions pin dead tuples and delay vacuum.
 - Monitor dead tuples, autovacuum frequency, and long-running queries before
@@ -812,7 +820,7 @@ Global options:
 | Option | Default | Meaning |
 |---|---|---|
 | `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
-| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
+| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1, "prefetch_multiplier": 2}]` | worker topology and queue selectors |
 | `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
 | `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
 | `recurring` | `{}` | static recurring task definitions keyed by name |
@@ -839,6 +847,7 @@ Worker entry options:
 | `threads` | `3` | worker threads per worker process |
 | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
 | `polling_interval` | `0.1` | seconds between worker polls |
+| `prefetch_multiplier` | `2` | upper bound on running plus queued claimed work; workers only poll when at least one thread is idle |
 
 Dispatcher entry options:
 
@@ -914,6 +923,7 @@ queues = ["default", "email*"]
 threads = 8
 processes = 1
 polling_interval = 0.1
+prefetch_multiplier = 2
 
 [[dispatchers]]
 batch_size = 500
@@ -1018,6 +1028,10 @@ Exported metric families:
 - `dj_queue_recurring_tasks{backend}`
 - `dj_queue_semaphores{queue_database}`
 - `dj_queue_process_rows{backend}`
+- `dj_queue_failed_jobs{backend}`
+- `dj_queue_failed_job_oldest_age_seconds{backend}`
+- `dj_queue_failed_job_retention_seconds{backend}`
+- `dj_queue_failed_jobs_over_retention{backend}`
 
 Both endpoints support bearer token authentication. Set
 `DJ_QUEUE_OBSERVABILITY_TOKEN` in `settings.py` and include it as

{dj_queue-0.11.0 → dj_queue-0.13.0}/README.md

@@ -491,9 +491,10 @@ python manage.py dj_queue_prune --failed-older-than 604800
 python manage.py dj_queue_prune --recurring-older-than 2592000
 python manage.py dj_queue_prune --task-path myapp.tasks.cleanup
 python manage.py dj_queue_prune --task-key nightly_cleanup
+python manage.py dj_queue_postgres_autovacuum
 ```
 
-The health and prune commands both accept `--backend` to target a non-default backend alias.
+The health, prune, and PostgreSQL autovacuum commands accept `--backend` to target a non-default backend alias.
 
 For `dj_queue_prune`, `--task-path` filters finished and failed job cleanup by
 task import path, while `--task-key` filters recurring execution cleanup by
@@ -517,9 +518,11 @@ from dj_queue.api import (
   discard_scheduled_jobs,
   retry_failed_job,
   retry_failed_jobs,
+  schedule_failed_job_retry,
 )
 
 retry_failed_job(job_id)
+schedule_failed_job_retry(job_id, retry_at=retry_at)
 discard_failed_job(job_id)
 retry_failed_jobs(job_ids=[job_id_a, job_id_b], batch_size=2)
 discard_ready_jobs(job_ids=[ready_job_id], batch_size=1)
@@ -528,7 +531,9 @@ discard_scheduled_jobs(job_ids=[scheduled_job_id], batch_size=1)
 discard_blocked_jobs(job_ids=[blocked_job_id], batch_size=1)
 ```
 
-Failures stay inspectable until you act on them.
+Failures stay inspectable until you act on them. Scheduled failed retries stay
+failed until a dispatcher promotes rows whose `retry_at` is due; `dj_queue` does
+not add an automatic retry policy or backoff engine.
 
 ## Errors When Enqueuing
 
@@ -688,6 +693,9 @@ policy, and autovacuum tuning.
 - Tune autovacuum for `dj_queue_jobs` and the high-churn
   `dj_queue_*_executions` tables, often default OLTP settings are too
   conservative for queue workloads.
+- Use `python manage.py dj_queue_postgres_autovacuum` to print reviewable
+  `ALTER TABLE ... SET (...)` SQL for queue-table autovacuum storage
+  parameters. `dj_queue` does not apply these settings in migrations.
 - Keep transactions short across workers and the rest of your app. Long-lived
   transactions pin dead tuples and delay vacuum.
 - Monitor dead tuples, autovacuum frequency, and long-running queries before
@@ -786,7 +794,7 @@ Global options:
 | Option | Default | Meaning |
 |---|---|---|
 | `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
-| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
+| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1, "prefetch_multiplier": 2}]` | worker topology and queue selectors |
 | `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
 | `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
 | `recurring` | `{}` | static recurring task definitions keyed by name |
@@ -813,6 +821,7 @@ Worker entry options:
 | `threads` | `3` | worker threads per worker process |
 | `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
 | `polling_interval` | `0.1` | seconds between worker polls |
+| `prefetch_multiplier` | `2` | upper bound on running plus queued claimed work; workers only poll when at least one thread is idle |
 
 Dispatcher entry options:
 
@@ -888,6 +897,7 @@ queues = ["default", "email*"]
 threads = 8
 processes = 1
 polling_interval = 0.1
+prefetch_multiplier = 2
 
 [[dispatchers]]
 batch_size = 500
@@ -992,6 +1002,10 @@ Exported metric families:
 - `dj_queue_recurring_tasks{backend}`
 - `dj_queue_semaphores{queue_database}`
 - `dj_queue_process_rows{backend}`
+- `dj_queue_failed_jobs{backend}`
+- `dj_queue_failed_job_oldest_age_seconds{backend}`
+- `dj_queue_failed_job_retention_seconds{backend}`
+- `dj_queue_failed_jobs_over_retention{backend}`
 
 Both endpoints support bearer token authentication. Set
 `DJ_QUEUE_OBSERVABILITY_TOKEN` in `settings.py` and include it as

{dj_queue-0.11.0 → dj_queue-0.13.0}/dj_queue/admin.py

@@ -82,9 +82,6 @@ def _install_dj_queue_admin_site(site):
   site._dj_queue_dashboard_installed = True
 
 
-_install_dj_queue_admin_site(admin.site)
-
-
 def _format_admin_datetime(value):
   if value is None:
     return None
@@ -530,7 +527,19 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
 
   def get_queryset(self, request):
     queryset = super().get_queryset(request)
-    return queryset.annotate(status_rank=status_rank_expression())
+    if self._orders_by_status(request):
+      return queryset.annotate(status_rank=status_rank_expression())
+    return queryset
+
+  def _orders_by_status(self, request):
+    ordering = request.GET.get("o")
+    if not ordering:
+      return False
+    try:
+      status_index = self.get_list_display(request).index("display_status") + 1
+    except ValueError:
+      return False
+    return any(part.lstrip("-") == str(status_index) for part in ordering.split("."))
 
   @admin.display(description="status", ordering="status_rank")
   def display_status(self, obj):
@@ -707,14 +716,14 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
 class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
   backend_filter_field = "job__backend_alias"
   ignored_filter_params = ("job__backend_alias",)
-  list_display = ("job", "exception_class", "message", "display_created_at")
+  list_display = ("job", "exception_class", "message", "display_retry_at", "display_created_at")
   list_filter = (
     ("job__queue_name", admin.AllValuesFieldListFilter),
     "exception_class",
   )
   list_select_related = ("job",)
   actions = ("retry_jobs", "discard_jobs")
-  readonly_fields = ("job", "exception_class", "message", "traceback", "created_at")
+  readonly_fields = ("job", "exception_class", "message", "traceback", "retry_at", "created_at")
   search_fields = ("job__id", "job__task_path", "message", "exception_class")
 
   @admin.action(description="Retry selected failed jobs")
@@ -747,6 +756,10 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
   def display_created_at(self, obj):
     return _format_admin_datetime(obj.created_at)
 
+  @admin.display(description="retry at", ordering="retry_at")
+  def display_retry_at(self, obj):
+    return _format_admin_datetime(obj.retry_at)
+
   def get_change_actions(self, request, obj):
     if obj is None:
       return ()

{dj_queue-0.11.0 → dj_queue-0.13.0}/dj_queue/api.py

@@ -15,6 +15,7 @@ from dj_queue.operations.jobs import (
   execute_claimed_job,
   retry_failed_job,
   retry_failed_jobs,
+  schedule_failed_job_retry,
 )
 from dj_queue.operations.queues import pause_queue, resume_queue
 from dj_queue.operations.recurring import schedule_recurring_task, unschedule_recurring_task
@@ -32,6 +33,7 @@ __all__ = [
   "execute_claimed_job",
   "retry_failed_job",
   "retry_failed_jobs",
+  "schedule_failed_job_retry",
   "schedule_recurring_task",
   "unschedule_recurring_task",
 ]

dj_queue-0.13.0/dj_queue/apps.py

@@ -0,0 +1,17 @@
+from django.apps import AppConfig
+from django.apps import apps
+
+
+class DjQueueConfig(AppConfig):
+  name = "dj_queue"
+  verbose_name = "dj_queue"
+
+  def ready(self):
+    if not apps.is_installed("django.contrib.admin"):
+      return
+
+    from django.contrib import admin
+
+    from dj_queue.admin import _install_dj_queue_admin_site
+
+    _install_dj_queue_admin_site(admin.site)

{dj_queue-0.11.0 → dj_queue-0.13.0}/dj_queue/backend.py

@@ -3,6 +3,7 @@ from django.db import close_old_connections, connections
 from django.tasks.backends.base import BaseTaskBackend
 from django.tasks.exceptions import TaskResultDoesNotExist
 
+from dj_queue.config import load_backend_config
 from dj_queue.db import get_database_alias
 from dj_queue.models import Job
 from dj_queue.operations.jobs import (
@@ -47,8 +48,13 @@ class DjQueueBackend(BaseTaskBackend):
     )
 
   async def aenqueue(self, task, args, kwargs):
-    return await sync_to_async(_async_backend_call, thread_sensitive=True)(
+    config = load_backend_config(self.alias)
+    return await sync_to_async(
+      _async_backend_call,
+      thread_sensitive=config.async_thread_sensitive,
+    )(
       self.enqueue,
+      close_connections=config.async_close_connections,
       task=task,
       args=args,
       kwargs=kwargs,
@@ -70,6 +76,17 @@ class DjQueueBackend(BaseTaskBackend):
       for job, task, dispatch_outcome in created_jobs
     ]
 
+  async def aenqueue_all(self, task_calls):
+    config = load_backend_config(self.alias)
+    return await sync_to_async(
+      _async_backend_call,
+      thread_sensitive=config.async_thread_sensitive,
+    )(
+      self.enqueue_all,
+      close_connections=config.async_close_connections,
+      task_calls=task_calls,
+    )
+
   def get_result(self, result_id):
     alias = get_database_alias(self.alias)
     try:
@@ -90,15 +107,23 @@ class DjQueueBackend(BaseTaskBackend):
     return task_result_from_job(job)
 
   async def aget_result(self, result_id):
-    return await sync_to_async(_async_backend_call, thread_sensitive=True)(
+    config = load_backend_config(self.alias)
+    return await sync_to_async(
+      _async_backend_call,
+      thread_sensitive=config.async_thread_sensitive,
+    )(
       self.get_result,
+      close_connections=config.async_close_connections,
       result_id=result_id,
     )
 
 
-def _async_backend_call(method, /, **kwargs):
+def _async_backend_call(method, /, *, close_connections=False, **kwargs):
   close_old_connections()
   try:
     return method(**kwargs)
   finally:
-    connections.close_all()
+    if close_connections:
+      connections.close_all()
+    else:
+      close_old_connections()

{dj_queue-0.11.0 → dj_queue-0.13.0}/dj_queue/config.py

@@ -19,6 +19,7 @@ DEFAULT_WORKER = {
   "threads": 3,
   "processes": 1,
   "polling_interval": 0.1,
+  "prefetch_multiplier": 2,
 }
 
 DEFAULT_DISPATCHER = {
@@ -52,6 +53,8 @@ DEFAULT_OPTIONS = {
   "use_skip_locked": True,
   "listen_notify": True,
   "silence_polling": True,
+  "async_thread_sensitive": False,
+  "async_close_connections": False,
   "on_thread_error": None,
 }
 
@@ -75,6 +78,7 @@ class WorkerConfig(ConfigValue):
   threads: int = 3
   processes: int = 1
   polling_interval: float = 0.1
+  prefetch_multiplier: int = 2
 
 
 @dataclass(frozen=True, slots=True)
@@ -125,6 +129,8 @@ class BackendConfig(ConfigValue):
   use_skip_locked: bool = True
   listen_notify: bool = True
   silence_polling: bool = True
+  async_thread_sensitive: bool = False
+  async_close_connections: bool = False
   on_thread_error: str | None = None
   skip_recurring: bool = False
   only_work: bool = False
@@ -151,21 +157,26 @@ def load_backend_config(
 
   ensure_dj_queue_backend_alias(tasks_settings, backend_alias)
   backend_block = _backend_block(tasks_settings, backend_alias)
-  env_values = {key: env.get(key) for key in CONFIG_ENV_KEYS if env.get(key) is not None}
+  env_key = tuple(env.get(key) for key in CONFIG_ENV_KEYS)
   cache_key = (
     backend_alias,
-    _cache_key(cli_overrides),
-    _cache_key(env_values),
+    _EMPTY_CLI_CACHE_KEY if not cli_overrides else _cache_key(cli_overrides),
+    env_key,
     _cache_key(backend_block),
   )
-  if cache_key not in _BACKEND_CONFIG_CACHE:
-    _BACKEND_CONFIG_CACHE[cache_key] = _load_backend_config_uncached(
-      backend_alias,
-      cli_overrides,
-      env_values,
-      tasks_settings,
-    )
-  return _BACKEND_CONFIG_CACHE[cache_key]
+  cached = _BACKEND_CONFIG_CACHE.get(cache_key)
+  if cached is not None:
+    return cached
+
+  env_values = {key: value for key, value in zip(CONFIG_ENV_KEYS, env_key) if value is not None}
+  config = _load_backend_config_uncached(
+    backend_alias,
+    cli_overrides,
+    env_values,
+    tasks_settings,
+  )
+  _BACKEND_CONFIG_CACHE[cache_key] = config
+  return config
 
 
 def load_allowed_queues(
@@ -272,6 +283,12 @@ def _load_backend_config_uncached(
     use_skip_locked=_bool_option(resolved_options["use_skip_locked"], "use_skip_locked"),
     listen_notify=_bool_option(resolved_options["listen_notify"], "listen_notify"),
     silence_polling=_bool_option(resolved_options["silence_polling"], "silence_polling"),
+    async_thread_sensitive=_bool_option(
+      resolved_options["async_thread_sensitive"], "async_thread_sensitive"
+    ),
+    async_close_connections=_bool_option(
+      resolved_options["async_close_connections"], "async_close_connections"
+    ),
     on_thread_error=on_thread_error,
     skip_recurring=skip_recurring,
     only_work=only_work,
@@ -370,6 +387,8 @@ def _load_toml_options(config_path: Any, *, backend_alias: str) -> dict[str, Any
 
   try:
     config_payload = tomllib.loads(Path(config_path).read_text(encoding="utf-8"))
+  except OSError as exc:
+    raise ImproperlyConfigured(f"DJ_QUEUE_CONFIG could not be read: {exc}") from exc
   except tomllib.TOMLDecodeError as exc:
     raise ImproperlyConfigured(f"DJ_QUEUE_CONFIG TOML is invalid: {exc}") from exc
   if not isinstance(config_payload, dict):
@@ -475,6 +494,10 @@ def _build_worker_configs(raw_workers: Any, mode: str) -> tuple[WorkerConfig, ..
         raw_worker.get("polling_interval", DEFAULT_WORKER["polling_interval"]),
         f"workers[{index}].polling_interval",
       ),
+      prefetch_multiplier=_positive_int(
+        raw_worker.get("prefetch_multiplier", DEFAULT_WORKER["prefetch_multiplier"]),
+        f"workers[{index}].prefetch_multiplier",
+      ),
     )
 
     if mode == "async" and worker.processes > 1:
@@ -758,3 +781,6 @@ def _cache_key(value: Any) -> str:
     return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
   except (TypeError, ValueError) as exc:
     raise ImproperlyConfigured("dj_queue config values must be JSON-serializable") from exc
+
+
+_EMPTY_CLI_CACHE_KEY = _cache_key({})