dj-queue 0.9.1__tar.gz → 0.10.0__tar.gz

{dj_queue-0.9.1 → dj_queue-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.9.1
+Version: 0.10.0
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE
@@ -713,9 +713,12 @@ from django.core.asgi import get_asgi_application
 from dj_queue.contrib.asgi import DjQueueLifespan
 
 django_application = get_asgi_application()
-application = DjQueueLifespan(django_application)
+application = DjQueueLifespan(django_application, forward_wrapped_lifespan=False)
 ```
 
+Set `forward_wrapped_lifespan=False` when the wrapped app does not implement the
+ASGI lifespan protocol itself, such as Django's plain `get_asgi_application()`.
+
 ### Gunicorn
 
 Import the provided hooks in your Gunicorn config:
@@ -780,26 +783,70 @@ concurrency-maintenance throughput.
 
 The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
 
-Start with these options:
-
-- `mode`: `"fork"` or `"async"`
-- `workers`: queue selectors, thread counts, and process counts
-- `dispatchers`: scheduled promotion and concurrency maintenance settings
-- `scheduler`: dynamic recurring polling settings
-- `database_alias`: database alias for queue tables and runtime activity
-- `preserve_finished_jobs` and `clear_finished_jobs_after`: successful result retention and cleanup
-- `clear_failed_jobs_after`: optional failed-job retention window
-- `clear_recurring_executions_after`: optional recurring reservation retention window
-
-Additional operational tuning is available when needed:
-
-- `use_skip_locked`: use `SKIP LOCKED` when the active backend supports it
-- `listen_notify`: PostgreSQL-only worker wakeup optimization layered on top of polling
-- `silence_polling`: suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger
-- `process_heartbeat_interval` and `process_alive_threshold`: process liveness reporting and stale-runner detection
-- `shutdown_timeout`: graceful drain window before standalone shutdown gives up on waiting
-- `supervisor_pidfile`: optional pidfile guard for standalone supervisors
-- `on_thread_error`: dotted callback path for runtime infrastructure exceptions
+No top-level `OPTIONS` key is required. Omit a key to use its default. Static
+`recurring` entries are the exception: each named recurring task requires
+`task_path` and `schedule`.
+
+Global options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
+| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
+| `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
+| `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
+| `recurring` | `{}` | static recurring task definitions keyed by name |
+| `database_alias` | `"default"` | database alias for queue tables and runtime activity |
+| `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
+| `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
+| `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
+| `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
+| `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
+| `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
+| `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
+| `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
+| `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
+| `process_alive_threshold` | `300` | seconds before a process row is stale |
+| `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
+| `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
+| `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
+
+Worker entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
+| `threads` | `3` | worker threads per worker process |
+| `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
+| `polling_interval` | `0.1` | seconds between worker polls |
+
+Dispatcher entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
+| `polling_interval` | `1` | seconds between dispatcher polls |
+| `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
+| `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
+
+Scheduler entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
+| `polling_interval` | `5` | seconds between scheduler polls |
+
+Recurring entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `task_path` | none | required dotted import path for the task to enqueue |
+| `schedule` | none | required cron or supported Fugit-style cronish schedule |
+| `args` | `[]` | positional arguments for the task |
+| `kwargs` | `{}` | keyword arguments for the task |
+| `queue_name` | `"default"` | queue used for jobs created from this recurring task |
+| `priority` | `0` | priority used for jobs created from this recurring task |
+| `description` | `""` | operator-facing description |
 
 On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
 configuration as the main database connection. Install a compatible driver in

{dj_queue-0.9.1 → dj_queue-0.10.0}/README.md

@@ -687,9 +687,12 @@ from django.core.asgi import get_asgi_application
 from dj_queue.contrib.asgi import DjQueueLifespan
 
 django_application = get_asgi_application()
-application = DjQueueLifespan(django_application)
+application = DjQueueLifespan(django_application, forward_wrapped_lifespan=False)
 ```
 
+Set `forward_wrapped_lifespan=False` when the wrapped app does not implement the
+ASGI lifespan protocol itself, such as Django's plain `get_asgi_application()`.
+
 ### Gunicorn
 
 Import the provided hooks in your Gunicorn config:
@@ -754,26 +757,70 @@ concurrency-maintenance throughput.
 
 The main configuration lives in `TASKS[backend_alias]["OPTIONS"]`.
 
-Start with these options:
-
-- `mode`: `"fork"` or `"async"`
-- `workers`: queue selectors, thread counts, and process counts
-- `dispatchers`: scheduled promotion and concurrency maintenance settings
-- `scheduler`: dynamic recurring polling settings
-- `database_alias`: database alias for queue tables and runtime activity
-- `preserve_finished_jobs` and `clear_finished_jobs_after`: successful result retention and cleanup
-- `clear_failed_jobs_after`: optional failed-job retention window
-- `clear_recurring_executions_after`: optional recurring reservation retention window
-
-Additional operational tuning is available when needed:
-
-- `use_skip_locked`: use `SKIP LOCKED` when the active backend supports it
-- `listen_notify`: PostgreSQL-only worker wakeup optimization layered on top of polling
-- `silence_polling`: suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger
-- `process_heartbeat_interval` and `process_alive_threshold`: process liveness reporting and stale-runner detection
-- `shutdown_timeout`: graceful drain window before standalone shutdown gives up on waiting
-- `supervisor_pidfile`: optional pidfile guard for standalone supervisors
-- `on_thread_error`: dotted callback path for runtime infrastructure exceptions
+No top-level `OPTIONS` key is required. Omit a key to use its default. Static
+`recurring` entries are the exception: each named recurring task requires
+`task_path` and `schedule`.
+
+Global options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `mode` | `"fork"` | standalone runtime mode, either `"fork"` or `"async"` |
+| `workers` | `[{"queues": "*", "threads": 3, "processes": 1, "polling_interval": 0.1}]` | worker topology and queue selectors |
+| `dispatchers` | `[{"batch_size": 500, "polling_interval": 1, "concurrency_maintenance": true, "concurrency_maintenance_interval": 600}]` | scheduled promotion and concurrency maintenance |
+| `scheduler` | `{"dynamic_tasks_enabled": false, "polling_interval": 5}` | dynamic recurring polling; the scheduler only starts when recurring or cleanup work exists |
+| `recurring` | `{}` | static recurring task definitions keyed by name |
+| `database_alias` | `"default"` | database alias for queue tables and runtime activity |
+| `preserve_finished_jobs` | `true` | keep successful jobs for result lookup and retention cleanup |
+| `clear_finished_jobs_after` | `86400` | seconds before finished successful jobs are cleaned up |
+| `clear_failed_jobs_after` | `null` | optional failed-job retention window in seconds |
+| `clear_recurring_executions_after` | `null` | optional recurring reservation retention window in seconds |
+| `default_concurrency_duration` | `180` | default semaphore TTL in seconds |
+| `use_skip_locked` | `true` | use `SKIP LOCKED` when the active backend supports it |
+| `listen_notify` | `true` | PostgreSQL-only worker wakeup optimization layered on top of polling |
+| `silence_polling` | `true` | suppress `dj_queue`'s own poll-cycle noise without mutating Django's global SQL logger |
+| `process_heartbeat_interval` | `60` | seconds between process heartbeat writes |
+| `process_alive_threshold` | `300` | seconds before a process row is stale |
+| `shutdown_timeout` | `5` | graceful drain window before standalone shutdown gives up on waiting |
+| `supervisor_pidfile` | `null` | optional pidfile guard for standalone supervisors |
+| `on_thread_error` | `null` | dotted callback path for runtime infrastructure exceptions |
+
+Worker entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `queues` | `"*"` | queue selector or selectors; `"*"` means all queues |
+| `threads` | `3` | worker threads per worker process |
+| `processes` | `1` | worker processes in `fork` mode; normalized to `1` in `async` mode |
+| `polling_interval` | `0.1` | seconds between worker polls |
+
+Dispatcher entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `batch_size` | `500` | maximum scheduled or blocked rows to promote per batch |
+| `polling_interval` | `1` | seconds between dispatcher polls |
+| `concurrency_maintenance` | `true` | run expired semaphore and blocked-job maintenance |
+| `concurrency_maintenance_interval` | `600` | seconds between maintenance passes; `0` means every poll |
+
+Scheduler entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `dynamic_tasks_enabled` | `false` | load dynamic recurring tasks from the database |
+| `polling_interval` | `5` | seconds between scheduler polls |
+
+Recurring entry options:
+
+| Option | Default | Meaning |
+|---|---|---|
+| `task_path` | none | required dotted import path for the task to enqueue |
+| `schedule` | none | required cron or supported Fugit-style cronish schedule |
+| `args` | `[]` | positional arguments for the task |
+| `kwargs` | `{}` | keyword arguments for the task |
+| `queue_name` | `"default"` | queue used for jobs created from this recurring task |
+| `priority` | `0` | priority used for jobs created from this recurring task |
+| `description` | `""` | operator-facing description |
 
 On PostgreSQL, `listen_notify` uses the same Django PostgreSQL driver
 configuration as the main database connection. Install a compatible driver in

{dj_queue-0.9.1 → dj_queue-0.10.0}/dj_queue/admin.py

@@ -29,13 +29,19 @@ from dj_queue.models import (
   Semaphore,
 )
 from dj_queue.operations.jobs import (
+  DispatchOutcome,
   discard_failed_job,
   dispatch_scheduled_job_now,
   enqueue_job_again,
   retry_failed_job,
   retry_failed_jobs,
 )
-from dj_queue.queue_state import status_rank_expression
+from dj_queue.queue_state import (
+  QUEUE_STATES,
+  filter_queue_state,
+  is_queue_state,
+  status_rank_expression,
+)
 
 
 class DjQueueFirstAdminSite(admin.AdminSite):
@@ -328,29 +334,12 @@ class JobStatusListFilter(admin.SimpleListFilter):
   parameter_name = "status"
 
   def lookups(self, request, model_admin):
-    return (
-      ("ready", "ready"),
-      ("scheduled", "scheduled"),
-      ("claimed", "claimed"),
-      ("blocked", "blocked"),
-      ("failed", "failed"),
-      ("finished", "finished"),
-    )
+    return QUEUE_STATES
 
   def queryset(self, request, queryset):
     value = self.value()
-    if value == "ready":
-      return queryset.ready()
-    if value == "scheduled":
-      return queryset.scheduled()
-    if value == "claimed":
-      return queryset.claimed()
-    if value == "blocked":
-      return queryset.blocked()
-    if value == "failed":
-      return queryset.failed()
-    if value == "finished":
-      return queryset.finished()
+    if is_queue_state(value):
+      return filter_queue_state(queryset, value)
     return queryset
 
 
@@ -400,8 +389,10 @@ class JobConcurrencyKeyListFilter(admin.SimpleListFilter):
 
   def lookups(self, request, model_admin):
     alias = model_admin._backend_database_alias(request)
+    backend_alias = model_admin._backend_alias(request)
     return tuple(
       Job.objects.using(alias)
+      .filter(backend_alias=backend_alias)
       .exclude(concurrency_key__isnull=True)
       .exclude(concurrency_key="")
       .order_by("concurrency_key")
@@ -590,15 +581,17 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
   def handle_change_action(self, request, obj, action):
     if action == "run_now":
       try:
-        _job, dispatched_as = dispatch_scheduled_job_now(obj.pk, backend_alias=obj.backend_alias)
+        _job, dispatch_outcome = dispatch_scheduled_job_now(
+          obj.pk, backend_alias=obj.backend_alias
+        )
       except (EnqueueError, ImportError, AttributeError) as exc:
         self.message_user(request, f"Could not dispatch job now: {exc}", level=messages.ERROR)
         return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
 
       message = "Dispatched scheduled job for immediate execution"
-      if dispatched_as == "blocked":
+      if dispatch_outcome is DispatchOutcome.BLOCKED:
         message = "Dispatched scheduled job immediately and it is now blocked"
-      if dispatched_as == "discarded":
+      if dispatch_outcome is DispatchOutcome.DISCARDED:
         message = "Dispatched scheduled job immediately and it was discarded"
       self.message_user(request, message, level=messages.SUCCESS)
       return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
@@ -722,6 +715,7 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
 
 @admin.register(Process)
 class ProcessAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
+  backend_filter_field = "backend_alias"
   list_display = (
     "name",
     "backend_alias",

{dj_queue-0.9.1 → dj_queue-0.10.0}/dj_queue/api.py

@@ -1,9 +1,12 @@
+from datetime import timedelta
 from functools import partial
 
 from django.db.models.functions import Coalesce
 from django.db import transaction
 from django.utils import timezone
 
+from dj_queue import observability
+from dj_queue.config import load_backend_config
 from dj_queue.db import get_database_alias
 from dj_queue.models import Pause, ReadyExecution
 from dj_queue.operations.jobs import (
@@ -50,6 +53,9 @@ class QueueInfo:
 
   @property
   def latency(self):
+    if self.paused:
+      return None
+
     oldest = (
       self._ready_queryset()
       .annotate(latency_at=Coalesce("latency_started_at", "created_at"))
@@ -59,7 +65,7 @@ class QueueInfo:
     )
     if oldest is None:
       return 0.0
-    return (timezone.now() - oldest).total_seconds()
+    return max((timezone.now() - oldest).total_seconds(), 0.0)
 
   @property
   def paused(self):
@@ -93,18 +99,15 @@ class QueueInfo:
 
   @classmethod
   def all(cls, *, backend_alias="default"):
-    alias = get_database_alias(backend_alias)
-    queue_names = (
-      ReadyExecution.objects.using(alias)
-      .filter(backend_alias=backend_alias)
-      .order_by("queue_name")
-      .values_list(
-        "queue_name",
-        flat=True,
-      )
-      .distinct()
+    now = timezone.now()
+    config = load_backend_config(backend_alias)
+    process_cutoff = now - timedelta(seconds=config.process_alive_threshold)
+    queue_rows = observability.queue_rows(
+      backend_alias=backend_alias,
+      now=now,
+      process_cutoff=process_cutoff,
     )
-    return [cls(queue_name, backend_alias=backend_alias) for queue_name in queue_names]
+    return [cls(row["name"], backend_alias=backend_alias) for row in queue_rows]
 
   def _ready_queryset(self):
     alias = get_database_alias(self.backend_alias)

dj_queue-0.10.0/dj_queue/backend.py

@@ -0,0 +1,93 @@
+from asgiref.sync import sync_to_async
+from django.db import close_old_connections, connections
+from django.tasks.backends.base import BaseTaskBackend
+from django.tasks.exceptions import TaskResultDoesNotExist
+
+from dj_queue.db import get_database_alias
+from dj_queue.models import Job
+from dj_queue.operations.jobs import (
+  DispatchOutcome,
+  enqueue_job_with_dispatch,
+  enqueue_jobs_bulk,
+  validate_priority,
+  validate_queue_allowed,
+)
+from dj_queue.task_results import task_result_from_enqueued_job, task_result_from_job
+
+
+class DjQueueBackend(BaseTaskBackend):
+  supports_async_task = True
+  supports_defer = True
+  supports_get_result = True
+  supports_priority = True
+
+  def validate_task(self, task):
+    validate_queue_allowed(task.queue_name, backend_alias=self.alias)
+    validate_priority(task.priority)
+    return super().validate_task(task)
+
+  def enqueue(self, task, args, kwargs):
+    self.validate_task(task)
+    job, dispatch_outcome = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
+    return task_result_from_enqueued_job(
+      job,
+      task,
+      successful=dispatch_outcome is DispatchOutcome.DISCARDED,
+    )
+
+  async def aenqueue(self, task, args, kwargs):
+    return await sync_to_async(_async_backend_call, thread_sensitive=True)(
+      self.enqueue,
+      task=task,
+      args=args,
+      kwargs=kwargs,
+    )
+
+  def enqueue_all(self, task_calls):
+    jobs = []
+    for task, args, kwargs in task_calls:
+      self.validate_task(task)
+      jobs.append((task, args, kwargs))
+
+    created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias)
+    return [
+      task_result_from_enqueued_job(
+        job,
+        task,
+        successful=dispatch_outcome is DispatchOutcome.DISCARDED,
+      )
+      for job, task, dispatch_outcome in created_jobs
+    ]
+
+  def get_result(self, result_id):
+    alias = get_database_alias(self.alias)
+    try:
+      job = (
+        Job.objects.using(alias)
+        .select_related(
+          "ready_execution",
+          "scheduled_execution",
+          "claimed_execution__process",
+          "blocked_execution",
+          "failed_execution",
+        )
+        .get(pk=result_id, backend_alias=self.alias)
+      )
+    except Job.DoesNotExist as exc:
+      raise TaskResultDoesNotExist(str(result_id)) from exc
+
+    return task_result_from_job(job)
+
+  async def aget_result(self, result_id):
+    return await sync_to_async(_async_backend_call, thread_sensitive=True)(
+      self.get_result,
+      result_id=result_id,
+    )
+
+
+def _async_backend_call(method, /, **kwargs):
+  close_old_connections()
+  try:
+    return method(**kwargs)
+  finally:
+    connections.close_all()