dj-queue 0.10.3__tar.gz → 0.10.5__tar.gz

{dj_queue-0.10.3 → dj_queue-0.10.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.10.3
+Version: 0.10.5
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE
@@ -61,6 +61,9 @@ It has a narrow, explicit shape:
 For detailed comparisons with Celery, RQ, Procrastinate, and other alternatives,
 see [COMPARISONS.md](docs/COMPARISONS.md).
 
+For benchmarks on enqueue, promotion, recurring, claiming, worker-drain, and concurrency-contention scenarios,
+see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
+
 ## Installation
 
 `dj_queue` requires Python 3.12+ and Django 6.0+.
@@ -640,8 +643,9 @@ Run your normal application migrations on `default`, then migrate `dj_queue`
 onto the queue database:
 
 ```bash
-python manage.py migrate
+# migrate dj_queue on its queue alias first so django doesn't mark it applied on default
 python manage.py migrate dj_queue --database queue
+python manage.py migrate
 ```
 
 With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
@@ -1012,11 +1016,6 @@ Both endpoints support bearer token authentication. Set
 `Authorization: Bearer <token>`. Leave it unset if you protect these URLs at
 the network or proxy layer.
 
-## Benchmarks
-
-The repository includes a standalone benchmark harness for enqueue, promotion, recurring, worker-drain, and concurrency-contention scenarios. 
-See [`docs/benchmarks/`](docs/benchmarks/) for the latest published reports.
-
 ## License
 
 MIT

{dj_queue-0.10.3 → dj_queue-0.10.5}/README.md

@@ -35,6 +35,9 @@ It has a narrow, explicit shape:
 For detailed comparisons with Celery, RQ, Procrastinate, and other alternatives,
 see [COMPARISONS.md](docs/COMPARISONS.md).
 
+For benchmarks on enqueue, promotion, recurring, claiming, worker-drain, and concurrency-contention scenarios,
+see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
+
 ## Installation
 
 `dj_queue` requires Python 3.12+ and Django 6.0+.
@@ -614,8 +617,9 @@ Run your normal application migrations on `default`, then migrate `dj_queue`
 onto the queue database:
 
 ```bash
-python manage.py migrate
+# migrate dj_queue on its queue alias first so django doesn't mark it applied on default
 python manage.py migrate dj_queue --database queue
+python manage.py migrate
 ```
 
 With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
@@ -986,11 +990,6 @@ Both endpoints support bearer token authentication. Set
 `Authorization: Bearer <token>`. Leave it unset if you protect these URLs at
 the network or proxy layer.
 
-## Benchmarks
-
-The repository includes a standalone benchmark harness for enqueue, promotion, recurring, worker-drain, and concurrency-contention scenarios. 
-See [`docs/benchmarks/`](docs/benchmarks/) for the latest published reports.
-
 ## License
 
 MIT

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/admin.py

@@ -44,7 +44,7 @@ from dj_queue.queue_state import (
 )
 
 
-class DjQueueFirstAdminSite(admin.AdminSite):
+class DjQueueAdminSiteMixin:
   def _dashboard_app_url(self):
     return reverse("admin:dj_queue_dashboard_changelist", current_app=self.name)
 
@@ -65,7 +65,18 @@ class DjQueueFirstAdminSite(admin.AdminSite):
     return super().app_index(request, app_label, extra_context=extra_context)
 
 
-admin.site.__class__ = DjQueueFirstAdminSite
+def _install_dj_queue_admin_site(site):
+  if isinstance(site, DjQueueAdminSiteMixin):
+    return
+
+  site.__class__ = type(
+    f"DjQueue{site.__class__.__name__}",
+    (DjQueueAdminSiteMixin, site.__class__),
+    {"__module__": __name__},
+  )
+
+
+_install_dj_queue_admin_site(admin.site)
 
 
 def _format_admin_datetime(value):

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/api.py

@@ -1,14 +1,13 @@
 from datetime import timedelta
 from functools import partial
 
-from django.db.models.functions import Coalesce
 from django.db import transaction
 from django.utils import timezone
 
 from dj_queue import observability
 from dj_queue.config import load_backend_config
 from dj_queue.db import get_database_alias
-from dj_queue.models import Pause, ReadyExecution
+from dj_queue.models import ReadyExecution
 from dj_queue.operations.jobs import (
   ClaimedJob,
   claim_ready_jobs,
@@ -53,30 +52,25 @@ class QueueInfo:
 
   @property
   def latency(self):
-    if self.paused:
+    paused = observability.queue_is_paused(
+      backend_alias=self.backend_alias,
+      queue_name=self.queue_name,
+    )
+    if paused:
       return None
 
-    oldest = (
-      self._ready_queryset()
-      .annotate(latency_at=Coalesce("latency_started_at", "created_at"))
-      .order_by("latency_at", "created_at")
-      .values_list("latency_at", flat=True)
-      .first()
+    latency = observability.queue_latency_seconds(
+      backend_alias=self.backend_alias,
+      queue_name=self.queue_name,
+      paused=False,
     )
-    if oldest is None:
-      return 0.0
-    return max((timezone.now() - oldest).total_seconds(), 0.0)
+    return 0.0 if latency is None else latency
 
   @property
   def paused(self):
-    alias = get_database_alias(self.backend_alias)
-    return (
-      Pause.objects.using(alias)
-      .filter(
-        backend_alias=self.backend_alias,
-        queue_name=self.queue_name,
-      )
-      .exists()
+    return observability.queue_is_paused(
+      backend_alias=self.backend_alias,
+      queue_name=self.queue_name,
     )
 
   def pause(self):

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/backend.py

@@ -21,6 +21,11 @@ class DjQueueBackend(BaseTaskBackend):
   supports_get_result = True
   supports_priority = True
 
+  def __init__(self, alias, params):
+    if not params.get("QUEUES"):
+      params = {**params, "QUEUES": []}
+    super().__init__(alias, params)
+
   def validate_task(self, task):
     validate_queue_allowed(task.queue_name, backend_alias=self.alias)
     validate_priority(task.priority)
@@ -28,7 +33,13 @@ class DjQueueBackend(BaseTaskBackend):
 
   def enqueue(self, task, args, kwargs):
     self.validate_task(task)
-    job, dispatch_outcome = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
+    job, dispatch_outcome = enqueue_job_with_dispatch(
+      task,
+      args,
+      kwargs,
+      backend_alias=self.alias,
+      validate=False,
+    )
     return task_result_from_enqueued_job(
       job,
       task,
@@ -49,7 +60,7 @@ class DjQueueBackend(BaseTaskBackend):
       self.validate_task(task)
       jobs.append((task, args, kwargs))
 
-    created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias)
+    created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias, validate=False)
     return [
       task_result_from_enqueued_job(
         job,

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/config.py

@@ -82,7 +82,7 @@ class DispatcherConfig(ConfigValue):
   batch_size: int = 500
   polling_interval: float = 1
   concurrency_maintenance: bool = True
-  concurrency_maintenance_interval: int = 600
+  concurrency_maintenance_interval: float = 600
 
 
 @dataclass(frozen=True, slots=True)
@@ -112,9 +112,9 @@ class BackendConfig(ConfigValue):
   dispatchers: tuple[DispatcherConfig, ...] = (DispatcherConfig(),)
   scheduler: SchedulerConfig | None = field(default_factory=SchedulerConfig)
   recurring: dict[str, RecurringTaskConfig] = field(default_factory=dict)
-  process_heartbeat_interval: int = 60
-  process_alive_threshold: int = 300
-  shutdown_timeout: int = 5
+  process_heartbeat_interval: float = 60
+  process_alive_threshold: float = 300
+  shutdown_timeout: float = 5
   supervisor_pidfile: str | None = None
   preserve_finished_jobs: bool = True
   clear_finished_jobs_after: int | None = 86400
@@ -248,7 +248,9 @@ def _load_backend_config_uncached(
       resolved_options["process_alive_threshold"], "process_alive_threshold"
     ),
     shutdown_timeout=_nonnegative_float(resolved_options["shutdown_timeout"], "shutdown_timeout"),
-    supervisor_pidfile=resolved_options["supervisor_pidfile"],
+    supervisor_pidfile=_optional_string_option(
+      resolved_options["supervisor_pidfile"], "supervisor_pidfile"
+    ),
     preserve_finished_jobs=preserve_finished_jobs,
     clear_finished_jobs_after=_optional_nonnegative_int(
       resolved_options["clear_finished_jobs_after"], "clear_finished_jobs_after"
@@ -264,7 +266,7 @@ def _load_backend_config_uncached(
       resolved_options["default_concurrency_duration"],
       "default_concurrency_duration",
     ),
-    database_alias=str(resolved_options["database_alias"]),
+    database_alias=_string_option(resolved_options["database_alias"], "database_alias"),
     use_skip_locked=_bool_option(resolved_options["use_skip_locked"], "use_skip_locked"),
     listen_notify=_bool_option(resolved_options["listen_notify"], "listen_notify"),
     silence_polling=_bool_option(resolved_options["silence_polling"], "silence_polling"),
@@ -428,7 +430,7 @@ def _validated_callback_path(callback_path: Any) -> str | None:
   if callback_path in (None, ""):
     return None
 
-  callback_path = str(callback_path)
+  callback_path = _string_option(callback_path, "on_thread_error")
   try:
     import_string(callback_path)
   except ImportError as exc:
@@ -546,24 +548,31 @@ def _build_recurring_config(
 
   recurring: dict[str, RecurringTaskConfig] = {}
   for key, raw_entry in raw_recurring.items():
+    key = _string_option(key, "recurring task key")
     if not isinstance(raw_entry, Mapping):
       raise ImproperlyConfigured("recurring entries must be mappings")
 
-    task_path = raw_entry.get("task_path")
-    schedule = raw_entry.get("schedule")
-    if not task_path or not schedule:
+    raw_task_path = raw_entry.get("task_path")
+    raw_schedule = raw_entry.get("schedule")
+    if raw_task_path in (None, "") or raw_schedule in (None, ""):
       raise ImproperlyConfigured(f"recurring task {key!r} requires task_path and schedule")
-    if not is_valid_cron(str(schedule)):
+    task_path = _string_option(raw_task_path, f"recurring task {key!r} task_path")
+    schedule = _string_option(raw_schedule, f"recurring task {key!r} schedule")
+    if not is_valid_cron(schedule):
       raise ImproperlyConfigured(f"recurring task {key!r} has an invalid cron schedule")
 
-    queue_name = str(raw_entry.get("queue_name", "default"))
+    queue_name = _string_option(
+      raw_entry.get("queue_name", "default"), f"recurring task {key!r} queue_name"
+    )
     priority = _priority_int(raw_entry.get("priority", 0), f"recurring task {key!r} priority")
     if allowed_queues and queue_name not in allowed_queues:
       raise ImproperlyConfigured(
         f"recurring task {key!r} is invalid: queue {queue_name!r} is not allowed for backend {backend_alias!r}"
       )
+    args = _tuple_option(raw_entry.get("args", []), f"recurring task {key!r} args")
+    kwargs = _dict_option(raw_entry.get("kwargs", {}), f"recurring task {key!r} kwargs")
     try:
-      task = import_string(str(task_path))
+      task = import_string(task_path)
     except ImportError as exc:
       raise ImproperlyConfigured(f"recurring task {key!r} is invalid: {exc}") from exc
     if not hasattr(task, "using"):
@@ -571,15 +580,17 @@ def _build_recurring_config(
         f"recurring task {key!r} is invalid: task_path must reference a Django task"
       )
 
-    recurring[str(key)] = RecurringTaskConfig(
-      key=str(key),
-      task_path=str(task_path),
-      schedule=str(schedule),
-      args=tuple(raw_entry.get("args", [])),
-      kwargs=dict(raw_entry.get("kwargs", {})),
+    recurring[key] = RecurringTaskConfig(
+      key=key,
+      task_path=task_path,
+      schedule=schedule,
+      args=args,
+      kwargs=kwargs,
       queue_name=queue_name,
       priority=priority,
-      description=str(raw_entry.get("description", "")),
+      description=_string_option(
+        raw_entry.get("description", ""), f"recurring task {key!r} description"
+      ),
     )
   return recurring
 
@@ -612,9 +623,36 @@ def _as_string_tuple(value: Any) -> tuple[str, ...]:
     return ()
   if isinstance(value, str):
     return (value,)
-  if not isinstance(value, Sequence):
+  if not isinstance(value, Sequence) or isinstance(value, (bytes, bytearray)):
+    raise ImproperlyConfigured("expected a string or a sequence of strings")
+  values = tuple(value)
+  if not all(isinstance(item, str) for item in values):
     raise ImproperlyConfigured("expected a string or a sequence of strings")
-  return tuple(str(item) for item in value)
+  return values
+
+
+def _string_option(value: Any, setting_name: str) -> str:
+  if not isinstance(value, str):
+    raise ImproperlyConfigured(f"dj_queue {setting_name} must be a string")
+  return value
+
+
+def _optional_string_option(value: Any, setting_name: str) -> str | None:
+  if value is None:
+    return None
+  return _string_option(value, setting_name)
+
+
+def _tuple_option(value: Any, setting_name: str) -> tuple[Any, ...]:
+  if isinstance(value, str) or not isinstance(value, Sequence):
+    raise ImproperlyConfigured(f"dj_queue {setting_name} must be a sequence")
+  return tuple(value)
+
+
+def _dict_option(value: Any, setting_name: str) -> dict[str, Any]:
+  if not isinstance(value, Mapping):
+    raise ImproperlyConfigured(f"dj_queue {setting_name} must be a mapping")
+  return dict(value)
 
 
 def _optional_nonnegative_int(value: Any, setting_name: str) -> int | None:

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/contrib/asgi.py

@@ -104,8 +104,10 @@ class DjQueueLifespan:
 
     receive_queue = asyncio.Queue()
     send_queue = asyncio.Queue()
-    app_task = await self._start_wrapped_lifespan_app(scope, receive_queue.get, send_queue.put)
     wrapped_app_supports_lifespan = self.forward_wrapped_lifespan
+    app_task = None
+    if wrapped_app_supports_lifespan:
+      app_task = await self._start_wrapped_lifespan_app(scope, receive_queue.get, send_queue.put)
 
     try:
       while True:
@@ -137,12 +139,12 @@ class DjQueueLifespan:
           if response is None:
             response = {"type": "lifespan.shutdown.complete"}
           await send(response)
-          if wrapped_app_supports_lifespan:
+          if wrapped_app_supports_lifespan and app_task is not None:
             await app_task
           return
     finally:
       await self._stop_supervisor()
-      if not app_task.done():
+      if app_task is not None and not app_task.done():
         app_task.cancel()
         with suppress(asyncio.CancelledError):
           await app_task

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/contrib/gunicorn.py

@@ -1,4 +1,5 @@
 import fcntl
+import hashlib
 import tempfile
 import threading
 from pathlib import Path
@@ -6,7 +7,7 @@ from pathlib import Path
 from dj_queue.runtime.errors import handle_thread_error
 from dj_queue.runtime.supervisor import AsyncSupervisor
 
-LOCK_PATH = Path(tempfile.gettempdir()) / "dj_queue_gunicorn_supervisor.lock"
+LOCK_PATH_PREFIX = "dj_queue_gunicorn_supervisor"
 LOCK_RETRY_INTERVAL = 1.0
 
 
@@ -20,7 +21,7 @@ def _set_supervisor_state(worker, **state):
 
 
 def _start_embedded_supervisor(worker, *, backend_alias="default"):
-  lock_file = _try_acquire_supervisor_lock()
+  lock_file = _try_acquire_supervisor_lock(backend_alias=backend_alias)
   if lock_file is None:
     return None
 
@@ -70,7 +71,12 @@ def _start_lock_retry_loop(worker, *, backend_alias="default"):
     while retry_stop.wait(LOCK_RETRY_INTERVAL) is False:
       if getattr(worker, "_dj_queue_supervisor", None) is not None:
         return
-      if _start_embedded_supervisor(worker, backend_alias=backend_alias) is not None:
+      try:
+        supervisor = _start_embedded_supervisor(worker, backend_alias=backend_alias)
+      except Exception as error:
+        handle_thread_error(error, context="gunicorn.supervisor", backend_alias=backend_alias)
+        continue
+      if supervisor is not None:
         retry_stop.set()
         return
 
@@ -81,7 +87,8 @@ def _start_lock_retry_loop(worker, *, backend_alias="default"):
   retry_thread.start()
 
 
-def post_fork(_server, worker):
+def post_fork(server, worker):
+  backend_alias = _backend_alias(server, worker)
   _set_supervisor_state(
     worker,
     supervisor=None,
@@ -91,11 +98,11 @@ def post_fork(_server, worker):
     supervisor_retry_stop=None,
     supervisor_retry_thread=None,
   )
-  supervisor = _start_embedded_supervisor(worker)
+  supervisor = _start_embedded_supervisor(worker, backend_alias=backend_alias)
   if supervisor is not None:
     return supervisor
 
-  _start_lock_retry_loop(worker)
+  _start_lock_retry_loop(worker, backend_alias=backend_alias)
   return None
 
 
@@ -133,9 +140,22 @@ def worker_exit(_server, worker):
   return None
 
 
-def _try_acquire_supervisor_lock():
-  LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
-  lock_file = LOCK_PATH.open("a+")
+def _backend_alias(server, worker):
+  return getattr(worker, "dj_queue_backend_alias", None) or getattr(
+    server, "dj_queue_backend_alias", "default"
+  )
+
+
+def _supervisor_lock_path(*, backend_alias):
+  lock_scope = f"{Path.cwd()}:{backend_alias}"
+  digest = hashlib.sha256(lock_scope.encode()).hexdigest()[:12]
+  return Path(tempfile.gettempdir()) / f"{LOCK_PATH_PREFIX}_{backend_alias}_{digest}.lock"
+
+
+def _try_acquire_supervisor_lock(*, backend_alias="default"):
+  lock_path = _supervisor_lock_path(backend_alias=backend_alias)
+  lock_path.parent.mkdir(parents=True, exist_ok=True)
+  lock_file = lock_path.open("a+")
   try:
     fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
   except BlockingIOError:

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/contrib/prometheus.py

@@ -8,19 +8,22 @@ except ImportError:
 else:
   from dj_queue.metrics import metric_families
 
+  METRIC_TYPES = {"gauge": GaugeMetricFamily}
+
   class DjQueueCollector:
     """Prometheus collector that exposes dj_queue metrics from the shared observability snapshot."""
 
     def collect(self):
       for family in metric_families():
-        gauge = GaugeMetricFamily(
+        metric_class = METRIC_TYPES[family.metric_type]
+        metric = metric_class(
           family.name,
           family.help_text,
           labels=list(family.labels),
         )
         for sample in family.samples:
-          gauge.add_metric(list(sample.labels), sample.value)
-        yield gauge
+          metric.add_metric(list(sample.labels), sample.value)
+        yield metric
 
   registry = CollectorRegistry(auto_describe=False)
   registry.register(DjQueueCollector())

{dj_queue-0.10.3 → dj_queue-0.10.5}/dj_queue/metrics.py

@@ -14,6 +14,7 @@ class MetricSample:
 class MetricFamily:
   name: str
   help_text: str
+  metric_type: str
   labels: tuple[str, ...]
   samples: tuple[MetricSample, ...]
 
@@ -109,54 +110,63 @@ def metric_families(*, snapshots=None):
     MetricFamily(
       name="dj_queue_queue_jobs",
       help_text="Current job count by backend, queue, and state",
+      metric_type="gauge",
       labels=("backend", "queue", "state"),
       samples=tuple(queue_jobs),
     ),
     MetricFamily(
       name="dj_queue_queue_paused",
       help_text="Whether a queue is paused for a backend",
+      metric_type="gauge",
       labels=("backend", "queue"),
       samples=tuple(queue_paused),
     ),
     MetricFamily(
       name="dj_queue_queue_latency_seconds",
       help_text="Latency of the oldest ready job in a backend queue",
+      metric_type="gauge",
       labels=("backend", "queue"),
       samples=tuple(queue_latency),
     ),
     MetricFamily(
       name="dj_queue_queue_live_workers",
       help_text="Live workers that can service a backend queue",
+      metric_type="gauge",
       labels=("backend", "queue"),
       samples=tuple(queue_workers),
     ),
     MetricFamily(
       name="dj_queue_runner_processes",
       help_text="Current runner process count by backend and liveness",
+      metric_type="gauge",
       labels=("backend", "status"),
       samples=tuple(runner_processes),
     ),
     MetricFamily(
       name="dj_queue_runner_processes_by_kind",
       help_text="Current runner process count by backend, kind, and liveness",
+      metric_type="gauge",
       labels=("backend", "kind", "status"),
       samples=tuple(runner_processes_by_kind),
     ),
     MetricFamily(
       name="dj_queue_recurring_tasks",
       help_text="Current recurring task count by backend",
+      metric_type="gauge",
       labels=("backend",),
       samples=tuple(recurring_tasks),
     ),
     MetricFamily(
       name="dj_queue_semaphores",
       help_text="Current semaphore count by queue database",
+      metric_type="gauge",
       labels=("queue_database",),
       samples=tuple(semaphores),
     ),
     MetricFamily(
       name="dj_queue_process_rows",
       help_text="Current process row count by backend",
+      metric_type="gauge",
       labels=("backend",),
       samples=tuple(process_rows),
     ),