dj-queue 0.6.0__tar.gz → 0.6.2__tar.gz

{dj_queue-0.6.0 → dj_queue-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.6.0
+Version: 0.6.2
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE

{dj_queue-0.6.0 → dj_queue-0.6.2}/dj_queue/config.py

@@ -1,4 +1,5 @@
 import json
+import math
 import os
 import warnings
 from collections.abc import Mapping, Sequence
@@ -393,7 +394,7 @@ def _build_worker_configs(raw_workers: Any, mode: str) -> tuple[WorkerConfig, ..
     raw_workers = [raw_workers]
 
   workers: list[WorkerConfig] = []
-  for raw_worker in raw_workers or []:
+  for index, raw_worker in enumerate(raw_workers or []):
     if not isinstance(raw_worker, Mapping):
       raise ImproperlyConfigured("worker entries must be mappings")
 
@@ -401,8 +402,9 @@ def _build_worker_configs(raw_workers: Any, mode: str) -> tuple[WorkerConfig, ..
       queues=_as_queue_selectors(raw_worker.get("queues", DEFAULT_WORKER["queues"])),
       threads=int(raw_worker.get("threads", DEFAULT_WORKER["threads"])),
       processes=int(raw_worker.get("processes", DEFAULT_WORKER["processes"])),
-      polling_interval=float(
-        raw_worker.get("polling_interval", DEFAULT_WORKER["polling_interval"])
+      polling_interval=_positive_float(
+        raw_worker.get("polling_interval", DEFAULT_WORKER["polling_interval"]),
+        f"workers[{index}].polling_interval",
       ),
     )
 
@@ -423,15 +425,16 @@ def _build_dispatcher_configs(raw_dispatchers: Any) -> tuple[DispatcherConfig, .
     raw_dispatchers = [raw_dispatchers]
 
   dispatchers: list[DispatcherConfig] = []
-  for raw_dispatcher in raw_dispatchers or []:
+  for index, raw_dispatcher in enumerate(raw_dispatchers or []):
     if not isinstance(raw_dispatcher, Mapping):
       raise ImproperlyConfigured("dispatcher entries must be mappings")
 
     dispatchers.append(
       DispatcherConfig(
         batch_size=int(raw_dispatcher.get("batch_size", DEFAULT_DISPATCHER["batch_size"])),
-        polling_interval=float(
-          raw_dispatcher.get("polling_interval", DEFAULT_DISPATCHER["polling_interval"])
+        polling_interval=_positive_float(
+          raw_dispatcher.get("polling_interval", DEFAULT_DISPATCHER["polling_interval"]),
+          f"dispatchers[{index}].polling_interval",
         ),
         concurrency_maintenance=bool(
           raw_dispatcher.get(
@@ -463,8 +466,9 @@ def _build_scheduler_config(raw_scheduler: Any) -> SchedulerConfig:
         DEFAULT_SCHEDULER["dynamic_tasks_enabled"],
       )
     ),
-    polling_interval=float(
-      raw_scheduler.get("polling_interval", DEFAULT_SCHEDULER["polling_interval"])
+    polling_interval=_positive_float(
+      raw_scheduler.get("polling_interval", DEFAULT_SCHEDULER["polling_interval"]),
+      "scheduler.polling_interval",
     ),
   )
 
@@ -539,5 +543,18 @@ def _optional_int(value: Any) -> int | None:
   return int(value)
 
 
+def _positive_float(value: Any, setting_name: str) -> float:
+  try:
+    number = float(value)
+  except (TypeError, ValueError) as exc:
+    raise ImproperlyConfigured(
+      f"dj_queue {setting_name} must be a positive number, got {value!r}"
+    ) from exc
+
+  if not math.isfinite(number) or number <= 0:
+    raise ImproperlyConfigured(f"dj_queue {setting_name} must be a positive number, got {value!r}")
+  return number
+
+
 def _cache_key(value: Any) -> str:
   return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)

dj_queue-0.6.2/dj_queue/contrib/gunicorn.py

@@ -0,0 +1,45 @@
+import threading
+
+from dj_queue.runtime.supervisor import AsyncSupervisor
+
+
+def build_supervisor(backend_alias="default"):
+  return AsyncSupervisor.from_backend_config(backend_alias=backend_alias, standalone=False)
+
+
+def post_fork(_server, worker):
+  if worker.age != 1:
+    return None
+
+  supervisor = build_supervisor()
+  worker._dj_queue_supervisor = supervisor
+  worker._dj_queue_supervisor_poll_stop = threading.Event()
+  supervisor.start()
+
+  def poll_supervisor():
+    stop_event = worker._dj_queue_supervisor_poll_stop
+    while stop_event.wait(supervisor.polling_interval) is False:
+      supervisor.poll_once()
+
+  poll_thread = threading.Thread(target=poll_supervisor, daemon=True)
+  worker._dj_queue_supervisor_poll_thread = poll_thread
+  poll_thread.start()
+  return supervisor
+
+
+def worker_exit(_server, worker):
+  supervisor = getattr(worker, "_dj_queue_supervisor", None)
+  stop_event = getattr(worker, "_dj_queue_supervisor_poll_stop", None)
+  poll_thread = getattr(worker, "_dj_queue_supervisor_poll_thread", None)
+  if stop_event is not None:
+    stop_event.set()
+  if poll_thread is not None:
+    poll_thread.join(timeout=1)
+    worker._dj_queue_supervisor_poll_thread = None
+  worker._dj_queue_supervisor_poll_stop = None
+  if supervisor is None:
+    return None
+
+  supervisor.stop()
+  worker._dj_queue_supervisor = None
+  return None

dj_queue-0.6.2/dj_queue/operations/_insert.py

@@ -0,0 +1,24 @@
+from django.db import connections
+from django.db.models import AutoField
+from django.db.models.constants import OnConflict
+from django.db.models.sql import InsertQuery
+
+
+def create_ignore_conflicts(model, /, *, using, **fields):
+  obj = model(**fields)
+  queryset = model.objects.using(using).all()
+  _objs_with_pk, objs_without_pk = queryset._prepare_for_bulk_create([obj])
+  insert_fields = [field for field in model._meta.concrete_fields if not field.generated]
+  if objs_without_pk:
+    insert_fields = [field for field in insert_fields if not isinstance(field, AutoField)]
+  query = InsertQuery(model, on_conflict=OnConflict.IGNORE)
+  query.insert_values(insert_fields, [obj], raw=False)
+  compiler = query.get_compiler(using=using)
+  rowcount = 0
+
+  with connections[using].cursor() as cursor:
+    for sql, params in compiler.as_sql():
+      cursor.execute(sql, params)
+      rowcount += cursor.rowcount
+
+  return rowcount > 0

{dj_queue-0.6.0 → dj_queue-0.6.2}/dj_queue/operations/concurrency.py

@@ -1,6 +1,7 @@
 from datetime import timedelta
 
-from django.db import IntegrityError, transaction
+from django.db import transaction
+from django.db.models import F
 from django.utils import timezone
 from django.utils.module_loading import import_string
 
@@ -8,6 +9,7 @@ from dj_queue.config import load_backend_config
 from dj_queue.db import get_database_alias, locked_queryset
 from dj_queue.log import log_event
 from dj_queue.models import BlockedExecution, Job, Pause, ReadyExecution, Semaphore
+from dj_queue.operations._insert import create_ignore_conflicts
 from dj_queue.runtime import notify as runtime_notify
 
 
@@ -19,36 +21,32 @@ def semaphore_acquire(
   backend_alias="default",
 ):
   alias = get_database_alias(backend_alias)
-  expires_at = timezone.now() + timedelta(seconds=duration_seconds)
+  now = timezone.now()
+  expires_at = now + timedelta(seconds=duration_seconds)
+
+  with transaction.atomic(using=alias):
+    if create_ignore_conflicts(
+      Semaphore,
+      using=alias,
+      key=key,
+      value=limit - 1,
+      limit=limit,
+      expires_at=expires_at,
+    ):
+      return True
 
-  for attempt in range(2):
-    try:
-      with transaction.atomic(using=alias):
-        semaphore = Semaphore.objects.using(alias).select_for_update().filter(key=key).first()
-        if semaphore is None:
-          Semaphore.objects.using(alias).create(
-            key=key,
-            value=limit - 1,
-            limit=limit,
-            expires_at=expires_at,
-          )
-          return True
-
-        if semaphore.value <= 0:
-          return False
-
-        semaphore.value -= 1
-        semaphore.expires_at = expires_at
-        semaphore.save(using=alias, update_fields=["value", "expires_at", "updated_at"])
-        return True
-    except IntegrityError:
-      # two workers can both miss the row, then race to create the unique key
-      # retry once so the loser can load the row created by the winner
-      if attempt == 0:
-        continue
-      continue
-
-  return False
+  # mysql-family backends can deadlock if a skipped insert and row lock happen in one tx
+  with transaction.atomic(using=alias):
+    updated = (
+      Semaphore.objects.using(alias)
+      .filter(key=key, value__gt=0)
+      .update(
+        value=F("value") - 1,
+        expires_at=expires_at,
+        updated_at=now,
+      )
+    )
+  return updated > 0
 
 
 def semaphore_release(key, *, duration_seconds, backend_alias="default"):

{dj_queue-0.6.0 → dj_queue-0.6.2}/dj_queue/operations/recurring.py

@@ -1,8 +1,9 @@
-from django.db import IntegrityError, transaction
+from django.db import transaction
 from django.utils.module_loading import import_string
 
 from dj_queue.db import get_database_alias
 from dj_queue.models import RecurringExecution, RecurringTask
+from dj_queue.operations._insert import create_ignore_conflicts
 from dj_queue.operations.jobs import enqueue_job
 
 
@@ -62,17 +63,24 @@ def fire_recurring_task(recurring_task, run_at, *, backend_alias="default"):
   alias = get_database_alias(backend_alias)
 
   with transaction.atomic(using=alias):
-    try:
-      execution = RecurringExecution.objects.using(alias).create(
-        backend_alias=backend_alias,
-        task_key=recurring_task.key,
-        run_at=run_at,
-      )
-    except IntegrityError:
+    created = create_ignore_conflicts(
+      RecurringExecution,
+      using=alias,
+      backend_alias=backend_alias,
+      task_key=recurring_task.key,
+      run_at=run_at,
+    )
+    if not created:
       # treat an existing reservation row as authoritative even if its job backfill
       # has not happened yet, so duplicate scheduler ticks never enqueue twice
       return None
 
+    execution = RecurringExecution.objects.using(alias).get(
+      backend_alias=backend_alias,
+      task_key=recurring_task.key,
+      run_at=run_at,
+    )
+
     task = import_string(recurring_task.task_path).using(
       queue_name=recurring_task.queue_name,
       priority=recurring_task.priority,

{dj_queue-0.6.0 → dj_queue-0.6.2}/dj_queue/runtime/base.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager, nullcontext
+import math
 import threading
 import time
 
@@ -24,6 +25,7 @@ def app_executor():
 
 
 _sqlite_process_write_lock = threading.Lock()
+_safe_polling_interval = 1.0
 
 
 def _process_write_context(alias):
@@ -82,7 +84,18 @@ class BaseRunner:
 
   @property
   def polling_interval(self):
-    return getattr(self.config, "polling_interval", 0)
+    return self._normalized_polling_interval(getattr(self.config, "polling_interval", None))
+
+  @staticmethod
+  def _normalized_polling_interval(value):
+    try:
+      polling_interval = float(value)
+    except (TypeError, ValueError):
+      return _safe_polling_interval
+
+    if not math.isfinite(polling_interval) or polling_interval <= 0:
+      return _safe_polling_interval
+    return polling_interval
 
   def start(self):
     if self.process is None:

{dj_queue-0.6.0 → dj_queue-0.6.2}/dj_queue/runtime/scheduler.py

@@ -21,6 +21,11 @@ class Scheduler(BaseRunner):
   process_kind = "Scheduler"
   hook_prefix = "scheduler"
 
+  @property
+  def polling_interval(self):
+    scheduler = getattr(self.config, "scheduler", None)
+    return self._normalized_polling_interval(getattr(scheduler, "polling_interval", None))
+
   def __init__(
     self,
     config,

{dj_queue-0.6.0 → dj_queue-0.6.2}/dj_queue/runtime/supervisor.py

@@ -2,6 +2,7 @@ import os
 import signal
 import socket
 import threading
+import time
 
 from django.utils import timezone
 from datetime import timedelta
@@ -48,6 +49,7 @@ class Supervisor(BaseRunner):
     )
     self.standalone = standalone
     self.pidfile = None
+    self._last_housekeeping_at = None
 
   @classmethod
   def from_backend_config(
@@ -84,7 +86,10 @@ class Supervisor(BaseRunner):
     return process
 
   def poll_once(self):
-    pruned_processes = self.prune_stale_process_rows()
+    pruned_processes = []
+    if self._housekeeping_due():
+      pruned_processes = self.prune_stale_process_rows()
+      self._last_housekeeping_at = time.monotonic()
     for process in pruned_processes:
       log_event(
         "process.pruned",
@@ -94,6 +99,18 @@ class Supervisor(BaseRunner):
       )
     return pruned_processes
 
+  @property
+  def housekeeping_interval(self):
+    heartbeat_interval = self.config.process_heartbeat_interval
+    if heartbeat_interval > 0:
+      return max(min(self.config.process_alive_threshold, max(heartbeat_interval, 1)), 1)
+    return max(min(self.config.process_alive_threshold, 60), 1)
+
+  def _housekeeping_due(self):
+    if self._last_housekeeping_at is None:
+      return True
+    return (time.monotonic() - self._last_housekeeping_at) >= self.housekeeping_interval
+
   def process_metadata(self):
     return {
       "mode": self.config.mode,

{dj_queue-0.6.0 → dj_queue-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "dj-queue"
-version = "0.6.0"
+version = "0.6.2"
 description = "Database-backed task queue backend for Django’s Tasks framework."
 readme = "README.md"
 license = "MIT"

dj_queue-0.6.0/dj_queue/contrib/gunicorn.py

@@ -1,25 +0,0 @@
-from dj_queue.runtime.supervisor import AsyncSupervisor
-
-
-def build_supervisor(backend_alias="default"):
-  return AsyncSupervisor.from_backend_config(backend_alias=backend_alias, standalone=False)
-
-
-def post_fork(_server, worker):
-  if worker.age != 1:
-    return None
-
-  supervisor = build_supervisor()
-  worker._dj_queue_supervisor = supervisor
-  supervisor.start()
-  return supervisor
-
-
-def worker_exit(_server, worker):
-  supervisor = getattr(worker, "_dj_queue_supervisor", None)
-  if supervisor is None:
-    return None
-
-  supervisor.stop()
-  worker._dj_queue_supervisor = None
-  return None