dj-queue 0.6.1__tar.gz → 0.6.3__tar.gz

{dj_queue-0.6.1 → dj_queue-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.6.1
+Version: 0.6.3
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/admin.py

@@ -28,7 +28,7 @@ from dj_queue.models import (
   RecurringTask,
   Semaphore,
 )
-from dj_queue.operations.jobs import enqueue_job_again
+from dj_queue.operations.jobs import dispatch_scheduled_job_now, enqueue_job_again
 
 
 class DjQueueFirstAdminSite(admin.AdminSite):
@@ -563,6 +563,16 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
   def get_change_actions(self, request, obj):
     if obj is None:
       return ()
+    if obj.status == "scheduled":
+      return (
+        {"name": "run_now", "label": "Run now", "css_class": "djq-object-action-retry"},
+        {
+          "name": "enqueue_copy_now",
+          "label": "Enqueue copy now",
+          "css_class": "djq-object-action-retry",
+        },
+      )
+
     actions = [{"name": "enqueue", "label": "Enqueue job", "css_class": "djq-object-action-retry"}]
     if obj.status == "failed":
       actions.extend(
@@ -582,6 +592,39 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
     return tuple(actions)
 
   def handle_change_action(self, request, obj, action):
+    if action == "run_now":
+      try:
+        _job, dispatched_as = dispatch_scheduled_job_now(obj.pk, backend_alias=obj.backend_alias)
+      except (EnqueueError, ImportError, AttributeError) as exc:
+        self.message_user(request, f"Could not dispatch job now: {exc}", level=messages.ERROR)
+        return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
+
+      message = "Dispatched scheduled job for immediate execution"
+      if dispatched_as == "blocked":
+        message = "Dispatched scheduled job immediately and it is now blocked"
+      if dispatched_as == "discarded":
+        message = "Dispatched scheduled job immediately and it was discarded"
+      self.message_user(request, message, level=messages.SUCCESS)
+      return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
+
+    if action == "enqueue_copy_now":
+      try:
+        new_job = enqueue_job_again(obj.pk, backend_alias=obj.backend_alias, run_after=None)
+      except (EnqueueError, ImportError, AttributeError) as exc:
+        self.message_user(request, f"Could not enqueue job: {exc}", level=messages.ERROR)
+        return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
+
+      self.message_user(
+        request,
+        format_html(
+          'Enqueued immediate copy <a href="{}">{}</a>.',
+          self._change_url(object_id=new_job.pk, backend_alias=new_job.backend_alias),
+          new_job.pk,
+        ),
+        level=messages.SUCCESS,
+      )
+      return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
+
     if action == "enqueue":
       try:
         new_job = enqueue_job_again(obj.pk, backend_alias=obj.backend_alias)

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/config.py

@@ -1,4 +1,5 @@
 import json
+import math
 import os
 import warnings
 from collections.abc import Mapping, Sequence
@@ -393,7 +394,7 @@ def _build_worker_configs(raw_workers: Any, mode: str) -> tuple[WorkerConfig, ..
     raw_workers = [raw_workers]
 
   workers: list[WorkerConfig] = []
-  for raw_worker in raw_workers or []:
+  for index, raw_worker in enumerate(raw_workers or []):
     if not isinstance(raw_worker, Mapping):
       raise ImproperlyConfigured("worker entries must be mappings")
 
@@ -401,8 +402,9 @@ def _build_worker_configs(raw_workers: Any, mode: str) -> tuple[WorkerConfig, ..
       queues=_as_queue_selectors(raw_worker.get("queues", DEFAULT_WORKER["queues"])),
       threads=int(raw_worker.get("threads", DEFAULT_WORKER["threads"])),
       processes=int(raw_worker.get("processes", DEFAULT_WORKER["processes"])),
-      polling_interval=float(
-        raw_worker.get("polling_interval", DEFAULT_WORKER["polling_interval"])
+      polling_interval=_positive_float(
+        raw_worker.get("polling_interval", DEFAULT_WORKER["polling_interval"]),
+        f"workers[{index}].polling_interval",
       ),
     )
 
@@ -423,15 +425,16 @@ def _build_dispatcher_configs(raw_dispatchers: Any) -> tuple[DispatcherConfig, .
     raw_dispatchers = [raw_dispatchers]
 
   dispatchers: list[DispatcherConfig] = []
-  for raw_dispatcher in raw_dispatchers or []:
+  for index, raw_dispatcher in enumerate(raw_dispatchers or []):
     if not isinstance(raw_dispatcher, Mapping):
       raise ImproperlyConfigured("dispatcher entries must be mappings")
 
     dispatchers.append(
       DispatcherConfig(
         batch_size=int(raw_dispatcher.get("batch_size", DEFAULT_DISPATCHER["batch_size"])),
-        polling_interval=float(
-          raw_dispatcher.get("polling_interval", DEFAULT_DISPATCHER["polling_interval"])
+        polling_interval=_positive_float(
+          raw_dispatcher.get("polling_interval", DEFAULT_DISPATCHER["polling_interval"]),
+          f"dispatchers[{index}].polling_interval",
         ),
         concurrency_maintenance=bool(
           raw_dispatcher.get(
@@ -463,8 +466,9 @@ def _build_scheduler_config(raw_scheduler: Any) -> SchedulerConfig:
         DEFAULT_SCHEDULER["dynamic_tasks_enabled"],
       )
     ),
-    polling_interval=float(
-      raw_scheduler.get("polling_interval", DEFAULT_SCHEDULER["polling_interval"])
+    polling_interval=_positive_float(
+      raw_scheduler.get("polling_interval", DEFAULT_SCHEDULER["polling_interval"]),
+      "scheduler.polling_interval",
     ),
   )
 
@@ -539,5 +543,18 @@ def _optional_int(value: Any) -> int | None:
   return int(value)
 
 
+def _positive_float(value: Any, setting_name: str) -> float:
+  try:
+    number = float(value)
+  except (TypeError, ValueError) as exc:
+    raise ImproperlyConfigured(
+      f"dj_queue {setting_name} must be a positive number, got {value!r}"
+    ) from exc
+
+  if not math.isfinite(number) or number <= 0:
+    raise ImproperlyConfigured(f"dj_queue {setting_name} must be a positive number, got {value!r}")
+  return number
+
+
 def _cache_key(value: Any) -> str:
   return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)

dj_queue-0.6.3/dj_queue/contrib/gunicorn.py

@@ -0,0 +1,45 @@
+import threading
+
+from dj_queue.runtime.supervisor import AsyncSupervisor
+
+
+def build_supervisor(backend_alias="default"):
+  return AsyncSupervisor.from_backend_config(backend_alias=backend_alias, standalone=False)
+
+
+def post_fork(_server, worker):
+  if worker.age != 1:
+    return None
+
+  supervisor = build_supervisor()
+  worker._dj_queue_supervisor = supervisor
+  worker._dj_queue_supervisor_poll_stop = threading.Event()
+  supervisor.start()
+
+  def poll_supervisor():
+    stop_event = worker._dj_queue_supervisor_poll_stop
+    while stop_event.wait(supervisor.polling_interval) is False:
+      supervisor.poll_once()
+
+  poll_thread = threading.Thread(target=poll_supervisor, daemon=True)
+  worker._dj_queue_supervisor_poll_thread = poll_thread
+  poll_thread.start()
+  return supervisor
+
+
+def worker_exit(_server, worker):
+  supervisor = getattr(worker, "_dj_queue_supervisor", None)
+  stop_event = getattr(worker, "_dj_queue_supervisor_poll_stop", None)
+  poll_thread = getattr(worker, "_dj_queue_supervisor_poll_thread", None)
+  if stop_event is not None:
+    stop_event.set()
+  if poll_thread is not None:
+    poll_thread.join(timeout=1)
+    worker._dj_queue_supervisor_poll_thread = None
+  worker._dj_queue_supervisor_poll_stop = None
+  if supervisor is None:
+    return None
+
+  supervisor.stop()
+  worker._dj_queue_supervisor = None
+  return None

dj_queue-0.6.3/dj_queue/operations/_insert.py

@@ -0,0 +1,24 @@
+from django.db import connections
+from django.db.models import AutoField
+from django.db.models.constants import OnConflict
+from django.db.models.sql import InsertQuery
+
+
+def create_ignore_conflicts(model, /, *, using, **fields):
+  obj = model(**fields)
+  queryset = model.objects.using(using).all()
+  _objs_with_pk, objs_without_pk = queryset._prepare_for_bulk_create([obj])
+  insert_fields = [field for field in model._meta.concrete_fields if not field.generated]
+  if objs_without_pk:
+    insert_fields = [field for field in insert_fields if not isinstance(field, AutoField)]
+  query = InsertQuery(model, on_conflict=OnConflict.IGNORE)
+  query.insert_values(insert_fields, [obj], raw=False)
+  compiler = query.get_compiler(using=using)
+  rowcount = 0
+
+  with connections[using].cursor() as cursor:
+    for sql, params in compiler.as_sql():
+      cursor.execute(sql, params)
+      rowcount += cursor.rowcount
+
+  return rowcount > 0

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/operations/concurrency.py

@@ -1,6 +1,7 @@
 from datetime import timedelta
 
-from django.db import IntegrityError, transaction
+from django.db import transaction
+from django.db.models import F
 from django.utils import timezone
 from django.utils.module_loading import import_string
 
@@ -8,6 +9,7 @@ from dj_queue.config import load_backend_config
 from dj_queue.db import get_database_alias, locked_queryset
 from dj_queue.log import log_event
 from dj_queue.models import BlockedExecution, Job, Pause, ReadyExecution, Semaphore
+from dj_queue.operations._insert import create_ignore_conflicts
 from dj_queue.runtime import notify as runtime_notify
 
 
@@ -19,36 +21,32 @@ def semaphore_acquire(
   backend_alias="default",
 ):
   alias = get_database_alias(backend_alias)
-  expires_at = timezone.now() + timedelta(seconds=duration_seconds)
+  now = timezone.now()
+  expires_at = now + timedelta(seconds=duration_seconds)
+
+  with transaction.atomic(using=alias):
+    if create_ignore_conflicts(
+      Semaphore,
+      using=alias,
+      key=key,
+      value=limit - 1,
+      limit=limit,
+      expires_at=expires_at,
+    ):
+      return True
 
-  for attempt in range(2):
-    try:
-      with transaction.atomic(using=alias):
-        semaphore = Semaphore.objects.using(alias).select_for_update().filter(key=key).first()
-        if semaphore is None:
-          Semaphore.objects.using(alias).create(
-            key=key,
-            value=limit - 1,
-            limit=limit,
-            expires_at=expires_at,
-          )
-          return True
-
-        if semaphore.value <= 0:
-          return False
-
-        semaphore.value -= 1
-        semaphore.expires_at = expires_at
-        semaphore.save(using=alias, update_fields=["value", "expires_at", "updated_at"])
-        return True
-    except IntegrityError:
-      # two workers can both miss the row, then race to create the unique key
-      # retry once so the loser can load the row created by the winner
-      if attempt == 0:
-        continue
-      continue
-
-  return False
+  # mysql-family backends can deadlock if a skipped insert and row lock happen in one tx
+  with transaction.atomic(using=alias):
+    updated = (
+      Semaphore.objects.using(alias)
+      .filter(key=key, value__gt=0)
+      .update(
+        value=F("value") - 1,
+        expires_at=expires_at,
+        updated_at=now,
+      )
+    )
+  return updated > 0
 
 
 def semaphore_release(key, *, duration_seconds, backend_alias="default"):

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/operations/jobs.py

@@ -351,6 +351,39 @@ def promote_scheduled_jobs(*, batch_size, backend_alias="default", use_skip_lock
     return jobs
 
 
+def dispatch_scheduled_job_now(job_id, *, backend_alias="default"):
+  alias = get_database_alias(backend_alias)
+  config = load_backend_config(backend_alias)
+
+  with transaction.atomic(using=alias):
+    scheduled = locked_queryset(
+      ScheduledExecution.objects.using(alias)
+      .select_related("job")
+      .filter(job_id=job_id, job__backend_alias=backend_alias),
+      use_skip_locked=config.use_skip_locked,
+    ).first()
+    if scheduled is None:
+      raise EnqueueError("job is not scheduled")
+
+    job = scheduled.job
+    scheduled.delete(using=alias)
+    job.scheduled_at = None
+    job.save(using=alias, update_fields=["scheduled_at", "updated_at"])
+    dispatched_as = _dispatch_existing_job(job)
+
+  if dispatched_as == "ready":
+    runtime_notify.notify_ready_queues((job.queue_name,), backend_alias=backend_alias)
+
+  log_event(
+    "job.dispatched_now",
+    job_id=str(job.id),
+    queue_name=job.queue_name,
+    priority=job.priority,
+    dispatched_as=dispatched_as,
+  )
+  return job, dispatched_as
+
+
 def retry_failed_job(job_id, *, backend_alias="default"):
   alias = get_database_alias(backend_alias)
 
@@ -370,15 +403,19 @@ def retry_failed_job(job_id, *, backend_alias="default"):
   return job
 
 
-def enqueue_job_again(job_id, *, backend_alias="default"):
+_KEEP_RUN_AFTER = object()
+
+
+def enqueue_job_again(job_id, *, backend_alias="default", run_after=_KEEP_RUN_AFTER):
   alias = get_database_alias(backend_alias)
   source_job = Job.objects.using(alias).get(pk=job_id)
   task = import_string(source_job.task_path)
+  source_run_after = source_job.scheduled_at if run_after is _KEEP_RUN_AFTER else run_after
   if hasattr(task, "using"):
     task = task.using(
       priority=source_job.priority,
       queue_name=source_job.queue_name,
-      run_after=source_job.scheduled_at,
+      run_after=source_run_after,
       backend=source_job.backend_alias,
     )
   args = list(source_job.payload.get("args", []))

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/operations/recurring.py

@@ -3,6 +3,7 @@ from django.utils.module_loading import import_string
 
 from dj_queue.db import get_database_alias
 from dj_queue.models import RecurringExecution, RecurringTask
+from dj_queue.operations._insert import create_ignore_conflicts
 from dj_queue.operations.jobs import enqueue_job
 
 
@@ -62,7 +63,9 @@ def fire_recurring_task(recurring_task, run_at, *, backend_alias="default"):
   alias = get_database_alias(backend_alias)
 
   with transaction.atomic(using=alias):
-    execution, created = RecurringExecution.objects.using(alias).get_or_create(
+    created = create_ignore_conflicts(
+      RecurringExecution,
+      using=alias,
       backend_alias=backend_alias,
       task_key=recurring_task.key,
       run_at=run_at,
@@ -72,6 +75,12 @@ def fire_recurring_task(recurring_task, run_at, *, backend_alias="default"):
       # has not happened yet, so duplicate scheduler ticks never enqueue twice
       return None
 
+    execution = RecurringExecution.objects.using(alias).get(
+      backend_alias=backend_alias,
+      task_key=recurring_task.key,
+      run_at=run_at,
+    )
+
     task = import_string(recurring_task.task_path).using(
       queue_name=recurring_task.queue_name,
       priority=recurring_task.priority,

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/runtime/base.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager, nullcontext
+import math
 import threading
 import time
 
@@ -24,6 +25,7 @@ def app_executor():
 
 
 _sqlite_process_write_lock = threading.Lock()
+_safe_polling_interval = 1.0
 
 
 def _process_write_context(alias):
@@ -82,7 +84,18 @@ class BaseRunner:
 
   @property
   def polling_interval(self):
-    return getattr(self.config, "polling_interval", 0)
+    return self._normalized_polling_interval(getattr(self.config, "polling_interval", None))
+
+  @staticmethod
+  def _normalized_polling_interval(value):
+    try:
+      polling_interval = float(value)
+    except (TypeError, ValueError):
+      return _safe_polling_interval
+
+    if not math.isfinite(polling_interval) or polling_interval <= 0:
+      return _safe_polling_interval
+    return polling_interval
 
   def start(self):
     if self.process is None:

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/runtime/scheduler.py

@@ -23,7 +23,8 @@ class Scheduler(BaseRunner):
 
   @property
   def polling_interval(self):
-    return self.config.scheduler.polling_interval
+    scheduler = getattr(self.config, "scheduler", None)
+    return self._normalized_polling_interval(getattr(scheduler, "polling_interval", None))
 
   def __init__(
     self,

{dj_queue-0.6.1 → dj_queue-0.6.3}/dj_queue/runtime/supervisor.py

@@ -2,6 +2,7 @@ import os
 import signal
 import socket
 import threading
+import time
 
 from django.utils import timezone
 from datetime import timedelta
@@ -48,6 +49,7 @@ class Supervisor(BaseRunner):
     )
     self.standalone = standalone
     self.pidfile = None
+    self._last_housekeeping_at = None
 
   @classmethod
   def from_backend_config(
@@ -84,7 +86,10 @@ class Supervisor(BaseRunner):
     return process
 
   def poll_once(self):
-    pruned_processes = self.prune_stale_process_rows()
+    pruned_processes = []
+    if self._housekeeping_due():
+      pruned_processes = self.prune_stale_process_rows()
+      self._last_housekeeping_at = time.monotonic()
     for process in pruned_processes:
       log_event(
         "process.pruned",
@@ -94,6 +99,18 @@ class Supervisor(BaseRunner):
       )
     return pruned_processes
 
+  @property
+  def housekeeping_interval(self):
+    heartbeat_interval = self.config.process_heartbeat_interval
+    if heartbeat_interval > 0:
+      return max(min(self.config.process_alive_threshold, max(heartbeat_interval, 1)), 1)
+    return max(min(self.config.process_alive_threshold, 60), 1)
+
+  def _housekeeping_due(self):
+    if self._last_housekeeping_at is None:
+      return True
+    return (time.monotonic() - self._last_housekeeping_at) >= self.housekeeping_interval
+
   def process_metadata(self):
     return {
       "mode": self.config.mode,

{dj_queue-0.6.1 → dj_queue-0.6.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "dj-queue"
-version = "0.6.1"
+version = "0.6.3"
 description = "Database-backed task queue backend for Django’s Tasks framework."
 readme = "README.md"
 license = "MIT"

dj_queue-0.6.1/dj_queue/contrib/gunicorn.py

@@ -1,25 +0,0 @@
-from dj_queue.runtime.supervisor import AsyncSupervisor
-
-
-def build_supervisor(backend_alias="default"):
-  return AsyncSupervisor.from_backend_config(backend_alias=backend_alias, standalone=False)
-
-
-def post_fork(_server, worker):
-  if worker.age != 1:
-    return None
-
-  supervisor = build_supervisor()
-  worker._dj_queue_supervisor = supervisor
-  supervisor.start()
-  return supervisor
-
-
-def worker_exit(_server, worker):
-  supervisor = getattr(worker, "_dj_queue_supervisor", None)
-  if supervisor is None:
-    return None
-
-  supervisor.stop()
-  worker._dj_queue_supervisor = None
-  return None