dj-queue 0.6.3__tar.gz → 0.6.4__tar.gz

{dj_queue-0.6.3 → dj_queue-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.6.3
+Version: 0.6.4
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/backend.py

@@ -54,7 +54,7 @@ class DjQueueBackend(BaseTaskBackend):
           "blocked_execution",
           "failed_execution",
         )
-        .get(pk=result_id)
+        .get(pk=result_id, backend_alias=self.alias)
       )
     except Job.DoesNotExist as exc:
       raise TaskResultDoesNotExist(str(result_id)) from exc

dj_queue-0.6.4/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py

@@ -0,0 +1,59 @@
+# Generated by Django 6.0.4 on 2026-05-06 05:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+  dependencies = [
+    (
+      "dj_queue",
+      "0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more",
+    ),
+  ]
+
+  operations = [
+    migrations.AddIndex(
+      model_name="blockedexecution",
+      index=models.Index(
+        fields=["concurrency_key", "-priority", "id"], name="dj_queue_bl_concurr_2d8393_idx"
+      ),
+    ),
+    migrations.AddIndex(
+      model_name="blockedexecution",
+      index=models.Index(
+        fields=["expires_at", "-priority", "id"], name="dj_queue_bl_expires_f4b090_idx"
+      ),
+    ),
+    migrations.AddIndex(
+      model_name="failedexecution",
+      index=models.Index(fields=["created_at", "job"], name="dj_queue_fa_created_343d7d_idx"),
+    ),
+    migrations.AddIndex(
+      model_name="job",
+      index=models.Index(
+        fields=["backend_alias", "finished_at", "id"], name="dj_queue_jo_backend_ee105e_idx"
+      ),
+    ),
+    migrations.AddIndex(
+      model_name="readyexecution",
+      index=models.Index(fields=["-priority", "id"], name="dj_queue_re_priorit_ee6ffe_idx"),
+    ),
+    migrations.AddIndex(
+      model_name="readyexecution",
+      index=models.Index(
+        fields=["queue_name", "-priority", "id"], name="dj_queue_re_queue_n_612755_idx"
+      ),
+    ),
+    migrations.AddIndex(
+      model_name="recurringexecution",
+      index=models.Index(
+        fields=["backend_alias", "run_at", "id"], name="dj_queue_re_backend_a68bf1_idx"
+      ),
+    ),
+    migrations.AddIndex(
+      model_name="scheduledexecution",
+      index=models.Index(
+        fields=["scheduled_at", "-priority", "id"], name="dj_queue_sc_schedul_edba95_idx"
+      ),
+    ),
+  ]

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/models/jobs.py

@@ -1,6 +1,6 @@
 import uuid
 
-from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.db.models import Q
 from django.utils.module_loading import import_string
@@ -65,6 +65,7 @@ class Job(models.Model):
       models.Index(fields=["queue_name", "finished_at"]),
       models.Index(fields=["scheduled_at", "finished_at"]),
       models.Index(fields=["finished_at"]),
+      models.Index(fields=["backend_alias", "finished_at", "id"]),
     ]
 
   @property
@@ -125,16 +126,10 @@ class ReadyExecution(models.Model):
     indexes = [
       models.Index(fields=["priority", "id"]),
       models.Index(fields=["queue_name", "priority", "id"]),
+      models.Index(fields=["-priority", "id"]),
+      models.Index(fields=["queue_name", "-priority", "id"]),
     ]
 
-  def clean(self):
-    super().clean()
-    _validate_live_state(self)
-
-  def save(self, *args, **kwargs):
-    self.full_clean()
-    return super().save(*args, **kwargs)
-
   @classmethod
   def discard_all_in_batches(cls, *, batch_size=500, backend_alias="default"):
     operation = import_string("dj_queue.operations.jobs.discard_ready_jobs")
@@ -159,15 +154,10 @@ class ScheduledExecution(models.Model):
 
   class Meta:
     db_table = "dj_queue_scheduled_executions"
-    indexes = [models.Index(fields=["scheduled_at", "priority", "id"])]
-
-  def clean(self):
-    super().clean()
-    _validate_live_state(self)
-
-  def save(self, *args, **kwargs):
-    self.full_clean()
-    return super().save(*args, **kwargs)
+    indexes = [
+      models.Index(fields=["scheduled_at", "priority", "id"]),
+      models.Index(fields=["scheduled_at", "-priority", "id"]),
+    ]
 
 
 class ClaimedExecution(models.Model):
@@ -189,14 +179,6 @@ class ClaimedExecution(models.Model):
     db_table = "dj_queue_claimed_executions"
     indexes = [models.Index(fields=["process", "job"])]
 
-  def clean(self):
-    super().clean()
-    _validate_live_state(self)
-
-  def save(self, *args, **kwargs):
-    self.full_clean()
-    return super().save(*args, **kwargs)
-
   @classmethod
   def discard_all_in_batches(cls, **_kwargs):
     raise UndiscardableError("cannot discard in-progress jobs")
@@ -219,16 +201,10 @@ class BlockedExecution(models.Model):
     indexes = [
       models.Index(fields=["concurrency_key", "priority", "id"]),
       models.Index(fields=["expires_at", "concurrency_key"]),
+      models.Index(fields=["concurrency_key", "-priority", "id"]),
+      models.Index(fields=["expires_at", "-priority", "id"]),
     ]
 
-  def clean(self):
-    super().clean()
-    _validate_live_state(self)
-
-  def save(self, *args, **kwargs):
-    self.full_clean()
-    return super().save(*args, **kwargs)
-
   @classmethod
   def discard_all_in_batches(cls, *, batch_size=500, backend_alias="default"):
     operation = import_string("dj_queue.operations.jobs.discard_blocked_jobs")
@@ -253,14 +229,7 @@ class FailedExecution(models.Model):
 
   class Meta:
     db_table = "dj_queue_failed_executions"
-
-  def clean(self):
-    super().clean()
-    _validate_live_state(self)
-
-  def save(self, *args, **kwargs):
-    self.full_clean()
-    return super().save(*args, **kwargs)
+    indexes = [models.Index(fields=["created_at", "job"])]
 
   def retry(self):
     return _retry_failed_job(self.job_id, backend_alias=self.job.backend_alias)
@@ -301,28 +270,11 @@ def _discard_jobs_for_state(model, operation, *, batch_size, backend_alias):
   alias = get_database_alias(backend_alias)
   deleted = 0
   while True:
-    job_ids = list(model.objects.using(alias).values_list("job_id", flat=True)[:batch_size])
+    job_ids = list(
+      model.objects.using(alias)
+      .filter(job__backend_alias=backend_alias)
+      .values_list("job_id", flat=True)[:batch_size]
+    )
     if not job_ids:
       return deleted
     deleted += operation(job_ids=job_ids, batch_size=batch_size, backend_alias=backend_alias)
-
-
-def _validate_live_state(instance):
-  if not instance.job_id:
-    return
-
-  for model in LIVE_STATE_MODELS:
-    queryset = model._default_manager.filter(job_id=instance.job_id)
-    if model is instance.__class__ and instance.pk is not None:
-      queryset = queryset.exclude(pk=instance.pk)
-    if queryset.exists():
-      raise ValidationError({"job": "job already has a live execution state"})
-
-
-LIVE_STATE_MODELS = (
-  ReadyExecution,
-  ScheduledExecution,
-  ClaimedExecution,
-  BlockedExecution,
-  FailedExecution,
-)

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/models/recurring.py

@@ -57,4 +57,7 @@ class RecurringExecution(models.Model):
         name="dj_queue_recur_exec_backend_run_at_unique",
       )
     ]
-    indexes = [models.Index(fields=["backend_alias", "task_key", "run_at"])]
+    indexes = [
+      models.Index(fields=["backend_alias", "task_key", "run_at"]),
+      models.Index(fields=["backend_alias", "run_at", "id"]),
+    ]

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/operations/cleanup.py

@@ -25,7 +25,11 @@ def clear_finished_jobs(
   if now is None:
     now = timezone.now()
   cutoff = now - timedelta(seconds=older_than)
-  queryset = Job.objects.using(alias).filter(finished_at__lt=cutoff).order_by("finished_at", "id")
+  queryset = (
+    Job.objects.using(alias)
+    .filter(backend_alias=backend_alias, finished_at__lt=cutoff)
+    .order_by("finished_at", "id")
+  )
   if task_path is not None:
     queryset = queryset.filter(task_path=task_path)
 
@@ -33,7 +37,7 @@ def clear_finished_jobs(
   if not job_ids:
     return 0
 
-  Job.objects.using(alias).filter(pk__in=job_ids).delete()
+  Job.objects.using(alias).filter(backend_alias=backend_alias, pk__in=job_ids).delete()
   return len(job_ids)
 
 
@@ -57,7 +61,7 @@ def clear_failed_jobs(
   cutoff = now - timedelta(seconds=older_than)
   queryset = (
     FailedExecution.objects.using(alias)
-    .filter(created_at__lt=cutoff)
+    .filter(job__backend_alias=backend_alias, created_at__lt=cutoff)
     .order_by("created_at", "job_id")
   )
   if task_path is not None:
@@ -67,7 +71,7 @@ def clear_failed_jobs(
   if not job_ids:
     return 0
 
-  Job.objects.using(alias).filter(pk__in=job_ids).delete()
+  Job.objects.using(alias).filter(backend_alias=backend_alias, pk__in=job_ids).delete()
   return len(job_ids)
 
 

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/operations/concurrency.py

@@ -8,7 +8,7 @@ from django.utils.module_loading import import_string
 from dj_queue.config import load_backend_config
 from dj_queue.db import get_database_alias, locked_queryset
 from dj_queue.log import log_event
-from dj_queue.models import BlockedExecution, Job, Pause, ReadyExecution, Semaphore
+from dj_queue.models import BlockedExecution, Pause, ReadyExecution, Semaphore
 from dj_queue.operations._insert import create_ignore_conflicts
 from dj_queue.runtime import notify as runtime_notify
 
@@ -77,7 +77,10 @@ def unblock_next_blocked_job(
 
   with transaction.atomic(using=alias):
     queryset = (
-      BlockedExecution.objects.using(alias).filter(concurrency_key=key).order_by("-priority", "id")
+      BlockedExecution.objects.using(alias)
+      .select_related("job")
+      .filter(concurrency_key=key, job__backend_alias=backend_alias)
+      .order_by("-priority", "id")
     )
     blocked = locked_queryset(queryset, use_skip_locked=use_skip_locked).first()
     if blocked is None:
@@ -91,7 +94,7 @@ def unblock_next_blocked_job(
     ):
       return None
 
-    job = Job.objects.using(alias).get(pk=blocked.job_id)
+    job = blocked.job
     queue_name = blocked.queue_name
     priority = blocked.priority
     blocked.delete(using=alias)
@@ -134,20 +137,16 @@ def promote_expired_blocked_jobs(*, batch_size=500, backend_alias="default", use
   with transaction.atomic(using=alias):
     queryset = (
       BlockedExecution.objects.using(alias)
-      .filter(expires_at__lte=now)
+      .select_related("job")
+      .filter(job__backend_alias=backend_alias, expires_at__lte=now)
       .order_by("expires_at", "-priority", "id")
     )
     blocked_rows = list(locked_queryset(queryset, use_skip_locked=use_skip_locked)[:batch_size])
     if not blocked_rows:
       return []
 
-    jobs_by_id = {
-      job.id: job
-      for job in Job.objects.using(alias).filter(pk__in=[b.job_id for b in blocked_rows])
-    }
-
     for blocked in blocked_rows:
-      job = jobs_by_id[blocked.job_id]
+      job = blocked.job
       limit, duration_seconds = task_settings.get(job.task_path, (None, None))
       if limit is None:
         task = import_string(job.task_path)

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/operations/jobs.py

@@ -219,7 +219,11 @@ def claim_ready_jobs(
   )
 
   with transaction.atomic(using=alias):
-    queryset = ReadyExecution.objects.using(alias).filter(job__backend_alias=backend_alias)
+    queryset = (
+      ReadyExecution.objects.using(alias)
+      .select_related("job")
+      .filter(job__backend_alias=backend_alias)
+    )
     if paused_queue_names:
       queryset = queryset.exclude(queue_name__in=paused_queue_names)
     ready_rows = _select_ready_rows(
@@ -231,9 +235,7 @@ def claim_ready_jobs(
     if not ready_rows:
       return []
 
-    job_ids = [row.job_id for row in ready_rows]
-    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=job_ids)}
-    jobs = [jobs_by_id[job_id] for job_id in job_ids]
+    jobs = [row.job for row in ready_rows]
 
     ReadyExecution.objects.using(alias).filter(pk__in=[row.pk for row in ready_rows]).delete()
     _bulk_create(
@@ -250,7 +252,9 @@ def claim_ready_jobs(
 def execute_claimed_job(job_id, *, backend_alias="default"):
   alias = get_database_alias(backend_alias)
   claimed = (
-    ClaimedExecution.objects.using(alias).select_related("job", "process").get(job_id=job_id)
+    ClaimedExecution.objects.using(alias)
+    .select_related("job", "process")
+    .get(job_id=job_id, job__backend_alias=backend_alias)
   )
   job = claimed.job
 
@@ -279,7 +283,12 @@ def complete_claimed_job(job_id, return_value, *, backend_alias="default"):
   alias = get_database_alias(backend_alias)
 
   with transaction.atomic(using=alias):
-    claimed = ClaimedExecution.objects.using(alias).select_related("job").get(job_id=job_id)
+    claimed = (
+      ClaimedExecution.objects.using(alias)
+      .select_for_update()
+      .select_related("job")
+      .get(job_id=job_id, job__backend_alias=backend_alias)
+    )
     job = claimed.job
     now = timezone.now()
     config = load_backend_config(job.backend_alias)
@@ -301,7 +310,12 @@ def fail_claimed_job(job_id, error, *, traceback_text="", backend_alias="default
   alias = get_database_alias(backend_alias)
 
   with transaction.atomic(using=alias):
-    claimed = ClaimedExecution.objects.using(alias).select_related("job").get(job_id=job_id)
+    claimed = (
+      ClaimedExecution.objects.using(alias)
+      .select_for_update()
+      .select_related("job")
+      .get(job_id=job_id, job__backend_alias=backend_alias)
+    )
     job = claimed.job
     claimed.delete(using=alias)
     FailedExecution.objects.using(alias).create(
@@ -330,16 +344,15 @@ def promote_scheduled_jobs(*, batch_size, backend_alias="default", use_skip_lock
   with transaction.atomic(using=alias):
     queryset = (
       ScheduledExecution.objects.using(alias)
-      .filter(scheduled_at__lte=now)
+      .select_related("job")
+      .filter(job__backend_alias=backend_alias, scheduled_at__lte=now)
       .order_by("scheduled_at", "-priority", "id")
     )
     scheduled_rows = list(locked_queryset(queryset, use_skip_locked=use_skip_locked)[:batch_size])
     if not scheduled_rows:
       return []
 
-    job_ids = [row.job_id for row in scheduled_rows]
-    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=job_ids)}
-    jobs = [jobs_by_id[job_id] for job_id in job_ids]
+    jobs = [row.job for row in scheduled_rows]
 
     ScheduledExecution.objects.using(alias).filter(
       pk__in=[row.pk for row in scheduled_rows]
@@ -388,7 +401,12 @@ def retry_failed_job(job_id, *, backend_alias="default"):
   alias = get_database_alias(backend_alias)
 
   with transaction.atomic(using=alias):
-    failed = FailedExecution.objects.using(alias).select_related("job").get(job_id=job_id)
+    failed = (
+      FailedExecution.objects.using(alias)
+      .select_for_update()
+      .select_related("job")
+      .get(job_id=job_id, job__backend_alias=backend_alias)
+    )
     job = failed.job
     failed.delete(using=alias)
     job.return_value = None
@@ -408,7 +426,7 @@ _KEEP_RUN_AFTER = object()
 
 def enqueue_job_again(job_id, *, backend_alias="default", run_after=_KEEP_RUN_AFTER):
   alias = get_database_alias(backend_alias)
-  source_job = Job.objects.using(alias).get(pk=job_id)
+  source_job = Job.objects.using(alias).get(pk=job_id, backend_alias=backend_alias)
   task = import_string(source_job.task_path)
   source_run_after = source_job.scheduled_at if run_after is _KEEP_RUN_AFTER else run_after
   if hasattr(task, "using"):
@@ -426,6 +444,7 @@ def enqueue_job_again(job_id, *, backend_alias="default", run_after=_KEEP_RUN_AF
 
 def discard_failed_jobs(*, job_ids=None, batch_size=500, backend_alias="default"):
   alias = get_database_alias(backend_alias)
+  config = load_backend_config(backend_alias)
 
   with transaction.atomic(using=alias):
     queryset = (
@@ -433,7 +452,9 @@ def discard_failed_jobs(*, job_ids=None, batch_size=500, backend_alias="default"
     )
     if job_ids is not None:
       queryset = queryset.filter(job_id__in=job_ids)
-    failed_rows = list(queryset[:batch_size])
+    failed_rows = list(
+      locked_queryset(queryset, use_skip_locked=config.use_skip_locked)[:batch_size]
+    )
     if not failed_rows:
       return 0
 
@@ -502,7 +523,6 @@ def discard_scheduled_jobs(*, job_ids=None, batch_size=500, backend_alias="defau
     Job.objects.using(alias).filter(pk__in=[row.job_id for row in scheduled_rows]).delete()
 
   for job in jobs:
-    _release_concurrency_slot(job)
     log_event("job.discarded", job_id=str(job.id), reason="scheduled")
   return len(jobs)
 

{dj_queue-0.6.3 → dj_queue-0.6.4}/dj_queue/runtime/supervisor.py

@@ -4,6 +4,7 @@ import socket
 import threading
 import time
 
+from django.db import connections
 from django.utils import timezone
 from datetime import timedelta
 
@@ -301,8 +302,11 @@ class AsyncSupervisor(Supervisor):
   def _fail_crashed_runner_jobs(self, runner):
     if runner.process is None:
       return
+    alias = get_database_alias(self.backend_alias)
     claimed_job_ids = list(
-      ClaimedExecution.objects.filter(process=runner.process).values_list("job_id", flat=True)
+      ClaimedExecution.objects.using(alias)
+      .filter(process=runner.process)
+      .values_list("job_id", flat=True)
     )
     with app_executor():
       for job_id in claimed_job_ids:
@@ -430,12 +434,21 @@ class ForkSupervisor(Supervisor):
     return process
 
   def stop(self):
+    timeout = max(float(self.config.shutdown_timeout), 0)
     for pid in tuple(self.children):
       try:
         self._killer(pid, signal.SIGTERM)
       except ProcessLookupError:
         pass
-    self.children.clear()
+
+    self._wait_for_children(timeout)
+    for pid in tuple(self.children):
+      try:
+        self._killer(pid, signal.SIGKILL)
+      except ProcessLookupError:
+        pass
+      self._fail_claimed_jobs_for_pid(pid)
+      self.children.pop(pid, None)
     return super().stop()
 
   def register_signal_handlers(self):
@@ -485,6 +498,22 @@ class ForkSupervisor(Supervisor):
     )
     return replacement_pid
 
+  def _wait_for_children(self, timeout):
+    deadline = time.monotonic() + timeout
+    while self.children and time.monotonic() < deadline:
+      try:
+        pid, _status = self._waitpid(-1, os.WNOHANG)
+      except ChildProcessError:
+        self.children.clear()
+        return None
+
+      if not pid:
+        time.sleep(min(0.05, max(deadline - time.monotonic(), 0)))
+        continue
+
+      self.children.pop(pid, None)
+    return None
+
   def poll_once(self):
     super().poll_once()
     return self.check_children()
@@ -566,9 +595,25 @@ class ForkSupervisor(Supervisor):
     return specs
 
   def _default_launcher(self, spec):
+    connections.close_all()
     pid = os.fork()
     if pid == 0:
-      runner = spec["runner_class"](**spec["kwargs"])
-      runner.run()
+      connections.close_all()
+      try:
+        runner = spec["runner_class"](**spec["kwargs"])
+        self._register_child_signal_handlers(runner)
+        runner.run()
+      finally:
+        connections.close_all()
       self._exit_fn(0)
+    connections.close_all()
     return pid
+
+  def _register_child_signal_handlers(self, runner):
+    def request_stop(*_args):
+      runner.request_stop()
+      return True
+
+    signal.signal(signal.SIGTERM, request_stop)
+    signal.signal(signal.SIGINT, request_stop)
+    signal.signal(signal.SIGQUIT, lambda *_args: self._exit_fn(1))

{dj_queue-0.6.3 → dj_queue-0.6.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "dj-queue"
-version = "0.6.3"
+version = "0.6.4"
 description = "Database-backed task queue backend for Django’s Tasks framework."
 readme = "README.md"
 license = "MIT"