dj-queue 0.8.0__tar.gz → 0.8.1__tar.gz

{dj_queue-0.8.0 → dj_queue-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.8.0
+Version: 0.8.1
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE

{dj_queue-0.8.0 → dj_queue-0.8.1}/dj_queue/operations/concurrency.py

@@ -1,6 +1,6 @@
 from datetime import timedelta
 
-from django.db import transaction
+from django.db import connections, transaction
 from django.db.models import F
 from django.utils import timezone
 from django.utils.module_loading import import_string
@@ -29,6 +29,16 @@ def semaphore_acquire(
   alias = get_database_alias(backend_alias)
   now = timezone.now()
   expires_at = now + timedelta(seconds=duration_seconds)
+  backend_family = database_capabilities(alias).backend_family
+
+  if backend_family in {"mysql", "mariadb"}:
+    return _mysql_family_semaphore_acquire(
+      alias,
+      key,
+      limit=limit,
+      expires_at=expires_at,
+      now=now,
+    )
 
   with transaction.atomic(using=alias):
     if create_ignore_conflicts(
@@ -41,7 +51,6 @@ def semaphore_acquire(
     ):
       return True
 
-  # mysql-family backends can deadlock if a skipped insert and row lock happen in one tx
   with transaction.atomic(using=alias):
     updated = (
       Semaphore.objects.using(alias)
@@ -55,6 +64,43 @@ def semaphore_acquire(
   return updated > 0
 
 
+def _mysql_family_semaphore_acquire(alias, key, *, limit, expires_at, now):
+  connection = connections[alias]
+  table = connection.ops.quote_name(Semaphore._meta.db_table)
+  key_column = connection.ops.quote_name("key")
+  value_column = connection.ops.quote_name("value")
+  limit_column = connection.ops.quote_name("limit")
+  expires_at_column = connection.ops.quote_name("expires_at")
+  created_at_column = connection.ops.quote_name("created_at")
+  updated_at_column = connection.ops.quote_name("updated_at")
+
+  # one upsert avoids mysql-family deadlocks from mixing ignored inserts and follow-up updates
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      INSERT INTO {table} (
+        {key_column},
+        {value_column},
+        {limit_column},
+        {expires_at_column},
+        {created_at_column},
+        {updated_at_column}
+      )
+      VALUES (%s, %s, %s, %s, %s, %s)
+      ON DUPLICATE KEY UPDATE
+        {expires_at_column} = IF({value_column} > 0, %s, {expires_at_column}),
+        {updated_at_column} = IF({value_column} > 0, %s, {updated_at_column}),
+        {value_column} = IF(
+          {value_column} > 0,
+          LAST_INSERT_ID({value_column}) - 1,
+          LAST_INSERT_ID(0) + {value_column}
+        )
+      """,
+      [key, limit - 1, limit, expires_at, now, now, expires_at, now],
+    )
+    return cursor.lastrowid != 0
+
+
 def semaphore_release(key, *, duration_seconds, backend_alias="default"):
   alias = get_database_alias(backend_alias)
   expires_at = timezone.now() + timedelta(seconds=duration_seconds)
@@ -144,16 +190,23 @@ def unblock_next_blocked_job(
 
 def cleanup_expired_semaphores(*, backend_alias="default"):
   alias = get_database_alias(backend_alias)
-  active_concurrency_keys = (
+  claimed_concurrency_keys = (
     ClaimedExecution.objects.using(alias)
     .exclude(job__concurrency_key__isnull=True)
     .exclude(job__concurrency_key="")
     .values_list("job__concurrency_key", flat=True)
   )
+  ready_concurrency_keys = (
+    ReadyExecution.objects.using(alias)
+    .exclude(job__concurrency_key__isnull=True)
+    .exclude(job__concurrency_key="")
+    .values_list("job__concurrency_key", flat=True)
+  )
   queryset = (
     Semaphore.objects.using(alias)
     .filter(expires_at__lte=timezone.now())
-    .exclude(key__in=active_concurrency_keys)
+    .exclude(key__in=claimed_concurrency_keys)
+    .exclude(key__in=ready_concurrency_keys)
   )
   deleted = queryset.count()
   if not deleted:

{dj_queue-0.8.0 → dj_queue-0.8.1}/dj_queue/operations/jobs.py

@@ -595,113 +595,85 @@ def enqueue_job_again(job_id, *, backend_alias="default", run_after=_KEEP_RUN_AF
   return job
 
 
-def discard_failed_jobs(*, job_ids=None, batch_size=500, backend_alias="default"):
+def _discard_state_jobs(
+  model,
+  reason,
+  *,
+  job_ids=None,
+  batch_size=500,
+  backend_alias="default",
+  release_concurrency=False,
+):
   alias = get_database_alias(backend_alias)
   config = load_backend_config(backend_alias)
 
   with transaction.atomic(using=alias):
-    queryset = (
-      FailedExecution.objects.using(alias).filter(job__backend_alias=backend_alias).order_by("id")
-    )
+    if model is FailedExecution:
+      queryset = model.objects.using(alias).filter(job__backend_alias=backend_alias).order_by("id")
+    else:
+      queryset = model.objects.using(alias).filter(backend_alias=backend_alias).order_by("id")
+
     if job_ids is not None:
       queryset = queryset.filter(job_id__in=job_ids)
-    failed_rows = list(
-      locked_queryset(queryset, use_skip_locked=config.use_skip_locked)[:batch_size]
-    )
-    if not failed_rows:
+    rows = list(locked_queryset(queryset, use_skip_locked=config.use_skip_locked)[:batch_size])
+    if not rows:
       return 0
 
-    job_ids = [row.job_id for row in failed_rows]
-    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=job_ids)}
-    jobs = [jobs_by_id[job_id] for job_id in job_ids]
-    Job.objects.using(alias).filter(pk__in=[row.job_id for row in failed_rows]).delete()
+    row_job_ids = [row.job_id for row in rows]
+    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=row_job_ids)}
+    jobs = [jobs_by_id[job_id] for job_id in row_job_ids]
+    Job.objects.using(alias).filter(pk__in=row_job_ids).delete()
 
   for job in jobs:
-    log_event("job.discarded", job_id=str(job.id), reason="failed")
+    if release_concurrency:
+      _release_concurrency_slot(job)
+    log_event("job.discarded", job_id=str(job.id), reason=reason)
   return len(jobs)
 
 
+def discard_failed_jobs(*, job_ids=None, batch_size=500, backend_alias="default"):
+  return _discard_state_jobs(
+    FailedExecution,
+    "failed",
+    job_ids=job_ids,
+    batch_size=batch_size,
+    backend_alias=backend_alias,
+  )
+
+
 def discard_failed_job(job_id, *, backend_alias="default"):
   return discard_failed_jobs(job_ids=[job_id], batch_size=1, backend_alias=backend_alias)
 
 
 def discard_ready_jobs(*, job_ids=None, batch_size=500, backend_alias="default"):
-  alias = get_database_alias(backend_alias)
-  config = load_backend_config(backend_alias)
-
-  with transaction.atomic(using=alias):
-    queryset = (
-      ReadyExecution.objects.using(alias).filter(backend_alias=backend_alias).order_by("id")
-    )
-    if job_ids is not None:
-      queryset = queryset.filter(job_id__in=job_ids)
-    ready_rows = list(
-      locked_queryset(queryset, use_skip_locked=config.use_skip_locked)[:batch_size]
-    )
-    if not ready_rows:
-      return 0
-
-    job_ids = [row.job_id for row in ready_rows]
-    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=job_ids)}
-    jobs = [jobs_by_id[job_id] for job_id in job_ids]
-    Job.objects.using(alias).filter(pk__in=[row.job_id for row in ready_rows]).delete()
-
-  for job in jobs:
-    _release_concurrency_slot(job)
-    log_event("job.discarded", job_id=str(job.id), reason="ready")
-  return len(jobs)
+  return _discard_state_jobs(
+    ReadyExecution,
+    "ready",
+    job_ids=job_ids,
+    batch_size=batch_size,
+    backend_alias=backend_alias,
+    release_concurrency=True,
+  )
 
 
 def discard_scheduled_jobs(*, job_ids=None, batch_size=500, backend_alias="default"):
-  alias = get_database_alias(backend_alias)
-  config = load_backend_config(backend_alias)
-
-  with transaction.atomic(using=alias):
-    queryset = (
-      ScheduledExecution.objects.using(alias).filter(backend_alias=backend_alias).order_by("id")
-    )
-    if job_ids is not None:
-      queryset = queryset.filter(job_id__in=job_ids)
-    scheduled_rows = list(
-      locked_queryset(queryset, use_skip_locked=config.use_skip_locked)[:batch_size]
-    )
-    if not scheduled_rows:
-      return 0
-
-    job_ids = [row.job_id for row in scheduled_rows]
-    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=job_ids)}
-    jobs = [jobs_by_id[job_id] for job_id in job_ids]
-    Job.objects.using(alias).filter(pk__in=[row.job_id for row in scheduled_rows]).delete()
-
-  for job in jobs:
-    log_event("job.discarded", job_id=str(job.id), reason="scheduled")
-  return len(jobs)
+  return _discard_state_jobs(
+    ScheduledExecution,
+    "scheduled",
+    job_ids=job_ids,
+    batch_size=batch_size,
+    backend_alias=backend_alias,
+  )
 
 
 def discard_blocked_jobs(*, job_ids=None, batch_size=500, backend_alias="default"):
-  alias = get_database_alias(backend_alias)
-  config = load_backend_config(backend_alias)
-
-  with transaction.atomic(using=alias):
-    queryset = (
-      BlockedExecution.objects.using(alias).filter(backend_alias=backend_alias).order_by("id")
-    )
-    if job_ids is not None:
-      queryset = queryset.filter(job_id__in=job_ids)
-    blocked_rows = list(
-      locked_queryset(queryset, use_skip_locked=config.use_skip_locked)[:batch_size]
-    )
-    if not blocked_rows:
-      return 0
-
-    job_ids = [row.job_id for row in blocked_rows]
-    jobs_by_id = {job.id: job for job in Job.objects.using(alias).filter(pk__in=job_ids)}
-    jobs = [jobs_by_id[job_id] for job_id in job_ids]
-    Job.objects.using(alias).filter(pk__in=[row.job_id for row in blocked_rows]).delete()
-
-  for job in jobs:
-    log_event("job.discarded", job_id=str(job.id), reason="blocked")
-  return len(jobs)
+  return _discard_state_jobs(
+    BlockedExecution,
+    "blocked",
+    job_ids=job_ids,
+    batch_size=batch_size,
+    backend_alias=backend_alias,
+  )
 
 
 def _dispatch_existing_job(job):

{dj_queue-0.8.0 → dj_queue-0.8.1}/dj_queue/runtime/base.py

@@ -73,6 +73,7 @@ class BaseRunner:
     self.hostname = hostname
     self.sleeper = sleeper or InterruptibleSleeper()
     self.supervisor = supervisor
+    self.supervisor_id = getattr(supervisor, "pk", supervisor)
     self.process = None
     self._stop_event = threading.Event()
     self._heartbeat_stop_event = threading.Event()
@@ -195,7 +196,7 @@ class BaseRunner:
           hostname=self.hostname,
           name=self.name,
           metadata=self.runtime_metadata(),
-          supervisor=self.supervisor,
+          supervisor_id=self.supervisor_id,
           last_heartbeat_at=timezone.now(),
         ),
         alias=alias,

{dj_queue-0.8.0 → dj_queue-0.8.1}/dj_queue/runtime/supervisor.py

@@ -278,9 +278,11 @@ class AsyncSupervisor(Supervisor):
 
         # runner crashed — fail its claimed jobs, then stop and replace
         self._fail_crashed_runner_jobs(runner)
-        runner.stop()
+        drained = runner.stop()
         if self._stop_event.is_set():
           return
+        if drained is False and not self._wait_for_runner_stop(runner):
+          return
         log_event(
           "process.replaced",
           backend_alias=self.backend_alias,
@@ -297,6 +299,11 @@ class AsyncSupervisor(Supervisor):
       if not self._stop_event.is_set():
         runner.stop()
 
+  def _wait_for_runner_stop(self, runner):
+    while runner.process is not None and not self._stop_event.is_set():
+      time.sleep(0.01)
+    return runner.process is None
+
   def _replace_runner(self, current, replacement):
     try:
       index = self.runners.index(current)
@@ -552,6 +559,7 @@ class ForkSupervisor(Supervisor):
 
   def _build_runner_specs(self):
     specs = []
+    supervisor_id = self.process.pk if self.process else None
 
     for index, worker_config in enumerate(self.config.workers, start=1):
       for process_index in range(worker_config.processes):
@@ -565,6 +573,7 @@ class ForkSupervisor(Supervisor):
               "backend_alias": self.backend_alias,
               "name": f"worker-{suffix}",
               "hostname": self.hostname,
+              "supervisor": supervisor_id,
             },
           }
         )
@@ -579,6 +588,7 @@ class ForkSupervisor(Supervisor):
             "backend_alias": self.backend_alias,
             "name": f"dispatcher-{index}",
             "hostname": self.hostname,
+            "supervisor": supervisor_id,
           },
         }
       )
@@ -593,6 +603,7 @@ class ForkSupervisor(Supervisor):
             "backend_alias": self.backend_alias,
             "name": "scheduler-1",
             "hostname": self.hostname,
+            "supervisor": supervisor_id,
           },
         }
       )

{dj_queue-0.8.0 → dj_queue-0.8.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "dj-queue"
-version = "0.8.0"
+version = "0.8.1"
 description = "Database-backed task queue backend for Django’s Tasks framework."
 readme = "README.md"
 license = "MIT"