dj-queue 0.13.0__tar.gz → 0.13.1__tar.gz

{dj_queue-0.13.0 → dj_queue-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.13.0
+Version: 0.13.1
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE

{dj_queue-0.13.0 → dj_queue-0.13.1}/dj_queue/db.py

@@ -1,3 +1,4 @@
+import time
 from collections.abc import Iterator
 from contextlib import contextmanager
 from dataclasses import dataclass
@@ -5,9 +6,36 @@ from typing import Literal
 
 from django.core.exceptions import ImproperlyConfigured
 from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.utils import OperationalError
 
 from dj_queue.config import load_backend_config
 
+TRANSIENT_DATABASE_RETRY_ATTEMPTS = 3
+TRANSIENT_DATABASE_RETRY_SLEEP_BASE = 0.01
+TRANSIENT_DATABASE_ERROR_SQLSTATES = {
+  "40001",  # serialization failure
+  "40P01",  # deadlock detected
+  "55P03",  # lock not available
+}
+TRANSIENT_DATABASE_ERROR_ERRNOS = {
+  1205,  # mysql lock wait timeout
+  1213,  # mysql deadlock
+}
+TRANSIENT_DATABASE_SQLITE_CODES = {
+  5,  # SQLITE_BUSY
+  6,  # SQLITE_LOCKED
+  261,  # SQLITE_BUSY_RECOVERY
+  262,  # SQLITE_LOCKED_SHAREDCACHE
+  517,  # SQLITE_BUSY_SNAPSHOT
+}
+TRANSIENT_DATABASE_ERROR_MESSAGES = (
+  "deadlock",
+  "lock wait timeout",
+  "try restarting transaction",
+  "could not serialize access",
+  "database is locked",
+)
+
 
 @dataclass(frozen=True, slots=True)
 class DatabaseCapabilities:
@@ -72,6 +100,52 @@ def queue_cursor(backend_alias: str = "default") -> Iterator:
     yield cursor
 
 
+def retry_transient_database_errors(operation):
+  for attempt in range(TRANSIENT_DATABASE_RETRY_ATTEMPTS):
+    try:
+      return operation()
+    except OperationalError as error:
+      if attempt == TRANSIENT_DATABASE_RETRY_ATTEMPTS - 1 or not is_transient_database_error(
+        error
+      ):
+        raise
+      time.sleep(TRANSIENT_DATABASE_RETRY_SLEEP_BASE * (attempt + 1))
+
+
+def is_transient_database_error(error):
+  for candidate in _exception_chain(error):
+    if _transient_sqlstate(candidate) or _transient_errno(candidate):
+      return True
+  message = str(error).lower()
+  return any(marker in message for marker in TRANSIENT_DATABASE_ERROR_MESSAGES)
+
+
+def _exception_chain(error):
+  seen = set()
+  while error is not None and id(error) not in seen:
+    seen.add(id(error))
+    yield error
+    error = error.__cause__ or error.__context__
+
+
+def _transient_sqlstate(error):
+  for name in ("pgcode", "sqlstate"):
+    value = getattr(error, name, None)
+    if value in TRANSIENT_DATABASE_ERROR_SQLSTATES:
+      return True
+  return False
+
+
+def _transient_errno(error):
+  sqlite_code = getattr(error, "sqlite_errorcode", None)
+  if sqlite_code in TRANSIENT_DATABASE_SQLITE_CODES:
+    return True
+  errno = getattr(error, "errno", None)
+  if errno in TRANSIENT_DATABASE_ERROR_ERRNOS:
+    return True
+  return bool(error.args and error.args[0] in TRANSIENT_DATABASE_ERROR_ERRNOS)
+
+
 def _backend_family(connection) -> Literal["postgresql", "mysql", "mariadb", "sqlite"]:
   if connection.vendor == "postgresql":
     return "postgresql"

{dj_queue-0.13.0 → dj_queue-0.13.1}/dj_queue/operations/_helpers.py

@@ -343,6 +343,12 @@ def _execution_priority(job, priority):
 
 
 def _consume_selected_rows(alias, model, rows):
+  if database_capabilities(alias).backend_family in {"mysql", "mariadb"}:
+    from dj_queue.sql.mysql import consume_ready_rows
+
+    if model is ReadyExecution:
+      return consume_ready_rows(alias, rows)
+
   if not database_capabilities(alias).uses_serialized_writes:
     model.objects.using(alias).filter(pk__in=[row.pk for row in rows]).delete()
     return rows

{dj_queue-0.13.0 → dj_queue-0.13.1}/dj_queue/operations/concurrency.py

@@ -10,13 +10,18 @@ from django.utils import timezone
 from django.utils.module_loading import import_string
 
 from dj_queue.config import load_backend_config
-from dj_queue.db import database_capabilities, get_database_alias, locked_queryset
+from dj_queue.db import (
+  database_capabilities,
+  get_database_alias,
+  locked_queryset,
+  retry_transient_database_errors,
+)
 from dj_queue.exceptions import EnqueueError
 from dj_queue.log import log_event
 from dj_queue.models import (
   BlockedExecution,
   ClaimedExecution,
-  ReadyExecution,
+  Job,
   Semaphore,
 )
 from dj_queue.operations._helpers import (
@@ -672,38 +677,35 @@ def _create_claimed_execution_after_blocked_consume(alias, *, job, process_id, c
 def cleanup_expired_semaphores(*, batch_size=500, backend_alias="default"):
   batch_size = _positive_int_option(batch_size, "batch_size")
   alias = get_database_alias(backend_alias)
-  now = timezone.now()
-  queryset = _expired_semaphore_cleanup_queryset(alias, now=now)
-  semaphore_ids = list(
-    queryset.order_by("expires_at", "key").values_list("pk", flat=True)[:batch_size]
-  )
-  if not semaphore_ids:
-    return 0
-  deleted, _ = (
-    _expired_semaphore_cleanup_queryset(alias, now=now).filter(pk__in=semaphore_ids).delete()
-  )
-  return deleted
-
+  use_skip_locked = load_backend_config(backend_alias).use_skip_locked
+
+  def cleanup_transition():
+    now = timezone.now()
+    with transaction.atomic(using=alias):
+      queryset = locked_queryset(
+        Semaphore.objects.using(alias).filter(expires_at__lte=now),
+        use_skip_locked=use_skip_locked,
+      )
+      semaphores = list(
+        queryset.order_by("expires_at", "key").values_list("pk", "key")[:batch_size]
+      )
+      if not semaphores:
+        return 0
+      semaphore_keys = [key for _semaphore_id, key in semaphores]
+      active_keys = set(
+        Job.objects.using(alias)
+        .filter(concurrency_key__in=semaphore_keys)
+        .filter(Q(claimed_execution__isnull=False) | Q(ready_execution__isnull=False))
+        .values_list("concurrency_key", flat=True)
+        .distinct()
+      )
+      semaphore_ids = [semaphore_id for semaphore_id, key in semaphores if key not in active_keys]
+      if not semaphore_ids:
+        return 0
+      deleted, _ = Semaphore.objects.using(alias).filter(pk__in=semaphore_ids).delete()
+      return deleted
 
-def _expired_semaphore_cleanup_queryset(alias, *, now):
-  claimed_concurrency_keys = (
-    ClaimedExecution.objects.using(alias)
-    .exclude(job__concurrency_key__isnull=True)
-    .exclude(job__concurrency_key="")
-    .values_list("job__concurrency_key", flat=True)
-  )
-  ready_concurrency_keys = (
-    ReadyExecution.objects.using(alias)
-    .exclude(job__concurrency_key__isnull=True)
-    .exclude(job__concurrency_key="")
-    .values_list("job__concurrency_key", flat=True)
-  )
-  return (
-    Semaphore.objects.using(alias)
-    .filter(expires_at__lte=now)
-    .exclude(key__in=claimed_concurrency_keys)
-    .exclude(key__in=ready_concurrency_keys)
-  )
+  return retry_transient_database_errors(cleanup_transition)
 
 
 def promote_expired_blocked_jobs(*, batch_size=500, backend_alias="default", use_skip_locked=None):