dj-queue 0.10.2__tar.gz → 0.10.4__tar.gz

{dj_queue-0.10.2 → dj_queue-0.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.10.2
+Version: 0.10.4
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE
@@ -61,6 +61,9 @@ It has a narrow, explicit shape:
 For detailed comparisons with Celery, RQ, Procrastinate, and other alternatives,
 see [COMPARISONS.md](docs/COMPARISONS.md).
 
+For benchmarks on enqueue, promotion, recurring, claiming, worker-drain, and concurrency-contention scenarios,
+see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
+
 ## Installation
 
 `dj_queue` requires Python 3.12+ and Django 6.0+.
@@ -640,8 +643,9 @@ Run your normal application migrations on `default`, then migrate `dj_queue`
 onto the queue database:
 
 ```bash
-python manage.py migrate
+# migrate dj_queue on its queue alias first so django doesn't mark it applied on default
 python manage.py migrate dj_queue --database queue
+python manage.py migrate
 ```
 
 With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
@@ -1012,11 +1016,6 @@ Both endpoints support bearer token authentication. Set
 `Authorization: Bearer <token>`. Leave it unset if you protect these URLs at
 the network or proxy layer.
 
-## Benchmarks
-
-The repository includes a standalone benchmark harness for enqueue, promotion, recurring, worker-drain, and concurrency-contention scenarios. 
-See [`docs/benchmarks/`](docs/benchmarks/) for the latest published reports.
-
 ## License
 
 MIT

{dj_queue-0.10.2 → dj_queue-0.10.4}/README.md

@@ -35,6 +35,9 @@ It has a narrow, explicit shape:
 For detailed comparisons with Celery, RQ, Procrastinate, and other alternatives,
 see [COMPARISONS.md](docs/COMPARISONS.md).
 
+For benchmarks on enqueue, promotion, recurring, claiming, worker-drain, and concurrency-contention scenarios,
+see [docs/benchmarks/](docs/benchmarks/) for the latest published reports.
+
 ## Installation
 
 `dj_queue` requires Python 3.12+ and Django 6.0+.
@@ -614,8 +617,9 @@ Run your normal application migrations on `default`, then migrate `dj_queue`
 onto the queue database:
 
 ```bash
-python manage.py migrate
+# migrate dj_queue on its queue alias first so django doesn't mark it applied on default
 python manage.py migrate dj_queue --database queue
+python manage.py migrate
 ```
 
 With this setup, `dj_queue`'s ORM queries and raw SQL helpers stay on the queue
@@ -986,11 +990,6 @@ Both endpoints support bearer token authentication. Set
 `Authorization: Bearer <token>`. Leave it unset if you protect these URLs at
 the network or proxy layer.
 
-## Benchmarks
-
-The repository includes a standalone benchmark harness for enqueue, promotion, recurring, worker-drain, and concurrency-contention scenarios. 
-See [`docs/benchmarks/`](docs/benchmarks/) for the latest published reports.
-
 ## License
 
 MIT

{dj_queue-0.10.2 → dj_queue-0.10.4}/dj_queue/backend.py

@@ -28,7 +28,13 @@ class DjQueueBackend(BaseTaskBackend):
 
   def enqueue(self, task, args, kwargs):
     self.validate_task(task)
-    job, dispatch_outcome = enqueue_job_with_dispatch(task, args, kwargs, backend_alias=self.alias)
+    job, dispatch_outcome = enqueue_job_with_dispatch(
+      task,
+      args,
+      kwargs,
+      backend_alias=self.alias,
+      validate=False,
+    )
     return task_result_from_enqueued_job(
       job,
       task,
@@ -49,7 +55,7 @@ class DjQueueBackend(BaseTaskBackend):
       self.validate_task(task)
       jobs.append((task, args, kwargs))
 
-    created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias)
+    created_jobs = enqueue_jobs_bulk(jobs, backend_alias=self.alias, validate=False)
     return [
       task_result_from_enqueued_job(
         job,

{dj_queue-0.10.2 → dj_queue-0.10.4}/dj_queue/operations/_helpers.py

@@ -170,8 +170,11 @@ def _scheduled_execution_row(job, *, backend_alias, scheduled_at=None, created_a
   )
 
 
-def _create_scheduled_execution(alias, job, *, backend_alias, scheduled_at=None):
-  _ensure_no_other_execution_state(alias, job)
+def _create_scheduled_execution(
+  alias, job, *, backend_alias, scheduled_at=None, check_conflicts=True
+):
+  if check_conflicts:
+    _ensure_no_other_execution_state(alias, job)
   return ScheduledExecution.objects.using(alias).create(
     **_scheduled_execution_fields(
       job,
@@ -190,8 +193,10 @@ def _create_blocked_execution(
   expires_at,
   queue_name=None,
   priority=None,
+  check_conflicts=True,
 ):
-  _ensure_no_other_execution_state(alias, job)
+  if check_conflicts:
+    _ensure_no_other_execution_state(alias, job)
   return BlockedExecution.objects.using(alias).create(
     **_blocked_execution_fields(
       job,

{dj_queue-0.10.2 → dj_queue-0.10.4}/dj_queue/operations/concurrency.py

@@ -10,11 +10,20 @@ from dj_queue.config import load_backend_config
 from dj_queue.db import database_capabilities, get_database_alias, locked_queryset
 from dj_queue.exceptions import EnqueueError
 from dj_queue.log import log_event
-from dj_queue.models import BlockedExecution, ClaimedExecution, ReadyExecution, Semaphore
+from dj_queue.models import (
+  BlockedExecution,
+  ClaimedExecution,
+  FailedExecution,
+  Job,
+  ReadyExecution,
+  ScheduledExecution,
+  Semaphore,
+)
 from dj_queue.operations._helpers import (
   _consume_selected_rows,
   _create_blocked_execution,
-  _create_ready_execution,
+  _create_ready_execution_locked,
+  _lock_active_pauses,
   _task_option,
 )
 from dj_queue.operations._insert import create_ignore_conflicts
@@ -42,7 +51,7 @@ def semaphore_acquire(
       now=now,
     )
 
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     if create_ignore_conflicts(
       Semaphore,
       using=alias,
@@ -54,7 +63,7 @@ def semaphore_acquire(
       return True
 
   reconciled_available = _reconciled_available_expression(limit)
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     updated = (
       Semaphore.objects.using(alias)
       .filter(key=key, value__gt=F("limit") - Value(limit))
@@ -177,6 +186,29 @@ def _consume_released_semaphore_slot(alias, key, *, limit, duration_seconds, now
   return updated > 0
 
 
+def _handoff_released_claimed_slot(alias, key, *, limit, duration_seconds, now):
+  expires_at = now + timedelta(seconds=duration_seconds)
+  released_available = _released_available_expression(limit)
+  updated = (
+    Semaphore.objects.using(alias)
+    .filter(key=key, value__gt=F("limit") - Value(limit) - Value(1))
+    .update(
+      value=released_available - Value(1),
+      limit=limit,
+      expires_at=expires_at,
+      updated_at=now,
+    )
+  )
+  return updated > 0
+
+
+def _released_available_expression(limit):
+  return Least(
+    Value(limit),
+    Greatest(Value(0), F("value") + Value(limit) - F("limit") + Value(1)),
+  )
+
+
 def _reconciled_available_expression(limit):
   return Least(Value(limit), Greatest(Value(0), F("value") + Value(limit) - F("limit")))
 
@@ -209,27 +241,31 @@ def unblock_next_blocked_job(
   backend_alias="default",
   use_skip_locked=True,
   handoff_released_slot=False,
+  release_slot=False,
 ):
   alias = get_database_alias(backend_alias)
   now = timezone.now()
 
-  with transaction.atomic(using=alias):
-    queryset = (
-      BlockedExecution.objects.using(alias)
-      .select_related("job")
-      .filter(backend_alias=backend_alias, concurrency_key=key)
-      .order_by("-priority", "id")
+  with _operation_atomic(alias):
+    blocked = _consume_next_blocked_job(
+      alias,
+      backend_alias=backend_alias,
+      key=key,
+      use_skip_locked=use_skip_locked,
     )
-    blocked = locked_queryset(queryset, use_skip_locked=use_skip_locked).first()
     if blocked is None:
       return None
 
-    consumed = _consume_selected_rows(alias, BlockedExecution, [blocked])
-    if not consumed:
-      return None
-
     slot_acquired = False
-    if handoff_released_slot:
+    if release_slot:
+      slot_acquired = _handoff_released_claimed_slot(
+        alias,
+        key,
+        limit=limit,
+        duration_seconds=duration_seconds,
+        now=now,
+      )
+    elif handoff_released_slot:
       slot_acquired = _consume_released_semaphore_slot(
         alias,
         key,
@@ -238,7 +274,7 @@ def unblock_next_blocked_job(
         now=now,
       )
 
-    if not slot_acquired:
+    if not slot_acquired and not release_slot:
       slot_acquired = semaphore_acquire(
         key,
         limit=limit,
@@ -246,37 +282,190 @@ def unblock_next_blocked_job(
         backend_alias=backend_alias,
       )
 
+    mock_job = Job(
+      id=blocked["job_id"],
+      queue_name=blocked["queue_name"],
+      priority=blocked["priority"],
+      concurrency_key=blocked["concurrency_key"],
+      backend_alias=backend_alias,
+    )
+
     if not slot_acquired:
       _create_blocked_execution(
         alias,
-        blocked.job,
+        mock_job,
         backend_alias=backend_alias,
-        queue_name=blocked.queue_name,
-        priority=blocked.priority,
-        concurrency_key=blocked.concurrency_key,
-        expires_at=blocked.expires_at,
+        queue_name=blocked["queue_name"],
+        priority=blocked["priority"],
+        concurrency_key=blocked["concurrency_key"],
+        expires_at=blocked["expires_at"],
+        check_conflicts=False,
       )
       return None
 
-    job = blocked.job
-    queue_name = blocked.queue_name
-    priority = blocked.priority
-    _create_ready_execution(
+    _create_ready_execution_after_blocked_consume(
       alias,
-      job=job,
+      job=mock_job,
       backend_alias=backend_alias,
-      queue_name=queue_name,
-      priority=priority,
+      queue_name=blocked["queue_name"],
+      priority=blocked["priority"],
       ready_at=now,
     )
 
   log_event(
     "job.unblocked",
-    job_id=str(job.id),
+    job_id=str(mock_job.id),
     concurrency_key=key,
   )
-  notify_ready_queues_on_commit((job.queue_name,), backend_alias=backend_alias)
-  return job
+  notify_ready_queues_on_commit((blocked["queue_name"],), backend_alias=backend_alias)
+  return mock_job
+
+
+def _consume_next_blocked_job(alias, *, backend_alias, key, use_skip_locked):
+  capabilities = database_capabilities(alias)
+  if capabilities.backend_family == "postgresql":
+    return _postgres_consume_next_blocked_job(
+      alias,
+      backend_alias=backend_alias,
+      key=key,
+      use_skip_locked=use_skip_locked and capabilities.supports_skip_locked,
+    )
+
+  queryset = (
+    BlockedExecution.objects.using(alias)
+    .filter(backend_alias=backend_alias, concurrency_key=key)
+    .order_by("-priority", "id")
+  )
+  blocked = locked_queryset(queryset, use_skip_locked=use_skip_locked).first()
+  if blocked is None:
+    return None
+
+  consumed = _consume_selected_rows(alias, BlockedExecution, [blocked])
+  if not consumed:
+    return None
+  return _blocked_execution_values(blocked)
+
+
+def _postgres_consume_next_blocked_job(alias, *, backend_alias, key, use_skip_locked):
+  connection = connections[alias]
+  quote = connection.ops.quote_name
+  table = quote(BlockedExecution._meta.db_table)
+  pk_column = quote(BlockedExecution._meta.pk.column)
+  job_id_column = quote(BlockedExecution._meta.get_field("job").column)
+  backend_alias_column = quote(BlockedExecution._meta.get_field("backend_alias").column)
+  queue_name_column = quote(BlockedExecution._meta.get_field("queue_name").column)
+  priority_column = quote(BlockedExecution._meta.get_field("priority").column)
+  concurrency_key_column = quote(BlockedExecution._meta.get_field("concurrency_key").column)
+  expires_at_column = quote(BlockedExecution._meta.get_field("expires_at").column)
+  skip_locked_sql = " SKIP LOCKED" if use_skip_locked else ""
+
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      DELETE FROM {table}
+      WHERE {table}.{pk_column} = (
+        SELECT {pk_column}
+        FROM {table}
+        WHERE {backend_alias_column} = %s AND {concurrency_key_column} = %s
+        ORDER BY {priority_column} DESC, {pk_column} ASC
+        LIMIT 1
+        FOR UPDATE{skip_locked_sql}
+      )
+      RETURNING
+        {job_id_column},
+        {queue_name_column},
+        {priority_column},
+        {concurrency_key_column},
+        {expires_at_column}
+      """,
+      [backend_alias, key],
+    )
+    row = cursor.fetchone()
+
+  if row is None:
+    return None
+  return {
+    "job_id": row[0],
+    "queue_name": row[1],
+    "priority": row[2],
+    "concurrency_key": row[3],
+    "expires_at": row[4],
+  }
+
+
+def _blocked_execution_values(blocked):
+  return {
+    "job_id": blocked.job_id,
+    "queue_name": blocked.queue_name,
+    "priority": blocked.priority,
+    "concurrency_key": blocked.concurrency_key,
+    "expires_at": blocked.expires_at,
+  }
+
+
+def _create_ready_execution_after_blocked_consume(
+  alias,
+  *,
+  job,
+  backend_alias,
+  queue_name,
+  priority,
+  ready_at,
+):
+  _lock_active_pauses(alias, backend_alias, {queue_name})
+  connection = connections[alias]
+  quote = connection.ops.quote_name
+  ready_table = quote(ReadyExecution._meta.db_table)
+  job_id_column = quote(ReadyExecution._meta.get_field("job").column)
+  backend_alias_column = quote(ReadyExecution._meta.get_field("backend_alias").column)
+  queue_name_column = quote(ReadyExecution._meta.get_field("queue_name").column)
+  priority_column = quote(ReadyExecution._meta.get_field("priority").column)
+  created_at_column = quote(ReadyExecution._meta.get_field("created_at").column)
+  latency_started_at_column = quote(ReadyExecution._meta.get_field("latency_started_at").column)
+  job_id = Job._meta.get_field("id").get_db_prep_value(
+    job.pk,
+    connection=connection,
+    prepared=False,
+  )
+  state_models = (ReadyExecution, ScheduledExecution, ClaimedExecution, FailedExecution)
+  state_checks = " AND ".join(
+    _state_absence_sql(model, job_id_column=job_id_column, quote=quote) for model in state_models
+  )
+
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      INSERT INTO {ready_table} (
+        {job_id_column},
+        {backend_alias_column},
+        {queue_name_column},
+        {priority_column},
+        {created_at_column},
+        {latency_started_at_column}
+      )
+      SELECT %s, %s, %s, %s, %s, %s
+      WHERE {state_checks}
+      """,
+      [
+        job_id,
+        backend_alias,
+        queue_name,
+        priority,
+        ready_at,
+        ready_at,
+        *([job_id] * len(state_models)),
+      ],
+    )
+    created = cursor.rowcount
+
+  if created != 1:
+    raise EnqueueError(f"job {job.id} already has an execution-state row")
+
+
+def _state_absence_sql(model, *, job_id_column, quote):
+  state_table = quote(model._meta.db_table)
+  state_job_id_column = quote(model._meta.get_field("job").column)
+  return f"NOT EXISTS (SELECT 1 FROM {state_table} WHERE {state_table}.{state_job_id_column} = %s)"
 
 
 def cleanup_expired_semaphores(*, backend_alias="default"):
@@ -349,13 +538,14 @@ def promote_expired_blocked_jobs(*, batch_size=500, backend_alias="default", use
         priority = blocked.priority
         if not uses_serialized_writes:
           blocked.delete(using=alias)
-        _create_ready_execution(
+        _create_ready_execution_locked(
           alias,
           job=job,
           backend_alias=backend_alias,
           queue_name=queue_name,
           priority=priority,
           ready_at=now,
+          check_conflicts=False,
         )
         promoted_jobs.append(job)
       else:
@@ -369,6 +559,7 @@ def promote_expired_blocked_jobs(*, batch_size=500, backend_alias="default", use
             priority=blocked.priority,
             concurrency_key=blocked.concurrency_key,
             expires_at=expires_at,
+            check_conflicts=False,
           )
         else:
           blocked.expires_at = expires_at
@@ -397,3 +588,7 @@ def _positive_int_option(value, name):
   if number <= 0:
     raise EnqueueError(f"{name} must be a positive integer")
   return number
+
+
+def _operation_atomic(alias):
+  return transaction.atomic(using=alias, savepoint=not connections[alias].in_atomic_block)