dj-queue 0.10.2__tar.gz → 0.10.3__tar.gz

{dj_queue-0.10.2 → dj_queue-0.10.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.10.2
+Version: 0.10.3
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE

{dj_queue-0.10.2 → dj_queue-0.10.3}/dj_queue/operations/concurrency.py

@@ -10,11 +10,20 @@ from dj_queue.config import load_backend_config
 from dj_queue.db import database_capabilities, get_database_alias, locked_queryset
 from dj_queue.exceptions import EnqueueError
 from dj_queue.log import log_event
-from dj_queue.models import BlockedExecution, ClaimedExecution, ReadyExecution, Semaphore
+from dj_queue.models import (
+  BlockedExecution,
+  ClaimedExecution,
+  FailedExecution,
+  Job,
+  ReadyExecution,
+  ScheduledExecution,
+  Semaphore,
+)
 from dj_queue.operations._helpers import (
   _consume_selected_rows,
   _create_blocked_execution,
   _create_ready_execution,
+  _lock_active_pauses,
   _task_option,
 )
 from dj_queue.operations._insert import create_ignore_conflicts
@@ -42,7 +51,7 @@ def semaphore_acquire(
       now=now,
     )
 
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     if create_ignore_conflicts(
       Semaphore,
       using=alias,
@@ -54,7 +63,7 @@ def semaphore_acquire(
       return True
 
   reconciled_available = _reconciled_available_expression(limit)
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     updated = (
       Semaphore.objects.using(alias)
       .filter(key=key, value__gt=F("limit") - Value(limit))
@@ -177,6 +186,29 @@ def _consume_released_semaphore_slot(alias, key, *, limit, duration_seconds, now
   return updated > 0
 
 
+def _handoff_released_claimed_slot(alias, key, *, limit, duration_seconds, now):
+  expires_at = now + timedelta(seconds=duration_seconds)
+  released_available = _released_available_expression(limit)
+  updated = (
+    Semaphore.objects.using(alias)
+    .filter(key=key, value__gt=F("limit") - Value(limit) - Value(1))
+    .update(
+      value=released_available - Value(1),
+      limit=limit,
+      expires_at=expires_at,
+      updated_at=now,
+    )
+  )
+  return updated > 0
+
+
+def _released_available_expression(limit):
+  return Least(
+    Value(limit),
+    Greatest(Value(0), F("value") + Value(limit) - F("limit") + Value(1)),
+  )
+
+
 def _reconciled_available_expression(limit):
   return Least(Value(limit), Greatest(Value(0), F("value") + Value(limit) - F("limit")))
 
@@ -209,11 +241,12 @@ def unblock_next_blocked_job(
   backend_alias="default",
   use_skip_locked=True,
   handoff_released_slot=False,
+  release_slot=False,
 ):
   alias = get_database_alias(backend_alias)
   now = timezone.now()
 
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     queryset = (
       BlockedExecution.objects.using(alias)
       .select_related("job")
@@ -229,7 +262,15 @@ def unblock_next_blocked_job(
       return None
 
     slot_acquired = False
-    if handoff_released_slot:
+    if release_slot:
+      slot_acquired = _handoff_released_claimed_slot(
+        alias,
+        key,
+        limit=limit,
+        duration_seconds=duration_seconds,
+        now=now,
+      )
+    elif handoff_released_slot:
       slot_acquired = _consume_released_semaphore_slot(
         alias,
         key,
@@ -238,7 +279,7 @@ def unblock_next_blocked_job(
         now=now,
       )
 
-    if not slot_acquired:
+    if not slot_acquired and not release_slot:
       slot_acquired = semaphore_acquire(
         key,
         limit=limit,
@@ -261,7 +302,7 @@ def unblock_next_blocked_job(
     job = blocked.job
     queue_name = blocked.queue_name
     priority = blocked.priority
-    _create_ready_execution(
+    _create_ready_execution_after_blocked_consume(
       alias,
       job=job,
       backend_alias=backend_alias,
@@ -279,6 +320,71 @@ def unblock_next_blocked_job(
   return job
 
 
+def _create_ready_execution_after_blocked_consume(
+  alias,
+  *,
+  job,
+  backend_alias,
+  queue_name,
+  priority,
+  ready_at,
+):
+  _lock_active_pauses(alias, backend_alias, {queue_name})
+  connection = connections[alias]
+  quote = connection.ops.quote_name
+  ready_table = quote(ReadyExecution._meta.db_table)
+  job_id_column = quote(ReadyExecution._meta.get_field("job").column)
+  backend_alias_column = quote(ReadyExecution._meta.get_field("backend_alias").column)
+  queue_name_column = quote(ReadyExecution._meta.get_field("queue_name").column)
+  priority_column = quote(ReadyExecution._meta.get_field("priority").column)
+  created_at_column = quote(ReadyExecution._meta.get_field("created_at").column)
+  latency_started_at_column = quote(ReadyExecution._meta.get_field("latency_started_at").column)
+  job_id = Job._meta.get_field("id").get_db_prep_value(
+    job.pk,
+    connection=connection,
+    prepared=False,
+  )
+  state_models = (ReadyExecution, ScheduledExecution, ClaimedExecution, FailedExecution)
+  state_checks = " AND ".join(
+    _state_absence_sql(model, job_id_column=job_id_column, quote=quote) for model in state_models
+  )
+
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      INSERT INTO {ready_table} (
+        {job_id_column},
+        {backend_alias_column},
+        {queue_name_column},
+        {priority_column},
+        {created_at_column},
+        {latency_started_at_column}
+      )
+      SELECT %s, %s, %s, %s, %s, %s
+      WHERE {state_checks}
+      """,
+      [
+        job_id,
+        backend_alias,
+        queue_name,
+        priority,
+        ready_at,
+        ready_at,
+        *([job_id] * len(state_models)),
+      ],
+    )
+    created = cursor.rowcount
+
+  if created != 1:
+    raise EnqueueError(f"job {job.id} already has an execution-state row")
+
+
+def _state_absence_sql(model, *, job_id_column, quote):
+  state_table = quote(model._meta.db_table)
+  state_job_id_column = quote(model._meta.get_field("job").column)
+  return f"NOT EXISTS (SELECT 1 FROM {state_table} WHERE {state_table}.{state_job_id_column} = %s)"
+
+
 def cleanup_expired_semaphores(*, backend_alias="default"):
   alias = get_database_alias(backend_alias)
   claimed_concurrency_keys = (
@@ -397,3 +503,7 @@ def _positive_int_option(value, name):
   if number <= 0:
     raise EnqueueError(f"{name} must be a positive integer")
   return number
+
+
+def _operation_atomic(alias):
+  return transaction.atomic(using=alias, savepoint=not connections[alias].in_atomic_block)

{dj_queue-0.10.2 → dj_queue-0.10.3}/dj_queue/operations/jobs.py

@@ -32,7 +32,6 @@ from dj_queue.operations._helpers import (
   _ensure_no_other_execution_state,
   _consume_selected_rows,
   _create_blocked_execution,
-  _create_ready_execution,
   _create_ready_execution_locked,
   _create_scheduled_execution,
   _exclude_active_pauses,
@@ -110,7 +109,12 @@ def enqueue_job_with_dispatch(task, args, kwargs, *, backend_alias="default"):
       scheduled_at=task.run_after,
       concurrency_key=concurrency_key,
     )
-    dispatch_outcome = _dispatch_job(job, task=task, backend_alias=backend_alias)
+    dispatch_outcome = _dispatch_job(
+      job,
+      task=task,
+      backend_alias=backend_alias,
+      check_conflicts=False,
+    )
 
   if dispatch_outcome.should_notify:
     notify_ready_queues_on_commit((job.queue_name,), backend_alias=backend_alias)
@@ -429,15 +433,13 @@ def _complete_claimed_job(job, return_value, *, backend_alias="default", task=No
 
   with transaction.atomic(using=alias):
     _delete_claimed_execution(alias, job.id)
-    _ensure_no_other_execution_state(alias, job, ignored_models=(ClaimedExecution,))
     now = timezone.now()
     config = load_backend_config(job.backend_alias)
 
     if config.preserve_finished_jobs:
-      job.finished_at = now
-      job.return_value = return_value
-      job.save(using=alias, update_fields=["finished_at", "return_value", "updated_at"])
+      _finish_job_if_no_execution_state(alias, job, return_value, finished_at=now)
     else:
+      _ensure_no_other_execution_state(alias, job, ignored_models=(ClaimedExecution,))
       job.delete(using=alias)
 
     _release_concurrency_slot(job, task=task)
@@ -910,7 +912,7 @@ def _dispatch_existing_job(job):
   return _dispatch_job(job, task=task, backend_alias=job.backend_alias)
 
 
-def _dispatch_job(job, *, task, backend_alias, now=None):
+def _dispatch_job(job, *, task, backend_alias, now=None, check_conflicts=True):
   alias = get_database_alias(backend_alias)
   if now is None:
     now = timezone.now()
@@ -925,11 +927,13 @@ def _dispatch_job(job, *, task, backend_alias, now=None):
     return DispatchOutcome.SCHEDULED
 
   if not job.concurrency_key:
-    _create_ready_execution(
+    _create_ready_execution_locked(
       alias,
       job=job,
       backend_alias=backend_alias,
+      queue_name=job.queue_name,
       ready_at=now,
+      check_conflicts=check_conflicts,
     )
     return DispatchOutcome.READY
 
@@ -979,6 +983,19 @@ def _release_concurrency_slot(job, *, task=None):
     limit = _semaphore_limit(job) or 1
     duration_seconds = config.default_concurrency_duration
 
+  if (
+    unblock_next_blocked_job(
+      job.concurrency_key,
+      limit=limit,
+      duration_seconds=duration_seconds,
+      backend_alias=job.backend_alias,
+      use_skip_locked=config.use_skip_locked,
+      release_slot=True,
+    )
+    is not None
+  ):
+    return
+
   semaphore_release(
     job.concurrency_key,
     limit=limit,
@@ -1064,11 +1081,19 @@ def _select_ready_rows(queryset, *, limit, queues, use_skip_locked):
   ordered_selectors = selectors if star_index is None else selectors[:star_index]
 
   if ordered_selectors:
-    ordered = _ordered_selector_rows_queryset(
-      queryset.exclude(pk__in=selected_ids),
-      ordered_selectors,
-    )
-    rows = list(locked_queryset(ordered, use_skip_locked=use_skip_locked)[:limit])
+    if _selectors_are_exact(ordered_selectors):
+      rows = _select_exact_selector_rows(
+        queryset.exclude(pk__in=selected_ids),
+        ordered_selectors,
+        limit=limit,
+        use_skip_locked=use_skip_locked,
+      )
+    else:
+      ordered = _ordered_selector_rows_queryset(
+        queryset.exclude(pk__in=selected_ids),
+        ordered_selectors,
+      )
+      rows = list(locked_queryset(ordered, use_skip_locked=use_skip_locked)[:limit])
     selected_rows.extend(rows)
     selected_ids.update(row.pk for row in rows)
 
@@ -1082,6 +1107,22 @@ def _select_ready_rows(queryset, *, limit, queues, use_skip_locked):
   return selected_rows
 
 
+def _select_exact_selector_rows(queryset, selectors, *, limit, use_skip_locked):
+  selected_rows = []
+  for selector in dict.fromkeys(selectors):
+    remaining = limit - len(selected_rows)
+    if remaining <= 0:
+      break
+    ordered = queryset.filter(queue_name=selector).order_by("-priority", "id")
+    rows = list(locked_queryset(ordered, use_skip_locked=use_skip_locked)[:remaining])
+    selected_rows.extend(rows)
+  return selected_rows
+
+
+def _selectors_are_exact(selectors):
+  return all(selector != "*" and not selector.endswith("*") for selector in selectors)
+
+
 def _ordered_selector_rows_queryset(queryset, selectors):
   filtered = _filter_queue_selectors(queryset, selectors)
   selector_rank = Case(
@@ -1141,6 +1182,69 @@ def _delete_claimed_execution(alias, job_id):
     raise ClaimedExecution.DoesNotExist
 
 
+def _finish_job_if_no_execution_state(alias, job, return_value, *, finished_at):
+  connection = connections[alias]
+  quote = connection.ops.quote_name
+  jobs_table = quote(Job._meta.db_table)
+  job_id_column = quote(Job._meta.get_field("id").column)
+  backend_alias_column = quote(Job._meta.get_field("backend_alias").column)
+  finished_at_column = quote(Job._meta.get_field("finished_at").column)
+  return_value_column = quote(Job._meta.get_field("return_value").column)
+  updated_at_column = quote(Job._meta.get_field("updated_at").column)
+  state_checks = " AND ".join(
+    _state_absence_sql(model, jobs_table=jobs_table, job_id_column=job_id_column, quote=quote)
+    for model in (
+      ReadyExecution,
+      ScheduledExecution,
+      BlockedExecution,
+      FailedExecution,
+    )
+  )
+  job_id = Job._meta.get_field("id").get_db_prep_value(
+    job.pk,
+    connection=connection,
+    prepared=False,
+  )
+  prepared_return_value = Job._meta.get_field("return_value").get_db_prep_save(
+    return_value,
+    connection=connection,
+  )
+
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      UPDATE {jobs_table}
+      SET
+        {finished_at_column} = %s,
+        {return_value_column} = %s,
+        {updated_at_column} = %s
+      WHERE
+        {jobs_table}.{job_id_column} = %s
+        AND {jobs_table}.{backend_alias_column} = %s
+        AND {state_checks}
+      """,
+      [finished_at, prepared_return_value, finished_at, job_id, job.backend_alias],
+    )
+    updated = cursor.rowcount
+
+  if updated != 1:
+    raise EnqueueError(f"job {job.id} already has an execution-state row")
+  job.finished_at = finished_at
+  job.return_value = return_value
+  job.updated_at = finished_at
+
+
+def _state_absence_sql(model, *, jobs_table, job_id_column, quote):
+  state_table = quote(model._meta.db_table)
+  state_job_id_column = quote(model._meta.get_field("job").column)
+  return (
+    f"NOT EXISTS ("
+    f"SELECT 1 FROM {state_table} "
+    f"WHERE {state_table}.{state_job_id_column} = {jobs_table}.{job_id_column}"
+    f")"
+  )
+
+
 def _bulk_create(alias, model, objects):
   if not objects:
     return None

{dj_queue-0.10.2 → dj_queue-0.10.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "dj-queue"
-version = "0.10.2"
+version = "0.10.3"
 description = "Database-backed task queue backend for Django’s Tasks framework."
 readme = "README.md"
 license = "MIT"