dj-queue 0.10.1__tar.gz → 0.10.3__tar.gz

{dj_queue-0.10.1 → dj_queue-0.10.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dj-queue
-Version: 0.10.1
+Version: 0.10.3
 Summary: Database-backed task queue backend for Django’s Tasks framework.
 License-Expression: MIT
 License-File: LICENSE
@@ -115,6 +115,11 @@ Run migrations:
 python manage.py migrate
 ```
 
+`python manage.py dj_queue` checks for pending `dj_queue` migrations before it
+starts workers. If another process is applying migrations during startup, the
+runtime waits up to 60 seconds by default instead of starting against a stale
+queue schema.
+
 ## Quick Start
 
 Define a task with Django's `@task` decorator:
@@ -274,11 +279,14 @@ python manage.py dj_queue --backend <alias>
 python manage.py dj_queue --only-work
 python manage.py dj_queue --only-dispatch
 python manage.py dj_queue --skip-recurring
+python manage.py dj_queue --migration-wait-timeout 120
 ```
 
 Notes:
 
 - `fork` is the default standalone mode
+- startup waits up to 60 seconds for pending `dj_queue` migrations to finish;
+  use `--migration-wait-timeout 0` to fail fast instead
 - `async` is also supported as a standalone mode and runs supervised actors in threads inside one process
 - `--backend` targets a non-default backend alias
 - `--only-work` starts workers without dispatchers or scheduler

{dj_queue-0.10.1 → dj_queue-0.10.3}/README.md

@@ -89,6 +89,11 @@ Run migrations:
 python manage.py migrate
 ```
 
+`python manage.py dj_queue` checks for pending `dj_queue` migrations before it
+starts workers. If another process is applying migrations during startup, the
+runtime waits up to 60 seconds by default instead of starting against a stale
+queue schema.
+
 ## Quick Start
 
 Define a task with Django's `@task` decorator:
@@ -248,11 +253,14 @@ python manage.py dj_queue --backend <alias>
 python manage.py dj_queue --only-work
 python manage.py dj_queue --only-dispatch
 python manage.py dj_queue --skip-recurring
+python manage.py dj_queue --migration-wait-timeout 120
 ```
 
 Notes:
 
 - `fork` is the default standalone mode
+- startup waits up to 60 seconds for pending `dj_queue` migrations to finish;
+  use `--migration-wait-timeout 0` to fail fast instead
 - `async` is also supported as a standalone mode and runs supervised actors in threads inside one process
 - `--backend` targets a non-default backend alias
 - `--only-work` starts workers without dispatchers or scheduler

dj_queue-0.10.3/dj_queue/management/commands/dj_queue.py

@@ -0,0 +1,106 @@
+import math
+import time
+
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+from django.db import connections
+from django.db.migrations.executor import MigrationExecutor
+
+from dj_queue.config import load_backend_config
+from dj_queue.runtime.supervisor import AsyncSupervisor, ForkSupervisor
+
+MIGRATION_WAIT_TIMEOUT = 60
+MIGRATION_WAIT_INTERVAL = 0.5
+
+
+def build_supervisor(*, backend_alias, cli_overrides):
+  mode = load_backend_config(backend_alias, cli_overrides=cli_overrides).mode
+  supervisor_class = AsyncSupervisor if mode == "async" else ForkSupervisor
+  return supervisor_class.from_backend_config(
+    backend_alias=backend_alias,
+    cli_overrides=cli_overrides,
+  )
+
+
+def wait_for_dj_queue_migrations(
+  database_alias,
+  *,
+  timeout=MIGRATION_WAIT_TIMEOUT,
+  interval=MIGRATION_WAIT_INTERVAL,
+  stdout=None,
+):
+  if not math.isfinite(timeout) or timeout < 0:
+    raise CommandError("dj_queue migration wait timeout must be non-negative")
+
+  deadline = time.monotonic() + timeout
+  announced = False
+  while True:
+    pending = pending_dj_queue_migrations(database_alias)
+    if not pending:
+      return
+
+    migration_names = ", ".join(
+      f"{migration.app_label}.{migration.name}" for migration, _backwards in pending
+    )
+    if stdout is not None and not announced:
+      stdout.write(
+        f"waiting for dj_queue migrations on database {database_alias!r}: {migration_names}"
+      )
+      announced = True
+
+    if time.monotonic() >= deadline:
+      raise CommandError(
+        f"dj_queue migrations are pending on database {database_alias!r}: {migration_names}; "
+        f"run manage.py migrate dj_queue --database {database_alias} before starting dj_queue"
+      )
+    time.sleep(min(interval, max(deadline - time.monotonic(), 0)))
+
+
+def pending_dj_queue_migrations(database_alias):
+  executor = MigrationExecutor(connections[database_alias])
+  targets = [node for node in executor.loader.graph.leaf_nodes() if node[0] == "dj_queue"]
+  return [
+    (migration, backwards)
+    for migration, backwards in executor.migration_plan(targets)
+    if migration.app_label == "dj_queue"
+  ]
+
+
+class Command(BaseCommand):
+  help = "Start the dj_queue supervisor"
+
+  def add_arguments(self, parser):
+    parser.add_argument("--backend", default="default")
+    parser.add_argument("--config")
+    parser.add_argument("--mode", choices=("fork", "async"))
+    parser.add_argument("--only-work", action="store_true")
+    parser.add_argument("--only-dispatch", action="store_true")
+    parser.add_argument("--skip-recurring", action="store_true")
+    parser.add_argument("--migration-wait-timeout", type=float, default=MIGRATION_WAIT_TIMEOUT)
+
+  def handle(self, *args, **options):
+    cli_overrides = {
+      "config": options["config"],
+      "mode": options["mode"],
+      "only_work": options["only_work"],
+      "only_dispatch": options["only_dispatch"],
+      "skip_recurring": options["skip_recurring"],
+    }
+    try:
+      config = load_backend_config(
+        options["backend"],
+        cli_overrides=cli_overrides,
+      )
+      wait_for_dj_queue_migrations(
+        config.database_alias,
+        timeout=options["migration_wait_timeout"],
+        stdout=self.stdout,
+      )
+      supervisor = build_supervisor(
+        backend_alias=options["backend"],
+        cli_overrides=cli_overrides,
+      )
+    except ImproperlyConfigured as exc:
+      raise CommandError(str(exc)) from exc
+    supervisor.run()

{dj_queue-0.10.1 → dj_queue-0.10.3}/dj_queue/operations/concurrency.py

@@ -10,11 +10,20 @@ from dj_queue.config import load_backend_config
 from dj_queue.db import database_capabilities, get_database_alias, locked_queryset
 from dj_queue.exceptions import EnqueueError
 from dj_queue.log import log_event
-from dj_queue.models import BlockedExecution, ClaimedExecution, ReadyExecution, Semaphore
+from dj_queue.models import (
+  BlockedExecution,
+  ClaimedExecution,
+  FailedExecution,
+  Job,
+  ReadyExecution,
+  ScheduledExecution,
+  Semaphore,
+)
 from dj_queue.operations._helpers import (
   _consume_selected_rows,
   _create_blocked_execution,
   _create_ready_execution,
+  _lock_active_pauses,
   _task_option,
 )
 from dj_queue.operations._insert import create_ignore_conflicts
@@ -42,7 +51,7 @@ def semaphore_acquire(
       now=now,
     )
 
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     if create_ignore_conflicts(
       Semaphore,
       using=alias,
@@ -54,7 +63,7 @@ def semaphore_acquire(
       return True
 
   reconciled_available = _reconciled_available_expression(limit)
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     updated = (
       Semaphore.objects.using(alias)
       .filter(key=key, value__gt=F("limit") - Value(limit))
@@ -177,6 +186,29 @@ def _consume_released_semaphore_slot(alias, key, *, limit, duration_seconds, now
   return updated > 0
 
 
+def _handoff_released_claimed_slot(alias, key, *, limit, duration_seconds, now):
+  expires_at = now + timedelta(seconds=duration_seconds)
+  released_available = _released_available_expression(limit)
+  updated = (
+    Semaphore.objects.using(alias)
+    .filter(key=key, value__gt=F("limit") - Value(limit) - Value(1))
+    .update(
+      value=released_available - Value(1),
+      limit=limit,
+      expires_at=expires_at,
+      updated_at=now,
+    )
+  )
+  return updated > 0
+
+
+def _released_available_expression(limit):
+  return Least(
+    Value(limit),
+    Greatest(Value(0), F("value") + Value(limit) - F("limit") + Value(1)),
+  )
+
+
 def _reconciled_available_expression(limit):
   return Least(Value(limit), Greatest(Value(0), F("value") + Value(limit) - F("limit")))
 
@@ -209,11 +241,12 @@ def unblock_next_blocked_job(
   backend_alias="default",
   use_skip_locked=True,
   handoff_released_slot=False,
+  release_slot=False,
 ):
   alias = get_database_alias(backend_alias)
   now = timezone.now()
 
-  with transaction.atomic(using=alias):
+  with _operation_atomic(alias):
     queryset = (
       BlockedExecution.objects.using(alias)
       .select_related("job")
@@ -229,7 +262,15 @@ def unblock_next_blocked_job(
       return None
 
     slot_acquired = False
-    if handoff_released_slot:
+    if release_slot:
+      slot_acquired = _handoff_released_claimed_slot(
+        alias,
+        key,
+        limit=limit,
+        duration_seconds=duration_seconds,
+        now=now,
+      )
+    elif handoff_released_slot:
       slot_acquired = _consume_released_semaphore_slot(
         alias,
         key,
@@ -238,7 +279,7 @@ def unblock_next_blocked_job(
         now=now,
       )
 
-    if not slot_acquired:
+    if not slot_acquired and not release_slot:
       slot_acquired = semaphore_acquire(
         key,
         limit=limit,
@@ -261,7 +302,7 @@ def unblock_next_blocked_job(
     job = blocked.job
     queue_name = blocked.queue_name
     priority = blocked.priority
-    _create_ready_execution(
+    _create_ready_execution_after_blocked_consume(
       alias,
       job=job,
       backend_alias=backend_alias,
@@ -279,6 +320,71 @@ def unblock_next_blocked_job(
   return job
 
 
+def _create_ready_execution_after_blocked_consume(
+  alias,
+  *,
+  job,
+  backend_alias,
+  queue_name,
+  priority,
+  ready_at,
+):
+  _lock_active_pauses(alias, backend_alias, {queue_name})
+  connection = connections[alias]
+  quote = connection.ops.quote_name
+  ready_table = quote(ReadyExecution._meta.db_table)
+  job_id_column = quote(ReadyExecution._meta.get_field("job").column)
+  backend_alias_column = quote(ReadyExecution._meta.get_field("backend_alias").column)
+  queue_name_column = quote(ReadyExecution._meta.get_field("queue_name").column)
+  priority_column = quote(ReadyExecution._meta.get_field("priority").column)
+  created_at_column = quote(ReadyExecution._meta.get_field("created_at").column)
+  latency_started_at_column = quote(ReadyExecution._meta.get_field("latency_started_at").column)
+  job_id = Job._meta.get_field("id").get_db_prep_value(
+    job.pk,
+    connection=connection,
+    prepared=False,
+  )
+  state_models = (ReadyExecution, ScheduledExecution, ClaimedExecution, FailedExecution)
+  state_checks = " AND ".join(
+    _state_absence_sql(model, job_id_column=job_id_column, quote=quote) for model in state_models
+  )
+
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      INSERT INTO {ready_table} (
+        {job_id_column},
+        {backend_alias_column},
+        {queue_name_column},
+        {priority_column},
+        {created_at_column},
+        {latency_started_at_column}
+      )
+      SELECT %s, %s, %s, %s, %s, %s
+      WHERE {state_checks}
+      """,
+      [
+        job_id,
+        backend_alias,
+        queue_name,
+        priority,
+        ready_at,
+        ready_at,
+        *([job_id] * len(state_models)),
+      ],
+    )
+    created = cursor.rowcount
+
+  if created != 1:
+    raise EnqueueError(f"job {job.id} already has an execution-state row")
+
+
+def _state_absence_sql(model, *, job_id_column, quote):
+  state_table = quote(model._meta.db_table)
+  state_job_id_column = quote(model._meta.get_field("job").column)
+  return f"NOT EXISTS (SELECT 1 FROM {state_table} WHERE {state_table}.{state_job_id_column} = %s)"
+
+
 def cleanup_expired_semaphores(*, backend_alias="default"):
   alias = get_database_alias(backend_alias)
   claimed_concurrency_keys = (
@@ -397,3 +503,7 @@ def _positive_int_option(value, name):
   if number <= 0:
     raise EnqueueError(f"{name} must be a positive integer")
   return number
+
+
+def _operation_atomic(alias):
+  return transaction.atomic(using=alias, savepoint=not connections[alias].in_atomic_block)

{dj_queue-0.10.1 → dj_queue-0.10.3}/dj_queue/operations/jobs.py

@@ -32,7 +32,6 @@ from dj_queue.operations._helpers import (
   _ensure_no_other_execution_state,
   _consume_selected_rows,
   _create_blocked_execution,
-  _create_ready_execution,
   _create_ready_execution_locked,
   _create_scheduled_execution,
   _exclude_active_pauses,
@@ -110,7 +109,12 @@ def enqueue_job_with_dispatch(task, args, kwargs, *, backend_alias="default"):
       scheduled_at=task.run_after,
       concurrency_key=concurrency_key,
     )
-    dispatch_outcome = _dispatch_job(job, task=task, backend_alias=backend_alias)
+    dispatch_outcome = _dispatch_job(
+      job,
+      task=task,
+      backend_alias=backend_alias,
+      check_conflicts=False,
+    )
 
   if dispatch_outcome.should_notify:
     notify_ready_queues_on_commit((job.queue_name,), backend_alias=backend_alias)
@@ -429,15 +433,13 @@ def _complete_claimed_job(job, return_value, *, backend_alias="default", task=No
 
   with transaction.atomic(using=alias):
     _delete_claimed_execution(alias, job.id)
-    _ensure_no_other_execution_state(alias, job, ignored_models=(ClaimedExecution,))
     now = timezone.now()
     config = load_backend_config(job.backend_alias)
 
     if config.preserve_finished_jobs:
-      job.finished_at = now
-      job.return_value = return_value
-      job.save(using=alias, update_fields=["finished_at", "return_value", "updated_at"])
+      _finish_job_if_no_execution_state(alias, job, return_value, finished_at=now)
     else:
+      _ensure_no_other_execution_state(alias, job, ignored_models=(ClaimedExecution,))
       job.delete(using=alias)
 
     _release_concurrency_slot(job, task=task)
@@ -910,7 +912,7 @@ def _dispatch_existing_job(job):
   return _dispatch_job(job, task=task, backend_alias=job.backend_alias)
 
 
-def _dispatch_job(job, *, task, backend_alias, now=None):
+def _dispatch_job(job, *, task, backend_alias, now=None, check_conflicts=True):
   alias = get_database_alias(backend_alias)
   if now is None:
     now = timezone.now()
@@ -925,11 +927,13 @@ def _dispatch_job(job, *, task, backend_alias, now=None):
     return DispatchOutcome.SCHEDULED
 
   if not job.concurrency_key:
-    _create_ready_execution(
+    _create_ready_execution_locked(
       alias,
       job=job,
       backend_alias=backend_alias,
+      queue_name=job.queue_name,
       ready_at=now,
+      check_conflicts=check_conflicts,
     )
     return DispatchOutcome.READY
 
@@ -979,6 +983,19 @@ def _release_concurrency_slot(job, *, task=None):
     limit = _semaphore_limit(job) or 1
     duration_seconds = config.default_concurrency_duration
 
+  if (
+    unblock_next_blocked_job(
+      job.concurrency_key,
+      limit=limit,
+      duration_seconds=duration_seconds,
+      backend_alias=job.backend_alias,
+      use_skip_locked=config.use_skip_locked,
+      release_slot=True,
+    )
+    is not None
+  ):
+    return
+
   semaphore_release(
     job.concurrency_key,
     limit=limit,
@@ -1064,11 +1081,19 @@ def _select_ready_rows(queryset, *, limit, queues, use_skip_locked):
   ordered_selectors = selectors if star_index is None else selectors[:star_index]
 
   if ordered_selectors:
-    ordered = _ordered_selector_rows_queryset(
-      queryset.exclude(pk__in=selected_ids),
-      ordered_selectors,
-    )
-    rows = list(locked_queryset(ordered, use_skip_locked=use_skip_locked)[:limit])
+    if _selectors_are_exact(ordered_selectors):
+      rows = _select_exact_selector_rows(
+        queryset.exclude(pk__in=selected_ids),
+        ordered_selectors,
+        limit=limit,
+        use_skip_locked=use_skip_locked,
+      )
+    else:
+      ordered = _ordered_selector_rows_queryset(
+        queryset.exclude(pk__in=selected_ids),
+        ordered_selectors,
+      )
+      rows = list(locked_queryset(ordered, use_skip_locked=use_skip_locked)[:limit])
     selected_rows.extend(rows)
     selected_ids.update(row.pk for row in rows)
 
@@ -1082,6 +1107,22 @@ def _select_ready_rows(queryset, *, limit, queues, use_skip_locked):
   return selected_rows
 
 
+def _select_exact_selector_rows(queryset, selectors, *, limit, use_skip_locked):
+  selected_rows = []
+  for selector in dict.fromkeys(selectors):
+    remaining = limit - len(selected_rows)
+    if remaining <= 0:
+      break
+    ordered = queryset.filter(queue_name=selector).order_by("-priority", "id")
+    rows = list(locked_queryset(ordered, use_skip_locked=use_skip_locked)[:remaining])
+    selected_rows.extend(rows)
+  return selected_rows
+
+
+def _selectors_are_exact(selectors):
+  return all(selector != "*" and not selector.endswith("*") for selector in selectors)
+
+
 def _ordered_selector_rows_queryset(queryset, selectors):
   filtered = _filter_queue_selectors(queryset, selectors)
   selector_rank = Case(
@@ -1141,6 +1182,69 @@ def _delete_claimed_execution(alias, job_id):
     raise ClaimedExecution.DoesNotExist
 
 
+def _finish_job_if_no_execution_state(alias, job, return_value, *, finished_at):
+  connection = connections[alias]
+  quote = connection.ops.quote_name
+  jobs_table = quote(Job._meta.db_table)
+  job_id_column = quote(Job._meta.get_field("id").column)
+  backend_alias_column = quote(Job._meta.get_field("backend_alias").column)
+  finished_at_column = quote(Job._meta.get_field("finished_at").column)
+  return_value_column = quote(Job._meta.get_field("return_value").column)
+  updated_at_column = quote(Job._meta.get_field("updated_at").column)
+  state_checks = " AND ".join(
+    _state_absence_sql(model, jobs_table=jobs_table, job_id_column=job_id_column, quote=quote)
+    for model in (
+      ReadyExecution,
+      ScheduledExecution,
+      BlockedExecution,
+      FailedExecution,
+    )
+  )
+  job_id = Job._meta.get_field("id").get_db_prep_value(
+    job.pk,
+    connection=connection,
+    prepared=False,
+  )
+  prepared_return_value = Job._meta.get_field("return_value").get_db_prep_save(
+    return_value,
+    connection=connection,
+  )
+
+  with connection.cursor() as cursor:
+    cursor.execute(
+      f"""
+      UPDATE {jobs_table}
+      SET
+        {finished_at_column} = %s,
+        {return_value_column} = %s,
+        {updated_at_column} = %s
+      WHERE
+        {jobs_table}.{job_id_column} = %s
+        AND {jobs_table}.{backend_alias_column} = %s
+        AND {state_checks}
+      """,
+      [finished_at, prepared_return_value, finished_at, job_id, job.backend_alias],
+    )
+    updated = cursor.rowcount
+
+  if updated != 1:
+    raise EnqueueError(f"job {job.id} already has an execution-state row")
+  job.finished_at = finished_at
+  job.return_value = return_value
+  job.updated_at = finished_at
+
+
+def _state_absence_sql(model, *, jobs_table, job_id_column, quote):
+  state_table = quote(model._meta.db_table)
+  state_job_id_column = quote(model._meta.get_field("job").column)
+  return (
+    f"NOT EXISTS ("
+    f"SELECT 1 FROM {state_table} "
+    f"WHERE {state_table}.{state_job_id_column} = {jobs_table}.{job_id_column}"
+    f")"
+  )
+
+
 def _bulk_create(alias, model, objects):
   if not objects:
     return None

{dj_queue-0.10.1 → dj_queue-0.10.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "dj-queue"
-version = "0.10.1"
+version = "0.10.3"
 description = "Database-backed task queue backend for Django’s Tasks framework."
 readme = "README.md"
 license = "MIT"

dj_queue-0.10.1/dj_queue/management/commands/dj_queue.py

@@ -1,44 +0,0 @@
-from django.core.exceptions import ImproperlyConfigured
-from django.core.management.base import BaseCommand
-from django.core.management.base import CommandError
-
-from dj_queue.config import load_backend_config
-from dj_queue.runtime.supervisor import AsyncSupervisor, ForkSupervisor
-
-
-def build_supervisor(*, backend_alias, cli_overrides):
-  mode = load_backend_config(backend_alias, cli_overrides=cli_overrides).mode
-  supervisor_class = AsyncSupervisor if mode == "async" else ForkSupervisor
-  return supervisor_class.from_backend_config(
-    backend_alias=backend_alias,
-    cli_overrides=cli_overrides,
-  )
-
-
-class Command(BaseCommand):
-  help = "Start the dj_queue supervisor"
-
-  def add_arguments(self, parser):
-    parser.add_argument("--backend", default="default")
-    parser.add_argument("--config")
-    parser.add_argument("--mode", choices=("fork", "async"))
-    parser.add_argument("--only-work", action="store_true")
-    parser.add_argument("--only-dispatch", action="store_true")
-    parser.add_argument("--skip-recurring", action="store_true")
-
-  def handle(self, *args, **options):
-    cli_overrides = {
-      "config": options["config"],
-      "mode": options["mode"],
-      "only_work": options["only_work"],
-      "only_dispatch": options["only_dispatch"],
-      "skip_recurring": options["skip_recurring"],
-    }
-    try:
-      supervisor = build_supervisor(
-        backend_alias=options["backend"],
-        cli_overrides=cli_overrides,
-      )
-    except ImproperlyConfigured as exc:
-      raise CommandError(str(exc)) from exc
-    supervisor.run()

{dj_queue-0.10.1 → dj_queue-0.10.3}/dj_queue/templates/admin/dj_queue/dashboard.html

@@ -586,8 +586,8 @@
                 <tr class="{% cycle 'row1' 'row2' %}">
                   <th class="djq-col-name"><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=ready">{{ queue.name }}</a></th>
                   <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=ready">{{ queue.ready_count }}</a></td>
-                  <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=claimed">{{ queue.claimed_count }}</a></td>
                   <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=scheduled">{{ queue.scheduled_count }}</a></td>
+                  <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=claimed">{{ queue.claimed_count }}</a></td>
                   <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=blocked">{{ queue.blocked_count }}</a></td>
                   <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=failed">{{ queue.failed_count }}</a></td>
                   <td><a href="{% url 'admin:dj_queue_dashboard_queue' queue.name %}?backend={{ backend_alias }}&state=finished">{{ queue.finished_count }}</a></td>