python-pq 0.7.0__py3-none-any.whl → 0.7.2__py3-none-any.whl

pq/__init__.py

@@ -5,7 +5,7 @@ from pq.models import Periodic, Task, TaskStatus
 from pq.priority import Priority
 from pq.worker import PostExecuteHook, PreExecuteHook, TaskTimeoutError
 
-__version__ = "0.7.0"
+__version__ = "0.7.2"
 
 __all__ = [
     "PQ",

pq/client.py

@@ -1,6 +1,7 @@
 """PQ client - main interface for task queue."""
 
 import importlib.resources
+import math
 from collections.abc import Callable, Set
 from contextlib import contextmanager
 from datetime import UTC, datetime, timedelta
@@ -9,7 +10,9 @@ from typing import Any, Self
 
 from croniter import croniter
 from croniter.croniter import CroniterBadCronError
-from sqlalchemy import create_engine, delete, func, select
+from loguru import logger
+import sqlalchemy as sa
+from sqlalchemy import create_engine, delete, func, select, update
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
@@ -20,6 +23,45 @@ from pq.registry import get_function_path
 from pq.serialization import serialize
 
 
+def _validate_max_runtime(max_runtime_seconds: float | None) -> None:
+    """Reject obviously-degenerate per-task ``max_runtime_seconds`` values.
+
+    ``None`` means "use the worker's configured default" and is the
+    common case — nothing to validate. ``0``, negative, NaN, and
+    infinity are all rejected because the downstream enforcement is
+    undefined for those:
+
+    - ``signal.alarm(int(max_runtime_seconds) + 1)`` would fire at 1 s for
+      ``0``, at 0 s (which disables the alarm entirely) for any
+      negative integer ≥ -1, and raise ``ValueError`` from inside
+      the worker for NaN / infinity.
+    - ``max_runtime_seconds * 2`` in the stale-reaper SQL would be ≤ 0 or
+      NaN, making the per-row threshold a no-op or NULL (the global
+      default would apply) — silent surprise vs. the call site's
+      intent.
+    - Periodic ``lock_duration`` already guards with a 3600 s
+      fallback for ``≤ 0`` (``worker.py``), but that fallback exists
+      for the worker-level default, not as a contract for callers.
+
+    None of these are useful behaviours to expose. Failing fast at
+    enqueue/upsert/schedule time keeps the bug at the call site,
+    where it's trivially fixable, instead of producing a confusing
+    runtime surprise hours later.
+    """
+    if max_runtime_seconds is None:
+        return
+    if math.isnan(max_runtime_seconds) or math.isinf(max_runtime_seconds):
+        raise ValueError(
+            f"max_runtime_seconds must be a finite positive number (got {max_runtime_seconds!r}); "
+            f"pass None to use the worker's configured default."
+        )
+    if max_runtime_seconds <= 0:
+        raise ValueError(
+            f"max_runtime_seconds must be > 0 (got {max_runtime_seconds!r}); pass None to "
+            f"use the worker's configured default."
+        )
+
+
 class PQ:
     """Postgres-backed task queue client."""
 
@@ -124,6 +166,7 @@ class PQ:
         run_at: datetime | None = None,
         priority: Priority = Priority.NORMAL,
         client_id: str | None = None,
+        max_runtime_seconds: float | None = None,
         **kwargs: Any,
     ) -> int:
         """Enqueue a one-off task.
@@ -134,15 +177,24 @@ class PQ:
             run_at: When to run the task. Defaults to now.
             priority: Task priority. Higher = higher priority. Defaults to NORMAL.
             client_id: Optional client-provided identifier. Must be unique if provided.
+            max_runtime_seconds: Per-task wall-clock cap in seconds. When ``None`` (the
+                common case), the worker that picks up this task uses its
+                configured default. When set, this specific task gets the
+                supplied ceiling AND the stale-reaper threshold scales with
+                it (``max_runtime_seconds * 2`` if larger than the reaper's global
+                default). Use this for occasionally-long tasks in a fleet
+                whose worker default is sized for typical short work.
             **kwargs: Keyword arguments to pass to the handler.
 
         Returns:
             Task ID.
 
         Raises:
-            ValueError: If task is a lambda, closure, or cannot be imported.
+            ValueError: If task is a lambda, closure, or cannot be imported,
+                or if ``max_runtime_seconds`` is not None and ``<= 0``.
             IntegrityError: If client_id already exists.
         """
+        _validate_max_runtime(max_runtime_seconds)
         name = get_function_path(task)
         payload = serialize(args, kwargs)
 
@@ -155,6 +207,7 @@ class PQ:
             run_at=run_at,
             priority=priority,
             client_id=client_id,
+            max_runtime_seconds=max_runtime_seconds,
         )
 
         with self.session() as session:
@@ -169,6 +222,7 @@ class PQ:
         run_at: datetime | None = None,
         priority: Priority = Priority.NORMAL,
         client_id: str,
+        max_runtime_seconds: float | None = None,
         **kwargs: Any,
     ) -> int:
         """Enqueue a task, updating if client_id already exists.
@@ -182,14 +236,19 @@ class PQ:
             run_at: When to run the task. Defaults to now.
             priority: Task priority. Higher = higher priority. Defaults to NORMAL.
             client_id: Client-provided identifier. Required for conflict resolution.
+            max_runtime_seconds: Per-task wall-clock cap in seconds. See ``enqueue`` for
+                details. ``None`` (the default) leaves the worker's own
+                configured ``max_runtime_seconds`` in effect.
             **kwargs: Keyword arguments to pass to the handler.
 
         Returns:
             Task ID.
 
         Raises:
-            ValueError: If task is a lambda, closure, or cannot be imported.
+            ValueError: If task is a lambda, closure, or cannot be imported,
+                or if ``max_runtime_seconds`` is not None and ``<= 0``.
         """
+        _validate_max_runtime(max_runtime_seconds)
         name = get_function_path(task)
         payload = serialize(args, kwargs)
 
@@ -205,6 +264,7 @@ class PQ:
                 priority=priority,
                 status=TaskStatus.PENDING,
                 run_at=run_at,
+                max_runtime_seconds=max_runtime_seconds,
             )
             .on_conflict_do_update(
                 index_elements=["client_id"],
@@ -214,6 +274,7 @@ class PQ:
                     "priority": priority,
                     "status": TaskStatus.PENDING,
                     "run_at": run_at,
+                    "max_runtime_seconds": max_runtime_seconds,
                     "attempts": 0,
                     "started_at": None,
                     "completed_at": None,
@@ -238,6 +299,7 @@ class PQ:
         max_concurrent: int | None = 1,
         key: str = "",
         active: bool = True,
+        max_runtime_seconds: float | None = None,
         **kwargs: Any,
     ) -> int:
         """Schedule a periodic task.
@@ -259,6 +321,14 @@ class PQ:
                 Defaults to "" (empty string).
             active: Whether the task is active. Inactive tasks are not executed.
                 Defaults to True.
+            max_runtime_seconds: Per-execution wall-clock cap in seconds. When ``None``
+                (the common case), each execution uses the worker's configured
+                default. When set, this schedule's executions get the
+                supplied ceiling, and the ``locked_until`` window (used when
+                ``max_concurrent`` is in effect) is extended to match so the
+                lock doesn't expire while the execution is legitimately still
+                running. Periodic tasks are not subject to the stale-task
+                reaper, so this knob is purely about the wall-clock cap.
             **kwargs: Keyword arguments to pass to the handler.
 
         Returns:
@@ -269,6 +339,7 @@ class PQ:
             ValueError: If cron expression is invalid.
             ValueError: If max_concurrent is greater than 1.
             ValueError: If task is a lambda, closure, or cannot be imported.
+            ValueError: If ``max_runtime_seconds`` is not None and ``<= 0``.
             IntegrityError: If client_id already exists.
         """
         if run_every is None and cron is None:
@@ -280,6 +351,7 @@ class PQ:
                 f"max_concurrent must be 1 or None, got {max_concurrent} "
                 "(values > 1 reserved for future use)"
             )
+        _validate_max_runtime(max_runtime_seconds)
 
         # Validate and normalize cron expression
         cron_expr: str | None = None
@@ -320,6 +392,7 @@ class PQ:
                     client_id=client_id,
                     max_concurrent=max_concurrent,
                     active=active,
+                    max_runtime_seconds=max_runtime_seconds,
                 )
                 .on_conflict_do_update(
                     index_elements=["name", "key"],
@@ -331,6 +404,7 @@ class PQ:
                         "next_run": next_run,
                         "max_concurrent": max_concurrent,
                         "active": active,
+                        "max_runtime_seconds": max_runtime_seconds,
                     },
                 )
                 .returning(Periodic.id)
@@ -507,6 +581,96 @@ class PQ:
             result = session.execute(stmt)
             return result.rowcount
 
+    def reap_stale_tasks(self, threshold: timedelta) -> int:
+        """Mark stale RUNNING tasks as FAILED.
+
+        When a worker dies mid-execution (e.g. pod restart, OOM on the worker
+        process), in-flight tasks stay RUNNING forever because no parent
+        process remains to update their status. This method detects those
+        orphaned rows and transitions them to FAILED.
+
+        Tasks enqueued with a per-task ``max_runtime_seconds`` override get a
+        proportionally larger reaper window: the effective threshold for
+        a row is ``max(threshold, max_runtime_seconds * 2)``. This avoids
+        reaping a legitimately long-running task whose declared budget
+        exceeds the global default. NULL ``max_runtime_seconds`` (the
+        common case) falls back to the supplied ``threshold`` unchanged.
+
+        Args:
+            threshold: Default per-task stale window for tasks that don't
+                have a ``max_runtime_seconds`` override (or whose override
+                is smaller than this default). A task is stale when
+                ``started_at + effective_threshold < now()``. Should be
+                at least 2x the worker's own ``max_runtime_seconds``, e.g.
+                ``timedelta(seconds=max_runtime_seconds * 2)``.
+
+        Returns:
+            Number of tasks reaped.
+        """
+        now = datetime.now(UTC)
+        threshold_seconds = threshold.total_seconds()
+        # Per-row effective stale window: the larger of the supplied
+        # default and 2x the row's per-task override (NULL → just the
+        # default). Computed in SQL with ``GREATEST`` + ``COALESCE``
+        # so the reaper stays a single round-trip even when the row
+        # set is heterogeneous in declared budget.
+        effective_threshold_seconds = sa.func.greatest(
+            threshold_seconds,
+            sa.func.coalesce(Task.max_runtime_seconds * 2, threshold_seconds),
+        )
+        stale_cutoff = sa.func.now() - sa.func.make_interval(
+            sa.literal(0),  # years
+            sa.literal(0),  # months
+            sa.literal(0),  # weeks
+            sa.literal(0),  # days
+            sa.literal(0),  # hours
+            sa.literal(0),  # mins
+            effective_threshold_seconds,  # secs
+        )
+        # Per-row ``error`` message includes the values that actually
+        # decided this row's fate: the row's own
+        # ``max_runtime_seconds`` (or ``NULL`` if unset) and the
+        # effective threshold the reaper applied. This avoids the
+        # operator needing a separate query to figure out "why did MY
+        # long-budget task get reaped?" — the answer is in the row.
+        # Built with Postgres ``format()`` so the message is computed
+        # per row in the same UPDATE round-trip.
+        per_row_error = sa.func.format(
+            "Reaped: task still RUNNING past its stale window "
+            "(default threshold=%s s, per-task max_runtime_seconds=%s, "
+            "effective threshold=%s s). Worker likely died.",
+            threshold_seconds,
+            sa.func.coalesce(sa.cast(Task.max_runtime_seconds, sa.String), "NULL"),
+            effective_threshold_seconds,
+        )
+        with self.session() as session:
+            stmt = (
+                update(Task)
+                .where(
+                    Task.status == TaskStatus.RUNNING,
+                    Task.started_at < stale_cutoff,
+                )
+                .values(
+                    status=TaskStatus.FAILED,
+                    completed_at=now,
+                    error=per_row_error,
+                )
+                .returning(
+                    Task.id,
+                    Task.name,
+                    Task.started_at,
+                    Task.max_runtime_seconds,
+                )
+            )
+            reaped = list(session.execute(stmt).all())
+            for task_id, name, started_at, max_runtime_seconds in reaped:
+                logger.warning(
+                    f"Reaped stale task '{name}' (id={task_id},"
+                    f" started_at={started_at},"
+                    f" max_runtime_seconds={max_runtime_seconds})"
+                )
+            return len(reaped)
+
     def run_worker(
         self,
         *,

pq/migrations/versions/20260527T170136Z_aee3e8e7e647_add_task_max_runtime_seconds.py

@@ -0,0 +1,50 @@
+"""add max_runtime_seconds to pq_tasks and pq_periodic
+
+Revision ID: aee3e8e7e647
+Revises: c3d4e5f6a7b8
+Create Date: 2026-05-27 17:01:36 Z
+
+Per-task override for the worker's wall-clock ``max_runtime`` cap, on
+both one-off tasks (``pq_tasks``) and periodic schedules
+(``pq_periodic``). Nullable on both so every existing row (and every
+existing enqueue / schedule call site that doesn't pass the new
+parameter) keeps the previous behaviour exactly — a NULL value tells
+the worker to use its own default.
+
+Schema impact on populated tables is intentionally minimal: PostgreSQL
+11+ treats ``ADD COLUMN ... DEFAULT NULL`` as a catalog-only operation
+(no table rewrite, brief ACCESS EXCLUSIVE lock on the system catalog
+row, typically < 100 ms even for tables with millions of rows). No
+backfill — NULL is the intended default.
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "aee3e8e7e647"
+down_revision: Union[str, Sequence[str], None] = "c3d4e5f6a7b8"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add the nullable ``max_runtime_seconds`` column to both task tables."""
+    op.add_column(
+        "pq_tasks",
+        sa.Column("max_runtime_seconds", sa.Float(), nullable=True),
+    )
+    op.add_column(
+        "pq_periodic",
+        sa.Column("max_runtime_seconds", sa.Float(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    """Remove the ``max_runtime_seconds`` column from both task tables."""
+    op.drop_column("pq_periodic", "max_runtime_seconds")
+    op.drop_column("pq_tasks", "max_runtime_seconds")

pq/models.py

@@ -9,6 +9,7 @@ from sqlalchemy import (
     Boolean,
     DateTime,
     Enum,
+    Float,
     Identity,
     Index,
     Integer,
@@ -70,6 +71,14 @@ class Task(Base):
     )
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
     attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    # Per-task override of the worker-level ``max_runtime``. When NULL
+    # (the common case), the worker uses its configured default. When
+    # set, it caps this specific task's wall-clock at the given number
+    # of seconds AND extends the stale-reaper threshold proportionally
+    # (the reaper picks the larger of its global default and
+    # ``max_runtime_seconds * 2``). Useful for occasionally-long tasks
+    # in a fleet whose default is sized for typical short work.
+    max_runtime_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
 
 
 class Periodic(Base):
@@ -94,6 +103,17 @@ class Periodic(Base):
     next_run: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
     max_concurrent: Mapped[int | None] = mapped_column(SmallInteger, nullable=True)
     active: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="true")
+    # Per-task override of the worker-level ``max_runtime``. Same semantics
+    # as ``Task.max_runtime_seconds`` — NULL means "use the worker's
+    # configured default", a set value caps this specific schedule's
+    # wall-clock at the given number of seconds and also extends the
+    # ``locked_until`` window (when ``max_concurrent`` is in effect) so
+    # the lock doesn't expire while the task is legitimately still
+    # running. Periodic tasks are not subject to the stale-task reaper
+    # (they're guarded by ``locked_until`` and the natural re-fire of
+    # the schedule), so this knob is purely about the per-execution
+    # wall-clock cap.
+    max_runtime_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
     last_run: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )

pq/worker.py

@@ -75,6 +75,14 @@ DEFAULT_RETENTION_DAYS: int = 7
 # Default cleanup interval: 1 hour
 DEFAULT_CLEANUP_INTERVAL: float = 3600
 
+# Default stale task timeout: 1 hour
+# Tasks RUNNING longer than this are assumed orphaned (worker died) and reaped.
+DEFAULT_STALE_TASK_TIMEOUT: timedelta = timedelta(hours=1)
+
+# Default reaper check interval: 5 minutes
+# How often the worker checks for stale RUNNING tasks.
+DEFAULT_REAPER_INTERVAL: float = 300
+
 # Exit codes for child process
 EXIT_SUCCESS = 0
 EXIT_FAILURE = 1
@@ -377,6 +385,35 @@ def _maybe_run_cleanup(
     last_cleanup[0] = now
 
 
+def _maybe_reap_stale(
+    pq: PQ,
+    stale_task_timeout: timedelta | None,
+    reaper_interval: float,
+    last_reap: list[float],
+) -> None:
+    """Reap stale RUNNING tasks if enabled and interval has passed.
+
+    Args:
+        pq: PQ client instance.
+        stale_task_timeout: RUNNING tasks older than this are marked FAILED.
+            ``None`` disables reaping.
+        reaper_interval: Seconds between reaper checks.
+        last_reap: Mutable list containing last reap timestamp.
+    """
+    if stale_task_timeout is None:
+        return
+
+    now = time.time()
+    if now - last_reap[0] < reaper_interval:
+        return
+
+    reaped = pq.reap_stale_tasks(stale_task_timeout)
+    if reaped:
+        logger.info(f"Reaped {reaped} stale RUNNING task(s)")
+
+    last_reap[0] = now
+
+
 def run_worker(
     pq: PQ,
     *,
@@ -386,6 +423,7 @@ def run_worker(
     priorities: Set[Priority] | None = None,
     retention_days: int = DEFAULT_RETENTION_DAYS,
     cleanup_interval: float = DEFAULT_CLEANUP_INTERVAL,
+    stale_task_timeout: timedelta | None = DEFAULT_STALE_TASK_TIMEOUT,
     pre_execute: PreExecuteHook | None = None,
     post_execute: PostExecuteHook | None = None,
 ) -> None:
@@ -404,6 +442,9 @@ def run_worker(
         retention_days: Days to keep completed/failed tasks. Default: 7.
             Set to 0 to disable automatic cleanup.
         cleanup_interval: Seconds between cleanup runs. Default: 3600 (1 hour).
+        stale_task_timeout: Mark RUNNING tasks older than this as FAILED.
+            Catches orphaned tasks whose worker died mid-execution.
+            Default: 1 hour. Set to ``None`` to disable.
         pre_execute: Called in forked child BEFORE task execution.
             Use for initializing fork-unsafe resources (OTel, DB connections).
         post_execute: Called in forked child AFTER task execution (success or failure).
@@ -432,10 +473,12 @@ def run_worker(
             post_execute=post_execute,
             retention_days=retention_days,
             cleanup_interval=cleanup_interval,
+            stale_task_timeout=stale_task_timeout,
         )
         return
 
-    last_cleanup: list[float] = [0.0]  # Mutable container for tracking
+    last_cleanup: list[float] = [0.0]
+    last_reap: list[float] = [0.0]
 
     try:
         while True:
@@ -447,6 +490,9 @@ def run_worker(
                 post_execute=post_execute,
             ):
                 _maybe_run_cleanup(pq, retention_days, cleanup_interval, last_cleanup)
+                _maybe_reap_stale(
+                    pq, stale_task_timeout, DEFAULT_REAPER_INTERVAL, last_reap
+                )
                 time.sleep(poll_interval)
     except KeyboardInterrupt:
         logger.info("Worker stopped.")
@@ -550,6 +596,10 @@ def _process_one_off_task(
             name = task.name
             payload = task.payload
             task_id = task.id
+            # Per-task ``max_runtime_seconds`` (NULL → use worker default).
+            # Read while still in the session because the row is expunged
+            # below for the forked-child handoff.
+            task_max_runtime = task.max_runtime_seconds
 
             # Flush to commit status changes, then expunge for forked process
             session.flush()
@@ -559,6 +609,14 @@ def _process_one_off_task(
         logger.error(f"Error claiming task: {e}")
         return False
 
+    # Effective per-task wall-clock ceiling: per-task value when set,
+    # otherwise the worker's configured default. The reaper applies the
+    # same rule on its own (see ``Client.reap_stale_tasks``), so the
+    # two stay in sync without the worker having to coordinate.
+    effective_max_runtime = (
+        task_max_runtime if task_max_runtime is not None else max_runtime
+    )
+
     # Phase 2: Execute handler in forked process
     start = time.perf_counter()
     status = TaskStatus.COMPLETED
@@ -571,7 +629,7 @@ def _process_one_off_task(
             handler,
             args,
             kwargs,
-            max_runtime=max_runtime,
+            max_runtime=effective_max_runtime,
             task=task,
             pre_execute=pre_execute,
             post_execute=post_execute,
@@ -595,15 +653,26 @@ def _process_one_off_task(
 
     elapsed = time.perf_counter() - start
 
-    # Phase 3: Update task status
+    # Phase 3: Update task status (guarded — only if still RUNNING, so we
+    # don't overwrite a reaper's FAILED verdict for an orphaned task)
     try:
         with pq.session() as session:
-            task = session.get(Task, task_id)
-            if task:
-                task.status = status
-                task.completed_at = datetime.now(UTC)
-                if error_msg:
-                    task.error = error_msg
+            values: dict[str, object] = {
+                "status": status,
+                "completed_at": datetime.now(UTC),
+            }
+            if error_msg:
+                values["error"] = error_msg
+            result = session.execute(
+                update(Task)
+                .where(Task.id == task_id, Task.status == TaskStatus.RUNNING)
+                .values(**values)
+            )
+            if result.rowcount == 0:
+                logger.warning(
+                    f"Task '{name}' (id={task_id}) was no longer RUNNING"
+                    " when Phase 3 tried to update — likely reaped"
+                )
     except Exception as e:
         logger.error(f"Error updating task status: {e}")
 
@@ -682,10 +751,27 @@ def _process_periodic_task(
             payload = periodic.payload
             periodic_id = periodic.id
             periodic_max_concurrent = periodic.max_concurrent
+            # Per-schedule ``max_runtime_seconds`` (NULL → use worker default).
+            # Read while still in the session because the row is expunged
+            # below for the forked-child handoff.
+            periodic_max_runtime = periodic.max_runtime_seconds
+
+            # Effective per-execution wall-clock ceiling: per-schedule value
+            # when set, otherwise the worker's configured default. Drives
+            # both the in-child timeout enforcement AND the
+            # ``locked_until`` window so the lock doesn't expire while a
+            # legitimately-long execution is still running.
+            effective_max_runtime = (
+                periodic_max_runtime
+                if periodic_max_runtime is not None
+                else max_runtime
+            )
 
             # Set lock before execution if concurrency is limited
             if periodic.max_concurrent is not None:
-                lock_duration = max_runtime if max_runtime > 0 else 3600
+                lock_duration = (
+                    effective_max_runtime if effective_max_runtime > 0 else 3600
+                )
                 periodic.locked_until = func.now() + timedelta(seconds=lock_duration)
 
             # Advance schedule BEFORE execution
@@ -712,7 +798,7 @@ def _process_periodic_task(
             handler,
             args,
             kwargs,
-            max_runtime=max_runtime,
+            max_runtime=effective_max_runtime,
             task=periodic,
             pre_execute=pre_execute,
             post_execute=post_execute,
@@ -794,6 +880,9 @@ def _claim_and_fork_one_off(
             name = task.name
             payload = task.payload
             task_id = task.id
+            # Per-task ``max_runtime_seconds`` (NULL → use worker default).
+            # Same handling as the sequential ``_process_one_off_task`` path.
+            task_max_runtime = task.max_runtime_seconds
 
             session.flush()
             session.expunge(task)
@@ -802,6 +891,10 @@ def _claim_and_fork_one_off(
         logger.error(f"Error claiming task: {e}")
         return None
 
+    effective_max_runtime = (
+        task_max_runtime if task_max_runtime is not None else max_runtime
+    )
+
     try:
         handler = resolve_function_path(name)
         args, kwargs = deserialize(payload)
@@ -809,7 +902,7 @@ def _claim_and_fork_one_off(
             handler,
             args,
             kwargs,
-            max_runtime=max_runtime,
+            max_runtime=effective_max_runtime,
             task=task,
             pre_execute=pre_execute,
             post_execute=post_execute,
@@ -877,9 +970,20 @@ def _claim_and_fork_periodic(
             payload = periodic.payload
             periodic_id = periodic.id
             periodic_max_concurrent = periodic.max_concurrent
+            # Per-schedule ``max_runtime_seconds`` (NULL → use worker default).
+            # Same handling as the sequential ``_process_periodic_task`` path.
+            periodic_max_runtime = periodic.max_runtime_seconds
+
+            effective_max_runtime = (
+                periodic_max_runtime
+                if periodic_max_runtime is not None
+                else max_runtime
+            )
 
             if periodic.max_concurrent is not None:
-                lock_duration = max_runtime if max_runtime > 0 else 3600
+                lock_duration = (
+                    effective_max_runtime if effective_max_runtime > 0 else 3600
+                )
                 periodic.locked_until = func.now() + timedelta(seconds=lock_duration)
 
             periodic.last_run = func.now()
@@ -902,7 +1006,7 @@ def _claim_and_fork_periodic(
             handler,
             args,
             kwargs,
-            max_runtime=max_runtime,
+            max_runtime=effective_max_runtime,
             task=periodic,
             pre_execute=pre_execute,
             post_execute=post_execute,
@@ -1016,12 +1120,25 @@ def _reap_and_update(pq: PQ, slot: _ChildSlot) -> None:
 
         try:
             with pq.session() as session:
-                task = session.get(Task, slot.task_id)
-                if task:
-                    task.status = result.task_status
-                    task.completed_at = datetime.now(UTC)
-                    if error_msg:
-                        task.error = error_msg
+                values: dict[str, object] = {
+                    "status": result.task_status,
+                    "completed_at": datetime.now(UTC),
+                }
+                if error_msg:
+                    values["error"] = error_msg
+                row_result = session.execute(
+                    update(Task)
+                    .where(
+                        Task.id == slot.task_id,
+                        Task.status == TaskStatus.RUNNING,
+                    )
+                    .values(**values)
+                )
+                if row_result.rowcount == 0:
+                    logger.warning(
+                        f"Task '{slot.name}' (id={slot.task_id}) was no longer"
+                        " RUNNING when reap tried to update — likely reaped"
+                    )
         except Exception as e:
             logger.error(f"Error updating task status: {e}")
 
@@ -1044,6 +1161,7 @@ def _run_concurrent(
     post_execute: PostExecuteHook | None,
     retention_days: int,
     cleanup_interval: float,
+    stale_task_timeout: timedelta | None = None,
 ) -> None:
     """Run the concurrent worker loop.
 
@@ -1060,10 +1178,12 @@ def _run_concurrent(
         post_execute: Called in forked child AFTER task execution.
         retention_days: Days to keep completed/failed tasks.
         cleanup_interval: Seconds between cleanup runs.
+        stale_task_timeout: If set, reap RUNNING tasks older than this.
     """
     children: dict[int, _ChildSlot] = {}  # pid -> slot
     fd_to_pid: dict[int, int] = {}  # read_fd -> pid
     last_cleanup: list[float] = [0.0]
+    last_reap: list[float] = [0.0]
 
     try:
         while True:
@@ -1092,8 +1212,11 @@ def _run_concurrent(
             else:
                 time.sleep(poll_interval)
 
-            # Cleanup runs on every iteration (rate-limited internally)
+            # Maintenance (each rate-limited independently)
             _maybe_run_cleanup(pq, retention_days, cleanup_interval, last_cleanup)
+            _maybe_reap_stale(
+                pq, stale_task_timeout, DEFAULT_REAPER_INTERVAL, last_reap
+            )
 
     except KeyboardInterrupt:
         if children:

{python_pq-0.7.0.dist-info → python_pq-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-pq
-Version: 0.7.0
+Version: 0.7.2
 Summary: Postgres-backed job queue for Python
 Keywords: postgres,job-queue,task-queue,background-jobs,worker
 Author: ricwo

{python_pq-0.7.0.dist-info → python_pq-0.7.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pq/__init__.py,sha256=MzrVLKualSjzTg3nVlpt9ooWEe-g8Vpc65H0zzorkBU,398
-pq/client.py,sha256=zZhcjq9dEHpB3elxJQDqylIhNlVY8BlUWc74YqyLkX8,19184
+pq/__init__.py,sha256=7j0ZYwlHBgi2fysJuYDAkNrFir6WC2m7irpoFtmu_lk,398
+pq/client.py,sha256=DBXZFEhgZ9Ynh554adtdvYK3vcFqHfwSHpb-s-h8j7U,27468
 pq/config.py,sha256=U7dW-LVHy4feLUDQAmxKSF8bavtwxMY4thYCNMR7nDc,293
 pq/logging.py,sha256=41kft5saLMLzzhHSstx27MSAHdFcd02_jxNGm8c-aoI,789
 pq/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
@@ -11,14 +11,15 @@ pq/migrations/versions/20260109T063747Z_2483bec70083_add_client_id.py,sha256=53h
 pq/migrations/versions/20260205T120000Z_a1b2c3d4e5f6_add_max_concurrent.py,sha256=gfhpPbcNb6JHnfPh3_K6An7EY5P2blYduQrp4khjJJg,1128
 pq/migrations/versions/20260205T180000Z_b7c8d9e0f1a2_add_periodic_key.py,sha256=W_sXMji-s6QObxxghDZf4nIeROWclc3QLpFaOIzDtH8,1137
 pq/migrations/versions/20260217T120000Z_c3d4e5f6a7b8_add_periodic_active.py,sha256=o5Wft7OkeXg6bAbLKwVSuwoIPBscuJr0ERcA1pVu_lI,758
+pq/migrations/versions/20260527T170136Z_aee3e8e7e647_add_task_max_runtime_seconds.py,sha256=DUAdNbVCR83eeZKIvqRg7nyT-fEU_HS7pR81yM5CMew,1708
 pq/migrations/versions/__init__.py,sha256=HtcCnjmL67QdG8QvzZXVOii80Q1cKNkuzNT3emD11oU,26
-pq/models.py,sha256=yoDf64MOLDzICF52AWI-oyTAIWj4hUWfR-ytTKehnlE,3659
+pq/models.py,sha256=rEboUf3hNPsf6acsIp_HU3xb5JeciVoLBAcfn-KmNLE,4972
 pq/priority.py,sha256=edqpQeGiJrxcmpJ5MZCvwTD9F6sIZbRoI1mx_LmUxAs,264
 pq/registry.py,sha256=1RFfB-dXXM2qKwLF6XeYeUm9buca1_fQvKQDnpfsXKs,2306
 pq/serialization.py,sha256=ZspG9qO4YYi-xCrtPGCg8yrzqRI2CGYnO_Ewjh8lE1E,3291
-pq/worker.py,sha256=Jf68AbTGCIo8vyAnEygd8Pakc2O-Pug9bd3SE3BaJX0,34664
-python_pq-0.7.0.dist-info/licenses/LICENSE,sha256=Bx9dUtjCpaluWGalMqkI3SGkpPxx_AYdJdmV5ptCM4g,1062
-python_pq-0.7.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-python_pq-0.7.0.dist-info/entry_points.txt,sha256=fpK4uA3gQaFpcDmdy5GFbP4VP7gaJcn4H4GJyMzRIM8,36
-python_pq-0.7.0.dist-info/METADATA,sha256=gk8x6hnGYVd7-k3tP1K6eQDsXfc7tldi7IcD8xCSc2Q,10141
-python_pq-0.7.0.dist-info/RECORD,,
+pq/worker.py,sha256=7TK27Du3elX1kHozNRG5-BUn2qOfz_W9X4t1YJDo4t0,39846
+python_pq-0.7.2.dist-info/licenses/LICENSE,sha256=Bx9dUtjCpaluWGalMqkI3SGkpPxx_AYdJdmV5ptCM4g,1062
+python_pq-0.7.2.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+python_pq-0.7.2.dist-info/entry_points.txt,sha256=fpK4uA3gQaFpcDmdy5GFbP4VP7gaJcn4H4GJyMzRIM8,36
+python_pq-0.7.2.dist-info/METADATA,sha256=PNIqmOY6EVbXSTvEF7Yd-c1IqWKh6NX1GgxIBaKWqoE,10141
+python_pq-0.7.2.dist-info/RECORD,,