simple-module-background-tasks 0.0.1__py3-none-any.whl

background_tasks/pages/retry.ts

@@ -0,0 +1,42 @@
+// Shared retry plumbing for the Index and Detail pages. Both POST to the
+// same endpoint; they differ only in what they do with the result (reload
+// the list vs. navigate to the new row), so the fetch + toast live here.
+
+import { toast } from 'sonner';
+import { API_BASE, type TaskStatus } from './constants';
+
+export interface Execution {
+  id: string;
+  celery_task_id: string | null;
+  task_name: string;
+  status: TaskStatus;
+  queue: string;
+  retries: number;
+  worker: string | null;
+  queued_at: string | null;
+  started_at: string | null;
+  finished_at: string | null;
+  exception_type: string | null;
+  retried_from_id: string | null;
+}
+
+export async function retryExecution(execution: {
+  id: string;
+  task_name: string;
+}): Promise<Execution | null> {
+  try {
+    const res = await fetch(`${API_BASE}/executions/${execution.id}/retry`, {
+      method: 'POST',
+    });
+    if (!res.ok) {
+      const body = await res.json().catch(() => ({}));
+      throw new Error(body.detail || `HTTP ${res.status}`);
+    }
+    const created = (await res.json()) as Execution;
+    toast.success(`Task "${execution.task_name}" re-enqueued`);
+    return created;
+  } catch (err) {
+    toast.error(err instanceof Error ? err.message : 'Failed to retry task');
+    return null;
+  }
+}

background_tasks/service.py

@@ -0,0 +1,138 @@
+"""BackgroundTaskService — admin listing, detail, and retry."""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING
+
+from fastapi import HTTPException, status
+from simple_module_core.events import EventBus
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from background_tasks.constants import RETRYABLE_STATUSES, TaskStatus
+from background_tasks.contracts.events import TaskRetried
+from background_tasks.contracts.schemas import (
+    TaskExecutionDetail,
+    TaskExecutionListItem,
+    TaskExecutionListResponse,
+)
+from background_tasks.models import TaskExecution
+
+if TYPE_CHECKING:
+    from celery import Celery
+
+
+class BackgroundTaskService:
+    """List, fetch, and retry task executions.
+
+    The service treats the DB as the system of record and the Celery app
+    as pure transport — ``retry`` loads the original row, enqueues a new
+    Celery task with the same args/kwargs, and inserts a fresh
+    ``TaskExecution`` linked to the original via ``retried_from_id``.
+    """
+
+    def __init__(
+        self,
+        db: AsyncSession,
+        celery: Celery,
+        event_bus: EventBus,
+    ) -> None:
+        self.db = db
+        self.celery = celery
+        self.event_bus = event_bus
+
+    async def list(
+        self,
+        *,
+        status: TaskStatus | None = None,
+        task_name: str | None = None,
+        page: int = 1,
+        per_page: int = 20,
+    ) -> TaskExecutionListResponse:
+        """Return a paginated listing, newest-first, with optional filters."""
+        page = max(page, 1)
+        per_page = max(1, min(per_page, 200))
+
+        # A window-function total lets us fetch the page and the count in a
+        # single round trip. SQLAlchemy's ``AsyncSession`` serialises calls on
+        # one connection, so ``asyncio.gather`` wouldn't help here.
+        total_col = func.count().over().label("_total")
+        query = select(TaskExecution, total_col)
+        if status is not None:
+            query = query.where(TaskExecution.status == status)
+        if task_name:
+            query = query.where(TaskExecution.task_name.ilike(f"%{task_name}%"))
+        query = (
+            query.order_by(TaskExecution.queued_at.desc().nulls_last())
+            .offset((page - 1) * per_page)
+            .limit(per_page)
+        )
+
+        result = (await self.db.execute(query)).all()
+        items = [TaskExecutionListItem.model_validate(row[0]) for row in result]
+        total = int(result[0][1]) if result else 0
+
+        return TaskExecutionListResponse(
+            items=items,
+            total=total,
+            page=page,
+            per_page=per_page,
+            status=status,
+            task_name=task_name,
+        )
+
+    async def get(self, execution_id: uuid.UUID) -> TaskExecutionDetail | None:
+        row = await self.db.get(TaskExecution, execution_id)
+        if row is None:
+            return None
+        return TaskExecutionDetail.model_validate(row)
+
+    async def retry(self, execution_id: uuid.UUID) -> TaskExecutionDetail:
+        """Re-enqueue a failed or stuck task.
+
+        The original row is immutable; we insert a new ``TaskExecution`` row
+        carrying ``retried_from_id`` so the detail page can show the chain.
+        """
+        row = await self.db.get(TaskExecution, execution_id)
+        if row is None:
+            raise HTTPException(status_code=404, detail="Task execution not found")
+
+        if row.status not in RETRYABLE_STATUSES:
+            raise HTTPException(
+                status_code=status.HTTP_409_CONFLICT,
+                detail=(
+                    f"Task execution status is {str(row.status)!r}; "
+                    "only failed or stuck tasks can be retried."
+                ),
+            )
+
+        async_result = self.celery.send_task(
+            row.task_name,
+            args=list(row.args or []),
+            kwargs=dict(row.kwargs or {}),
+            queue=row.queue,
+        )
+
+        new_row = TaskExecution(
+            celery_task_id=async_result.id,
+            task_name=row.task_name,
+            status=TaskStatus.PENDING,
+            queue=row.queue,
+            args=list(row.args or []),
+            kwargs=dict(row.kwargs or {}),
+            retried_from_id=row.id,
+        )
+        self.db.add(new_row)
+        await self.db.flush()
+        await self.db.refresh(new_row)
+
+        await self.event_bus.publish(
+            TaskRetried(
+                original_id=row.id,
+                new_id=new_row.id,
+                task_name=row.task_name,
+            )
+        )
+
+        return TaskExecutionDetail.model_validate(new_row)

background_tasks/services.py

@@ -0,0 +1,25 @@
+"""Module-scoped state container for background_tasks.
+
+Stored as ``app.state.background_tasks`` by
+:meth:`BackgroundTasksModule.register_settings`. Not frozen — ``on_startup``
+populates :attr:`celery` once the Celery app is built. Convention: set once
+during boot, treat as read-only after.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from celery import Celery
+
+    from background_tasks.settings import BackgroundTasksSettings
+
+
+@dataclass
+class BackgroundTasksServices:
+    """BackgroundTasks singletons. Single slot at ``app.state.background_tasks``."""
+
+    settings: BackgroundTasksSettings
+    celery: Celery | None = None

background_tasks/settings.py

@@ -0,0 +1,83 @@
+"""BackgroundTasks module settings (DB-backed).
+
+Construction no longer reads ``SM_BG_TASKS_*`` environment variables. Values
+come from pydantic defaults at boot, then get hydrated from the DB by the
+hosting lifespan before module ``on_startup`` runs. Runtime changes go
+through ``settings.reload.apply_changes_and_reload``.
+
+The one remaining env read is ``SM_ENVIRONMENT``, consulted by the
+``@model_validator`` to refuse a localhost broker in production — that's a
+host-level setting, not a background_tasks-module field.
+
+The Celery-critical fields (``broker_url``, ``result_backend``,
+``task_default_queue``) are marked ``requires_restart=True`` via
+``json_schema_extra`` because workers read these once at process start, so
+DB changes can't be hot-reloaded: the admin UI should surface that bumping
+these values requires a worker restart.
+"""
+
+from __future__ import annotations
+
+import os
+
+from pydantic import Field, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from simple_module_core.environments import NON_PROD_ENVIRONMENTS
+
+from background_tasks.constants import (
+    DEFAULT_BROKER_URL,
+    DEFAULT_MAX_RETRIES,
+    DEFAULT_PURGE_INTERVAL_SECONDS,
+    DEFAULT_QUEUE,
+    DEFAULT_RESULT_BACKEND,
+    DEFAULT_RETENTION_DAYS,
+    DEFAULT_STUCK_AFTER_SECONDS,
+    DEFAULT_STUCK_SWEEP_INTERVAL_SECONDS,
+)
+
+_CELERY_RESTART = {"requires_restart": True, "group": "Celery"}
+
+
+class BackgroundTasksSettings(BaseSettings):
+    """Configuration for the Celery + Redis task runner."""
+
+    model_config = SettingsConfigDict(extra="ignore")
+
+    broker_url: str = Field(default=DEFAULT_BROKER_URL, json_schema_extra=_CELERY_RESTART)
+    result_backend: str = Field(default=DEFAULT_RESULT_BACKEND, json_schema_extra=_CELERY_RESTART)
+    task_default_queue: str = Field(default=DEFAULT_QUEUE, json_schema_extra=_CELERY_RESTART)
+
+    # A task that has been ``running`` longer than this without a heartbeat is
+    # flipped to ``stuck`` by the beat sweep. 5 min is long enough to cover
+    # normal slow jobs while still surfacing wedged workers within one UI
+    # refresh.
+    stuck_after_seconds: int = DEFAULT_STUCK_AFTER_SECONDS
+    stuck_sweep_interval_seconds: int = DEFAULT_STUCK_SWEEP_INTERVAL_SECONDS
+    purge_interval_seconds: int = DEFAULT_PURGE_INTERVAL_SECONDS
+
+    retention_days: int = DEFAULT_RETENTION_DAYS
+    max_retries: int = DEFAULT_MAX_RETRIES
+
+    @model_validator(mode="after")
+    def _forbid_localhost_broker_in_production(self) -> BackgroundTasksSettings:
+        """Fail boot if production is still pointed at the dev default broker.
+
+        A localhost broker in prod means the web container is talking to its
+        own 6379 instead of the shared Redis service — tasks would silently
+        queue to a broker no worker reads.
+        """
+        env = os.environ.get("SM_ENVIRONMENT", "development")
+        if env in NON_PROD_ENVIRONMENTS:
+            return self
+        bad = []
+        if "localhost" in self.broker_url or "127.0.0.1" in self.broker_url:
+            bad.append("broker_url")
+        if "localhost" in self.result_backend or "127.0.0.1" in self.result_backend:
+            bad.append("result_backend")
+        if bad:
+            names = ", ".join(bad)
+            raise ValueError(
+                f"{names} must not point at localhost when SM_ENVIRONMENT={env!r}. "
+                "Set these to the Redis service host (e.g. redis://redis:6379/0)."
+            )
+        return self

background_tasks/signals.py

@@ -0,0 +1,286 @@
+"""Celery signal handlers that keep the ``TaskExecution`` table in sync.
+
+Each signal writes one row: publish creates ``pending``, prerun flips to
+``running`` with a heartbeat, postrun/success/failure/retry/revoked update
+the terminal columns. We update an existing row (matched by
+``celery_task_id``) so a single task's lifecycle stays on one row instead
+of spawning a new row per signal.
+
+Signals are sync — see :mod:`.sync_db` for why we maintain a separate sync
+engine, and :mod:`._signal_support` for the shared helpers.
+
+``TaskFailed`` is also dispatched from :func:`on_task_failure` when an event
+bus has been bound via :func:`bind_event_bus` — typically from the web
+process's ``on_startup``. In a standalone Celery worker (separate process)
+no bus is bound and the publish becomes a no-op; subscribers that need
+cross-process notification should consume Celery events or poll the DB
+directly.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from celery import signals
+from simple_module_core.events import EventBus
+
+from background_tasks._signal_support import (
+    coerce_args_kwargs,
+    jsonable_result,
+    now_utc,
+    render_traceback,
+    task_id_from,
+    task_name_of,
+    upsert_by_celery_id,
+)
+from background_tasks.constants import DEFAULT_QUEUE, TaskStatus
+from background_tasks.contracts.events import TaskFailed
+from background_tasks.models import TaskExecution
+from background_tasks.sync_db import sync_session
+
+logger = logging.getLogger(__name__)
+
+
+_bus: EventBus | None = None
+_loop: asyncio.AbstractEventLoop | None = None
+
+
+def bind_event_bus(bus: EventBus, loop: asyncio.AbstractEventLoop) -> None:
+    """Bind an event bus + its running loop so signals can publish events.
+
+    Signals fire on the Celery sync thread; `run_coroutine_threadsafe`
+    bridges back to ``loop`` so handlers run on the API event loop
+    regardless of which thread triggered the signal.
+    """
+    global _bus, _loop
+    _bus = bus
+    _loop = loop
+
+
+def unbind_event_bus() -> None:
+    """Drop the bound bus — called from ``on_shutdown`` so tests stay isolated."""
+    global _bus, _loop
+    _bus = None
+    _loop = None
+
+
+def _publish_from_signal(event: Any) -> None:
+    """Dispatch ``event`` onto the bound bus without blocking the signal thread."""
+    if _bus is None or _loop is None:
+        return
+    try:
+        future = asyncio.run_coroutine_threadsafe(_bus.publish(event), _loop)
+    except RuntimeError:
+        # Loop has stopped (shutdown race). The DB row is already written.
+        logger.debug("Event bus loop is not running; skipping %s", type(event).__name__)
+        return
+    # Surface subscriber exceptions — run_coroutine_threadsafe otherwise only
+    # logs them when the Future is GC'd, which happens far from the failure.
+    future.add_done_callback(_log_publish_failure)
+
+
+def _log_publish_failure(future: asyncio.Future[Any]) -> None:
+    if future.cancelled():
+        return
+    exc = future.exception()
+    if exc is not None:
+        logger.error("Event publish raised: %s", exc, exc_info=exc)
+
+
+def _apply(
+    handler: str,
+    *,
+    celery_task_id: str | None,
+    defaults: dict[str, Any],
+    after: Callable[[TaskExecution], None] | None = None,
+) -> TaskExecution | None:
+    """Open a sync session, upsert by celery_task_id, log on failure.
+
+    Returns the upserted row (or ``None`` if the session raised) so callers
+    can read DB-assigned fields like ``id`` without a second round trip.
+    """
+    try:
+        with sync_session() as session:
+            row = upsert_by_celery_id(session, celery_task_id=celery_task_id, defaults=defaults)
+            if after is not None:
+                after(row)
+            return row
+    except Exception:
+        logger.exception("%s failed for task_id=%s", handler, celery_task_id)
+        return None
+
+
+# ── Enqueue ─────────────────────────────────────────────────────
+
+
+@signals.before_task_publish.connect
+def on_task_publish(
+    sender: str | None = None,
+    headers: dict[str, Any] | None = None,
+    body: Any = None,
+    routing_key: str | None = None,
+    **_kwargs: Any,
+) -> None:
+    """Record a row the moment a task is pushed onto the broker."""
+    task_id = (headers or {}).get("id")
+    task_name = sender or (headers or {}).get("task") or "unknown"
+
+    # ``body`` on the publish signal is ``(args, kwargs, options)`` for the
+    # standard Celery protocol; guard against edge shapes.
+    args_in, kwargs_in = [], {}
+    if isinstance(body, list | tuple) and len(body) >= 2:
+        args_in, kwargs_in = body[0], body[1]
+    args, kwargs = coerce_args_kwargs(args_in, kwargs_in)
+
+    _apply(
+        "on_task_publish",
+        celery_task_id=task_id,
+        defaults={
+            "task_name": task_name,
+            "status": TaskStatus.PENDING,
+            "queue": routing_key or DEFAULT_QUEUE,
+            "args": args,
+            "kwargs": kwargs,
+            "queued_at": now_utc(),
+        },
+    )
+
+
+# ── Execution lifecycle ─────────────────────────────────────────
+
+
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any = None,
+    task_id: str | None = None,
+    task: Any = None,
+    args: Any = None,
+    kwargs: Any = None,
+    **_k: Any,
+) -> None:
+    """Flip the row to ``running`` and start the heartbeat."""
+    args_n, kwargs_n = coerce_args_kwargs(args, kwargs)
+    now = now_utc()
+    _apply(
+        "on_task_prerun",
+        celery_task_id=task_id,
+        defaults={
+            "task_name": task_name_of(sender, task),
+            "status": TaskStatus.RUNNING,
+            "args": args_n,
+            "kwargs": kwargs_n,
+            "started_at": now,
+            "heartbeat_at": now,
+        },
+    )
+
+
+@signals.task_postrun.connect
+def on_task_postrun(
+    sender: Any = None,
+    task_id: str | None = None,
+    task: Any = None,
+    **_k: Any,
+) -> None:
+    """Refresh the heartbeat on normal completion.
+
+    Terminal status is written by ``task_success`` / ``task_failure`` /
+    ``task_retry`` which fire *before* postrun; postrun only refreshes the
+    heartbeat so the sweep doesn't immediately flip a just-finished row.
+    """
+    _apply(
+        "on_task_postrun",
+        celery_task_id=task_id,
+        defaults={"task_name": task_name_of(sender, task), "heartbeat_at": now_utc()},
+    )
+
+
+@signals.task_success.connect
+def on_task_success(sender: Any = None, result: Any = None, **_k: Any) -> None:
+    _apply(
+        "on_task_success",
+        celery_task_id=task_id_from(sender=sender),
+        defaults={
+            "task_name": task_name_of(sender),
+            "status": TaskStatus.SUCCESS,
+            "result": jsonable_result(result),
+            "finished_at": now_utc(),
+            "traceback": None,
+            "exception_type": None,
+        },
+    )
+
+
+@signals.task_failure.connect
+def on_task_failure(
+    sender: Any = None,
+    task_id: str | None = None,
+    exception: BaseException | None = None,
+    einfo: Any = None,
+    **_k: Any,
+) -> None:
+    task_name = task_name_of(sender)
+    exception_type = type(exception).__name__ if exception is not None else None
+
+    row = _apply(
+        "on_task_failure",
+        celery_task_id=task_id,
+        defaults={
+            "task_name": task_name,
+            "status": TaskStatus.FAILED,
+            "traceback": render_traceback(einfo, exception),
+            "exception_type": exception_type,
+            "finished_at": now_utc(),
+        },
+    )
+
+    if row is not None:
+        _publish_from_signal(
+            TaskFailed(
+                task_execution_id=row.id,
+                task_name=task_name,
+                exception_type=exception_type,
+            )
+        )
+
+
+@signals.task_retry.connect
+def on_task_retry(
+    sender: Any = None,
+    request: Any = None,
+    reason: Any = None,
+    **_k: Any,
+) -> None:
+    def _stamp_reason(row: TaskExecution) -> None:
+        # Preserve the latest reason for the UI without nulling a
+        # previously-captured traceback.
+        if reason is not None:
+            row.traceback = f"retry: {reason!r}"
+
+    _apply(
+        "on_task_retry",
+        celery_task_id=task_id_from(request=request),
+        defaults={
+            "task_name": task_name_of(sender),
+            "status": TaskStatus.RETRYING,
+            "retries": int(getattr(request, "retries", 0) or 0),
+            "heartbeat_at": now_utc(),
+        },
+        after=_stamp_reason,
+    )
+
+
+@signals.task_revoked.connect
+def on_task_revoked(sender: Any = None, request: Any = None, **_k: Any) -> None:
+    _apply(
+        "on_task_revoked",
+        celery_task_id=task_id_from(request=request),
+        defaults={
+            "task_name": task_name_of(sender),
+            "status": TaskStatus.REVOKED,
+            "finished_at": now_utc(),
+        },
+    )

background_tasks/sync_db.py

@@ -0,0 +1,82 @@
+"""Sync SQLAlchemy session factory for Celery signal handlers.
+
+Celery signals are called synchronously from inside the worker / web-process
+hot path. Building an event-loop just to await a query is both ugly and
+deadlock-prone (web-process signals can fire from inside an already-running
+loop). We instead maintain a second, sync engine pointed at the same DB URL
+and borrow a short-lived session from it whenever a signal fires.
+
+The engine is lazily built on first use and process-global. It's cheap —
+SQLAlchemy's connection pool is per-process so the signal path reuses
+connections after the first signal.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session, sessionmaker
+
+logger = logging.getLogger(__name__)
+
+_engine: Engine | None = None
+_session_factory: sessionmaker[Session] | None = None
+
+
+def _sync_url(async_url: str) -> str:
+    """Convert an async SQLAlchemy URL to its sync driver equivalent.
+
+    Mirrors ``host/migrations/env.py`` so signals use the same URL shape
+    Alembic does.
+    """
+    return async_url.replace("+aiosqlite", "").replace("+asyncpg", "+psycopg2")
+
+
+def _build_engine() -> Engine:
+    url = os.environ.get("SM_DATABASE_URL", "sqlite:///./app.db")
+    sync_url = _sync_url(url)
+    # Small pool — signals fire sequentially per worker process.
+    return create_engine(sync_url, pool_pre_ping=True, pool_size=2, max_overflow=3)
+
+
+def get_sync_session_factory() -> sessionmaker[Session]:
+    """Return the process-global sync session factory, building it once."""
+    global _engine, _session_factory
+    if _session_factory is None:
+        _engine = _build_engine()
+        _session_factory = sessionmaker(bind=_engine, expire_on_commit=False)
+    return _session_factory
+
+
+def dispose_sync_engine() -> None:
+    """Release pooled connections and drop the cached engine.
+
+    Called from :meth:`BackgroundTasksModule.on_shutdown` so lifespan
+    restarts within one process (test runners, uvicorn dev reload) don't
+    accumulate engines against the old DB URL.
+    """
+    global _engine, _session_factory
+    if _engine is not None:
+        _engine.dispose()
+    _engine = None
+    _session_factory = None
+
+
+@contextmanager
+def sync_session() -> Iterator[Session]:
+    """Open a short-lived sync session; commit on success, rollback on error."""
+    factory = get_sync_session_factory()
+    session = factory()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()