celery-manager 0.1.0a2__py3-none-any.whl

celery_manager/__init__.py

@@ -0,0 +1,47 @@
+"""celery-manager — a dashboard and control API for Celery background tasks.
+
+The public surface spec §4 assumes, importable straight from the package
+root::
+
+    import celery_manager
+    from celery_manager import CeleryManagerConfig, setup, track, report_progress
+
+    celery_manager.setup(config)        # wire this process (worker or API)
+
+    @app.task(bind=True)
+    @celery_manager.track()
+    def send_report(self): ...
+
+`report_progress(current, total=None, message=None)` is called from *inside*
+a task body to publish live progress to the dashboard while the task is still
+running. It is throttled, tolerant of bad arguments, and — like everything
+else here — never raises into the task that called it::
+
+    @app.task(bind=True)
+    @celery_manager.track()
+    def import_rows(self, rows):
+        for i, row in enumerate(rows):
+            handle(row)
+            celery_manager.report_progress(i + 1, len(rows), f"row {i + 1}")
+
+`celery_manager.control` (task/schedule/audit operations) is re-exported here
+too, so `celery_manager.control.list_task_runs(...)` works without a separate
+import of the submodule.
+"""
+
+from celery_manager import control
+from celery_manager.config import CeleryManagerConfig
+from celery_manager.decorator import track
+from celery_manager.progress import report_progress
+from celery_manager.runtime import setup
+
+__version__ = "0.1.0a2"
+
+__all__ = [
+    "CeleryManagerConfig",
+    "__version__",
+    "control",
+    "report_progress",
+    "setup",
+    "track",
+]

celery_manager/api/app.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi import Depends, FastAPI, Request
+from fastapi.responses import StreamingResponse
+from fastapi.staticfiles import StaticFiles
+
+from celery_manager.api.deps import get_current_user
+from celery_manager.api.routes import (
+    audit as audit_routes,
+)
+from celery_manager.api.routes import (
+    auth as auth_routes,
+)
+from celery_manager.api.routes import (
+    schedules as schedule_routes,
+)
+from celery_manager.api.routes import (
+    settings as settings_routes,
+)
+from celery_manager.api.routes import (
+    tasks as task_routes,
+)
+from celery_manager.api.security import SecurityMiddleware
+from celery_manager.api.sse import EventBus, sse_stream
+from celery_manager.config import CeleryManagerConfig
+from celery_manager.control.schedules import sync_periodic_schedules
+from celery_manager.runtime import setup as setup_runtime
+
+# The placeholder `CeleryManagerConfig.secret_key` ships with. It is public —
+# it's in this file and the docs — so a session cookie signed with it can be
+# forged for any username by anyone.
+_DEFAULT_SECRET_KEY = "change-me"
+
+
+def create_app(config: CeleryManagerConfig):
+    """Build the celery-manager FastAPI app. Returns (app, session_factory)
+    — callers that need direct DB access (tests, the events listener CLI)
+    use the session_factory; HTTP consumers just use the app."""
+    # Checked here rather than in CeleryManagerConfig: worker and listener
+    # processes build the same config but never sign or verify a session, so
+    # they're free to leave secret_key alone. Only a process serving HTTP is
+    # a forgery risk, and it should refuse to start, not start insecurely.
+    if not config.secret_key or not config.secret_key.strip() or config.secret_key == _DEFAULT_SECRET_KEY:
+        raise ValueError(
+            "CeleryManagerConfig.secret_key is unset or still the default "
+            f"{_DEFAULT_SECRET_KEY!r}; anyone could forge a dashboard session with it. "
+            "Set it to a private random value, e.g. the output of "
+            '`python -c "import secrets; print(secrets.token_urlsafe(32))"`.'
+        )
+
+    # Single shared entry point for "wire celery-manager into this process"
+    # (init_db + session factory + decorator configure) rather than an inline
+    # copy of it, so the API process and the worker process — which must call
+    # `celery_manager.setup(config)` itself, see runtime.setup's docstring —
+    # can never drift out of sync.
+    session_factory = setup_runtime(config)
+
+    with session_factory() as session:
+        sync_periodic_schedules(session, config.celery_app)
+
+    app = FastAPI(title="celery-manager")
+    app.state.cm_config = config
+    app.state.cm_session_factory = session_factory
+    app.state.cm_event_bus = EventBus()
+    app.add_middleware(SecurityMiddleware)
+
+    app.include_router(auth_routes.router)
+    app.include_router(task_routes.router)
+    app.include_router(schedule_routes.router)
+    app.include_router(audit_routes.router)
+    app.include_router(settings_routes.router)
+
+    @app.get("/api/stream")
+    def stream(request: Request, username: str = Depends(get_current_user)):
+        return StreamingResponse(sse_stream(app.state.cm_event_bus), media_type="text/event-stream")
+
+    static_dir = Path(__file__).parent.parent / "dashboard" / "static"
+    if static_dir.exists():
+        app.mount("/", StaticFiles(directory=static_dir, html=True), name="dashboard")
+
+    return app, session_factory

celery_manager/api/deps.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from fastapi import Cookie, HTTPException, Request
+
+from celery_manager.auth.sessions import verify_session
+from celery_manager.auth.users import user_exists
+
+# Not plain `session`: celery-manager is usually mounted inside a host app,
+# and Flask's and Starlette's own session cookies are both called `session`
+# — two cookies of the same name on overlapping paths clobber each other.
+SESSION_COOKIE = "cm_session"
+
+
+def cookie_path(request: Request) -> str:
+    """Scope the session cookie to wherever this app is mounted, so a
+    dashboard under `/celery` doesn't send its credential to every other
+    path on the host. Must be identical on set and delete — a browser only
+    removes a cookie whose path matches the one it was set with."""
+    return request.scope.get("root_path") or "/"
+
+
+def get_current_user(request: Request, session: str | None = Cookie(default=None, alias=SESSION_COOKIE)) -> str:
+    config = request.app.state.cm_config
+    if session is None:
+        raise HTTPException(status_code=401, detail="not authenticated")
+    username = verify_session(config.secret_key, session)
+    if username is None:
+        raise HTTPException(status_code=401, detail="invalid or expired session")
+    # Sessions are stateless signed tokens, so a valid signature alone would
+    # keep a deleted account working for the rest of its 7 days.
+    with request.app.state.cm_session_factory() as db_session:
+        if not user_exists(db_session, username):
+            raise HTTPException(status_code=401, detail="invalid or expired session")
+    return username

celery_manager/api/routes/audit.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Request
+
+from celery_manager.api.deps import get_current_user
+from celery_manager.api.serialization import utc
+from celery_manager.control.tasks import MAX_PAGE_SIZE
+from celery_manager.db.models import AuditLog
+
+router = APIRouter(prefix="/api/audit")
+
+
+@router.get("")
+def list_audit(request: Request, limit: int = 50, username: str = Depends(get_current_user)):
+    """The most recent audit entries, newest first.
+
+    `limit` is clamped the same way `GET /api/tasks` clamps its own — and for
+    a sharper reason here: SQLite reads `LIMIT -1` as *no limit*, so
+    `?limit=-1` handed back the entire audit log in one response. Same
+    reasoning as the task list: clamp rather than reject, because a stale
+    bookmark deserves a capped page rather than a 400.
+    """
+    limit = max(1, min(limit, MAX_PAGE_SIZE))
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        rows = session.query(AuditLog).order_by(AuditLog.id.desc()).limit(limit).all()
+        return [
+            {
+                "username": r.username,
+                "action": r.action,
+                "target_type": r.target_type,
+                "target_id": r.target_id,
+                "details": r.details,
+                # Marked UTC for the same reason the task routes mark theirs:
+                # SQLite hands the value back naive, and the audit screen
+                # renders it with `new Date(...).toLocaleString()`, which reads
+                # an unmarked timestamp as local time and shifts every entry by
+                # the viewer's offset.
+                "created_at": utc(r.created_at),
+            }
+            for r in rows
+        ]

celery_manager/api/routes/auth.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from pydantic import BaseModel
+
+from celery_manager.api.deps import SESSION_COOKIE, cookie_path, get_current_user
+from celery_manager.auth.sessions import sign_session
+from celery_manager.auth.users import verify_password
+
+router = APIRouter(prefix="/api")
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+@router.post("/auth/login")
+def login(payload: LoginRequest, request: Request, response: Response):
+    config = request.app.state.cm_config
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as db_session:
+        user = verify_password(db_session, payload.username, payload.password)
+    if user is None:
+        raise HTTPException(status_code=401, detail="invalid credentials")
+    token = sign_session(config.secret_key, user.username)
+    response.set_cookie(
+        SESSION_COOKIE,
+        token,
+        path=cookie_path(request),
+        httponly=True,
+        samesite="lax",
+        secure=config.cookie_secure,
+    )
+    return {"username": user.username}
+
+
+@router.post("/auth/logout")
+def logout(request: Request, response: Response):
+    config = request.app.state.cm_config
+    # Same attributes as the login cookie: some browsers ignore a deletion
+    # whose Secure/SameSite don't match, and a mismatched path deletes nothing.
+    response.delete_cookie(
+        SESSION_COOKIE,
+        path=cookie_path(request),
+        httponly=True,
+        samesite="lax",
+        secure=config.cookie_secure,
+    )
+    return {"ok": True}
+
+
+@router.get("/whoami")
+def whoami(username: str = Depends(get_current_user)):
+    return {"username": username}

celery_manager/api/routes/schedules.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+
+from celery_manager.api.deps import get_current_user
+from celery_manager.control.schedules import (
+    list_periodic_schedules,
+    list_upcoming_with_overrides,
+    pause_occurrence,
+)
+
+router = APIRouter(prefix="/api/schedules")
+
+
+class PauseRequest(BaseModel):
+    target_time: datetime
+    # Deliberately narrower than ScheduleOverride.action's column, which also
+    # permits "reschedule". Nothing downstream implements reschedule — the
+    # decorator only ever acts on "skip"/"cancel" — so accepting the value
+    # here persisted an override that silently did nothing while the
+    # dashboard cheerfully displayed the occurrence as paused. Rejecting it
+    # at the boundary (Pydantic returns 422) is honest; storing a lie is not.
+    # Widen this the day reschedule is actually implemented, not before.
+    action: Literal["skip", "cancel"]
+
+
+@router.post("/{name}/occurrences/pause")
+def pause(name: str, payload: PauseRequest, request: Request, username: str = Depends(get_current_user)):
+    session_factory = request.app.state.cm_session_factory
+    eta_tolerance_seconds = request.app.state.cm_config.eta_tolerance_seconds
+    with session_factory() as session:
+        override = pause_occurrence(
+            session, name, payload.target_time, payload.action,
+            created_by=username, eta_tolerance_seconds=eta_tolerance_seconds,
+        )
+    return {"id": override.id, "action": override.action, "target_time": override.target_time}
+
+
+@router.get("/{name}/occurrences")
+def occurrences(name: str, request: Request, username: str = Depends(get_current_user)):
+    """Every upcoming occurrence for this schedule (not just already-paused
+    ones) so the dashboard has something to act on — see
+    list_upcoming_with_overrides in Task 6.
+
+    `list_upcoming_with_overrides` recomputes occurrence timestamps
+    relative to "now" on every call, so a `target_time` a user paused a
+    few seconds ago won't be byte-identical to the same logical
+    occurrence recomputed on this request. It handles that itself via
+    `eta_tolerance_seconds`-window matching (same concept as the
+    decorator's own skip-check window) rather than exact equality — we
+    just need to pass the configured tolerance through.
+    """
+    celery_app = request.app.state.cm_config.celery_app
+    session_factory = request.app.state.cm_session_factory
+    eta_tolerance_seconds = request.app.state.cm_config.eta_tolerance_seconds
+    with session_factory() as session:
+        return list_upcoming_with_overrides(session, celery_app, name, eta_tolerance_seconds=eta_tolerance_seconds)
+
+
+@router.get("")
+def list_schedules(request: Request, username: str = Depends(get_current_user)):
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        schedules = list_periodic_schedules(session)
+        return [
+            {"name": s.name, "task_name": s.task_name, "schedule_spec": s.schedule_spec, "enabled": s.enabled}
+            for s in schedules
+        ]

celery_manager/api/routes/settings.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, ConfigDict
+
+from celery_manager.api.deps import get_current_user
+from celery_manager.control.settings import get_settings, update_settings
+
+router = APIRouter(prefix="/api/settings")
+
+
+@router.get("")
+def read_settings(request: Request):
+    """The deployment's branding. **Deliberately unauthenticated.**
+
+    The login screen has to render this deployment's branding — its wordmark,
+    its accent, its logo — and the login screen is by definition pre-auth. An
+    authenticated endpoint here would mean the dashboard shows stock branding
+    until the moment you sign in, which defeats the feature for the one screen
+    an operator's users see most often.
+
+    That is safe only because the payload is non-sensitive by construction: a
+    product name, a hex colour and an image URL, and nothing else, because
+    `control.settings.DEFAULT_SETTINGS` is a closed allowlist. It is why that
+    allowlist must stay closed — see the comment on it before adding a key.
+
+    `updated_by` and `updated_at` are deliberately *not* returned. They are
+    the one genuinely sensitive thing this table holds: `updated_by` is a
+    valid username, and handing an unauthenticated visitor a real account name
+    turns a password-guessing attack from two unknowns into one. Nothing in
+    the UI needs them, so they are omitted rather than served behind a second
+    authenticated route that would exist only to be tidy.
+    """
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        return get_settings(session)
+
+
+class SettingsUpdate(BaseModel):
+    """PUT body. Every field optional, so a caller can send just the one it
+    is changing rather than read-modify-writing the whole set.
+
+    `extra="forbid"` so an unrecognized key is a 422 from pydantic, matching
+    what `update_settings` does with unknown keys when it is called directly
+    from Python. Silently dropping an unknown field would let a typo look like
+    a successful save.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    brand_name: str | None = None
+    brand_accent: str | None = None
+    brand_logo_url: str | None = None
+
+
+@router.put("")
+def write_settings(payload: SettingsUpdate, request: Request, username: str = Depends(get_current_user)):
+    """Update branding. Requires a session; returns the new effective settings.
+
+    Any authenticated user may rebrand the dashboard. There is no role system
+    in this app — `get_current_user` yields a username and nothing else — so
+    "admin only" is not expressible today without inventing one. That is an
+    accepted v1 limitation, not an oversight: the blast radius is cosmetic and
+    every change is attributed in the audit log, which is how you find out who
+    turned the nav pink.
+    """
+    # `exclude_unset` is what makes a partial update possible: it keeps the
+    # difference between "brand_accent was not in the JSON" (leave it alone)
+    # and "brand_accent was sent as null" (an explicit value, which the
+    # validators reject as a non-string rather than silently ignoring).
+    values = payload.model_dump(exclude_unset=True)
+
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        try:
+            return update_settings(session, values, username)
+        except ValueError as exc:
+            # 422, not 400: this is a well-formed request whose *content*
+            # failed validation, which is the same code FastAPI/pydantic
+            # already returns for a bad body — so a client has one error shape
+            # to handle, not two. The message is human-readable on purpose;
+            # it is rendered straight into the settings form.
+            raise HTTPException(status_code=422, detail=str(exc))

celery_manager/api/routes/tasks.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel
+
+from celery_manager.api.deps import get_current_user
+from celery_manager.api.serialization import utc, utc_bound
+from celery_manager.control.tasks import (
+    MAX_PAGE_SIZE,
+    TaskRunNotFound,
+    bulk_retry,
+    cancel_task,
+    count_task_runs,
+    get_task_run,
+    list_retries_of,
+    list_task_runs,
+    retry_task,
+    start_task,
+)
+
+router = APIRouter(prefix="/api/tasks")
+
+
+@router.get("")
+def list_tasks(
+    request: Request,
+    status: str | None = None,
+    task_name: str | None = None,
+    q: str | None = None,
+    date_from: datetime | None = None,
+    date_to: datetime | None = None,
+    limit: int = 50,
+    offset: int = 0,
+    username: str = Depends(get_current_user),
+):
+    """One page of task runs, plus the total matching the same filters.
+
+    Returns an envelope (`items`/`total`/`limit`/`offset`) rather than a bare
+    array. The array shape had no room for a total, which left the UI unable
+    to tell "50 results" from "the first 50 of 1,284" — and with the old hard
+    50-row cap and no offset, older history was simply unreachable. Changing
+    the shape is a deliberate break, taken while the package is unreleased at
+    0.1.0 and its only consumers are our own SPA and docs/rest-api.md.
+    """
+    # Clamp rather than reject: `?limit=10000000` is more often a stale
+    # bookmark than an attack, but either way it must not turn one HTTP
+    # request into a full-table load, and a 400 here would only give the
+    # caller a broken page instead of a capped one. Negative values are
+    # meaningless to OFFSET/LIMIT, so floor them too.
+    limit = max(1, min(limit, MAX_PAGE_SIZE))
+    offset = max(0, offset)
+
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        filters = {
+            "status": status, "task_name": task_name, "q": q,
+            "date_from": utc_bound(date_from), "date_to": utc_bound(date_to),
+        }
+        # The same filter dict feeds both calls, so `total` can never be a
+        # count of some other query than the one `items` came from.
+        runs = list_task_runs(session, limit=limit, offset=offset, **filters)
+        total = count_task_runs(session, **filters)
+        return {
+            "items": [
+                {
+                    "task_id": r.task_id,
+                    "task_name": r.task_name,
+                    "status": r.status,
+                    "started_at": utc(r.started_at),
+                    "finished_at": utc(r.finished_at),
+                    "runtime_seconds": r.runtime_seconds,
+                    # Progress is opt-in (only tasks that report it have any),
+                    # so these are usually null. Shipped raw: the frontend
+                    # derives percentage and ETA, because it is the side that
+                    # knows how much space it has to say it in.
+                    "progress_current": r.progress_current,
+                    "progress_total": r.progress_total,
+                    "progress_message": r.progress_message,
+                    "progress_updated_at": utc(r.progress_updated_at),
+                }
+                for r in runs
+            ],
+            "total": total,
+            "limit": limit,
+            "offset": offset,
+        }
+
+
+@router.get("/{task_id}")
+def get_task(task_id: str, request: Request, username: str = Depends(get_current_user)):
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        run = get_task_run(session, task_id)
+        if run is None:
+            raise HTTPException(status_code=404, detail="not found")
+        return {
+            "task_id": run.task_id,
+            "task_name": run.task_name,
+            "status": run.status,
+            "args": run.args,
+            "kwargs": run.kwargs,
+            "exception": run.exception,
+            "traceback": run.traceback,
+            "log_output": run.log_output,
+            # Operational detail already stored on TaskRun (by the events
+            # listener for worker/queue, by the decorator for the timings) but
+            # never surfaced here — so the dashboard's detail view could not
+            # answer "which worker ran this, off which queue, and how long did
+            # it take". The list endpoint returned started_at/finished_at while
+            # this one didn't, which also made the frontend's TaskDetail type
+            # (extending TaskSummary) claim fields the response never sent.
+            "queue": run.queue,
+            "worker": run.worker,
+            "eta": utc(run.eta),
+            "started_at": utc(run.started_at),
+            "finished_at": utc(run.finished_at),
+            "runtime_seconds": run.runtime_seconds,
+            "progress_current": run.progress_current,
+            "progress_total": run.progress_total,
+            "progress_message": run.progress_message,
+            "progress_updated_at": utc(run.progress_updated_at),
+            # Both directions of the retry chain: the run this one was
+            # retried from, and the runs retried from it. `retried_by` is a
+            # list because an operator can retry the same failed run more than
+            # once, and each attempt is its own run.
+            "retried_from_id": run.retried_from_id,
+            "retried_by": [r.task_id for r in list_retries_of(session, task_id)],
+        }
+
+
+class StartRequest(BaseModel):
+    task_name: str
+    args: list = []
+    kwargs: dict = {}
+
+
+@router.post("/start")
+def start(payload: StartRequest, request: Request, username: str = Depends(get_current_user)):
+    celery_app = request.app.state.cm_config.celery_app
+    session_factory = request.app.state.cm_session_factory
+    # session + username are what make this action land in the audit log,
+    # same as the retry route below.
+    with session_factory() as session:
+        task_id = start_task(
+            celery_app, payload.task_name, payload.args, payload.kwargs,
+            session=session, username=username,
+        )
+    return {"task_id": task_id}
+
+
+@router.post("/{task_id}/cancel")
+def cancel(task_id: str, request: Request, terminate: bool = False, username: str = Depends(get_current_user)):
+    celery_app = request.app.state.cm_config.celery_app
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        cancel_task(celery_app, task_id, terminate=terminate, session=session, username=username)
+    return {"ok": True}
+
+
+@router.post("/{task_id}/retry")
+def retry(task_id: str, request: Request, username: str = Depends(get_current_user)):
+    """Re-dispatch a stored run. 404 if there is no such run.
+
+    A retry of an unknown task id used to escape as an unhandled `ValueError`
+    and reach the client as a 500 — which reads as "the dashboard is broken"
+    rather than "that task id doesn't exist", and is indistinguishable from a
+    genuine server fault to anyone reading logs or alerting on 5xx. It is the
+    same condition `GET /api/tasks/{task_id}` already answers with a 404, so
+    it gets the same status. Only `TaskRunNotFound` is translated: a
+    `ValueError` from undecodable stored args is a real server-side problem
+    and must keep surfacing as one.
+    """
+    celery_app = request.app.state.cm_config.celery_app
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        try:
+            new_id = retry_task(celery_app, session, task_id, username=username)
+        except TaskRunNotFound:
+            raise HTTPException(status_code=404, detail="not found")
+    return {"new_task_id": new_id}
+
+
+class BulkRetryRequest(BaseModel):
+    task_ids: list[str]
+
+
+@router.post("/retry-bulk")
+def retry_bulk(payload: BulkRetryRequest, request: Request, username: str = Depends(get_current_user)):
+    """Re-dispatch several stored runs. 404 if any of them is unknown.
+
+    `bulk_retry` stops at the first unknown id (documented limitation — it is
+    not partial-failure aware), so an unknown id in the batch is reported the
+    same way a single unknown id is, rather than as a 500.
+    """
+    celery_app = request.app.state.cm_config.celery_app
+    session_factory = request.app.state.cm_session_factory
+    with session_factory() as session:
+        try:
+            new_ids = bulk_retry(celery_app, session, payload.task_ids, username=username)
+        except TaskRunNotFound as exc:
+            raise HTTPException(status_code=404, detail=str(exc))
+    return {"new_task_ids": new_ids}