percival-server 0.1.0__py3-none-any.whl

percival_server/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

percival_server/__main__.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import os
+import sys
+
+import uvicorn
+
+from percival.naming import ENV_PREFIX
+from percival_server import logs
+from percival_server.config import ConfigError, ServerConfig
+from percival_server.main import create_app
+
+
+def main() -> None:
+    logs.setup(os.environ.get(ENV_PREFIX + "LOG_LEVEL", "INFO"))
+    try:
+        app = create_app(ServerConfig.from_env())
+    except ConfigError as exc:
+        print(f"percival-server: {exc}", file=sys.stderr)
+        sys.exit(2)
+
+    uvicorn.run(
+        app,
+        host=os.environ.get(ENV_PREFIX + "HOST", "127.0.0.1"),
+        port=int(os.environ.get(ENV_PREFIX + "PORT", "8080")),
+        log_config=None,
+        proxy_headers=True,
+    )
+
+
+if __name__ == "__main__":
+    main()

percival_server/api/accounts.py

@@ -0,0 +1,228 @@
+"""Users, API tokens and connection settings. Logged-in users only."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends, HTTPException, Request, status
+from pydantic import BaseModel, Field
+from sqlalchemy import delete, func, select
+from sqlalchemy.orm import Session
+
+from percival_server import audit
+from percival_server.auth.deps import ALL_SCOPES, Principal, require_user
+from percival_server.auth.security import (
+    TOKEN_PREFIX,
+    hash_password,
+    hash_token,
+    new_api_token,
+    password_problem,
+    verify_password,
+)
+from percival_server.connections import ENV_FIELDS, ConnectionSettings
+from percival_server.db import get_db
+from percival_server.models import ApiToken, User
+from percival_server.models import Session as UserSession
+
+router = APIRouter(tags=["accounts"])
+
+
+# users
+
+
+class UserOut(BaseModel):
+    id: int
+    username: str
+    created_at: datetime
+    last_login_at: datetime | None
+
+
+class NewUser(BaseModel):
+    username: str = Field(min_length=1, max_length=150)
+    password: str
+
+
+class PasswordChange(BaseModel):
+    current_password: str
+    new_password: str
+
+
+def _check_password(password: str) -> None:
+    problem = password_problem(password)
+    if problem:
+        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, problem)
+
+
+@router.get("/users", response_model=list[UserOut])
+def list_users(db: Session = Depends(get_db), _: Principal = Depends(require_user)) -> list[User]:
+    return list(db.scalars(select(User).order_by(User.username)))
+
+
+@router.post("/users", response_model=UserOut, status_code=status.HTTP_201_CREATED)
+def create_user(
+    body: NewUser, db: Session = Depends(get_db), who: Principal = Depends(require_user)
+) -> User:
+    _check_password(body.password)
+    username = body.username.strip()
+    if db.scalar(select(User).where(User.username == username)):
+        raise HTTPException(status.HTTP_409_CONFLICT, "that username is taken")
+    user = User(username=username, password_hash=hash_password(body.password))
+    db.add(user)
+    audit.record(db, who, "user.create", username)
+    db.commit()
+    return user
+
+
+@router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT)
+def delete_user(
+    user_id: int, db: Session = Depends(get_db), who: Principal = Depends(require_user)
+) -> None:
+    user = db.get(User, user_id)
+    if user is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "no such user")
+    if user.id == who.user_id:
+        raise HTTPException(status.HTTP_409_CONFLICT, "you can't delete yourself")
+    if (db.scalar(select(func.count()).select_from(User)) or 0) <= 1:
+        raise HTTPException(status.HTTP_409_CONFLICT, "can't delete the last user")
+    audit.record(db, who, "user.delete", user.username)
+    db.delete(user)
+    db.commit()
+
+
+@router.post("/users/me/password", status_code=status.HTTP_204_NO_CONTENT)
+def change_password(
+    body: PasswordChange, db: Session = Depends(get_db), who: Principal = Depends(require_user)
+) -> None:
+    user = db.get(User, who.user_id)
+    if user is None or not verify_password(user.password_hash, body.current_password):
+        raise HTTPException(status.HTTP_403_FORBIDDEN, "current password is wrong")
+    _check_password(body.new_password)
+    user.password_hash = hash_password(body.new_password)
+    # log out everywhere else
+    db.execute(delete(UserSession).where(UserSession.user_id == user.id))
+    audit.record(db, who, "user.password", user.username)
+    db.commit()
+
+
+# tokens
+
+
+class TokenOut(BaseModel):
+    id: int
+    name: str
+    prefix: str
+    scopes: list[str]
+    created_at: datetime
+    last_used_at: datetime | None
+    revoked_at: datetime | None
+
+
+class NewToken(BaseModel):
+    name: str = Field(min_length=1, max_length=100)
+    scopes: list[str] = Field(default_factory=lambda: ["read"])
+
+
+class CreatedToken(TokenOut):
+    token: str
+
+
+def _token_out(t: ApiToken) -> dict[str, object]:
+    return {
+        "id": t.id,
+        "name": t.name,
+        "prefix": t.prefix,
+        "scopes": sorted(t.scope_set),
+        "created_at": t.created_at,
+        "last_used_at": t.last_used_at,
+        "revoked_at": t.revoked_at,
+    }
+
+
+@router.get("/tokens", response_model=list[TokenOut])
+def list_tokens(
+    db: Session = Depends(get_db), _: Principal = Depends(require_user)
+) -> list[dict[str, object]]:
+    rows = db.scalars(select(ApiToken).order_by(ApiToken.created_at.desc()))
+    return [_token_out(t) for t in rows]
+
+
+@router.post("/tokens", response_model=CreatedToken, status_code=status.HTTP_201_CREATED)
+def create_token(
+    body: NewToken, db: Session = Depends(get_db), who: Principal = Depends(require_user)
+) -> dict[str, object]:
+    scopes = set(body.scopes)
+    if not scopes or not scopes <= set(ALL_SCOPES):
+        raise HTTPException(
+            status.HTTP_422_UNPROCESSABLE_ENTITY, "scopes must be read and/or write"
+        )
+    if "write" in scopes:
+        scopes.add("read")
+
+    raw = new_api_token()
+    token = ApiToken(
+        name=body.name.strip(),
+        prefix=raw[: len(TOKEN_PREFIX) + 6],
+        token_hash=hash_token(raw),
+        scopes=",".join(sorted(scopes)),
+        created_by_id=who.user_id,
+    )
+    db.add(token)
+    audit.record(db, who, "token.create", token.name, after={"scopes": sorted(scopes)})
+    db.commit()
+    # the only time the full token is ever returned
+    return {**_token_out(token), "token": raw}
+
+
+@router.delete("/tokens/{token_id}", status_code=status.HTTP_204_NO_CONTENT)
+def revoke_token(
+    token_id: int, db: Session = Depends(get_db), who: Principal = Depends(require_user)
+) -> None:
+    token = db.get(ApiToken, token_id)
+    if token is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "no such token")
+    if token.revoked_at is None:
+        token.revoked_at = datetime.now(timezone.utc)
+        audit.record(db, who, "token.revoke", token.name)
+        db.commit()
+
+
+# connection settings
+
+
+class ConnectionsIn(BaseModel):
+    # leave a url out to keep the stored one, so the masked value never has
+    # to round-trip through the browser
+    broker_url: str | None = None
+    redis_url: str | None = None
+    redbeat_key_prefix: str | None = None
+    key_prefix: str | None = None
+    retention_days: int | None = Field(default=None, ge=1, le=3650)
+
+
+@router.get("/settings/connections")
+def get_connections(request: Request, _: Principal = Depends(require_user)) -> dict[str, object]:
+    conns = request.app.state.connections
+    return {**conns.settings.masked(), "from_env": conns.from_env}
+
+
+@router.put("/settings/connections")
+def update_connections(
+    body: ConnectionsIn,
+    request: Request,
+    db: Session = Depends(get_db),
+    who: Principal = Depends(require_user),
+) -> dict[str, object]:
+    conns = request.app.state.connections
+    current: ConnectionSettings = conns.settings
+    changes = body.model_dump(exclude_none=True)
+    locked = sorted(set(changes) & ENV_FIELDS) if conns.from_env else []
+    if locked:
+        raise HTTPException(
+            status.HTTP_409_CONFLICT,
+            f"{', '.join(locked)} come from environment variables, change them there",
+        )
+    updated = ConnectionSettings(**{**current.__dict__, **changes})
+    conns.save(db, updated)
+    audit.record(db, who, "settings.update", before=current.masked(), after=updated.masked())
+    db.commit()
+    return {**updated.masked(), "from_env": conns.from_env}

percival_server/api/auth.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from percival_server.auth.deps import Principal, current_principal
+from percival_server.auth.ratelimit import FailureLimiter
+from percival_server.auth.security import hash_password, needs_rehash, verify_password
+from percival_server.auth.sessions import end_session, start_session
+from percival_server.db import get_db
+from percival_server.models import User
+
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+
+class LoginLimits:
+    def __init__(self) -> None:
+        self.by_ip = FailureLimiter(max_failures=20, window_seconds=15 * 60)
+        self.by_username = FailureLimiter(max_failures=5, window_seconds=15 * 60)
+
+
+# checked against when the username doesn't exist, so a wrong username takes
+# as long as a wrong password and doesn't reveal which accounts exist
+_DUMMY_HASH = hash_password("not-a-real-password-just-burning-time")
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+class Me(BaseModel):
+    username: str
+    kind: str
+    csrf_token: str | None
+    scopes: list[str]
+
+
+def _client_ip(request: Request) -> str:
+    return request.client.host if request.client else "unknown"
+
+
+@router.post("/login", response_model=Me)
+def login(
+    body: LoginRequest, request: Request, response: Response, db: Session = Depends(get_db)
+) -> Me:
+    limits: LoginLimits = request.app.state.login_limits
+    ip, name = _client_ip(request), body.username.strip().lower()
+    wait = max(limits.by_ip.retry_after(ip), limits.by_username.retry_after(name))
+    if wait:
+        raise HTTPException(
+            status.HTTP_429_TOO_MANY_REQUESTS,
+            "too many failed logins, try again later",
+            headers={"Retry-After": str(int(wait) + 1)},
+        )
+
+    user = db.scalar(select(User).where(User.username == body.username.strip()))
+    valid = verify_password(user.password_hash if user else _DUMMY_HASH, body.password)
+    if user is None or not valid:
+        limits.by_ip.fail(ip)
+        limits.by_username.fail(name)
+        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "wrong username or password")
+
+    limits.by_username.reset(name)
+    if needs_rehash(user.password_hash):
+        user.password_hash = hash_password(body.password)
+    csrf = start_session(db, user, request.app.state.config, response)
+    db.commit()
+    return Me(username=user.username, kind="user", csrf_token=csrf, scopes=["read", "write"])
+
+
+@router.post("/logout", status_code=status.HTTP_204_NO_CONTENT)
+def logout(request: Request, response: Response, db: Session = Depends(get_db)) -> None:
+    end_session(db, request, response)
+    db.commit()
+
+
+@router.get("/me", response_model=Me)
+def me(who: Principal = Depends(current_principal)) -> Me:
+    return Me(
+        username=who.name, kind=who.kind, csrf_token=who.csrf_token, scopes=sorted(who.scopes)
+    )

percival_server/api/cluster.py

@@ -0,0 +1,175 @@
+"""Workers, beat and registered tasks: the state of the user's Celery setup."""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request, status
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from percival_server import live
+from percival_server.api.history import run_summary
+from percival_server.auth.deps import Principal, require_read
+from percival_server.connections import Connections
+from percival_server.db import get_db
+from percival_server.models import Run, WorkerSeen
+from percival_server.schedules import store
+
+router = APIRouter(tags=["cluster"])
+
+# dead workers stay listed this long after their last heartbeat
+FORGET_AFTER = timedelta(hours=24)
+
+
+def _conns(request: Request) -> Connections:
+    conns: Connections = request.app.state.connections
+    return conns
+
+
+def all_workers(conns: Connections, db: Session) -> list[dict[str, Any]]:
+    alive = live.read_workers(conns)
+    seen = {w["hostname"] for w in alive}
+    cutoff = datetime.now(timezone.utc) - FORGET_AFTER
+    dead: list[dict[str, Any]] = [
+        {
+            "hostname": row.hostname,
+            "code_version": row.code_version,
+            "queues": row.queues or [],
+            "pid": row.pid,
+            "started_at": row.started_at,
+            "last_seen": row.last_seen,
+            "alive": False,
+        }
+        for row in db.scalars(select(WorkerSeen).where(WorkerSeen.last_seen >= cutoff))
+        if row.hostname not in seen
+    ]
+    return sorted(alive, key=lambda w: w["hostname"]) + sorted(dead, key=lambda w: w["hostname"])
+
+
+@router.get("/workers")
+def workers(
+    request: Request, db: Session = Depends(get_db), _: Principal = Depends(require_read)
+) -> dict[str, Any]:
+    conns = _conns(request)
+    everyone = all_workers(conns, db)
+    for w in everyone:
+        w.pop("tasks", None)  # can be long, /tasks has it
+    return {
+        "workers": everyone,
+        "versions": live.version_summary(everyone, _expected_version(conns)),
+    }
+
+
+def _expected_version(conns: Connections) -> str | None:
+    required = [
+        store.serialize(entry, count=0)["required_version"]
+        for entry in store.list_entries(conns.celery())
+    ]
+    return live.expected_version(required, live.read_beat(conns))
+
+
+@router.get("/status")
+def overview(
+    request: Request, db: Session = Depends(get_db), _: Principal = Depends(require_read)
+) -> dict[str, Any]:
+    """Everything the header bar needs in one request."""
+    conns = _conns(request)
+    workers = live.read_workers(conns)
+    versions = live.version_summary(workers, _expected_version(conns))
+
+    affected = []
+    if versions["mismatch"]:
+        stale = {v["version"] for v in versions["versions"] if v["stale"]}
+        for entry in store.list_entries(conns.celery()):
+            data = store.serialize(entry, count=0)
+            queue = data["queue"] or live.DEFAULT_QUEUE
+            on_queue = [w for w in workers if queue in (w.get("queues") or [])]
+            if any(w.get("code_version") in stale for w in on_queue):
+                affected.append(
+                    {
+                        "name": entry.name,
+                        "queue": queue,
+                        "required_version": data["required_version"],
+                        # without a required version a stale worker will just run it
+                        "protected": bool(data["required_version"]),
+                    }
+                )
+
+    since = datetime.now(timezone.utc) - timedelta(hours=24)
+    counts = dict(
+        db.execute(
+            select(Run.status, func.count()).where(Run.ts >= since).group_by(Run.status)
+        ).all()
+    )
+    return {
+        "beat": live.read_beat(conns),
+        "lock": live.read_lock(conns),
+        "workers_alive": len(workers),
+        "versions": versions,
+        "affected_schedules": affected,
+        "runs_24h": counts,
+    }
+
+
+@router.get("/tasks")
+def tasks(
+    request: Request, db: Session = Depends(get_db), _: Principal = Depends(require_read)
+) -> list[dict[str, Any]]:
+    conns = _conns(request)
+    meta = live.read_task_meta(conns)
+    workers = live.read_workers(conns)
+
+    names = set(meta)
+    hosts: dict[str, list[dict[str, Any]]] = {}
+    for w in workers:
+        for name in w.get("tasks") or []:
+            names.add(name)
+            hosts.setdefault(name, []).append(w)
+
+    latest = select(func.max(Run.id)).where(Run.status != "running").group_by(Run.task)
+    last = {r.task: r for r in db.scalars(select(Run).where(Run.id.in_(latest)))}
+    scheduled: dict[str, int] = {}
+    for entry in store.list_entries(conns.celery()):
+        scheduled[entry.task] = scheduled.get(entry.task, 0) + 1
+
+    out = []
+    for name in sorted(names):
+        m = meta.get(name, {})
+        on = hosts.get(name, [])
+        out.append(
+            {
+                "name": name,
+                "module": m.get("module"),
+                "signature": m.get("signature"),
+                "doc": (m.get("doc") or "").split("\n\n")[0],
+                "workers": sorted(w["hostname"] for w in on),
+                "versions": sorted({w.get("code_version") or "unknown" for w in on}),
+                "schedules": scheduled.get(name, 0),
+                "last_run": (
+                    {"status": last[name].status, "ts": last[name].ts} if name in last else None
+                ),
+            }
+        )
+    return out
+
+
+@router.get("/tasks/{name}")
+def task_detail(
+    name: str, request: Request, db: Session = Depends(get_db), _: Principal = Depends(require_read)
+) -> dict[str, Any]:
+    conns = _conns(request)
+    meta = live.read_task_meta(conns).get(name)
+    on = [w for w in live.read_workers(conns) if name in (w.get("tasks") or [])]
+    if meta is None and not on:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "no worker has published this task")
+    runs = db.scalars(select(Run).where(Run.task == name).order_by(Run.id.desc()).limit(20))
+    schedules = [e.name for e in store.list_entries(conns.celery()) if e.task == name]
+    return {
+        "name": name,
+        "meta": meta,
+        "workers": [{"hostname": w["hostname"], "code_version": w.get("code_version")} for w in on],
+        "schedules": schedules,
+        "recent_runs": [run_summary(r) for r in runs],
+    }

percival_server/api/history.py

@@ -0,0 +1,146 @@
+"""Run history and the audit log. Both are append-mostly and can get big, so
+they page with ``before_id`` (newest first) instead of offsets."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from sqlalchemy import Select, select
+from sqlalchemy.orm import Session
+
+from percival_server.auth.deps import Principal, require_read
+from percival_server.db import get_db
+from percival_server.models import AuditLog, Run
+
+router = APIRouter(tags=["history"])
+
+RUN_LIST_FIELDS = (
+    "id",
+    "task_id",
+    "task",
+    "schedule",
+    "status",
+    "worker",
+    "code_version",
+    "required_version",
+    "queue",
+    "args",
+    "kwargs",
+    "started_at",
+    "finished_at",
+    "duration",
+    "exc_type",
+    "exc_message",
+    "retries",
+    "ts",
+)
+
+
+def run_summary(run: Run) -> dict[str, Any]:
+    return {f: getattr(run, f) for f in RUN_LIST_FIELDS}
+
+
+def _page(query: Select[Any], model: Any, before_id: int | None, limit: int) -> Select[Any]:
+    if before_id is not None:
+        query = query.where(model.id < before_id)
+    return query.order_by(model.id.desc()).limit(limit)
+
+
+@router.get("/runs")
+def list_runs(
+    schedule: str | None = None,
+    task: str | None = None,
+    run_status: list[str] | None = Query(None, alias="status"),
+    worker: str | None = None,
+    since: datetime | None = Query(None, alias="from"),
+    until: datetime | None = Query(None, alias="to"),
+    before_id: int | None = None,
+    limit: int = Query(50, ge=1, le=500),
+    db: Session = Depends(get_db),
+    _: Principal = Depends(require_read),
+) -> dict[str, Any]:
+    query = select(Run)
+    if schedule:
+        query = query.where(Run.schedule == schedule)
+    if task:
+        query = query.where(Run.task == task)
+    if run_status:
+        query = query.where(Run.status.in_(run_status))
+    if worker:
+        query = query.where(Run.worker == worker)
+    if since:
+        query = query.where(Run.ts >= since)
+    if until:
+        query = query.where(Run.ts < until)
+
+    rows = list(db.scalars(_page(query, Run, before_id, limit)))
+    return {
+        "items": [run_summary(r) for r in rows],
+        "next_before_id": rows[-1].id if len(rows) == limit else None,
+    }
+
+
+@router.get("/runs/{run_id}")
+def get_run(
+    run_id: int, db: Session = Depends(get_db), _: Principal = Depends(require_read)
+) -> dict[str, Any]:
+    run = db.get(Run, run_id)
+    if run is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND, "no such run")
+    attempts = []
+    if run.task_id:
+        # the same message can show up more than once, e.g. blocked on a stale
+        # worker and then run on a fresh one
+        attempts = [
+            run_summary(r)
+            for r in db.scalars(select(Run).where(Run.task_id == run.task_id).order_by(Run.id))
+        ]
+    return {
+        **run_summary(run),
+        "traceback": run.traceback,
+        "extra": run.extra,
+        "attempts": attempts,
+    }
+
+
+@router.get("/audit-log")
+def audit_log(
+    action: str | None = None,
+    actor: str | None = None,
+    source: str | None = None,
+    target: str | None = None,
+    before_id: int | None = None,
+    limit: int = Query(50, ge=1, le=500),
+    db: Session = Depends(get_db),
+    _: Principal = Depends(require_read),
+) -> dict[str, Any]:
+    query = select(AuditLog)
+    if action:
+        # "schedule" matches schedule.create, schedule.pause, ...
+        query = query.where(AuditLog.action.startswith(action))
+    if actor:
+        query = query.where(AuditLog.actor == actor)
+    if source:
+        query = query.where(AuditLog.source == source)
+    if target:
+        query = query.where(AuditLog.target == target)
+    rows = list(db.scalars(_page(query, AuditLog, before_id, limit)))
+    return {
+        "items": [
+            {
+                "id": r.id,
+                "ts": r.ts,
+                "actor": r.actor,
+                "actor_type": r.actor_type,
+                "source": r.source,
+                "action": r.action,
+                "target": r.target,
+                "before": r.before,
+                "after": r.after,
+            }
+            for r in rows
+        ],
+        "next_before_id": rows[-1].id if len(rows) == limit else None,
+    }