evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/api/auth.py

@@ -0,0 +1,225 @@
+import secrets
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from functools import cache
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from evalkit.api.deps import SessionDep
+from evalkit.contracts.enums import TokenKind
+from evalkit.db.models import AuthToken, User, Workspace, WorkspaceMember
+from evalkit.db.tokens import token_hash
+from evalkit.db.users import PASSWORD_MAX_BYTES, UserStore, hash_password, verify_password
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+DOWNLOAD_TICKET_TTL = timedelta(seconds=60)
+
+
+@cache
+def _dummy_password_hash() -> str:
+    return hash_password(secrets.token_urlsafe(16))
+
+
+async def issue_token(
+    session: AsyncSession, user: User, ttl: timedelta, kind: TokenKind = TokenKind.USER
+) -> str:
+    token = secrets.token_urlsafe(32)
+    session.add(
+        AuthToken(
+            user_id=user.id,
+            token_hash=token_hash(token),
+            expires_at=datetime.now(UTC) + ttl,
+            kind=kind,
+        )
+    )
+    await session.commit()
+    return token
+
+
+def _bearer(request: Request) -> str:
+    header = request.headers.get("authorization") or ""
+    if not header.startswith("Bearer "):
+        raise HTTPException(status_code=401, detail="missing bearer token")
+    return header.removeprefix("Bearer ")
+
+
+async def _valid_token(session: AsyncSession, token: str) -> AuthToken:
+    row = (
+        await session.execute(select(AuthToken).where(AuthToken.token_hash == token_hash(token)))
+    ).scalar_one_or_none()
+    if row is None:
+        raise HTTPException(status_code=401, detail="invalid token")
+    expires = token_expiry(row)
+    if expires is not None and expires < datetime.now(UTC):
+        raise HTTPException(status_code=401, detail="token expired")
+    return row
+
+
+async def _valid_bearer_token(session: AsyncSession, token: str) -> AuthToken:
+    row = await _valid_token(session, token)
+    if row.kind == TokenKind.TICKET:
+        raise HTTPException(status_code=401, detail="a download ticket is not a bearer token")
+    row.last_used_at = datetime.now(UTC)
+    return row
+
+
+async def _valid_bearer(request: Request, session: AsyncSession) -> AuthToken:
+    return await _valid_bearer_token(session, _bearer(request))
+
+
+# A person's bearer token, by the rules every non-collector route applies; /mcp
+# calls it too, so the two surfaces accept exactly the same tokens.
+async def bearer_login(session: AsyncSession, token: str) -> tuple[AuthToken, User]:
+    row = await _valid_bearer_token(session, token)
+    if row.kind == TokenKind.AGENT:
+        raise HTTPException(status_code=403, detail="agent tokens may only export traces")
+    return row, await _active_user(session, row.user_id)
+
+
+def token_expiry(row: AuthToken) -> datetime | None:
+    if row.expires_at is None:
+        return None
+    return row.expires_at if row.expires_at.tzinfo else row.expires_at.replace(tzinfo=UTC)
+
+
+# A download ticket is a single-use token carried in the query string so the
+# browser can save a response straight to disk. URLs leak into logs and
+# history, so a ticket only ever authorises a GET; a bearer header always wins.
+async def current_user(request: Request, session: SessionDep) -> User:
+    ticket = request.query_params.get("ticket")
+    if ticket is not None and request.method == "GET" and "authorization" not in request.headers:
+        row = await _valid_token(session, ticket)
+        if row.kind != TokenKind.TICKET:
+            raise HTTPException(status_code=401, detail="not a download ticket")
+        await session.delete(row)
+        user = await _active_user(session, row.user_id)
+    else:
+        row, user = await bearer_login(session, _bearer(request))
+        request.state.token_expires_at = token_expiry(row)
+    await session.commit()
+    return user
+
+
+async def _active_user(session: AsyncSession, user_id: int) -> User:
+    user = (await session.execute(select(User).where(User.id == user_id))).scalar_one()
+    if user.archived_at is not None:
+        raise HTTPException(status_code=401, detail="user is archived")
+    return user
+
+
+CurrentUser = Annotated[User, Depends(current_user)]
+
+
+@dataclass(frozen=True)
+class Principal:
+    user: User | None
+    agent_id: int | None
+
+
+async def collector_principal(request: Request, session: SessionDep) -> Principal:
+    row = await _valid_bearer(request, session)
+    if row.kind == TokenKind.AGENT:
+        await session.commit()
+        return Principal(user=None, agent_id=row.agent_id)
+    user = await _active_user(session, row.user_id)
+    await session.commit()
+    return Principal(user=user, agent_id=None)
+
+
+CollectorPrincipal = Annotated[Principal, Depends(collector_principal)]
+
+
+async def require_admin(user: CurrentUser) -> User:
+    if not user.is_admin:
+        raise HTTPException(status_code=403, detail="admin only")
+    return user
+
+
+AdminUser = Annotated[User, Depends(require_admin)]
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+@router.post("/login")
+async def login(body: LoginRequest, request: Request, session: SessionDep) -> dict:
+    user = (
+        await session.execute(select(User).where(User.username == body.username))
+    ).scalar_one_or_none()
+    if len(body.password.encode()) > PASSWORD_MAX_BYTES:
+        raise HTTPException(
+            status_code=400, detail=f"password is longer than {PASSWORD_MAX_BYTES} bytes"
+        )
+    if user is None:
+        verify_password(body.password, _dummy_password_hash())
+        raise HTTPException(status_code=401, detail="invalid username or password")
+    if not verify_password(body.password, user.password_hash):
+        raise HTTPException(status_code=401, detail="invalid username or password")
+    if user.archived_at is not None:
+        raise HTTPException(status_code=401, detail="user is archived")
+    ttl = timedelta(hours=request.app.state.cfg.auth.token_ttl_hours)
+    token = await issue_token(session, user, ttl)
+    return {"token": token, "username": user.username, "is_admin": user.is_admin}
+
+
+@router.post("/download-ticket")
+async def download_ticket(session: SessionDep, user: CurrentUser) -> dict:
+    await session.execute(delete(AuthToken).where(AuthToken.expires_at < datetime.now(UTC)))
+    return {"ticket": await issue_token(session, user, DOWNLOAD_TICKET_TTL, TokenKind.TICKET)}
+
+
+@router.post("/logout")
+async def logout(request: Request, session: SessionDep, user: CurrentUser) -> dict:
+    token = _bearer(request)
+    await session.execute(delete(AuthToken).where(AuthToken.token_hash == token_hash(token)))
+    await session.commit()
+    return {"ok": True}
+
+
+@router.get("/me")
+async def me(request: Request, session: SessionDep, user: CurrentUser) -> dict:
+    rows = (
+        await session.execute(
+            select(Workspace.name, WorkspaceMember.role)
+            .join(Workspace, Workspace.id == WorkspaceMember.workspace_id)
+            .where(WorkspaceMember.user_id == user.id)
+            .order_by(Workspace.name, WorkspaceMember.role)
+        )
+    ).all()
+    roles: dict[str, list[str]] = {}
+    for name, role in rows:
+        roles.setdefault(name, []).append(role)
+    personal = await session.scalar(
+        select(Workspace.name).where(
+            Workspace.owner_user_id == user.id, Workspace.name == user.username
+        )
+    )
+    return {
+        "username": user.username,
+        "is_admin": user.is_admin,
+        "personal_workspace": personal,
+        "roles": roles,
+        "token_expires_at": getattr(request.state, "token_expires_at", None),
+    }
+
+
+async def bootstrap_admin(
+    session_factory: async_sessionmaker[AsyncSession], password: str | None
+) -> None:
+    async with session_factory() as session:
+        existing = await session.scalar(select(User.id).where(User.is_admin.is_(True)))
+    if existing is not None:
+        return
+    if not password:
+        raise RuntimeError(
+            "no admin user exists and EVALKIT_ADMIN_PASSWORD is not set; "
+            "set it and start the server again to create the admin user"
+        )
+    await UserStore(session_factory).create("admin", password, is_admin=True)

evalkit/api/cli_dist.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+
+from evalkit import version
+
+router = APIRouter(prefix="/api/cli", tags=["cli"])
+
+
+@router.get("")
+async def cli_info() -> dict:
+    return {"version": version.current(), "package": version.DISTRIBUTION}

evalkit/api/datasets.py

@@ -0,0 +1,169 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from evalkit.api.access import ActorDep, ensure_active, require, workspace_ref
+from evalkit.api.deps import PgStoreDep, ReviewLogDep, SessionDep
+from evalkit.api.experiments import LIST_LIMIT, with_progress
+from evalkit.contracts.enums import ExperimentStatus
+from evalkit.core.access import can_write
+from evalkit.core.review import CaseVersion, is_reviewable, review_summary
+from evalkit.db import experiments as db_experiments
+from evalkit.db.agents import agent_id_for
+from evalkit.db.cases import create_dataset, current_cases, dataset_by_ref, list_datasets, splits
+from evalkit.db.promotion import twin_status
+from evalkit.db.review import ReviewLog
+from evalkit.db.scorecard import case_history
+from evalkit.store.base import CaseView, DatasetRow, Selection
+
+router = APIRouter(prefix="/api/datasets", tags=["datasets"])
+
+
+class CaseSelection(BaseModel):
+    selection: Selection
+    agent: str | None = None
+
+
+def _dataset(row: DatasetRow) -> dict:
+    return {**row.model_dump(mode="json"), "ref": row.ref}
+
+
+@router.get("")
+async def list_all(session: SessionDep, workspace: str | None = None) -> list[dict]:
+    return [_dataset(row) for row in await list_datasets(session, workspace)]
+
+
+class CreateDataset(BaseModel):
+    name: str
+    workspace: str
+
+
+@router.post("")
+async def create(body: CreateDataset, session: SessionDep, actor: ActorDep) -> dict:
+    ws = await workspace_ref(session, body.workspace)
+    require(can_write(actor, ws), f"you cannot create datasets in @{body.workspace}")
+    ensure_active(ws)
+    row = await create_dataset(session, ws, body.name)
+    await session.commit()
+    return _dataset(row)
+
+
+@router.get("/{ref}")
+async def get_dataset(ref: str, session: SessionDep) -> dict:
+    return _dataset(await dataset_by_ref(session, ref))
+
+
+async def _with_review(
+    session: SessionDep, review_log: ReviewLog, dataset: DatasetRow, cases: list[CaseView]
+) -> list[dict]:
+    ws = await workspace_ref(session, dataset.workspace)
+    twins = await twin_status(session, ws, dataset, cases)
+    if not is_reviewable(dataset):
+        return [
+            case.model_dump(mode="json") | {"review": None, "twin": twins[case.case_id]}
+            for case in cases
+        ]
+    reviews = await review_log.load(dataset.ref, [CaseVersion.of(case) for case in cases])
+    return [
+        case.model_dump(mode="json")
+        | {"review": review_summary(reviews[case.case_id]), "twin": twins[case.case_id]}
+        for case in cases
+    ]
+
+
+@router.get("/{ref}/cases")
+async def list_cases(
+    ref: str,
+    session: SessionDep,
+    review_log: ReviewLogDep,
+    tag: str | None = None,
+    q: str | None = None,
+) -> list[dict]:
+    dataset = await dataset_by_ref(session, ref)
+    cases = await current_cases(session, dataset)
+    if tag:
+        cases = [case for case in cases if tag in case.split_tags]
+    if q:
+        needle = q.lower()
+        cases = [
+            case
+            for case in cases
+            if needle in case.case_id.lower() or needle in str(case.input).lower()
+        ]
+    return await _with_review(session, review_log, dataset, cases)
+
+
+@router.get("/{ref}/cases/{case_id}")
+async def get_case(ref: str, case_id: str, session: SessionDep, review_log: ReviewLogDep) -> dict:
+    dataset = await dataset_by_ref(session, ref)
+    case = next((c for c in await current_cases(session, dataset) if c.case_id == case_id), None)
+    if case is None:
+        raise HTTPException(status_code=404, detail=f"unknown case: {case_id}")
+    [row] = await _with_review(session, review_log, dataset, [case])
+    return row
+
+
+@router.post("/{ref}/cases/select")
+async def select(ref: str, body: CaseSelection, store: PgStoreDep) -> list[dict]:
+    cases = await store.cases_for(ref, body.selection, agent=body.agent)
+    return [case.model_dump(mode="json") for case in cases]
+
+
+@router.get("/{ref}/splits")
+async def list_splits(ref: str, session: SessionDep) -> list[dict]:
+    dataset = await dataset_by_ref(session, ref)
+    return [{"name": tag, "count": count} for tag, count in await splits(session, dataset)]
+
+
+@router.get("/{ref}/experiments")
+async def list_experiments(
+    ref: str,
+    session: SessionDep,
+    status: ExperimentStatus | None = None,
+    agent: str | None = None,
+    limit: int = 50,
+    offset: int = 0,
+) -> list[dict]:
+    dataset = await dataset_by_ref(session, ref)
+    found = await db_experiments.list_experiments(
+        session,
+        dataset_id=dataset.id,
+        status=status,
+        agent_id=await agent_id_for(session, agent) if agent else None,
+        limit=min(limit, LIST_LIMIT),
+        offset=offset,
+    )
+    return await with_progress(session, found)
+
+
+@router.get("/{ref}/cases/{case_id}/history")
+async def case_revisions(ref: str, case_id: str, session: SessionDep) -> dict:
+    dataset = await dataset_by_ref(session, ref)
+    current = next((c for c in await current_cases(session, dataset) if c.case_id == case_id), None)
+    if current is None:
+        raise HTTPException(status_code=404, detail=f"unknown case: {case_id}")
+    return {
+        "case_id": case_id,
+        "current": current.model_dump(mode="json"),
+        "revisions": [{"content_hash": current.content_hash, "kind": "current"}],
+    }
+
+
+@router.get("/{ref}/cases/{case_id}/scores")
+async def case_scores(
+    ref: str, case_id: str, session: SessionDep, limit: int = 10, offset: int = 0
+) -> dict:
+    dataset = await dataset_by_ref(session, ref)
+    points = await case_history(session, dataset.id, [case_id], limit=limit, offset=offset)
+    return {
+        "case_id": case_id,
+        "points": [p.model_dump(mode="json") for p in points.get(case_id, [])],
+    }
+
+
+@router.get("/{ref}/case-scores")
+async def dataset_case_scores(ref: str, session: SessionDep, experiments: int = 5) -> dict:
+    dataset = await dataset_by_ref(session, ref)
+    by_case = await case_history(session, dataset.id, None, limit=experiments)
+    return {
+        case_id: [p.model_dump(mode="json") for p in points] for case_id, points in by_case.items()
+    }

evalkit/api/deps.py

@@ -0,0 +1,79 @@
+from collections.abc import AsyncIterator
+from typing import Annotated
+
+from fastapi import Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from evalkit.config import Config
+from evalkit.core.runner import RunnerDeps
+from evalkit.db.agents import AgentStore
+from evalkit.db.keys import KeyLoader
+from evalkit.db.mappings import MappingStore
+from evalkit.db.review import ReviewLog
+from evalkit.db.sessions import DbSessionReader
+from evalkit.db.spans import SpanStore
+from evalkit.db.store import PgStore
+from evalkit.db.tokens import AgentTokens
+
+
+def get_config(request: Request) -> Config:
+    return request.app.state.cfg
+
+
+def get_runner_deps(request: Request) -> RunnerDeps:
+    return request.app.state.runner_deps
+
+
+def get_keys(request: Request) -> KeyLoader:
+    return request.app.state.keys
+
+
+def get_review_log(request: Request) -> ReviewLog:
+    return request.app.state.review_log
+
+
+def get_store(request: Request) -> PgStore:
+    return request.app.state.store
+
+
+def get_holder(request: Request) -> str:
+    return request.app.state.holder
+
+
+def get_agents(request: Request) -> AgentStore:
+    return request.app.state.agents
+
+
+def get_tokens(request: Request) -> AgentTokens:
+    return request.app.state.tokens
+
+
+def get_spans(request: Request) -> SpanStore:
+    return request.app.state.spans
+
+
+def get_mappings(request: Request) -> MappingStore:
+    return request.app.state.mappings
+
+
+def get_sessions(request: Request) -> DbSessionReader:
+    return request.app.state.sessions
+
+
+async def get_session(request: Request) -> AsyncIterator[AsyncSession]:
+    async with request.app.state.session_factory() as session:
+        yield session
+
+
+ConfigDep = Annotated[Config, Depends(get_config)]
+RunnerDepsDep = Annotated[RunnerDeps, Depends(get_runner_deps)]
+SessionDep = Annotated[AsyncSession, Depends(get_session)]
+ReviewLogDep = Annotated[ReviewLog, Depends(get_review_log)]
+PgStoreDep = Annotated[PgStore, Depends(get_store)]
+HolderDep = Annotated[str, Depends(get_holder)]
+AgentStoreDep = Annotated[AgentStore, Depends(get_agents)]
+AgentTokensDep = Annotated[AgentTokens, Depends(get_tokens)]
+SpanStoreDep = Annotated[SpanStore, Depends(get_spans)]
+MappingStoreDep = Annotated[MappingStore, Depends(get_mappings)]
+SessionReaderDep = Annotated[DbSessionReader, Depends(get_sessions)]
+KeysDep = Annotated[KeyLoader, Depends(get_keys)]

evalkit/api/download.py

@@ -0,0 +1,15 @@
+import re
+from urllib.parse import quote
+
+# Agent-controlled filenames flow into Content-Disposition; Starlette encodes
+# headers as latin-1, so anything outside this set goes into filename* only.
+_ASCII_SAFE = re.compile(r"[^A-Za-z0-9._-]")
+
+
+def content_disposition(name: str) -> dict[str, str]:
+    ascii_name = _ASCII_SAFE.sub("_", name).strip("._") or "file"
+    return {
+        "Content-Disposition": (
+            f"attachment; filename=\"{ascii_name}\"; filename*=UTF-8''{quote(name, safe='')}"
+        )
+    }

evalkit/api/evaluators.py

@@ -0,0 +1,121 @@
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
+
+from evalkit.api.access import ActorDep, writable, writable_endpoint
+from evalkit.api.deps import ConfigDep, KeysDep, SessionDep
+from evalkit.contracts.enums import EvaluatorKind
+from evalkit.contracts.settings import RemoteEvaluatorSettings
+from evalkit.core.remote_evaluators import RemoteEvaluator
+from evalkit.db.evaluators import RemoteEvaluatorStore
+
+router = APIRouter(prefix="/api/evaluators", tags=["evaluators"])
+
+
+def _store(request: Request) -> RemoteEvaluatorStore:
+    return RemoteEvaluatorStore(request.app.state.session_factory)
+
+
+def _view(evaluator: RemoteEvaluator) -> dict:
+    return {**evaluator.model_dump(mode="json"), "ref": evaluator.ref}
+
+
+@router.get("")
+async def list_evaluators(
+    request: Request, workspace: str | None = None, include_archived: bool = False
+) -> list[dict]:
+    rows = await _store(request).list_evaluators(
+        workspace=workspace, include_archived=include_archived
+    )
+    return [_view(e) for e in rows]
+
+
+class CreateEvaluator(BaseModel):
+    workspace: str
+    name: str
+    kind: EvaluatorKind
+    settings: RemoteEvaluatorSettings
+    api_key: str | None = None
+
+
+@router.post("")
+async def create_evaluator(
+    body: CreateEvaluator,
+    request: Request,
+    keys: KeysDep,
+    session: SessionDep,
+    actor: ActorDep,
+    cfg: ConfigDep,
+) -> dict:
+    await writable_endpoint(
+        session,
+        actor,
+        cfg,
+        body.workspace,
+        api_key="api_key" in body.model_fields_set,
+        url=body.settings.endpoint,
+    )
+    created = await _store(request).create(
+        body.workspace,
+        body.name,
+        body.kind,
+        body.settings,
+        keys.seal(body.api_key),
+    )
+    return _view(created)
+
+
+class EditEvaluator(BaseModel):
+    settings: RemoteEvaluatorSettings | None = None
+    api_key: str | None = None
+
+
+@router.patch("/{ref}")
+async def edit_evaluator(
+    ref: str,
+    body: EditEvaluator,
+    request: Request,
+    keys: KeysDep,
+    session: SessionDep,
+    actor: ActorDep,
+    cfg: ConfigDep,
+) -> dict:
+    current = await _store(request).get(ref)
+    moved = body.settings is not None and body.settings.endpoint != current.settings.endpoint
+    await writable_endpoint(
+        session,
+        actor,
+        cfg,
+        current.workspace,
+        api_key="api_key" in body.model_fields_set,
+        url=body.settings.endpoint if moved and body.settings is not None else None,
+    )
+    edited = await _store(request).edit(
+        ref,
+        body.settings,
+        api_key_encrypted=keys.seal(body.api_key),
+        clear_api_key="api_key" in body.model_fields_set and not body.api_key,
+    )
+    return _view(edited)
+
+
+@router.get("/{ref}")
+async def get_evaluator(ref: str, request: Request) -> dict:
+    return _view(await _store(request).get(ref))
+
+
+@router.post("/{ref}/archive")
+async def archive_evaluator(
+    ref: str, request: Request, session: SessionDep, actor: ActorDep
+) -> dict:
+    current = await _store(request).get(ref)
+    await writable(session, actor, current.workspace)
+    return _view(await _store(request).archive(ref))
+
+
+@router.post("/{ref}/restore")
+async def restore_evaluator(
+    ref: str, request: Request, session: SessionDep, actor: ActorDep
+) -> dict:
+    current = await _store(request).get(ref)
+    await writable(session, actor, current.workspace)
+    return _view(await _store(request).restore(ref))