evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/api/fixtures.py

@@ -0,0 +1,95 @@
+import hashlib
+import mimetypes
+import re
+import uuid
+from collections.abc import AsyncIterator
+
+from fastapi import APIRouter, HTTPException, Request
+from starlette.responses import Response, StreamingResponse
+
+from evalkit.api.access import ActorDep, workspace_ref
+from evalkit.api.deps import RunnerDepsDep, SessionDep
+from evalkit.api.download import content_disposition
+from evalkit.core.blob import fixture_blob_key
+from evalkit.db.fixture_blobs import blob_size, known_blobs, record_blob, workspace_fixtures
+
+router = APIRouter(prefix="/api/fixtures", tags=["fixtures"])
+
+FIXTURE_STAGING_PREFIX = "fixtures/staging/"
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+def _sha(value: str) -> str:
+    if not _SHA256.match(value):
+        raise HTTPException(status_code=400, detail=f"{value!r} is not a lowercase sha256 digest")
+    return value
+
+
+@router.get("")
+async def list_fixtures(
+    session: SessionDep,
+    actor: ActorDep,
+    workspace: str | None = None,
+    q: str = "",
+    limit: int = 50,
+    offset: int = 0,
+) -> dict:
+    ws = await workspace_ref(session, workspace or actor.username)
+    page = await workspace_fixtures(
+        session, ws.id, text=q or None, limit=max(1, min(limit, 200)), offset=max(0, offset)
+    )
+    return page.model_dump(mode="json")
+
+
+async def _known(session: SessionDep, sha256: str) -> None:
+    if not await known_blobs(session, [_sha(sha256)]):
+        raise HTTPException(status_code=404, detail=f"no fixture with sha256 {sha256}")
+
+
+@router.head("/sha256/{sha256}")
+async def head(sha256: str, session: SessionDep) -> Response:
+    await _known(session, sha256)
+    return Response(status_code=200)
+
+
+@router.get("/sha256/{sha256}")
+async def download(
+    sha256: str, deps: RunnerDepsDep, session: SessionDep, name: str | None = None
+) -> Response:
+    await _known(session, sha256)
+    blob = await deps.blob_store.get_stream(fixture_blob_key(sha256))
+    media_type = (mimetypes.guess_type(name)[0] if name else None) or "application/octet-stream"
+    headers = {"Content-Length": str(blob.size), **(content_disposition(name) if name else {})}
+    return StreamingResponse(blob.chunks, media_type=media_type, headers=headers)
+
+
+@router.put("/sha256/{sha256}")
+async def upload(
+    sha256: str, request: Request, deps: RunnerDepsDep, session: SessionDep, actor: ActorDep
+) -> dict:
+    expected = _sha(sha256)
+    stored = await blob_size(session, expected)
+    if stored is not None:
+        return {"sha256": expected, "size": stored}
+    hasher = hashlib.sha256()
+    size = 0
+
+    async def hashed() -> AsyncIterator[bytes]:
+        nonlocal size
+        async for chunk in request.stream():
+            hasher.update(chunk)
+            size += len(chunk)
+            yield chunk
+
+    staging = f"{FIXTURE_STAGING_PREFIX}{uuid.uuid4().hex}"
+    await deps.blob_store.put_stream(staging, hashed())
+    actual = hasher.hexdigest()
+    if actual != expected:
+        await deps.blob_store.delete_object(staging)
+        raise HTTPException(status_code=400, detail=f"bytes hash to {actual}, not {sha256}")
+    await deps.blob_store.copy_object(staging, fixture_blob_key(sha256))
+    await deps.blob_store.delete_object(staging)
+    await record_blob(session, sha256, size, actor.id)
+    await session.commit()
+    return {"sha256": sha256, "size": size}

evalkit/api/harness_check.py

@@ -0,0 +1,48 @@
+import asyncio
+
+from fastapi import APIRouter, HTTPException, Request
+
+from evalkit.api.access import ActorDep, require, workspace_ref
+from evalkit.api.deps import AgentStoreDep, RunnerDepsDep, SessionDep
+from evalkit.api.server_keys import server_keys
+from evalkit.core.access import can_use_agent
+from evalkit.core.agents import explicit, server_may_drive
+from evalkit.core.harness_check import check_harness
+from evalkit.core.workspace import parse_slug
+
+router = APIRouter(prefix="/api/agents", tags=["agents"])
+
+
+@router.post("/{agent}/check")
+async def run_check(
+    agent: str,
+    request: Request,
+    deps: RunnerDepsDep,
+    agents: AgentStoreDep,
+    session: SessionDep,
+    actor: ActorDep,
+) -> dict:
+    definition = await agents.get(explicit(parse_slug(agent)))
+    ws = await workspace_ref(session, definition.workspace)
+    require(can_use_agent(actor, ws), f"only the owner of @{ws.name} can check {definition.ref}")
+    if definition.archived:
+        raise HTTPException(status_code=409, detail=f"agent {definition.ref} is archived")
+    if definition.harness_url is None:
+        raise HTTPException(status_code=400, detail=f"{definition.ref} has no harness_url")
+    if not server_may_drive(definition):
+        raise HTTPException(
+            status_code=400,
+            detail=f"the server does not drive {definition.ref}; its owner checks it with "
+            f"evalkit harness check {definition.ref}",
+        )
+    report = await asyncio.wait_for(
+        check_harness(
+            definition,
+            (await server_keys(request, definition, None))[0].key_for("agent", definition.ref),
+            deps.blob_store,
+            transport=deps.harness_transport,
+            sessions=request.app.state.sessions,
+        ),
+        timeout=120,
+    )
+    return {"agent": agent, "checks": report}

evalkit/api/imports.py

@@ -0,0 +1,180 @@
+import json
+import tempfile
+from pathlib import Path
+from typing import Annotated
+
+import httpx
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
+
+from evalkit.api.access import ensure_active, load_actor, require, workspace_ref
+from evalkit.api.auth import current_user
+from evalkit.api.deps import PgStoreDep, RunnerDepsDep, SessionDep
+from evalkit.api.runs import harness_client
+from evalkit.contracts.cases import REGISTRY_FILENAME
+from evalkit.core.access import can_write
+from evalkit.core.agents import explicit, server_may_drive
+from evalkit.core.fixtures import load_registries
+from evalkit.core.importer import import_dataset, load_files
+from evalkit.core.lint import fixture_findings
+from evalkit.core.lint import lint as lint_cases
+from evalkit.core.runner import RunnerDeps
+from evalkit.core.workspace import parse_slug
+from evalkit.db.evaluators import RemoteEvaluatorStore
+from evalkit.db.models import User
+from evalkit.db.workspaces import WorkspaceStore
+
+router = APIRouter(prefix="/api", tags=["imports"])
+
+
+def _safe_relative(name: str) -> Path:
+    parts = [p for p in Path(name).parts if p not in ("", ".")]
+    if not parts or any(p == ".." for p in parts) or Path(name).is_absolute():
+        raise HTTPException(status_code=400, detail=f"unsafe upload path: {name!r}")
+    return Path(*parts)
+
+
+async def _save_uploads(files: list[UploadFile], directory: Path) -> list[Path]:
+    saved = []
+    for upload in files:
+        path = directory / _safe_relative(upload.filename or "cases.yaml")
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(await upload.read())
+        saved.append(path)
+    return saved
+
+
+def _input_schema(raw: str) -> dict:
+    try:
+        schema = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise HTTPException(
+            status_code=400, detail=f"input_schema is not valid JSON: {exc}"
+        ) from exc
+    if not isinstance(schema, dict):
+        raise HTTPException(status_code=400, detail="input_schema must be a JSON object")
+    return schema
+
+
+async def _agent_input_schema(deps: RunnerDeps, agent: str) -> dict:
+    definition = await deps.agents.get(explicit(parse_slug(agent)))
+    if not server_may_drive(definition):
+        raise HTTPException(
+            status_code=400,
+            detail=f"the server does not drive {definition.ref}; evalkit lint --agent "
+            "reads its input schema from your laptop and sends it as input_schema",
+        )
+    client = await harness_client(deps, definition)
+    try:
+        caps = await client.capabilities()
+    except (httpx.HTTPError, ValueError) as exc:
+        raise HTTPException(status_code=502, detail=f"harness {agent}: {exc}") from exc
+    finally:
+        await client.aclose()
+    return caps.effective_input_schema
+
+
+@router.post("/lint")
+async def lint_endpoint(
+    files: list[UploadFile],
+    deps: RunnerDepsDep,
+    store: PgStoreDep,
+    session: SessionDep,
+    request: Request,
+    dataset: str | None = None,
+    agent: str | None = None,
+    input_schema: Annotated[str | None, Form()] = None,
+    warn_unused_fixtures: bool = False,
+) -> dict:
+    with tempfile.TemporaryDirectory() as tmp:
+        paths = await _save_uploads(files, Path(tmp))
+        sourced_cases, overlays, findings = load_files(paths)
+        registry = Path(tmp) / REGISTRY_FILENAME
+        if registry in paths:
+            entries, registry_findings = load_registries([registry])
+            findings.extend(registry_findings)
+            findings.extend(
+                fixture_findings(
+                    [case for case, _ in sourced_cases],
+                    entries,
+                    warn_unused=warn_unused_fixtures,
+                )
+            )
+        other_cases: dict[str, str] | None = None
+        if dataset is not None:
+            ref = parse_slug(dataset)
+            other_cases = await store.workspace_cases(
+                await workspace_ref(session, ref.workspace), exclude_dataset=ref.name
+            )
+        schema = await _agent_input_schema(deps, agent) if agent is not None else None
+        if input_schema is not None:
+            schema = _input_schema(input_schema)
+        known = await RemoteEvaluatorStore(request.app.state.session_factory).names()
+        findings.extend(
+            lint_cases(
+                [case for case, _ in sourced_cases],
+                [overlay for overlay, _ in overlays],
+                known_custom_evaluators=known,
+                other_cases=other_cases,
+                input_schema=schema,
+            )
+        )
+    return {
+        "findings": [f.__dict__ for f in findings],
+        "errors": sum(1 for f in findings if f.severity == "error"),
+        "warnings": sum(1 for f in findings if f.severity == "warning"),
+    }
+
+
+@router.post("/sync")
+async def sync(
+    request: Request,
+    session: SessionDep,
+    store: PgStoreDep,
+    dataset: str,
+    user: Annotated[User, Depends(current_user)],
+    files: list[UploadFile] = File(default_factory=list),
+    workspace: str | None = None,
+    prune: bool = False,
+    sha: str | None = None,
+    branch: str | None = None,
+    dirty: bool | None = None,
+    actor: str | None = None,
+) -> dict:
+    target = workspace or user.username
+    target_ws = await workspace_ref(session, target)
+    require(can_write(await load_actor(session, user), target_ws), f"you cannot sync to @{target}")
+    ensure_active(target_ws)
+    if target_ws.is_target:
+        assert target_ws.promoted_from is not None
+        source_ws = await WorkspaceStore(request.app.state.session_factory).get_by_id(
+            target_ws.promoted_from
+        )
+        raise HTTPException(
+            status_code=409,
+            detail=f"{target} is the promotion target of {source_ws.name}; "
+            f"sync {source_ws.name} instead",
+        )
+    if actor is not None and not user.is_admin:
+        raise HTTPException(status_code=403, detail="actor override is admin only")
+    detail = {
+        k: v
+        for k, v in {"sha": sha, "branch": branch, "dirty": dirty, "actor": actor}.items()
+        if v is not None
+    }
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        paths = await _save_uploads(files, root)
+        report = await import_dataset(
+            store,
+            workspace=target_ws,
+            dataset=dataset,
+            files=paths,
+            full=prune,
+            known_custom_evaluators=await RemoteEvaluatorStore(
+                request.app.state.session_factory
+            ).names(),
+            actor_id=user.id,
+            detail=detail,
+            root=root,
+        )
+    return report.model_dump()

evalkit/api/limits.py

@@ -0,0 +1,84 @@
+import logging
+import re
+from dataclasses import dataclass
+
+from fastapi import HTTPException
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from evalkit.config import MB, Config
+
+LOGGER = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class BodyLimit:
+    what: str
+    key: str
+    limit: int
+
+    def detail(self) -> str:
+        return (
+            f"{self.what} is larger than the {_size(self.limit)} limit (server config {self.key})"
+        )
+
+
+def _size(n: int) -> str:
+    return f"{n // MB} MB" if n % MB == 0 else f"{n} bytes"
+
+
+def body_limit(cfg: Config, method: str, path: str) -> BodyLimit:
+    limits = cfg.limits
+    if method == "PUT" and re.fullmatch(r"/api/fixtures/sha256/[^/]+", path):
+        return BodyLimit(
+            "fixture upload", "limits.fixture_upload_bytes", limits.fixture_upload_bytes
+        )
+    if method == "PUT" and re.fullmatch(r"/api/experiments/\d+/blob/.+", path):
+        return BodyLimit(
+            "experiment blob", "limits.experiment_blob_bytes", limits.experiment_blob_bytes
+        )
+    if method == "POST" and path in ("/api/sync", "/api/lint"):
+        return BodyLimit("case upload", "limits.case_upload_bytes", limits.case_upload_bytes)
+    if method == "POST" and path == "/api/otel/v1/traces":
+        return BodyLimit("trace batch", "otel.max_batch_bytes", cfg.otel.max_batch_bytes)
+    return BodyLimit("request body", "limits.request_bytes", limits.request_bytes)
+
+
+def _log_rejection(limit: BodyLimit, method: str, path: str, received: int) -> None:
+    LOGGER.warning(
+        "request body too large: route=%s %s limit=%s key=%s received=%s",
+        method,
+        path,
+        limit.limit,
+        limit.key,
+        received,
+    )
+
+
+class BodyLimitMiddleware:
+    def __init__(self, app: ASGIApp, cfg: Config) -> None:
+        self.app = app
+        self.cfg = cfg
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+        method, path = scope["method"], scope["path"]
+        limit = body_limit(self.cfg, method, path)
+        declared = dict(scope["headers"]).get(b"content-length", b"")
+        received = 0
+
+        async def capped() -> Message:
+            nonlocal received
+            if declared.isdigit() and int(declared) > limit.limit:
+                _log_rejection(limit, method, path, int(declared))
+                raise HTTPException(status_code=413, detail=limit.detail())
+            message = await receive()
+            if message["type"] == "http.request":
+                received += len(message.get("body", b""))
+                if received > limit.limit:
+                    _log_rejection(limit, method, path, received)
+                    raise HTTPException(status_code=413, detail=limit.detail())
+            return message
+
+        await self.app(scope, capped, send)

evalkit/api/llm_providers.py

@@ -0,0 +1,120 @@
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel
+
+from evalkit.api.auth import AdminUser
+from evalkit.api.deps import ConfigDep, KeysDep
+from evalkit.contracts.settings import JudgeSettings
+from evalkit.core.endpoints import check_endpoint
+from evalkit.db.llm_providers import LlmProviderStore
+
+router = APIRouter(tags=["llm providers"])
+
+NO_DEFAULT_JUDGE = "no default judge; an admin sets one with evalkit judge-default set"
+
+
+def _store(request: Request) -> LlmProviderStore:
+    return request.app.state.llm_providers
+
+
+async def require_registered_judge(providers: LlmProviderStore, base_url: str) -> None:
+    if await providers.active_by_base_url(base_url) is None:
+        registered = ", ".join(p.base_url for p in await providers.list_providers())
+        raise HTTPException(
+            status_code=400,
+            detail=f"{base_url} is not a registered LLM provider; "
+            f"registered: {registered or 'none'}",
+        )
+
+
+@router.get("/api/llm-providers")
+async def list_llm_providers(request: Request, include_archived: bool = False) -> list[dict]:
+    rows = await _store(request).list_providers(include_archived=include_archived)
+    return [row.model_dump(mode="json") for row in rows]
+
+
+@router.get("/api/llm-providers/{name}")
+async def get_llm_provider(name: str, request: Request) -> dict:
+    return (await _store(request).get(name)).model_dump(mode="json")
+
+
+class CreateLlmProvider(BaseModel):
+    name: str
+    base_url: str
+    api_key: str | None = None
+
+
+@router.post("/api/llm-providers")
+async def create_llm_provider(
+    body: CreateLlmProvider, request: Request, keys: KeysDep, cfg: ConfigDep, admin: AdminUser
+) -> dict:
+    await check_endpoint(body.base_url, allow_private=cfg.allow_private_endpoints)
+    created = await _store(request).create(body.name, body.base_url, keys.seal(body.api_key))
+    return created.model_dump(mode="json")
+
+
+class EditLlmProvider(BaseModel):
+    base_url: str | None = None
+    api_key: str | None = None
+
+
+@router.patch("/api/llm-providers/{name}")
+async def edit_llm_provider(
+    name: str,
+    body: EditLlmProvider,
+    request: Request,
+    keys: KeysDep,
+    cfg: ConfigDep,
+    admin: AdminUser,
+) -> dict:
+    current = await _store(request).get(name)
+    if body.base_url is not None and body.base_url != current.base_url:
+        await check_endpoint(body.base_url, allow_private=cfg.allow_private_endpoints)
+    edited = await _store(request).edit(
+        name,
+        base_url=body.base_url,
+        api_key_encrypted=keys.seal(body.api_key),
+        clear_api_key="api_key" in body.model_fields_set and not body.api_key,
+    )
+    return edited.model_dump(mode="json")
+
+
+@router.post("/api/llm-providers/{name}/archive")
+async def archive_llm_provider(name: str, request: Request, admin: AdminUser) -> dict:
+    return (await _store(request).archive(name)).model_dump(mode="json")
+
+
+@router.post("/api/llm-providers/{name}/restore")
+async def restore_llm_provider(name: str, request: Request, admin: AdminUser) -> dict:
+    return (await _store(request).restore(name)).model_dump(mode="json")
+
+
+@router.get("/api/judge-default")
+async def get_judge_default(request: Request) -> dict:
+    judge = await _store(request).default_judge()
+    if judge is None:
+        raise HTTPException(status_code=404, detail=NO_DEFAULT_JUDGE)
+    return judge.model_dump()
+
+
+@router.put("/api/judge-default")
+async def set_judge_default(body: JudgeSettings, request: Request, admin: AdminUser) -> dict:
+    return (await _store(request).set_default_judge(body)).model_dump()
+
+
+class JudgePatch(BaseModel):
+    base_url: str | None = None
+    model: str | None = None
+    reasoning_effort: str | None = None
+    max_tokens: int | None = None
+    concurrency: int | None = None
+
+
+@router.patch("/api/judge-default")
+async def patch_judge_default(body: JudgePatch, request: Request, admin: AdminUser) -> dict:
+    current = await _store(request).default_judge()
+    merged = {
+        **(current.model_dump() if current is not None else {}),
+        **body.model_dump(exclude_none=True),
+    }
+    judge = JudgeSettings.model_validate(merged)
+    return (await _store(request).set_default_judge(judge)).model_dump()

evalkit/api/mcp/auth.py

@@ -0,0 +1,100 @@
+from fastapi import HTTPException
+from fastmcp.server.auth import (
+    AccessToken,
+    AuthProvider,
+    MultiAuth,
+    RemoteAuthProvider,
+    TokenVerifier,
+)
+from fastmcp.server.auth.providers.jwt import JWTVerifier
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from evalkit.api.auth import bearer_login, token_expiry
+from evalkit.config import Config
+
+SCOPE_READ = "evalkit:read"
+MCP_PATH = "/mcp"
+
+
+def resource_url(cfg: Config) -> str:
+    return f"{cfg.server.base_url.rstrip('/')}{MCP_PATH}"
+
+
+# An evalkit login token reaches MCP with the read scope; agent tokens stay
+# collector-only here as on /api.
+class EvalkitTokenVerifier(TokenVerifier):
+    def __init__(
+        self,
+        session_factory: async_sessionmaker[AsyncSession],
+        base_url: str,
+        required_scopes: list[str] | None = None,
+    ):
+        super().__init__(base_url=base_url, required_scopes=required_scopes)
+        self._session_factory = session_factory
+
+    async def verify_token(self, token: str) -> AccessToken | None:
+        async with self._session_factory() as session:
+            try:
+                row, user = await bearer_login(session, token)
+            except HTTPException:
+                return None
+            await session.commit()
+        expires = token_expiry(row)
+        return AccessToken(
+            token=token,
+            client_id=f"evalkit:{user.username}",
+            scopes=[SCOPE_READ],
+            expires_at=int(expires.timestamp()) if expires is not None else None,
+            claims={"sub": user.username, "evalkit_user": user.username},
+        )
+
+
+# The identity provider names the scope (`mcp.oauth.scope`); a token carrying it
+# gets evalkit's own read scope, which the tools check.
+class ProviderScopeVerifier(TokenVerifier):
+    def __init__(self, inner: TokenVerifier, scope: str, base_url: str):
+        super().__init__(base_url=base_url)
+        self._inner = inner
+        self._scope = scope
+
+    async def verify_token(self, token: str) -> AccessToken | None:
+        found = await self._inner.verify_token(token)
+        if found is None:
+            return None
+        scopes = [s for s in found.scopes if s != SCOPE_READ]
+        if self._scope in found.scopes:
+            scopes.append(SCOPE_READ)
+        return found.model_copy(update={"scopes": scopes})
+
+
+def build_auth(
+    cfg: Config,
+    session_factory: async_sessionmaker[AsyncSession],
+    jwt_verifier: TokenVerifier | None = None,
+) -> AuthProvider:
+    oauth = cfg.mcp.oauth
+    if oauth is None:
+        return EvalkitTokenVerifier(
+            session_factory, cfg.server.base_url, required_scopes=[SCOPE_READ]
+        )
+    verifier = jwt_verifier or JWTVerifier(
+        jwks_uri=oauth.jwks_uri,
+        issuer=oauth.issuer,
+        audience=oauth.audience or resource_url(cfg),
+        algorithm=oauth.algorithm,
+    )
+    remote = RemoteAuthProvider(
+        token_verifier=ProviderScopeVerifier(verifier, oauth.scope, cfg.server.base_url),
+        # As configured: AnyHttpUrl would turn https://host into https://host/, which clients
+        # cannot match to the provider's issuer (RFC 8414).
+        authorization_servers=[oauth.issuer],  # type: ignore[list-item]
+        base_url=cfg.server.base_url,
+        scopes_supported=[oauth.scope],
+        resource_name="evalkit",
+    )
+    return MultiAuth(
+        server=remote,
+        verifiers={"evalkit": EvalkitTokenVerifier(session_factory, cfg.server.base_url)},
+        server_source_id="oauth",
+        required_scopes=[SCOPE_READ],
+    )