agent-auditor 0.13.0__py3-none-any.whl

agent_auditor/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.13.0"

agent_auditor/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

agent_auditor/api/app.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import secrets
+from dataclasses import replace
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+from sqlalchemy import text
+
+from ..config import get_settings
+from ..db import Base, make_engine, make_session_factory
+from .. import models  # noqa: F401  (register tables on Base.metadata)
+from .routes import router
+from .evals_routes import router as evals_router
+from .drift_routes import router as drift_router
+from .regress_routes import router as regress_router
+from .rootcause_routes import router as rootcause_router
+from .canary_routes import router as canary_router
+from .release_routes import router as release_router
+from .assistant_routes import router as assistant_router
+from .fixes_routes import router as fixes_router
+from .insight_routes import feedback_router, router as insight_router
+from ..assistant import RateLimiter
+
+_DASH_DIR = Path(__file__).resolve().parent.parent / "dashboard"
+
+
+def _dashboard_page() -> str:
+    """The page is stored as a shell, one stylesheet and several script parts (kept apart so each is easy to edit); join them here."""
+    shell = (_DASH_DIR / "index.html").read_text(encoding="utf-8")
+    css = (_DASH_DIR / "app.css").read_text(encoding="utf-8")
+    js = "\n".join(p.read_text(encoding="utf-8") for p in sorted((_DASH_DIR / "js").glob("*.js")))
+    return shell.replace("/*__CSS__*/", css).replace("/*__JS__*/", js)
+
+
+def create_app(database_url: str | None = None, api_key: str | None = None,
+               anthropic_api_key: str | None = None) -> FastAPI:
+    settings = get_settings()
+    if database_url is not None:
+        settings = replace(settings, database_url=database_url)
+    if api_key is not None:
+        settings = replace(settings, api_key=api_key)
+
+    if anthropic_api_key is not None:
+        settings = replace(settings, anthropic_api_key=anthropic_api_key)
+
+    engine = make_engine(settings.database_url)
+    if settings.auto_create_tables:  # with Alembic (Docker Compose) set AUDITOR_AUTO_CREATE_TABLES=false
+        Base.metadata.create_all(engine)
+
+    app = FastAPI(title="Agent Auditor", version="0.13.0")
+    app.state.settings = settings
+    app.state.engine = engine
+    app.state.session_factory = make_session_factory(engine)
+    app.state.assistant_transport = None  # tests inject an httpx transport here
+    app.state.rate_limiter = RateLimiter()
+    app.state.assistant_dead = {}  # AI services that failed recently, skipped for a few minutes
+
+    @app.get("/", include_in_schema=False)
+    def root() -> RedirectResponse:
+        return RedirectResponse(url="/docs")
+
+    @app.get("/healthz", include_in_schema=False)
+    def healthz() -> dict:
+        """Liveness: the process is up. No database access, no authentication."""
+        return {"status": "ok", "version": app.version}
+
+    @app.get("/readyz", include_in_schema=False)
+    def readyz():
+        """Readiness: the database answers. Used by Docker Compose and load balancers."""
+        try:
+            with engine.connect() as conn:
+                conn.execute(text("SELECT 1"))
+        except Exception:  # noqa: BLE001 - report not-ready instead of crashing
+            return JSONResponse({"status": "database unavailable"}, status_code=503)
+        return {"status": "ok"}
+
+    @app.get("/dashboard", include_in_schema=False)
+    def dashboard() -> HTMLResponse:
+        """Read-only dashboard. The page itself is public; every data call it makes
+        carries the API key the person types in, so it exposes nothing by itself."""
+        nonce = secrets.token_urlsafe(16)
+        page = _dashboard_page().replace("__NONCE__", nonce)
+        csp = (
+            "default-src 'none'; "
+            f"script-src 'nonce-{nonce}'; style-src 'nonce-{nonce}'; "
+            "connect-src 'self'; img-src 'self' data:; "
+            "base-uri 'none'; form-action 'none'; frame-ancestors 'none'"
+        )
+        return HTMLResponse(page, headers={
+            "Content-Security-Policy": csp,
+            "Cache-Control": "no-store",
+            "X-Content-Type-Options": "nosniff",
+        })
+
+    app.include_router(router)
+    app.include_router(evals_router)
+    app.include_router(drift_router)
+    app.include_router(regress_router)
+    app.include_router(rootcause_router)
+    app.include_router(canary_router)
+    app.include_router(release_router)
+    app.include_router(assistant_router)
+    app.include_router(fixes_router)
+    app.include_router(insight_router)
+    app.include_router(feedback_router)
+    return app

agent_auditor/api/assistant_routes.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .. import assistant
+from ..insights import service as insights
+from ..models import Agent, AgentVersion
+from ..rootcause import service as rootcause
+from .deps import get_db, require_api_key
+
+router = APIRouter(prefix="/v1", dependencies=[Depends(require_api_key)])
+
+
+class StatusOut(BaseModel):
+    enabled: bool  # an AI service is available (the built-in guide always is)
+    model: str | None
+    providers: list[str] = Field(default_factory=list)
+    guide: bool = True
+
+
+class Turn(BaseModel):
+    role: Literal["user", "assistant"]
+    content: str = Field(max_length=8000)
+
+
+class AskIn(BaseModel):
+    question: str = Field(min_length=1, max_length=assistant.MAX_QUESTION_CHARS)
+    context: dict[str, Any] = Field(default_factory=dict, description="What the dashboard is showing right now")
+    history: list[Turn] = Field(default_factory=list, max_length=20)
+
+
+class AnswerOut(BaseModel):
+    answer: str
+    source: str = "guide"  # the AI service that wrote it ("Gemini"), or "guide" for the built-in guide
+
+
+class SuggestFixIn(BaseModel):
+    environment: Literal["production", "canary", "test"] = "production"
+    since_hours: float | None = Field(default=None, gt=0, le=24 * 365)
+
+
+class FixOut(BaseModel):
+    suggestion: str
+    version: str | None
+    failed_runs: int
+    note: str
+
+
+def _call(request: Request, system: str, messages: list[dict], max_tokens: int) -> str:
+    s = request.app.state.settings
+    try:
+        request.app.state.rate_limiter.check()
+        return assistant.ask_any(system=system, messages=messages, max_tokens=max_tokens, anthropic_key=s.anthropic_api_key,
+                                 anthropic_model=s.assistant_model, transport=request.app.state.assistant_transport,
+                                 dead=request.app.state.assistant_dead)[0]
+    except assistant.RateLimited as exc:
+        raise HTTPException(429, str(exc)) from exc
+    except assistant.AssistantUnavailable as exc:
+        raise HTTPException(503, "an AI key is needed for this: add GEMINI_API_KEY (free) or another key to .env and restart") from exc
+    except assistant.AssistantError as exc:
+        raise HTTPException(502, str(exc)) from exc
+
+
+@router.get("/assistant/status", response_model=StatusOut)
+def status(request: Request) -> StatusOut:
+    s = request.app.state.settings
+    names = assistant.configured_providers(anthropic_key=s.anthropic_api_key)
+    return StatusOut(enabled=bool(names), model=s.assistant_model if "anthropic" in names else None, providers=names)
+
+
+@router.post("/assistant/ask", response_model=AnswerOut)
+def ask(body: AskIn, request: Request, db: Session = Depends(get_db)) -> AnswerOut:
+    context = dict(body.context)
+    agent = context.get("agent")
+    if isinstance(agent, str) and agent:
+        try:  # live numbers for the agent on screen, so "why are there so many failures?" can be answered
+            context["live_overview"] = insights.overview(db, agent, 168)
+        except Exception:  # noqa: BLE001 - the answer is still useful without them
+            pass
+    msgs = assistant.build_messages(body.question, context, [t.model_dump() for t in body.history])
+    s = request.app.state.settings
+    system = assistant.SYSTEM + "\n\nGuide to every page of the dashboard:\n" + assistant.guide_text()
+    try:
+        request.app.state.rate_limiter.check()
+    except assistant.RateLimited as exc:
+        raise HTTPException(429, str(exc)) from exc
+    try:
+        text, source = assistant.ask_any(system=system, messages=msgs, anthropic_key=s.anthropic_api_key, anthropic_model=s.assistant_model,
+                                         transport=request.app.state.assistant_transport, dead=request.app.state.assistant_dead)
+        return AnswerOut(answer=text, source=source)
+    except assistant.AssistantUnavailable:
+        return AnswerOut(answer=assistant.answer_from_guide(body.question), source="guide")
+    except assistant.AssistantError as exc:
+        note = "\n\n(The AI services did not answer, so this comes from the built-in guide. " + str(exc)[:300] + ")"
+        return AnswerOut(answer=assistant.answer_from_guide(body.question) + note, source="guide")
+
+
+@router.post("/agents/{name}/suggest-fix", response_model=FixOut)
+def suggest_fix(name: str, body: SuggestFixIn, request: Request, db: Session = Depends(get_db)) -> FixOut:
+    agent = db.scalar(select(Agent).where(Agent.name == name))
+    if agent is None:
+        raise HTTPException(404, f"agent '{name}' not found")
+    try:
+        report = rootcause.root_cause_report(db, agent=name, environment=body.environment, since_hours=body.since_hours,
+                                             diagnose_first=False)
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    active = db.scalar(select(AgentVersion).where(AgentVersion.agent_id == agent.id, AgentVersion.is_active.is_(True)))
+    msgs = assistant.fix_messages(active.label if active else None, active.prompt if active else None,
+                                  active.config if active else None, report)
+    text = _call(request, assistant.FIX_SYSTEM, msgs, 900)
+    return FixOut(suggestion=text, version=active.label if active else None, failed_runs=int(report.get("failed_runs") or 0),
+                  note="A draft written by an AI model. Register it as a new version and run it through a canary before release.")

agent_auditor/api/canary_routes.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from ..canary import service
+from .canary_schemas import CanaryEvaluateIn, CanaryReportOut, CanarySummaryOut
+from .deps import get_db, require_api_key
+
+router = APIRouter(prefix="/v1", dependencies=[Depends(require_api_key)])
+
+
+def _wrap(call):
+    try:
+        return call()
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+
+
+@router.post("/agents/{name}/canary/evaluate", response_model=CanaryReportOut)
+def evaluate(name: str, body: CanaryEvaluateIn, db: Session = Depends(get_db)) -> dict:
+    payload = body.model_dump()
+    data = _wrap(lambda: service.canary_report(db, agent=name, **payload))
+    db.commit()
+    return data
+
+
+@router.get("/agents/{name}/canary", response_model=list[CanarySummaryOut])
+def list_reports(
+    name: str, candidate: str | None = None, limit: int = Query(default=20, ge=1, le=100),
+    db: Session = Depends(get_db),
+) -> list[dict]:
+    return _wrap(lambda: service.list_reports(db, name, limit, candidate))
+
+
+@router.get("/canary/{report_id}", response_model=CanaryReportOut)
+def get_report(report_id: str, db: Session = Depends(get_db)) -> dict:
+    return _wrap(lambda: service.get_report(db, report_id))

agent_auditor/api/canary_schemas.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+Environment = Literal["production", "canary", "test"]
+
+
+class CanaryPolicyIn(BaseModel):
+    min_canary_runs: int = Field(default=30, ge=1, le=1_000_000)
+    min_baseline_runs: int = Field(default=30, ge=1, le=1_000_000)
+    min_eval_samples: int = Field(default=20, ge=1, le=1_000_000)
+    min_effect: float = Field(default=0.05, ge=0, le=1)
+    rollback_effect: float = Field(default=0.10, ge=0, le=1)
+    alpha: float = Field(default=0.01, gt=0, lt=1)
+    latency_warn_ratio: float = Field(default=1.3, ge=1)
+    latency_fail_ratio: float = Field(default=2.0, ge=1)
+    latency_rollback_ratio: float = Field(default=4.0, ge=1)
+    require_replay: bool = False
+    replay_rollback_min: int = Field(default=2, ge=1)
+    replay_rollback_share: float = Field(default=0.10, ge=0, le=1)
+    warnings_block: bool = False
+    include_drift: bool = True
+
+
+class CanaryEvaluateIn(BaseModel):
+    baseline: str = Field(description="Version label running in production")
+    candidate: str = Field(description="Version label being canaried")
+    baseline_environment: Environment = "production"
+    candidate_environment: Environment = "canary"
+    since_hours: float = Field(default=24.0, gt=0, le=24 * 365)
+    policy: CanaryPolicyIn = Field(default_factory=CanaryPolicyIn)
+    evaluate_first: bool = Field(default=True, description="Score unscored runs on both sides first")
+    diagnose_first: bool = Field(default=True, description="Diagnose undiagnosed runs on both sides first")
+    save: bool = True
+
+
+class CheckOut(BaseModel):
+    name: str
+    status: Literal["pass", "warn", "fail", "skipped"]
+    severity: Literal["none", "minor", "major"]
+    reason: str
+    details: dict[str, Any]
+
+
+class SideOut(BaseModel):
+    version: str
+    environment: str
+    runs: int
+
+
+class CanaryReportOut(BaseModel):
+    id: str | None = None
+    created_at: datetime | None = None
+    agent: str
+    baseline: SideOut
+    candidate: SideOut
+    window: dict[str, Any]
+    verdict: Literal["promote", "hold", "rollback", "continue"]
+    reasons: list[str]
+    blocking: list[str]
+    warnings: list[str]
+    checks: list[CheckOut]
+    policy: dict[str, Any]
+    caveat: str
+
+
+class CanarySummaryOut(BaseModel):
+    id: str
+    created_at: datetime
+    agent: str
+    baseline: str
+    candidate: str
+    verdict: str
+    blocking: list[str]
+    reasons: list[str]

agent_auditor/api/deps.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import hmac
+from collections.abc import Iterator
+
+from fastapi import HTTPException, Request, Security
+from fastapi.security import APIKeyHeader
+from sqlalchemy.orm import Session
+
+
+def get_db(request: Request) -> Iterator[Session]:
+    factory = request.app.state.session_factory
+    with factory() as session:
+        yield session
+
+
+# Declared as a security scheme so the Swagger page (/docs) shows one Authorize button.
+_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+
+def require_api_key(request: Request, api_key: str | None = Security(_api_key_header)) -> None:
+    expected = request.app.state.settings.api_key
+    if expected and not hmac.compare_digest((api_key or "").encode(), expected.encode()):
+        raise HTTPException(status_code=401, detail="invalid or missing API key")

agent_auditor/api/drift_routes.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from ..drift import service
+from ..models import DriftReport
+from .deps import get_db, require_api_key
+from .drift_schemas import DriftCheckIn, DriftReportOut, DriftReportSummary
+
+router = APIRouter(prefix="/v1", dependencies=[Depends(require_api_key)])
+
+
+def _full(row: DriftReport) -> DriftReportOut:
+    return DriftReportOut(id=row.id, created_at=row.created_at, **row.result)
+
+
+@router.post("/drift/check", response_model=DriftReportOut)
+def check_drift(body: DriftCheckIn, db: Session = Depends(get_db)) -> DriftReportOut:
+    try:
+        data, row = service.run_drift_check(db, **body.model_dump())
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    db.commit()
+    if row is None:
+        return DriftReportOut(**data)
+    return _full(row)
+
+
+@router.get("/agents/{name}/drift", response_model=list[DriftReportSummary])
+def list_drift_reports(name: str, limit: int = Query(default=10, ge=1, le=100), db: Session = Depends(get_db)) -> list[DriftReportSummary]:
+    try:
+        rows = service.list_reports(db, name, limit)
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    return [
+        DriftReportSummary(
+            id=r.id, created_at=r.created_at, as_of=r.as_of, environment=r.environment, version=r.version_label,
+            overall=r.overall, reference_runs=r.result.get("reference_runs", 0), current_runs=r.result.get("current_runs", 0),
+            flagged=[f"{s['name']}:{s['severity']}" for s in r.result.get("signals", []) if s["severity"] in ("medium", "high")],
+        )
+        for r in rows
+    ]
+
+
+@router.get("/drift/reports/{report_id}", response_model=DriftReportOut)
+def get_drift_report(report_id: str, db: Session = Depends(get_db)) -> DriftReportOut:
+    row = db.get(DriftReport, report_id)
+    if row is None:
+        raise HTTPException(404, "drift report not found")
+    return _full(row)

agent_auditor/api/drift_schemas.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class DriftCheckIn(BaseModel):
+    agent: str
+    environment: Literal["production", "canary", "test"] = "production"
+    version: str | None = Field(default=None, description="Only look at runs of this version label")
+    current_hours: float = Field(default=24.0, gt=0, le=24 * 365, description="Length of the recent window")
+    reference_hours: float = Field(default=168.0, gt=0, le=24 * 365 * 5, description="Length of the baseline window that ends where the recent one starts")
+    as_of: datetime | None = Field(default=None, description="End of the recent window; defaults to now")
+    min_samples: int = Field(default=30, ge=5, le=100_000, description="Minimum observations per window for a signal to be tested")
+    alpha: float = Field(default=0.01, gt=0, lt=1, description="Family-wise false alarm rate, split across signals")
+    correct_multiple: bool = True
+    include_evals: bool = Field(default=True, description="Also compare eval pass rates (needs runs scored by /v1/evals/run)")
+    max_runs: int = Field(default=2000, ge=50, le=5000, description="Newest runs analysed per window")
+    save: bool = True
+
+
+class DriftSignalOut(BaseModel):
+    name: str
+    kind: str
+    severity: str
+    significant: bool
+    p_value: float | None
+    alpha_used: float
+    effect_name: str
+    effect: float | None
+    reference_n: int
+    current_n: int
+    reference: dict[str, Any]
+    current: dict[str, Any]
+    reason: str
+    details: dict[str, Any]
+
+
+class DriftReportOut(BaseModel):
+    id: str | None = None
+    created_at: datetime | None = None
+    agent: str
+    environment: str
+    version: str | None
+    as_of: datetime
+    windows: dict[str, datetime]
+    overall: str
+    reference_runs: int
+    current_runs: int
+    reference_total_runs: int
+    current_total_runs: int
+    alpha: float
+    min_samples: int
+    notes: list[str]
+    signals: list[DriftSignalOut]
+
+
+class DriftReportSummary(BaseModel):
+    id: str
+    created_at: datetime
+    as_of: datetime
+    environment: str
+    version: str | None
+    overall: str
+    reference_runs: int
+    current_runs: int
+    flagged: list[str]

agent_auditor/api/evals_routes.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import Literal
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from ..evals import compare_counts, get_scorers, wilson_interval
+from ..evals import service
+from ..models import EvalResult, Run
+from .deps import get_db, require_api_key
+from .evals_schemas import (
+    CompareOut, EvalBatchOut, EvalResultOut, EvalRunIn, ScorerComparisonOut, ScorerInfo, ScorerStat,
+    SummaryOut, VersionSummary,
+)
+
+router = APIRouter(prefix="/v1", dependencies=[Depends(require_api_key)])
+
+Environment = Literal["production", "canary", "test"]
+
+
+def _utc(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
+
+
+@router.get("/evals/scorers", response_model=list[ScorerInfo])
+def list_scorers() -> list[ScorerInfo]:
+    return [
+        ScorerInfo(name=s.name, description=s.description, defaults=s.defaults, default_enabled=s.default_enabled)
+        for s in get_scorers().values()
+    ]
+
+
+@router.post("/evals/run", response_model=EvalBatchOut)
+def run_evals(body: EvalRunIn, db: Session = Depends(get_db)) -> EvalBatchOut:
+    if body.scorers:
+        unknown = [n for n in body.scorers if n not in get_scorers()]
+        if unknown:
+            raise HTTPException(422, f"unknown scorers: {', '.join(unknown)}")
+    try:
+        batch = service.evaluate_runs(
+            db, agent=body.agent, version=body.version, environment=body.environment, status=body.status,
+            limit=body.limit, only_unevaluated=body.only_unevaluated, scorers=body.scorers, params=body.params,
+        )
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    db.commit()
+    return EvalBatchOut(**asdict(batch))
+
+
+@router.get("/runs/{run_id}/evals", response_model=list[EvalResultOut])
+def get_run_evals(run_id: str, db: Session = Depends(get_db)) -> list[EvalResultOut]:
+    if db.get(Run, run_id) is None:
+        raise HTTPException(404, "run not found")
+    rows = db.scalars(select(EvalResult).where(EvalResult.run_id == run_id).order_by(EvalResult.scorer)).all()
+    return [
+        EvalResultOut(
+            scorer=r.scorer, applicable=r.applicable, score=r.score, passed=r.passed, reason=r.reason,
+            details=r.details, evaluated_at=r.evaluated_at,
+        )
+        for r in rows
+    ]
+
+
+@router.get("/agents/{name}/eval-summary", response_model=SummaryOut)
+def eval_summary(
+    name: str,
+    environment: Environment | None = None,
+    since: datetime | None = None,
+    db: Session = Depends(get_db),
+) -> SummaryOut:
+    try:
+        agent = service.get_agent(db, name)
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    data = service.agent_summary(db, agent, environment=environment, since=_utc(since))
+    versions = []
+    for label in sorted(data, key=lambda v: (v == service.UNVERSIONED, v)):
+        entry = data[label]
+        stats = []
+        for scorer, c in sorted(entry["scorers"].items()):
+            lo, hi = wilson_interval(c.passed, c.n)
+            stats.append(ScorerStat(
+                scorer=scorer, n=c.n, passed=c.passed, pass_rate=(c.passed / c.n) if c.n else None,
+                ci_low=lo if c.n else None, ci_high=hi if c.n else None, mean_score=c.mean_score,
+            ))
+        versions.append(VersionSummary(version=label, runs=entry["runs"], evaluated_runs=entry["evaluated_runs"], scorers=stats))
+    return SummaryOut(agent=agent.name, versions=versions)
+
+
+@router.get("/agents/{name}/compare", response_model=CompareOut)
+def compare_versions(
+    name: str,
+    baseline: str = Query(description="Version label of the known-good version"),
+    candidate: str = Query(description="Version label under test"),
+    baseline_environment: Environment | None = None,
+    candidate_environment: Environment | None = None,
+    since: datetime | None = None,
+    min_samples: int = Query(default=20, ge=1),
+    min_effect: float = Query(default=0.05, ge=0.0, le=1.0),
+    alpha: float = Query(default=0.05, gt=0.0, lt=1.0),
+    correct_multiple: bool = True,
+    db: Session = Depends(get_db),
+) -> CompareOut:
+    if baseline == candidate and baseline_environment == candidate_environment:
+        raise HTTPException(422, "compare two different versions, or the same version in different environments")
+    try:
+        agent = service.get_agent(db, name)
+        since_utc = _utc(since)
+        base = service.version_counts(db, agent, baseline, environment=baseline_environment, since=since_utc)
+        cand = service.version_counts(db, agent, candidate, environment=candidate_environment, since=since_utc)
+    except LookupError as exc:
+        raise HTTPException(404, str(exc)) from exc
+    result = compare_counts(
+        base, cand, min_samples=min_samples, min_effect=min_effect, alpha=alpha, correct_multiple=correct_multiple
+    )
+    return CompareOut(
+        agent=agent.name, baseline=baseline, candidate=candidate, overall_verdict=result.overall_verdict,
+        alpha=result.alpha, min_samples=result.min_samples, min_effect=result.min_effect, notes=result.notes,
+        scorers=[ScorerComparisonOut(**asdict(row)) for row in result.scorers],
+    )