selfevals 0.2.2__py3-none-any.whl

selfevals/api/queries.py

@@ -0,0 +1,447 @@
+"""Read queries over the SQLite store, shaped for the web UI.
+
+We don't add an ORM. We open a `WorkspaceScope` per request, list
+entities, and project them into the view models in
+`selfevals.api.schemas`. The single non-trivial bit is rebuilding
+the `OptimizationResult` JSON via the existing reconstruction helper
+in `cli.commands` so the web reuses the exact same shape the
+reporter emits.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel
+
+from selfevals.api.schemas import (
+    ExperimentDetailResponse,
+    ExperimentSummary,
+    IterationSummary,
+    SpanSummary,
+    ThreadResponse,
+    ThreadTurn,
+    TraceResponse,
+    WorkspaceResponse,
+    WorkspaceSummary,
+)
+from selfevals.cli.commands import (
+    _experiment_decisions,
+    _experiment_iterations,
+    _reconstruct_result,
+)
+from selfevals.reporter import render_json
+from selfevals.schemas.experiment import Experiment
+from selfevals.schemas.iteration import DecisionRecord, IterationRecord
+from selfevals.schemas.trace import Trace
+from selfevals.schemas.workspace import Workspace
+from selfevals.storage.interface import ListFilter
+from selfevals.storage.sqlite import SQLiteStorage
+
+
+class AnchorPoint(BaseModel):
+    experiment_id: str
+    experiment_name: str
+    iteration: int
+    primary_metric_name: str
+    primary_metric_value: float
+    decision_outcome: str
+    created_at: str
+
+
+def list_workspaces(storage: SQLiteStorage) -> list[WorkspaceSummary]:
+    """Cross-workspace listing. Direct SQL because the typed interface is
+    intentionally scoped — no way to list without a workspace_id."""
+    rows = storage.connection.execute(
+        "SELECT payload FROM entities WHERE entity_type = 'Workspace' ORDER BY created_at DESC"
+    ).fetchall()
+    summaries: list[WorkspaceSummary] = []
+    for (payload,) in rows:
+        ws = Workspace.model_validate(json.loads(payload))
+        exp_count = storage.connection.execute(
+            "SELECT COUNT(1) FROM entities WHERE entity_type = 'Experiment' AND workspace_id = ?",
+            (ws.id,),
+        ).fetchone()[0]
+        last_run = storage.connection.execute(
+            "SELECT MAX(updated_at) FROM entities "
+            "WHERE entity_type = 'IterationRecord' AND workspace_id = ?",
+            (ws.id,),
+        ).fetchone()[0]
+        summaries.append(
+            WorkspaceSummary(
+                id=ws.id,
+                slug=ws.slug,
+                name=ws.name,
+                description=ws.description,
+                owner_id=ws.owner_id,
+                created_at=ws.created_at,
+                experiment_count=int(exp_count or 0),
+                last_run_at=last_run,
+            )
+        )
+    return summaries
+
+
+def workspace_detail(storage: SQLiteStorage, *, workspace_id: str) -> WorkspaceResponse | None:
+    try:
+        with storage.open(workspace_id) as scope:
+            ws = scope.get_entity(Workspace, workspace_id)
+            assert isinstance(ws, Workspace)
+            experiments: Sequence[Experiment] = [
+                e
+                for e in scope.list_entities(Experiment, ListFilter())
+                if isinstance(e, Experiment)
+            ]
+            recent_iterations = [
+                it
+                for it in scope.list_entities(
+                    IterationRecord,
+                    ListFilter(order_by="updated_at", limit=20),
+                )
+                if isinstance(it, IterationRecord)
+            ]
+    except Exception:
+        return None
+    keep_count = sum(
+        1
+        for it in recent_iterations
+        if it.decision is not None and str(it.decision.outcome) == "keep_candidate"
+    )
+    recent_health: float | None = None
+    if recent_iterations:
+        recent_health = round(keep_count / len(recent_iterations), 3)
+    return WorkspaceResponse(
+        id=ws.id,
+        slug=ws.slug,
+        name=ws.name,
+        description=ws.description,
+        owner_id=ws.owner_id,
+        created_at=ws.created_at,
+        experiment_count=len(experiments),
+        recent_health=recent_health,
+    )
+
+
+def list_experiments(
+    storage: SQLiteStorage, *, workspace_id: str, limit: int = 100
+) -> list[dict[str, Any]]:
+    with storage.open(workspace_id) as scope:
+        experiments = [
+            e
+            for e in scope.list_entities(Experiment, ListFilter(order_by="updated_at", limit=limit))
+            if isinstance(e, Experiment)
+        ]
+        out: list[dict[str, Any]] = []
+        for exp in experiments:
+            it_count = sum(
+                1
+                for it in scope.list_entities(IterationRecord, ListFilter())
+                if isinstance(it, IterationRecord) and it.experiment_id == exp.id
+            )
+            out.append(_experiment_summary_dict(exp, iteration_count=it_count))
+        return out
+
+
+def experiment_detail(
+    storage: SQLiteStorage, *, workspace_id: str, experiment_id: str
+) -> ExperimentDetailResponse | None:
+    with storage.open(workspace_id) as scope:
+        try:
+            exp = scope.get_entity(Experiment, experiment_id)
+        except Exception:
+            return None
+        assert isinstance(exp, Experiment)
+        iterations = _experiment_iterations(scope, exp.id)
+        decisions = _experiment_decisions(scope, exp.id)
+
+    result_dict: dict[str, Any] | None = None
+    if iterations:
+        result = _reconstruct_result(exp, iterations, decisions)
+        result_dict = json.loads(render_json(result))
+
+    summary = ExperimentSummary(**_experiment_summary_dict(exp, iteration_count=len(iterations)))
+    return ExperimentDetailResponse(
+        summary=summary,
+        result=result_dict,
+        iterations=_iteration_summaries(iterations, decisions),
+    )
+
+
+def experiment_iterations(
+    storage: SQLiteStorage, *, workspace_id: str, experiment_id: str
+) -> list[IterationSummary]:
+    with storage.open(workspace_id) as scope:
+        iterations = _experiment_iterations(scope, experiment_id)
+        decisions = _experiment_decisions(scope, experiment_id)
+    return _iteration_summaries(iterations, decisions)
+
+
+def experiment_decisions(
+    storage: SQLiteStorage, *, workspace_id: str, experiment_id: str
+) -> list[dict[str, Any]]:
+    with storage.open(workspace_id) as scope:
+        decisions = _experiment_decisions(scope, experiment_id)
+    out: list[dict[str, Any]] = []
+    for iteration in sorted(decisions):
+        d = decisions[iteration]
+        out.append(
+            {
+                "id": d.id,
+                "iteration": d.iteration,
+                "outcome": str(d.outcome),
+                "automated_rationale": d.rationale.automated,
+                "human_rationale": (d.rationale.human.notes if d.rationale.human else None),
+                "metrics_snapshot": d.metrics_snapshot,
+                "created_at": d.created_at.isoformat(),
+            }
+        )
+    return out
+
+
+def iteration_detail(
+    storage: SQLiteStorage, *, workspace_id: str, iteration_id: str
+) -> dict[str, Any] | None:
+    with storage.open(workspace_id) as scope:
+        try:
+            it = scope.get_entity(IterationRecord, iteration_id)
+        except Exception:
+            return None
+        assert isinstance(it, IterationRecord)
+        decisions = _experiment_decisions(scope, it.experiment_id)
+    decision = decisions.get(it.iteration)
+    return {
+        "iteration": it.model_dump(mode="json"),
+        "decision": decision.model_dump(mode="json") if decision else None,
+    }
+
+
+def load_trace(storage: SQLiteStorage, *, workspace_id: str, trace_id: str) -> TraceResponse | None:
+    """Look up a Trace by either its entity id (`tr_...`) or its run_id
+    (`run_...`). Both are common navigation targets — IterationRecord
+    persists `run_id`s while internal storage keys by entity id."""
+    with storage.open(workspace_id) as scope:
+        try:
+            trace = scope.get_entity(Trace, trace_id)
+        except Exception:
+            trace = None
+        if trace is None:
+            # Fall back to a run_id lookup. The generic entities table
+            # does not index json_extract, but the workspace-scoped
+            # table is small enough that a single scan is fine.
+            row = storage.connection.execute(
+                "SELECT payload FROM entities "
+                "WHERE workspace_id = ? AND entity_type = 'Trace' "
+                "AND json_extract(payload, '$.run.run_id') = ? LIMIT 1",
+                (workspace_id, trace_id),
+            ).fetchone()
+            if row is None:
+                return None
+            trace = Trace.model_validate(json.loads(row[0]))
+    assert isinstance(trace, Trace)
+    return TraceResponse(
+        id=trace.id,
+        run_id=trace.run.run_id,
+        experiment_id=trace.run.experiment_id,
+        iteration=trace.run.iteration,
+        thread_id=trace.run.thread_id,
+        thread_position=trace.run.thread_position,
+        final_state=str(trace.final_state.status),
+        started_at=trace.environment.started_at,
+        ended_at=trace.environment.ended_at,
+        spans=[_span_summary(s) for s in trace.spans],
+        metrics=trace.metrics.model_dump(mode="json"),
+    )
+
+
+def load_thread(
+    storage: SQLiteStorage, *, workspace_id: str, thread_id: str
+) -> ThreadResponse | None:
+    """Assemble every Trace sharing `thread_id` into an ordered conversation.
+
+    Traces are ordered by `run.thread_position` when set, falling back to
+    `environment.started_at` so a thread without explicit turn indices still
+    reads in chronological order. Each turn carries its grader results so the
+    thread view shows the grade per turn, not just the transcript.
+    Returns None when no trace carries the thread_id.
+    """
+    rows = storage.connection.execute(
+        "SELECT payload FROM entities "
+        "WHERE workspace_id = ? AND entity_type = 'Trace' "
+        "AND json_extract(payload, '$.run.thread_id') = ?",
+        (workspace_id, thread_id),
+    ).fetchall()
+    if not rows:
+        return None
+
+    traces = [Trace.model_validate(json.loads(payload)) for (payload,) in rows]
+
+    def _sort_key(t: Trace) -> tuple[int, int, datetime]:
+        # Explicitly-positioned turns first (by position), then the rest by
+        # start time. The leading int makes positioned turns sort ahead of
+        # unpositioned ones deterministically.
+        pos = t.run.thread_position
+        has_pos = 0 if pos is not None else 1
+        return (has_pos, pos if pos is not None else 0, t.environment.started_at)
+
+    traces.sort(key=_sort_key)
+
+    turns: list[ThreadTurn] = []
+    for idx, trace in enumerate(traces):
+        primary_grade = trace.grader_results[0].label if trace.grader_results else None
+        turns.append(
+            ThreadTurn(
+                trace_id=trace.id,
+                run_id=trace.run.run_id,
+                position=trace.run.thread_position if trace.run.thread_position is not None else idx,
+                experiment_id=trace.run.experiment_id,
+                iteration=trace.run.iteration,
+                final_state=str(trace.final_state.status),
+                started_at=trace.environment.started_at,
+                ended_at=trace.environment.ended_at,
+                primary_grade=primary_grade,
+                grader_results=[g.model_dump(mode="json") for g in trace.grader_results],
+                metrics=trace.metrics.model_dump(mode="json"),
+            )
+        )
+    return ThreadResponse(thread_id=thread_id, turn_count=len(turns), turns=turns)
+
+
+def anchor_set_history(storage: SQLiteStorage, *, workspace_id: str) -> list[AnchorPoint]:
+    """Longitudinal view: latest primary-metric value per experiment.
+
+    Anchor-set proper requires repeated reruns of a canonical case
+    set; until that lands, we expose the per-experiment latest
+    completed iteration so the chart has shape.
+    """
+    with storage.open(workspace_id) as scope:
+        experiments = [
+            e for e in scope.list_entities(Experiment, ListFilter()) if isinstance(e, Experiment)
+        ]
+        points: list[AnchorPoint] = []
+        for exp in experiments:
+            iterations = _experiment_iterations(scope, exp.id)
+            decisions = _experiment_decisions(scope, exp.id)
+            for it in iterations:
+                if it.metrics is None:
+                    continue
+                decision = decisions.get(it.iteration)
+                outcome = str(decision.outcome) if decision else "unknown"
+                points.append(
+                    AnchorPoint(
+                        experiment_id=exp.id,
+                        experiment_name=exp.name,
+                        iteration=it.iteration,
+                        primary_metric_name=it.metrics.primary.name,
+                        primary_metric_value=it.metrics.primary.value,
+                        decision_outcome=outcome,
+                        created_at=it.created_at.isoformat(),
+                    )
+                )
+    points.sort(key=lambda p: p.created_at)
+    return points
+
+
+def _experiment_summary_dict(exp: Experiment, *, iteration_count: int) -> dict[str, Any]:
+    return {
+        "id": exp.id,
+        "name": exp.name,
+        "goal": exp.goal,
+        "mode": str(exp.mode),
+        "state": str(exp.state),
+        "primary_metric": exp.target.primary.name,
+        "primary_target": {
+            "operator": exp.target.primary.operator,
+            "value": exp.target.primary.value,
+        },
+        "proposer_strategy": str(exp.proposer.strategy),
+        "max_iterations": exp.run.max_iterations,
+        "created_at": exp.created_at.isoformat(),
+        "updated_at": exp.updated_at.isoformat(),
+        "iteration_count": iteration_count,
+    }
+
+
+def _iteration_summaries(
+    iterations: Sequence[IterationRecord],
+    decisions: dict[int, DecisionRecord],
+) -> list[IterationSummary]:
+    best_so_far: float | None = None
+    out: list[IterationSummary] = []
+    for it in iterations:
+        primary = it.metrics.primary if it.metrics else None
+        delta: float | None = None
+        if primary is not None:
+            delta = 0.0 if best_so_far is None else primary.value - best_so_far
+            if best_so_far is None or primary.value > best_so_far:
+                best_so_far = primary.value
+        decision = decisions.get(it.iteration)
+        out.append(
+            IterationSummary(
+                id=it.id,
+                iteration=it.iteration,
+                state=str(it.state),
+                hypothesis=it.hypothesis,
+                proposed_parameters=dict(it.proposed_parameters),
+                primary_metric_name=primary.name if primary else None,
+                primary_metric_value=primary.value if primary else None,
+                delta_vs_best=delta,
+                decision_outcome=(str(decision.outcome) if decision is not None else None),
+                decision_rationale=(decision.rationale.automated if decision is not None else None),
+                cost_usd=it.cost_usd,
+                duration_seconds=it.duration_seconds,
+                trace_run_ids=list(it.execution.trace_run_ids),
+                created_at=it.created_at,
+            )
+        )
+    return out
+
+
+def _span_summary(span: Any) -> SpanSummary:
+    """Project any Span subclass into the trimmed view shape.
+
+    We surface kind + name + parent + timing on every span, and copy
+    the kind-specific high-value fields into `detail` for the trace
+    inspector to render without fetching the full payload.
+    """
+    detail: dict[str, Any] = {}
+    payload = span.model_dump(mode="json")
+    keep_keys = {
+        "provider",
+        "model",
+        "params",
+        "tokens",
+        "cost_usd",
+        "output",
+        "reasoning",
+        "tool_name",
+        "tool_use_id",
+        "status",
+        "error",
+        "retriever",
+        "top_k_requested",
+        "top_k_returned",
+        "retrieved",
+        "decision_type",
+        "chosen",
+        "alternatives_considered",
+        "guardrail",
+        "passed",
+        "error_type",
+        "message",
+        "recoverable",
+    }
+    for key, value in payload.items():
+        if key in keep_keys:
+            detail[key] = value
+    return SpanSummary(
+        id=span.id,
+        parent_id=span.parent_id,
+        kind=str(span.kind),
+        name=span.name,
+        started_at=span.started_at,
+        duration_ms=span.duration_ms,
+        detail=detail,
+    )

selfevals/api/schemas.py

@@ -0,0 +1,151 @@
+"""Pydantic response models for the HTTP bridge.
+
+These are *view* shapes — denormalized snapshots of the canonical
+entities that the web UI needs. The canonical schemas in
+`selfevals.schemas` stay the source of truth; this module simply
+chooses what to expose and in what shape.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class HealthResponse(BaseModel):
+    status: str
+    db_path: str
+
+
+class WorkspaceSummary(BaseModel):
+    id: str
+    slug: str
+    name: str
+    description: str | None = None
+    owner_id: str | None = None
+    created_at: datetime
+    experiment_count: int = 0
+    last_run_at: datetime | None = None
+
+
+class WorkspaceListResponse(BaseModel):
+    workspaces: list[WorkspaceSummary]
+
+
+class WorkspaceResponse(BaseModel):
+    id: str
+    slug: str
+    name: str
+    description: str | None = None
+    owner_id: str | None = None
+    created_at: datetime
+    experiment_count: int
+    recent_health: float | None = Field(
+        default=None,
+        description="Fraction of recent experiments that landed on keep_candidate.",
+    )
+
+
+class CreateWorkspaceRequest(BaseModel):
+    slug: str = Field(min_length=1, max_length=63)
+    name: str | None = None
+    description: str | None = None
+
+
+class IterationSummary(BaseModel):
+    id: str
+    iteration: int
+    state: str
+    hypothesis: str
+    proposed_parameters: dict[str, Any] = Field(default_factory=dict)
+    primary_metric_name: str | None = None
+    primary_metric_value: float | None = None
+    delta_vs_best: float | None = None
+    decision_outcome: str | None = None
+    decision_rationale: str | None = None
+    cost_usd: float | None = None
+    duration_seconds: float | None = None
+    trace_run_ids: list[str] = Field(default_factory=list)
+    created_at: datetime
+
+
+class IterationListResponse(BaseModel):
+    iterations: list[IterationSummary]
+
+
+class ExperimentSummary(BaseModel):
+    id: str
+    name: str
+    goal: str
+    mode: str
+    state: str
+    primary_metric: str
+    primary_target: dict[str, Any]
+    proposer_strategy: str
+    max_iterations: int
+    created_at: datetime
+    updated_at: datetime
+    iteration_count: int = 0
+
+
+class ExperimentDetailResponse(BaseModel):
+    """Shape returned by GET /workspaces/{ws}/experiments/{id}.
+
+    `result` is the JSON shape from `selfevals.reporter.render_json`
+    when there is at least one completed iteration to reconstruct;
+    `None` when the experiment has not run yet.
+    """
+
+    summary: ExperimentSummary
+    result: dict[str, Any] | None = None
+    iterations: list[IterationSummary] = Field(default_factory=list)
+
+
+class SpanSummary(BaseModel):
+    id: str
+    parent_id: str | None
+    kind: str
+    name: str
+    started_at: datetime
+    duration_ms: int
+    detail: dict[str, Any] = Field(default_factory=dict)
+
+
+class TraceResponse(BaseModel):
+    id: str
+    run_id: str
+    experiment_id: str | None
+    iteration: int | None
+    thread_id: str | None = None
+    thread_position: int | None = None
+    final_state: str
+    started_at: datetime
+    ended_at: datetime | None
+    spans: list[SpanSummary]
+    metrics: dict[str, Any]
+
+
+class ThreadTurn(BaseModel):
+    """One trace within a thread, projected as a turn for the thread view."""
+
+    trace_id: str
+    run_id: str
+    position: int
+    experiment_id: str | None = None
+    iteration: int | None = None
+    final_state: str
+    started_at: datetime
+    ended_at: datetime | None = None
+    primary_grade: str | None = None
+    grader_results: list[dict[str, Any]] = Field(default_factory=list)
+    metrics: dict[str, Any] = Field(default_factory=dict)
+
+
+class ThreadResponse(BaseModel):
+    """All traces sharing a thread_id, assembled into an ordered conversation."""
+
+    thread_id: str
+    turn_count: int
+    turns: list[ThreadTurn] = Field(default_factory=list)

selfevals/api/sse.py

@@ -0,0 +1,114 @@
+"""SSE stream of trace spans, with a heartbeat and a snapshot prelude.
+
+Wire format (one frame is `<lines>\\n\\n`):
+
+    event: snapshot
+    data: {... full Trace JSON ...}
+
+    event: span
+    data: {... one SpanSummary ...}
+
+    event: ping
+    data: 1
+
+    event: complete
+    data: {"final_state": "completed"}
+
+The client subscribes via `new EventSource(url)`. Heartbeat every 15s
+keeps proxies from idle-closing the connection.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from collections.abc import AsyncIterator, Callable
+from typing import Any
+
+from fastapi.responses import StreamingResponse
+
+from selfevals.api.broker import SpanBroker, _Closed
+from selfevals.api.queries import load_trace
+from selfevals.storage.sqlite import SQLiteStorage
+
+logger = logging.getLogger(__name__)
+
+_HEARTBEAT_SECONDS = 15.0
+_HEADERS = {
+    "Cache-Control": "no-cache, no-transform",
+    "Connection": "keep-alive",
+    "X-Accel-Buffering": "no",
+}
+
+
+def _encode(event: str, data: Any) -> bytes:
+    payload = data if isinstance(data, str) else json.dumps(data, default=str)
+    return f"event: {event}\ndata: {payload}\n\n".encode()
+
+
+async def stream_trace(
+    *,
+    workspace_id: str,
+    run_id: str,
+    broker: SpanBroker,
+    storage_factory: Callable[[], SQLiteStorage],
+) -> StreamingResponse:
+    """Build a StreamingResponse that emits snapshot + live spans."""
+
+    async def gen() -> AsyncIterator[bytes]:
+        # 1. Initial snapshot from SQLite (may be None if the run hasn't
+        #    persisted yet; that's fine — the client gets an empty
+        #    snapshot and waits for live spans).
+        storage = storage_factory()
+        try:
+            snapshot = load_trace(storage, workspace_id=workspace_id, trace_id=run_id)
+        finally:
+            storage.close()
+        if snapshot is not None:
+            yield _encode("snapshot", snapshot.model_dump(mode="json"))
+        else:
+            yield _encode("snapshot", {"run_id": run_id, "spans": []})
+
+        # 2. Live subscription.
+        sub = broker.subscribe(workspace_id, run_id)
+        agen = sub.__aiter__()
+
+        async def _next() -> dict[str, Any] | _Closed:
+            return await agen.__anext__()
+
+        heartbeat_task: asyncio.Task[None] | None = None
+        next_event_task: asyncio.Task[dict[str, Any] | _Closed] | None = None
+        try:
+            while True:
+                if next_event_task is None:
+                    next_event_task = asyncio.create_task(_next())
+                if heartbeat_task is None:
+                    heartbeat_task = asyncio.create_task(asyncio.sleep(_HEARTBEAT_SECONDS))
+                done, _pending = await asyncio.wait(
+                    {next_event_task, heartbeat_task},
+                    return_when=asyncio.FIRST_COMPLETED,
+                )
+                if heartbeat_task in done:
+                    heartbeat_task = None
+                    yield _encode("ping", "1")
+                if next_event_task in done:
+                    try:
+                        event = next_event_task.result()
+                    except StopAsyncIteration:
+                        return
+                    next_event_task = None
+                    if isinstance(event, _Closed):
+                        yield _encode("complete", {"final_state": event.final_state})
+                        return
+                    yield _encode("span", event)
+        finally:
+            for task in (heartbeat_task, next_event_task):
+                if task is not None and not task.done():
+                    task.cancel()
+
+    return StreamingResponse(
+        gen(),
+        media_type="text/event-stream",
+        headers=_HEADERS,
+    )