selfevals 0.2.2__py3-none-any.whl

selfevals/api/app.py

@@ -0,0 +1,351 @@
+"""FastAPI app — read-mostly HTTP bridge over the SQLite store.
+
+Mounted on `/` (no version prefix; this is a single internal service).
+Endpoints map 1:1 to the pages of the web UI; payload shapes match
+the existing Pydantic models so the web side can validate against the
+same canonical JSON.
+
+Auth: stubbed via a single `X-SelfEvals-User` header (default
+`"local"`). Real auth lands later; everything else is forward-compat.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Annotated, Any
+
+from fastapi import Depends, FastAPI, Header, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+from selfevals.api.broker import get_broker
+from selfevals.api.queries import (
+    AnchorPoint,
+    anchor_set_history,
+    experiment_decisions,
+    experiment_detail,
+    experiment_iterations,
+    iteration_detail,
+    list_experiments,
+    list_workspaces,
+    load_thread,
+    load_trace,
+    workspace_detail,
+)
+from selfevals.api.schemas import (
+    CreateWorkspaceRequest,
+    ExperimentDetailResponse,
+    HealthResponse,
+    IterationListResponse,
+    ThreadResponse,
+    TraceResponse,
+    WorkspaceListResponse,
+    WorkspaceResponse,
+)
+from selfevals.api.sse import stream_trace
+from selfevals.storage.sqlite import SQLiteStorage
+
+DEFAULT_DB_PATH = "./selfevals.sqlite"
+_USER_HEADER = "X-SelfEvals-User"
+
+UserHeader = Annotated[
+    str | None,
+    Header(alias=_USER_HEADER, description="Stubbed user id (auth is post-MVP)."),
+]
+
+
+def _resolve_db_path(db_path: str | None) -> str:
+    return db_path or os.environ.get("SELFEVALS_DB", DEFAULT_DB_PATH)
+
+
+def build_app(*, db_path: str | None = None) -> FastAPI:
+    """Construct the FastAPI app, parameterized on the SQLite db path."""
+    resolved = _resolve_db_path(db_path)
+    Path(resolved).parent.mkdir(parents=True, exist_ok=True)
+
+    @asynccontextmanager
+    async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
+        # Capture the running event loop so the OTLP receiver thread
+        # (which runs sync) can schedule span publishes onto it.
+        get_broker().bind_loop(asyncio.get_running_loop())
+        yield
+
+    app = FastAPI(
+        title="selfevals",
+        description="HTTP bridge for the selfevals evals framework.",
+        version="0.0.1",
+        docs_url="/api/docs",
+        redoc_url=None,
+        openapi_url="/api/openapi.json",
+        lifespan=lifespan,
+    )
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"],
+        allow_credentials=False,
+        allow_methods=["GET", "POST", "OPTIONS"],
+        allow_headers=["*"],
+    )
+
+    def _storage() -> SQLiteStorage:
+        return SQLiteStorage(resolved)
+
+    def _storage_factory() -> SQLiteStorage:
+        return SQLiteStorage(resolved)
+
+    @app.get("/api/health", response_model=HealthResponse, tags=["meta"])
+    def health() -> HealthResponse:
+        return HealthResponse(status="ok", db_path=resolved)
+
+    @app.get(
+        "/api/workspaces",
+        response_model=WorkspaceListResponse,
+        tags=["workspaces"],
+    )
+    def workspaces_index(
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> WorkspaceListResponse:
+        try:
+            return WorkspaceListResponse(workspaces=list_workspaces(storage))
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}",
+        response_model=WorkspaceResponse,
+        tags=["workspaces"],
+    )
+    def workspaces_show(
+        workspace_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> WorkspaceResponse:
+        try:
+            ws = workspace_detail(storage, workspace_id=workspace_id)
+            if ws is None:
+                raise HTTPException(status_code=404, detail="workspace not found")
+            return ws
+        finally:
+            storage.close()
+
+    @app.post(
+        "/api/workspaces",
+        response_model=WorkspaceResponse,
+        status_code=201,
+        tags=["workspaces"],
+    )
+    def workspaces_create(
+        body: CreateWorkspaceRequest,
+        storage: SQLiteStorage = Depends(_storage),
+        user: UserHeader = None,
+    ) -> WorkspaceResponse:
+        from selfevals.storage.seed import seed_workspace
+
+        try:
+            seeded = seed_workspace(
+                storage,
+                slug=body.slug,
+                name=body.name or body.slug,
+                user_id=user or "local",
+                description=body.description,
+            )
+            ws = seeded.workspace
+            return WorkspaceResponse(
+                id=ws.id,
+                slug=ws.slug,
+                name=ws.name,
+                description=ws.description,
+                owner_id=ws.owner_id,
+                created_at=ws.created_at,
+                experiment_count=0,
+                recent_health=None,
+            )
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/experiments",
+        response_model=list[dict[str, Any]],
+        tags=["experiments"],
+    )
+    def experiments_index(
+        workspace_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        limit: Annotated[int, Query(ge=1, le=500)] = 100,
+        _user: UserHeader = None,
+    ) -> list[dict[str, Any]]:
+        try:
+            return list_experiments(storage, workspace_id=workspace_id, limit=limit)
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/experiments/{experiment_id}",
+        response_model=ExperimentDetailResponse,
+        tags=["experiments"],
+    )
+    def experiments_show(
+        workspace_id: str,
+        experiment_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> ExperimentDetailResponse:
+        try:
+            detail = experiment_detail(
+                storage,
+                workspace_id=workspace_id,
+                experiment_id=experiment_id,
+            )
+            if detail is None:
+                raise HTTPException(
+                    status_code=404,
+                    detail=f"experiment {experiment_id} not found",
+                )
+            return detail
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/experiments/{experiment_id}/iterations",
+        response_model=IterationListResponse,
+        tags=["experiments"],
+    )
+    def experiments_iterations(
+        workspace_id: str,
+        experiment_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> IterationListResponse:
+        try:
+            return IterationListResponse(
+                iterations=experiment_iterations(
+                    storage,
+                    workspace_id=workspace_id,
+                    experiment_id=experiment_id,
+                )
+            )
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/experiments/{experiment_id}/decisions",
+        tags=["experiments"],
+    )
+    def experiments_decisions(
+        workspace_id: str,
+        experiment_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> list[dict[str, Any]]:
+        try:
+            return experiment_decisions(
+                storage,
+                workspace_id=workspace_id,
+                experiment_id=experiment_id,
+            )
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/iterations/{iteration_id}",
+        tags=["experiments"],
+    )
+    def iterations_show(
+        workspace_id: str,
+        iteration_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> dict[str, Any]:
+        try:
+            detail = iteration_detail(
+                storage,
+                workspace_id=workspace_id,
+                iteration_id=iteration_id,
+            )
+            if detail is None:
+                raise HTTPException(status_code=404, detail="iteration not found")
+            return detail
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/traces/{trace_id}",
+        response_model=TraceResponse,
+        tags=["traces"],
+    )
+    def traces_show(
+        workspace_id: str,
+        trace_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> TraceResponse:
+        try:
+            trace = load_trace(storage, workspace_id=workspace_id, trace_id=trace_id)
+            if trace is None:
+                raise HTTPException(status_code=404, detail="trace not found")
+            return trace
+        finally:
+            storage.close()
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/threads/{thread_id}",
+        response_model=ThreadResponse,
+        tags=["traces"],
+    )
+    def threads_show(
+        workspace_id: str,
+        thread_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> ThreadResponse:
+        try:
+            thread = load_thread(storage, workspace_id=workspace_id, thread_id=thread_id)
+            if thread is None:
+                raise HTTPException(status_code=404, detail="thread not found")
+            return thread
+        finally:
+            storage.close()
+
+    @app.get("/api/runs/active", tags=["traces"])
+    def runs_active(_user: UserHeader = None) -> list[dict[str, str]]:
+        return [{"workspace_id": ws, "run_id": run} for (ws, run) in get_broker().active_runs()]
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/traces/{run_id}/stream",
+        tags=["traces"],
+        response_class=StreamingResponse,
+    )
+    async def traces_stream(
+        workspace_id: str,
+        run_id: str,
+        _user: UserHeader = None,
+    ) -> StreamingResponse:
+        return await stream_trace(
+            workspace_id=workspace_id,
+            run_id=run_id,
+            broker=get_broker(),
+            storage_factory=_storage_factory,
+        )
+
+    @app.get(
+        "/api/workspaces/{workspace_id}/anchor-set",
+        response_model=list[AnchorPoint],
+        tags=["anchor-set"],
+    )
+    def anchor_set(
+        workspace_id: str,
+        storage: SQLiteStorage = Depends(_storage),
+        _user: UserHeader = None,
+    ) -> list[AnchorPoint]:
+        try:
+            return anchor_set_history(storage, workspace_id=workspace_id)
+        finally:
+            storage.close()
+
+    return app

selfevals/api/broker.py

@@ -0,0 +1,210 @@
+"""In-process pub/sub for live trace streaming.
+
+The OTLP receiver thread calls `publish_threadsafe()` when a span lands.
+FastAPI SSE handlers call `subscribe()` to get an async generator of
+events for a given `(workspace_id, run_id)` pair.
+
+Why this lives in `selfevals.api`: the broker is a *transport* concern,
+not a capture concern. The receiver doesn't know what it's for; it
+just calls a callback. Coupling the broker to the API package keeps
+the capture pipeline import-graph clean (the SDK / OTLP receiver
+don't need to know FastAPI exists).
+
+Scaling note: this is a single-process in-memory broker. The contract
+(`publish` + `subscribe`) is intentionally narrow so a Redis-backed
+implementation can drop in later without touching callers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import suppress
+from dataclasses import dataclass, field
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+# Sentinel objects on the queue — clearer than magic dicts.
+@dataclass(frozen=True)
+class _Closed:
+    final_state: str = "completed"
+
+
+_QUEUE_MAXSIZE = 256
+"""Per-subscriber queue depth. If we exceed it, the slowest subscriber
+gets disconnected — it's the wrong behaviour to backpressure the
+receiver thread for one stuck browser tab."""
+
+
+@dataclass
+class _Subscriber:
+    queue: asyncio.Queue[dict[str, Any] | _Closed]
+    workspace_id: str
+    run_id: str
+    closed: bool = False
+
+
+@dataclass
+class _Channel:
+    """All subscribers for one (workspace_id, run_id) pair."""
+
+    subscribers: list[_Subscriber] = field(default_factory=list)
+    closed: bool = False
+    final_state: str | None = None
+
+
+class SpanBroker:
+    """In-proc fan-out from the OTLP receiver to SSE subscribers."""
+
+    def __init__(self) -> None:
+        self._channels: dict[tuple[str, str], _Channel] = {}
+        self._lock = asyncio.Lock()
+        self._loop: asyncio.AbstractEventLoop | None = None
+
+    def bind_loop(self, loop: asyncio.AbstractEventLoop) -> None:
+        """Capture the FastAPI event loop so the receiver thread can
+        schedule publishes onto it. Call once at app startup."""
+        self._loop = loop
+
+    def active_runs(self) -> list[tuple[str, str]]:
+        """Snapshot of (workspace_id, run_id) channels that are open.
+
+        Used by `GET /api/runs/active` so the web shell can show a
+        live pill for in-flight runs. Includes runs whose channel was
+        opened by `mark_run_active` even before any spans arrived."""
+        return [(ws, run) for (ws, run), ch in self._channels.items() if not ch.closed]
+
+    def mark_run_active_threadsafe(self, workspace_id: str, run_id: str) -> None:
+        """Open the channel for a run before any spans flow. Lets the
+        web's "active runs" pill light up the moment a run starts."""
+        loop = self._loop
+        if loop is None:
+            return
+        with suppress(RuntimeError):
+            loop.call_soon_threadsafe(self._mark_active_sync, workspace_id, run_id)
+
+    def _mark_active_sync(self, workspace_id: str, run_id: str) -> None:
+        key = (workspace_id, run_id)
+        self._channels.setdefault(key, _Channel())
+
+    async def subscribe(
+        self, workspace_id: str, run_id: str
+    ) -> AsyncIterator[dict[str, Any] | _Closed]:
+        """Async-iterate events for one run. Caller is responsible for
+        cancelling the iteration when the client disconnects.
+
+        Note: this does NOT replay history. The SSE handler emits a
+        snapshot of the current Trace state *before* calling
+        subscribe(), so the subscriber only needs new spans from here.
+        """
+        key = (workspace_id, run_id)
+        sub = _Subscriber(
+            queue=asyncio.Queue(maxsize=_QUEUE_MAXSIZE),
+            workspace_id=workspace_id,
+            run_id=run_id,
+        )
+        async with self._lock:
+            channel = self._channels.setdefault(key, _Channel())
+            channel.subscribers.append(sub)
+            # If the channel is already closed, emit the close event and
+            # return without ever blocking.
+            already_closed = channel.closed
+            final_state = channel.final_state
+        if already_closed:
+            yield _Closed(final_state=final_state or "completed")
+            return
+        try:
+            while True:
+                event = await sub.queue.get()
+                if isinstance(event, _Closed):
+                    yield event
+                    return
+                yield event
+        finally:
+            sub.closed = True
+            async with self._lock:
+                ch = self._channels.get(key)
+                if ch is not None:
+                    ch.subscribers = [s for s in ch.subscribers if not s.closed]
+                    if not ch.subscribers and ch.closed:
+                        self._channels.pop(key, None)
+
+    def publish_threadsafe(
+        self, workspace_id: str, run_id: str, span_payload: dict[str, Any]
+    ) -> None:
+        """Called from the OTLP receiver's background thread.
+
+        Hops onto the FastAPI event loop via call_soon_threadsafe.
+        If no loop is bound, drops silently — the broker is best-effort,
+        not the source of truth (SQLite is)."""
+        loop = self._loop
+        if loop is None:
+            return
+        # Loop may be closed during process shutdown — best-effort.
+        with suppress(RuntimeError):
+            loop.call_soon_threadsafe(self._publish_sync, workspace_id, run_id, span_payload)
+
+    def close_run_threadsafe(
+        self, workspace_id: str, run_id: str, final_state: str = "completed"
+    ) -> None:
+        loop = self._loop
+        if loop is None:
+            return
+        with suppress(RuntimeError):
+            loop.call_soon_threadsafe(self._close_sync, workspace_id, run_id, final_state)
+
+    def _publish_sync(self, workspace_id: str, run_id: str, span_payload: dict[str, Any]) -> None:
+        key = (workspace_id, run_id)
+        channel = self._channels.get(key)
+        if channel is None or channel.closed:
+            # No live subscribers and channel hasn't been opened — drop.
+            # A late subscriber will start from the SQLite snapshot.
+            return
+        for sub in list(channel.subscribers):
+            try:
+                sub.queue.put_nowait(span_payload)
+            except asyncio.QueueFull:
+                logger.warning(
+                    "SpanBroker: dropping slow subscriber ws=%s run=%s",
+                    workspace_id,
+                    run_id,
+                )
+                sub.closed = True
+                with suppress(asyncio.QueueFull):
+                    sub.queue.put_nowait(_Closed(final_state="disconnected"))
+
+    def _close_sync(self, workspace_id: str, run_id: str, final_state: str) -> None:
+        key = (workspace_id, run_id)
+        channel = self._channels.get(key)
+        if channel is None:
+            # Subscribers may attach later; record the closed state so
+            # subscribe() can emit _Closed immediately.
+            self._channels[key] = _Channel(closed=True, final_state=final_state)
+            return
+        channel.closed = True
+        channel.final_state = final_state
+        close_event = _Closed(final_state=final_state)
+        for sub in channel.subscribers:
+            with suppress(asyncio.QueueFull):
+                sub.queue.put_nowait(close_event)
+
+
+# Module-level singleton bound at build_app() time.
+_broker: SpanBroker | None = None
+
+
+def get_broker() -> SpanBroker:
+    """Return the process-wide broker, lazily constructed."""
+    global _broker
+    if _broker is None:
+        _broker = SpanBroker()
+    return _broker
+
+
+def reset_for_tests() -> None:
+    """Drop the singleton — used by test fixtures to keep state clean."""
+    global _broker
+    _broker = None

selfevals/api/broker_bridge.py

@@ -0,0 +1,29 @@
+"""Adapter that lets the OTLP receiver feed the SSE broker.
+
+Lives in `selfevals.api` so the `runner/` package stays unaware that
+SSE / FastAPI exist. Only `selfevals serve` imports this; CLI-only
+runs never load it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from selfevals.api.broker import SpanBroker
+from selfevals.runner.otlp_receiver import SpanPublisher
+
+
+class BrokerPublisher(SpanPublisher):
+    """SpanPublisher impl that forwards to a SpanBroker."""
+
+    def __init__(self, broker: SpanBroker) -> None:
+        self._broker = broker
+
+    def mark_active(self, workspace_id: str, run_id: str) -> None:
+        self._broker.mark_run_active_threadsafe(workspace_id, run_id)
+
+    def publish(self, workspace_id: str, run_id: str, span_payload: dict[str, Any]) -> None:
+        self._broker.publish_threadsafe(workspace_id, run_id, span_payload)
+
+    def close(self, workspace_id: str, run_id: str, final_state: str = "completed") -> None:
+        self._broker.close_run_threadsafe(workspace_id, run_id, final_state)