evalrx 0.1.2__py3-none-any.whl

evalrx/reporting/langfuse_exporter.py

@@ -0,0 +1,38 @@
+"""Langfuse observability & exporter facade for backward compatibility."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalrx.observability.tracer import (
+    DiagnosticTracer,
+    backfill_run_to_langfuse,
+    export_to_langfuse_bundle,
+    sync_to_langfuse_live,
+)
+
+__all__ = [
+    "DiagnosticTracer",
+    "backfill_run_to_langfuse",
+    "export_to_langfuse_bundle",
+    "sync_to_langfuse_live",
+]
+
+
+def main() -> None:
+    import argparse
+    ap = argparse.ArgumentParser(description="Export an EvalRX run into a Langfuse ingestion bundle.")
+    ap.add_argument("run_dir", nargs="?", default="outputs", help="Run directory.")
+    ap.add_argument("--out", "-o", default=None, help="Output JSON path.")
+    ap.add_argument("--sync", action="store_true", help="Push live to Langfuse API.")
+    args = ap.parse_args()
+
+    if args.sync:
+        sync_to_langfuse_live(args.run_dir)
+    else:
+        out_p = args.out or (Path(args.run_dir) / "langfuse_trace.json")
+        export_to_langfuse_bundle(args.run_dir, out_p)
+
+
+if __name__ == "__main__":
+    main()

evalrx/reporting/langfuse_source.py

@@ -0,0 +1,155 @@
+"""Langfuse-backed run reader for the EvalRX HTML report.
+
+Langfuse's observations API is row-oriented.  A run whose durable events were
+delivered through the outbox queue arrives as one EVENT observation per event
+(``input={"event": {...}}``); this adapter recovers the ordered stream from
+those without making the renderer understand Langfuse-specific response
+types, then re-buckets it into a V2-shaped cache (:func:`materialize`) so the
+normal ``run.json``/``M<n>/log.json`` read path can render it.
+
+Caveat: :class:`~evalrx.eval_agent.run_logger_v2.RunLoggerV2` does not
+currently push structured events through that outbox (it relies on live
+spans/generations instead), so a V2 run's recovered events presently carry
+no ``stage`` tag and land in ``run.json``'s ``unrouted`` bucket rather than
+a real ``M<n>/log.json`` — full V2 support here is unfinished, not something
+this module claims to solve.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from pathlib import Path
+from typing import Any
+
+
+class LangfuseRunSource:
+    """Read EvalRX event envelopes from Langfuse Observations API v2."""
+
+    def __init__(self, client: Any | None = None) -> None:
+        if client is None:
+            try:
+                from langfuse import Langfuse
+            except ImportError as exc:
+                raise ImportError(
+                    "Langfuse report input needs `pip install evalrx[observability]`."
+                ) from exc
+            client = Langfuse()
+        self.client = client
+
+    def events(self, trace_id: str) -> list[dict[str, Any]]:
+        """Return the exact EvalRX events for *trace_id*, ordered by sequence."""
+        return [_public_event(event) for event in self._raw_events(trace_id)]
+
+    def _raw_events(self, trace_id: str) -> list[dict[str, Any]]:
+        cursor: str | None = None
+        recovered: list[tuple[int, dict[str, Any]]] = []
+        # The SDK uses the canonical 32-hex Langfuse trace id.  Our local UUID
+        # carries dashes, so accept either spelling at the CLI boundary.
+        langfuse_trace_id = trace_id.replace("-", "")
+        while True:
+            try:
+                response = self.client.api.observations.get_many(
+                    trace_id=langfuse_trace_id,
+                    fields="io,metadata,time",
+                    expand_metadata="event_id,event_seq,stage,cycle,artifact_refs",
+                    limit=1000,
+                    cursor=cursor,
+                )
+            except Exception as exc:
+                raise RuntimeError(
+                    "Could not query Langfuse observations. Set LANGFUSE_PUBLIC_KEY, "
+                    "LANGFUSE_SECRET_KEY, and LANGFUSE_HOST (if self-hosted)."
+                ) from exc
+            for observation in response.data:
+                event = _event_from_observation(observation)
+                if event is not None:
+                    metadata = _as_dict(getattr(observation, "metadata", None))
+                    recovered.append((int(metadata.get("event_seq") or event.get("event_seq") or 0), event))
+            cursor = getattr(getattr(response, "meta", None), "cursor", None)
+            if not cursor:
+                break
+        recovered.sort(key=lambda item: item[0])
+        return [event for _, event in recovered]
+
+    def materialize(self, trace_id: str, destination: str | Path) -> Path:
+        """Write a renderer-compatible cache; it is not a source of truth."""
+        from evalrx.reporting.run_events import write_v2_bundle
+
+        root = Path(destination)
+        root.mkdir(parents=True, exist_ok=True)
+        root.chmod(0o700)
+        events = self._raw_events(trace_id)
+        if not events:
+            raise LookupError(f"No EvalRX events found for Langfuse trace {trace_id!r}")
+        for event in events:
+            self._materialize_artifacts(event, root)
+        write_v2_bundle(root, [_public_event(event) for event in events])
+        return root
+
+    def _materialize_artifacts(self, event: dict[str, Any], root: Path) -> None:
+        attachments = event.pop("_langfuse_artifacts", [])
+        refs = event.pop("_langfuse_artifact_refs", [])
+        if not attachments or not refs or not hasattr(self.client, "resolve_media_references"):
+            return
+        resolved = self.client.resolve_media_references(
+            obj=attachments, resolve_with="base64_data_uri",
+        )
+        ref_by_id = {str(ref.get("artifact_id")): ref for ref in refs if isinstance(ref, dict)}
+        for attachment in resolved:
+            if not isinstance(attachment, dict):
+                continue
+            ref = ref_by_id.get(str(attachment.get("artifact_id")))
+            content = attachment.get("content")
+            if ref is None or not isinstance(content, str):
+                continue
+            _write_data_uri(root, str(ref.get("path") or ""), content)
+
+
+def _event_from_observation(observation: Any) -> dict[str, Any] | None:
+    """Extract our payload from an ObservationV2 or a lightweight test double."""
+    input_data = _as_dict(getattr(observation, "input", None))
+    event = input_data.get("event")
+    if not isinstance(event, dict):
+        return None
+    result = dict(event)
+    attachments = input_data.get("artifacts")
+    metadata = _as_dict(getattr(observation, "metadata", None))
+    refs = metadata.get("artifact_refs")
+    if isinstance(attachments, list):
+        result["_langfuse_artifacts"] = attachments
+    if isinstance(refs, list):
+        result["_langfuse_artifact_refs"] = refs
+    return result
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except json.JSONDecodeError:
+            return {}
+        return parsed if isinstance(parsed, dict) else {}
+    return {}
+
+
+def _public_event(event: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in event.items() if not key.startswith("_langfuse_")}
+
+
+def _write_data_uri(root: Path, relative_path: str, data_uri: str) -> None:
+    """Write a resolved Langfuse media value inside the renderer cache only."""
+    if not data_uri.startswith("data:") or "," not in data_uri:
+        return
+    target = (root / relative_path).resolve()
+    if root.resolve() not in target.parents:
+        return
+    try:
+        payload = base64.b64decode(data_uri.split(",", 1)[1])
+    except ValueError:
+        return
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_bytes(payload)
+    target.chmod(0o600)

evalrx/reporting/model.py

@@ -0,0 +1,151 @@
+"""Structured diagnostic report model for claim-first dashboards.
+
+The raw EvalRX artifacts are optimized for auditability: JSONL events,
+candidate signal dicts, CSV tables, and rendered chart files.  This module
+defines the semantic layer above those artifacts.  The dashboard should render
+this model instead of re-interpreting raw logs ad hoc.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+ClaimStatus = Literal["supported", "inconclusive", "refuted", "descriptive"]
+Confidence = Literal["high", "medium", "low", "unknown"]
+
+
+@dataclass
+class Evidence:
+    id: str
+    kind: str
+    title: str
+    summary: str = ""
+    artifact: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "kind": self.kind,
+            "title": self.title,
+            "summary": self.summary,
+            "artifact": self.artifact,
+        }
+
+
+@dataclass
+class Claim:
+    id: str
+    text: str
+    status: ClaimStatus
+    evidence_ids: list[str] = field(default_factory=list)
+    counter_evidence_ids: list[str] = field(default_factory=list)
+    interpretation: str = ""
+    do_not_infer: str = ""
+    downstream: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "text": self.text,
+            "status": self.status,
+            "evidence_ids": self.evidence_ids,
+            "counter_evidence_ids": self.counter_evidence_ids,
+            "interpretation": self.interpretation,
+            "do_not_infer": self.do_not_infer,
+            "downstream": self.downstream,
+        }
+
+
+@dataclass
+class ReportStep:
+    stage: str
+    title: str
+    summary: str
+    artifact_ids: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "stage": self.stage,
+            "title": self.title,
+            "summary": self.summary,
+            "artifact_ids": self.artifact_ids,
+        }
+
+
+@dataclass
+class DiagnosticReport:
+    question: str = ""
+    answer: str = ""
+    confidence: Confidence = "unknown"
+    claims: list[Claim] = field(default_factory=list)
+    evidence: list[Evidence] = field(default_factory=list)
+    timeline: list[ReportStep] = field(default_factory=list)
+    visual_decisions: list[dict[str, Any]] = field(default_factory=list)
+    chart_readings: list[dict[str, Any]] = field(default_factory=list)
+    dashboard_storyboard: list[dict[str, Any]] = field(default_factory=list)
+    critique: list[str] = field(default_factory=list)
+    caveats: list[str] = field(default_factory=list)
+    next_actions: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "question": self.question,
+            "answer": self.answer,
+            "confidence": self.confidence,
+            "claims": [c.to_dict() for c in self.claims],
+            "evidence": [e.to_dict() for e in self.evidence],
+            "timeline": [s.to_dict() for s in self.timeline],
+            "visual_decisions": self.visual_decisions,
+            "chart_readings": self.chart_readings,
+            "dashboard_storyboard": self.dashboard_storyboard,
+            "critique": self.critique,
+            "caveats": self.caveats,
+            "next_actions": self.next_actions,
+        }
+
+
+@dataclass
+class ReaderFinding:
+    """One claim phrased for a reader who does not know the pipeline."""
+
+    title: str
+    summary: str
+    why_it_matters: str
+    evidence_level: str
+    limitation: str = ""
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "title": self.title,
+            "summary": self.summary,
+            "why_it_matters": self.why_it_matters,
+            "evidence_level": self.evidence_level,
+            "limitation": self.limitation,
+        }
+
+
+@dataclass
+class ReaderReport:
+    """Stable, audience-first view model rendered before technical evidence."""
+
+    headline: str
+    question: str
+    answer: str
+    confidence: str
+    what_we_did: list[str] = field(default_factory=list)
+    key_findings: list[ReaderFinding] = field(default_factory=list)
+    open_questions: list[str] = field(default_factory=list)
+    next_steps: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "headline": self.headline,
+            "question": self.question,
+            "answer": self.answer,
+            "confidence": self.confidence,
+            "what_we_did": self.what_we_did,
+            "key_findings": [finding.to_dict() for finding in self.key_findings],
+            "open_questions": self.open_questions,
+            "next_steps": self.next_steps,
+        }

evalrx/reporting/run_events.py

@@ -0,0 +1,184 @@
+"""Read tidy RunLoggerV2 documents through the existing event-view contract."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+_STAGES = ("M1", "M2", "M3", "M4", "M5")
+
+
+def resolve_v2_root(root: str | Path) -> Path | None:
+    """Return the directory containing V2 ``run.json`` and stage folders."""
+    root = Path(root).resolve()
+    candidates = (root, root / "logs")
+    for candidate in candidates:
+        if (candidate / "run.json").is_file() and any(
+            (candidate / stage / "log.json").is_file() for stage in _STAGES
+        ):
+            return candidate
+    return None
+
+
+def read_v2_events(root: str | Path) -> list[dict[str, Any]]:
+    """Flatten V2 bucketed documents into the legacy read-only event view.
+
+    The V2 files remain authoritative and are never rewritten by this adapter.
+    It exists so current report/UI code can migrate independently from the
+    storage layout.
+    """
+    v2_root = resolve_v2_root(root)
+    if v2_root is None:
+        return []
+    try:
+        run = json.loads((v2_root / "run.json").read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return []
+
+    trace_id = str((run.get("run_start") or {}).get("trace_id") or run.get("trace_id") or "")
+    events: list[dict[str, Any]] = []
+
+    def append(event: str, payload: Any, *, stage: str = "RUN") -> None:
+        if not isinstance(payload, dict):
+            return
+        record = dict(payload)
+        record.setdefault("event", event)
+        record.setdefault("stage", stage)
+        # Older V2 bundles encode this only in the folder name. Existing
+        # case-study/report consumers still use the V1 module discriminator.
+        if event == "surgery":
+            record.setdefault("module", stage.lower())
+        elif event == "fix":
+            record.setdefault("module", "fix")
+        if trace_id:
+            record.setdefault("trace_id", trace_id)
+        events.append(record)
+
+    append("run_start", run.get("run_start"))
+    for row in run.get("cases") or []:
+        append("case_record", row)
+    for key in (
+        "report_published", "diagnose_reports", "loop_end",
+        "agent_decisions", "agent_tool_calls", "unrouted",
+    ):
+        event_name = {
+            "diagnose_reports": "diagnose_report",
+            "agent_decisions": "agent_decision",
+            "agent_tool_calls": "agent_tool",
+            "unrouted": "unrouted",
+        }.get(key, key)
+        for row in run.get(key) or []:
+            append(event_name, row)
+
+    for stage in _STAGES:
+        path = v2_root / stage / "log.json"
+        if not path.is_file():
+            continue
+        try:
+            doc = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        if not isinstance(doc, dict):
+            continue
+        for event, rows in doc.items():
+            if not isinstance(rows, list):
+                continue
+            for row in rows:
+                append("model_call" if event == "model_calls" else event, row, stage=stage)
+
+    # New bundles persist a global sequence at write time, even when calls
+    # finish concurrently or the wall clock moves backwards. Legacy bundles
+    # retain timestamp ordering; never replace a persisted event identity.
+    if all(isinstance(event.get("event_seq"), int) for event in events):
+        events.sort(key=lambda event: event["event_seq"])
+    else:
+        events.sort(key=lambda event: str(event.get("ts") or ""))
+        seq = max((event.get("event_seq", 0) for event in events), default=0)
+        for event in events:
+            if "event_seq" not in event:
+                seq += 1
+                event["event_seq"] = seq
+    return events
+
+
+# Inverse of the RUN-stage bucket mapping `read_v2_events` flattens — event
+# name -> run.json key. These names are always RUN-level regardless of any
+# "stage" tag on the recovered event.
+_RUN_EVENT_TO_KEY = {
+    "case_record": "cases",
+    "diagnose_report": "diagnose_reports",
+    "agent_decision": "agent_decisions",
+    "agent_tool": "agent_tool_calls",
+    "report_published": "report_published",
+    "loop_end": "loop_end",
+}
+
+# A source that recovers events out-of-band (e.g. Langfuse observations
+# reconstructed from a durable-outbox delivery, which predates per-event
+# "stage" tagging) may hand back events with no "stage" field at all. Route
+# by the event's own name to its canonical stage rather than dumping it into
+# run.json's "unrouted" bucket, where `resolve_v2_root` would never find it
+# (it requires at least one real M<n>/log.json to recognize a V2 root).
+_NAME_TO_STAGE = {
+    "probe": "M1",
+    "analysis": "M2", "explore": "M2",
+    "diagnosis": "M3",
+    "experiment": "M5",  # module=... on the row disambiguates M4 vs M5 below
+    "fix": "M5",
+}
+
+
+def _fallback_stage(name: "str | None", row: dict[str, Any]) -> "str | None":
+    """Best-effort stage for an event with no "stage" tag of its own."""
+    if name == "surgery" or name == "tool_codegen" or name == "experiment":
+        module = str(row.get("module") or "").lower()
+        if module.startswith("m") and module[1:].isdigit():
+            return f"M{module[1:]}"
+        return "M5" if name == "experiment" else "M4"
+    return _NAME_TO_STAGE.get(name or "")
+
+
+def write_v2_bundle(root: str | Path, events: list[dict[str, Any]]) -> None:
+    """Reconstruct ``run.json`` + ``M<n>/log.json`` from a flat event list.
+
+    The exact inverse of :func:`read_v2_events`'s bucketing, for callers that
+    recover events from an out-of-band source (e.g. Langfuse) and need to
+    write a V2-shaped cache directory back for the report/dashboard's
+    normal V2-only read path to find.
+    """
+    root = Path(root)
+    run_doc: dict[str, Any] = {}
+    stage_docs: dict[str, dict[str, list[dict[str, Any]]]] = {}
+
+    for event in events:
+        row = {k: v for k, v in event.items() if k not in ("event", "stage")}
+        stage = event.get("stage")
+        name = event.get("event")
+        if name == "run_start":
+            run_doc["run_start"] = row
+            continue
+        if name in _RUN_EVENT_TO_KEY:
+            run_doc.setdefault(_RUN_EVENT_TO_KEY[name], []).append(row)
+            continue
+        if stage not in _STAGES:
+            stage = _fallback_stage(name, row) or stage
+        if stage not in _STAGES:
+            run_doc.setdefault("unrouted", []).append(row)
+            continue
+        key = "model_calls" if name == "model_call" else (name or "unrouted")
+        stage_docs.setdefault(stage, {}).setdefault(key, []).append(row)
+
+    root.mkdir(parents=True, exist_ok=True)
+    (root / "run.json").write_text(
+        json.dumps(run_doc, ensure_ascii=False, default=str), encoding="utf-8",
+    )
+    for stage, doc in stage_docs.items():
+        stage_dir = root / stage
+        stage_dir.mkdir(parents=True, exist_ok=True)
+        (stage_dir / "log.json").write_text(
+            json.dumps(doc, ensure_ascii=False, default=str), encoding="utf-8",
+        )
+
+
+__all__ = ["read_v2_events", "resolve_v2_root", "write_v2_bundle"]