evalrx 0.1.2__py3-none-any.whl

evalrx/analysis/trajectory_records.py

@@ -0,0 +1,193 @@
+"""Trajectory → flat records — the M1→M2 bridge for agent runs.
+
+`evalrx explore` and :func:`~evalrx.analysis.stats_tools.build_stats_input_from_records`
+consume flat rows (``case_id`` + ``label`` + numeric signal columns).  This
+module flattens each agent :class:`~evalrx.core.case.Trajectory` into such
+a row of trajectory-level features, so agent runs plug into the SAME M2/M3
+statistical machinery as single-step probes.
+
+Feature semantics (all deterministic, recomputed from steps when metrics are
+missing):
+
+``n_steps`` / ``n_turns`` / ``n_tool_calls``
+    loop volume — how much work the agent did.
+``n_calls_<tool>``
+    per-tool call counts (tool-selection profile), one column per tool seen
+    anywhere in the batch (0-filled elsewhere).
+``n_tool_errors`` / ``tool_error_rate``
+    observations matching the executor's ``"[tool error"`` / ``"[error"``
+    envelope.
+``n_repeated_calls`` / ``repeated_call_frac`` / ``max_consecutive_repeat``
+    duplicate ``(tool, args)`` signatures — the loop-failure family
+    (``max_consecutive_repeat >= 2`` is what LoopDetector flags).
+``n_images_returned``
+    images tools handed back (visual-evidence volume).
+``first_call_turn``
+    turn of the first tool call (0 = never called a tool).
+``distinct_tools``, ``terminated_final``, ``answer_len_chars``
+    coverage, clean termination (1/0), and answer volume.
+
+No imports from ``eval_agent`` (the analysis layer stays standalone).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Iterable, Optional
+
+from evalrx.core.case import FailureCase, Label, StepRole, Trajectory
+
+_ERROR_MARKERS = ("[tool error", "[error")
+
+#: The standard M2 question for agent-trajectory records.  It declares the
+#: column-family semantics up front so the analysis treats each family for
+#: what it is (causal attribution vs stability vs judge labels vs cost) and
+#: steers explanations toward the black-box fix surface (prompt/tool changes).
+#: Use as-is or as a skeleton; the workbench and docs reference it.
+AGENT_QUESTION_TEMPLATE = (
+    "What predicts failures (label=fail) in this agent run? Records are one row "
+    "per case with trajectory features: tool-use volume (n_tool_calls, "
+    "n_calls_<tool>), loop structure (max_consecutive_repeat, "
+    "repeated_call_frac), tool errors (tool_error_rate), cost "
+    "(total_completion_tokens, total_latency_ms), stability from repeated runs "
+    "(success_rate, flaky, pass_at_k vs pass_all_k), causal tool attribution "
+    "from subset ablation (shap_outcome_<tool>, no_tools_pass), and a "
+    "judge-assigned failure_mode (a hypothesis label, not ground truth). "
+    "Compare FAIL vs PASS, say which signal families carry independent "
+    "information, and prefer explanations that point at fixable causes — "
+    "prompt wording, tool descriptions, tool availability, or loop policy."
+)
+
+
+def _observation_text(observation: Any) -> str:
+    if isinstance(observation, dict):
+        return str(observation.get("text", ""))
+    return "" if observation is None else str(observation)
+
+
+def _call_signature(tool_call: dict) -> "tuple[str, str]":
+    name = str(tool_call.get("name", ""))
+    try:
+        args = json.dumps(tool_call.get("args", {}), sort_keys=True, default=str)
+    except (TypeError, ValueError):
+        args = str(tool_call.get("args"))
+    return name, args
+
+
+def trajectory_features(trajectory: Trajectory) -> dict[str, Any]:
+    """Flatten one trajectory into a dict of scalar features (see module doc)."""
+    tool_calls: list[tuple[str, str]] = []
+    n_tool_errors = 0
+    n_images = 0
+    first_call_turn = 0
+    max_consecutive = 0
+    consecutive = 0
+    prev_sig: Optional[tuple[str, str]] = None
+
+    for step in trajectory.steps:
+        if step.tool_call:
+            sig = _call_signature(step.tool_call)
+            tool_calls.append(sig)
+            if first_call_turn == 0:
+                first_call_turn = int(step.span.get("turn", len(tool_calls)))
+            if sig == prev_sig:
+                consecutive += 1
+            else:
+                consecutive = 1
+            prev_sig = sig
+            max_consecutive = max(max_consecutive, consecutive)
+        if step.role is StepRole.TOOL:
+            text = _observation_text(step.observation)
+            if text.startswith(_ERROR_MARKERS):
+                n_tool_errors += 1
+            if isinstance(step.observation, dict):
+                n_images += int(step.observation.get("n_images") or 0)
+
+    n_calls = len(tool_calls)
+    seen: set[tuple[str, str]] = set()
+    n_repeated = 0
+    for sig in tool_calls:
+        if sig in seen:
+            n_repeated += 1
+        seen.add(sig)
+
+    metrics = trajectory.metrics or {}
+    actor_spans = [s.span for s in trajectory.steps if s.role is StepRole.ACTOR and s.span]
+    latencies = [float(sp["latency_ms"]) for sp in actor_spans if sp.get("latency_ms")]
+    total_latency = sum(latencies)
+    total_prompt_toks = sum(int(sp.get("prompt_tokens") or 0) for sp in actor_spans)
+    total_completion_toks = sum(int(sp.get("completion_tokens") or 0) for sp in actor_spans)
+
+    features: dict[str, Any] = {
+        "n_steps": int(metrics.get("n_steps", len(trajectory.steps))),
+        "n_turns": int(metrics.get("n_turns", 0)) or sum(
+            1 for s in trajectory.steps if s.role is StepRole.ACTOR
+        ),
+        "n_tool_calls": n_calls,
+        "distinct_tools": len({name for name, _ in tool_calls}),
+        "n_tool_errors": n_tool_errors,
+        "tool_error_rate": round(n_tool_errors / n_calls, 4) if n_calls else 0.0,
+        "n_repeated_calls": n_repeated,
+        "repeated_call_frac": round(n_repeated / n_calls, 4) if n_calls else 0.0,
+        "max_consecutive_repeat": max_consecutive,
+        "n_images_returned": n_images,
+        "first_call_turn": first_call_turn,
+        "terminated": str(metrics.get("terminated", "unknown")),
+        "terminated_final": 1 if metrics.get("terminated") == "final" else 0,
+        "answer_len_chars": len(str(trajectory.final_answer)) if trajectory.final_answer else 0,
+        # cost accounting (0 when the backend reports no usage/latency)
+        "total_latency_ms": round(float(metrics.get("total_latency_ms") or total_latency), 1),
+        "mean_turn_latency_ms": round(total_latency / len(latencies), 1) if latencies else 0.0,
+        "total_prompt_tokens": int(metrics.get("total_prompt_tokens") or total_prompt_toks),
+        "total_completion_tokens": int(
+            metrics.get("total_completion_tokens") or total_completion_toks
+        ),
+    }
+    for name in sorted({name for name, _ in tool_calls}):
+        features[f"n_calls_{name}"] = sum(1 for n, _ in tool_calls if n == name)
+    return features
+
+
+def trajectories_to_records(
+    items: Iterable["FailureCase | Trajectory"],
+    *,
+    id_col: str = "case_id",
+    label_col: str = "label",
+) -> list[dict[str, Any]]:
+    """Flatten cases/trajectories into ``records.json``-shaped rows.
+
+    Each row carries *id_col*, *label_col* (the case label, falling back to
+    the trajectory outcome), the trajectory features, and any scalar
+    ``case.metadata`` entries (provenance columns like model/source).  Per-tool
+    count columns are 0-filled across the batch so every row has the same
+    schema.  Items without a trajectory are skipped.
+    """
+    rows: list[dict[str, Any]] = []
+    tool_cols: set[str] = set()
+    for item in items:
+        if isinstance(item, Trajectory):
+            case, trajectory = None, item
+        else:
+            case, trajectory = item, item.trajectory
+        if trajectory is None:
+            continue
+        label = trajectory.outcome
+        if case is not None and case.label is not Label.UNKNOWN:
+            label = case.label
+        row: dict[str, Any] = {
+            id_col: case.id if case is not None else trajectory.sample_id,
+            label_col: label.value if isinstance(label, Label) else str(label),
+        }
+        if case is not None:
+            for key, value in (case.metadata or {}).items():
+                if isinstance(value, (str, int, float, bool)) and key not in (id_col, label_col):
+                    row.setdefault(key, value)
+        features = trajectory_features(trajectory)
+        tool_cols.update(k for k in features if k.startswith("n_calls_"))
+        row.update(features)
+        rows.append(row)
+
+    for row in rows:  # uniform schema: 0-fill per-tool columns
+        for col in tool_cols:
+            row.setdefault(col, 0)
+    return rows

evalrx/analysis/workbench.py

@@ -0,0 +1,431 @@
+"""Durable primitives for the browser workbench.
+
+The original upload app treated an archive as a directory of JSON files.  This
+module is deliberately independent from Streamlit: it validates and extracts
+an archive, turns the supported tabular and media files into an auditable
+``DatasetBundle``, and writes append-only events/messages for a data thread.
+Keeping these operations here makes the browser UI a thin observer rather than
+the source of truth for an analysis run.
+"""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import io
+import json
+import mimetypes
+import os
+import shutil
+import subprocess
+import threading
+import zipfile
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+TABULAR_SUFFIXES = {".json", ".jsonl", ".ndjson", ".csv", ".tsv", ".parquet", ".xlsx", ".xls"}
+IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff"}
+PDF_SUFFIXES = {".pdf"}
+AUDIO_SUFFIXES = {".mp3", ".wav", ".m4a", ".aac", ".flac", ".ogg"}
+VIDEO_SUFFIXES = {".mp4", ".mov", ".webm", ".mkv", ".avi"}
+SUPPORTED_SUFFIXES = TABULAR_SUFFIXES | IMAGE_SUFFIXES | PDF_SUFFIXES | AUDIO_SUFFIXES | VIDEO_SUFFIXES
+
+
+@dataclass(frozen=True)
+class UploadLimits:
+    """Conservative local-workbench limits; callers may explicitly raise them."""
+
+    max_archive_bytes: int = 1024 * 1024 * 1024
+    max_extracted_bytes: int = 4 * 1024 * 1024 * 1024
+    max_files: int = 10_000
+    max_compression_ratio: float = 100.0
+    max_path_depth: int = 32
+
+
+@dataclass
+class Event:
+    seq: int
+    timestamp: str
+    thread_id: str
+    turn_id: str
+    stage: str
+    status: str
+    message: str
+    attempt: int | None = None
+    artifact_refs: list[str] = field(default_factory=list)
+    metrics: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class EventSink:
+    """Append-only, restart-safe progress events for one data thread."""
+
+    def __init__(self, path: str | Path, *, thread_id: str, turn_id: str) -> None:
+        self.path = Path(path)
+        self.thread_id = thread_id
+        self.turn_id = turn_id
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        self._seq = _last_event_seq(self.path)
+
+    def emit(
+        self,
+        stage: str,
+        status: str,
+        message: str,
+        *,
+        attempt: int | None = None,
+        artifact_refs: Iterable[str | Path] = (),
+        metrics: dict[str, Any] | None = None,
+    ) -> Event:
+        with self._lock:
+            self._seq += 1
+            event = Event(
+                seq=self._seq,
+                timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+                thread_id=self.thread_id,
+                turn_id=self.turn_id,
+                stage=stage,
+                status=status,
+                message=message,
+                attempt=attempt,
+                artifact_refs=[str(p) for p in artifact_refs],
+                metrics=metrics or {},
+            )
+            _append_jsonl(self.path, event.to_dict())
+            return event
+
+
+@dataclass
+class DatasetBundle:
+    """Normalized view of a local archive plus references to original media."""
+
+    root: str
+    source_dir: str
+    records_path: str
+    media_units_path: str
+    manifest_path: str
+    n_records: int
+    n_media_units: int
+    files: list[dict[str, Any]]
+    warnings: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class ThreadStore:
+    """Small filesystem store for a local, single-user analysis conversation."""
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+
+    def create(self, *, name: str, provider: str, model: str = "") -> Path:
+        slug = _safe_name(name)
+        thread_id = f"{slug}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        thread = self.root / thread_id
+        thread.mkdir(parents=True, exist_ok=False)
+        meta = {
+            "id": thread_id,
+            "name": name,
+            "provider": provider,
+            "model": model,
+            "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            "current_turn": "",
+        }
+        (thread / "thread.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+        return thread
+
+    @staticmethod
+    def append_message(thread: str | Path, role: str, content: str, *, turn_id: str = "") -> None:
+        _append_jsonl(Path(thread) / "messages.jsonl", {
+            "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            "role": role,
+            "content": content,
+            "turn_id": turn_id,
+        })
+
+    @staticmethod
+    def set_current_turn(thread: str | Path, turn_id: str) -> None:
+        path = Path(thread) / "thread.json"
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception:
+            data = {}
+        data["current_turn"] = turn_id
+        path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+
+def extract_archive(payload: bytes, destination: str | Path, *, limits: UploadLimits | None = None) -> Path:
+    """Safely extract a ZIP and return its data root.
+
+    In addition to zip-slip, this rejects symlinks, duplicate targets, overly
+    deep paths, excessive file counts, and archives with suspicious expansion.
+    """
+    limits = limits or UploadLimits()
+    if len(payload) > limits.max_archive_bytes:
+        raise ValueError("archive exceeds the configured upload size limit")
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    root = destination.resolve()
+    seen: set[Path] = set()
+    total = 0
+    n_files = 0
+    with zipfile.ZipFile(io.BytesIO(payload)) as zf:
+        infos = [i for i in zf.infolist() if not i.is_dir()]
+        if len(infos) > limits.max_files:
+            raise ValueError("archive contains too many files")
+        for info in infos:
+            name = info.filename.replace("\\", "/")
+            parts = Path(name).parts
+            if "__MACOSX" in parts or parts[-1] in {".DS_Store", "Thumbs.db"}:
+                continue
+            is_link = (info.external_attr >> 16) & 0o170000 == 0o120000
+            if (not name or name.startswith("/") or ".." in parts or is_link
+                    or len(parts) > limits.max_path_depth):
+                raise ValueError(f"unsafe path in archive: {info.filename!r}")
+            if info.file_size and info.compress_size and info.file_size / info.compress_size > limits.max_compression_ratio:
+                raise ValueError(f"archive member has an unsafe compression ratio: {info.filename!r}")
+            total += info.file_size
+            if total > limits.max_extracted_bytes:
+                raise ValueError("archive exceeds the configured extracted size limit")
+            target = (destination / name).resolve()
+            if not target.is_relative_to(root) or target in seen:
+                raise ValueError(f"unsafe or duplicate path in archive: {info.filename!r}")
+            seen.add(target)
+            target.parent.mkdir(parents=True, exist_ok=True)
+            with zf.open(info) as source, target.open("wb") as out:
+                shutil.copyfileobj(source, out)
+            n_files += 1
+    if not n_files:
+        raise ValueError("archive contains no files")
+    children = [p for p in destination.iterdir() if p.name not in {"__MACOSX", ".DS_Store"}]
+    return children[0] if len(children) == 1 and children[0].is_dir() else destination
+
+
+def ingest_directory(
+    source_dir: str | Path,
+    bundle_dir: str | Path,
+    *,
+    media_unit_limit: int = 500,
+    sink: EventSink | None = None,
+) -> DatasetBundle:
+    """Create an auditable normalized bundle without sending data off-machine."""
+    source = Path(source_dir).resolve()
+    bundle = Path(bundle_dir).resolve()
+    bundle.mkdir(parents=True, exist_ok=True)
+    if sink:
+        sink.emit("discover", "started", "Discovering uploaded files")
+    files = [p for p in sorted(source.rglob("*")) if p.is_file() and not _is_junk(p)]
+    file_entries = [_file_entry(p, source) for p in files]
+    unsupported = [e["path"] for e in file_entries if e["suffix"] not in SUPPORTED_SUFFIXES]
+    if sink:
+        sink.emit("discover", "completed", f"Discovered {len(files)} files", metrics={"files": len(files)})
+        sink.emit("normalize", "started", "Normalizing tabular records and media metadata")
+
+    records: list[dict[str, Any]] = []
+    media: list[dict[str, Any]] = []
+    warnings: list[str] = []
+    for path, entry in zip(files, file_entries):
+        suffix = entry["suffix"]
+        if suffix in TABULAR_SUFFIXES:
+            try:
+                for row in _read_tabular(path):
+                    row.setdefault("_source_file", entry["path"])
+                    records.append(row)
+            except Exception as exc:  # data quality is reported, not fatal for a mixed archive
+                warnings.append(f"could not read {entry['path']}: {exc}")
+        elif suffix in IMAGE_SUFFIXES | PDF_SUFFIXES | AUDIO_SUFFIXES | VIDEO_SUFFIXES:
+            bundled_media = _materialize_media(path, source, bundle)
+            entry["bundle_path"] = str(bundled_media.relative_to(bundle))
+            media.extend(_media_units(path, source, entry, media_unit_limit - len(media)))
+            if len(media) >= media_unit_limit:
+                warnings.append(f"media unit limit ({media_unit_limit}) reached; remaining media was not profiled")
+                break
+
+    records_path = bundle / "records.json"
+    media_path = bundle / "media_units.jsonl"
+    manifest_path = bundle / "bundle.json"
+    records_path.write_text(json.dumps(records, indent=2, default=str), encoding="utf-8")
+    for unit in media:
+        _append_jsonl(media_path, unit)
+    bundle_data = DatasetBundle(
+        root=str(bundle), source_dir=str(source), records_path=str(records_path),
+        media_units_path=str(media_path), manifest_path=str(manifest_path),
+        n_records=len(records), n_media_units=len(media), files=file_entries,
+        warnings=warnings + ([f"unsupported files skipped: {', '.join(unsupported[:8])}"] if unsupported else []),
+    )
+    manifest_path.write_text(json.dumps(bundle_data.to_dict(), indent=2), encoding="utf-8")
+    if sink:
+        sink.emit("normalize", "completed", "Dataset bundle is ready", artifact_refs=[manifest_path, records_path, media_path], metrics={"records": len(records), "media_units": len(media)})
+    return bundle_data
+
+
+def load_bundle(path: str | Path) -> DatasetBundle:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    return DatasetBundle(**data)
+
+
+def _read_tabular(path: Path) -> list[dict[str, Any]]:
+    suffix = path.suffix.lower()
+    if suffix in {".json", ".jsonl", ".ndjson"}:
+        return _read_json_rows(path)
+    if suffix in {".csv", ".tsv"}:
+        with path.open("r", encoding="utf-8-sig", errors="replace", newline="") as f:
+            return [dict(row) for row in csv.DictReader(f, delimiter="\t" if suffix == ".tsv" else ",")]
+    try:
+        import pandas as pd
+    except ImportError as exc:  # pragma: no cover - dashboard extra supplies pandas
+        raise RuntimeError("pandas is required to read Parquet and Excel files") from exc
+    frame = pd.read_parquet(path) if suffix == ".parquet" else pd.read_excel(path)
+    return [dict(r) for r in frame.to_dict(orient="records")]
+
+
+def _read_json_rows(path: Path) -> list[dict[str, Any]]:
+    text = path.read_text(encoding="utf-8", errors="replace").strip()
+    if not text:
+        return []
+    try:
+        value = json.loads(text)
+        if isinstance(value, list):
+            return [dict(v) if isinstance(v, dict) else {"value": v} for v in value]
+        if isinstance(value, dict):
+            for key in ("cases", "records", "results", "rows", "items", "data", "examples", "samples"):
+                if isinstance(value.get(key), list):
+                    meta = {k: v for k, v in value.items() if k != key and not isinstance(v, (list, dict))}
+                    return [{**meta, **dict(v)} if isinstance(v, dict) else {**meta, "value": v} for v in value[key]]
+            return [value]
+    except json.JSONDecodeError:
+        pass
+    rows = []
+    for line in text.splitlines():
+        try:
+            value = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        rows.append(dict(value) if isinstance(value, dict) else {"value": value})
+    return rows
+
+
+def _media_units(path: Path, source: Path, entry: dict[str, Any], remaining: int) -> list[dict[str, Any]]:
+    if remaining <= 0:
+        return []
+    parent_id = hashlib.sha256(entry["path"].encode()).hexdigest()[:16]
+    unit = {"id": parent_id, "parent_id": None, "kind": _media_kind(entry["suffix"]), **entry}
+    suffix = entry["suffix"]
+    if suffix in IMAGE_SUFFIXES:
+        try:
+            from PIL import Image
+            with Image.open(path) as image:
+                unit.update(width=image.width, height=image.height, mode=image.mode)
+        except Exception as exc:
+            unit["warning"] = f"image metadata unavailable: {exc}"
+    elif suffix in PDF_SUFFIXES:
+        return _pdf_units(path, unit, remaining)
+    elif suffix in AUDIO_SUFFIXES | VIDEO_SUFFIXES:
+        unit.update(_ffprobe_metadata(path))
+        duration = unit.get("duration_sec")
+        if isinstance(duration, (float, int)) and duration > 0 and remaining > 1:
+            segment_sec = 30.0 if suffix in AUDIO_SUFFIXES else 10.0
+            children = []
+            start = 0.0
+            while start < float(duration) and len(children) < remaining - 1:
+                end = min(start + segment_sec, float(duration))
+                children.append({
+                    "id": f"{parent_id}:{len(children):04d}", "parent_id": parent_id,
+                    "kind": f"{unit['kind']}_segment", **entry,
+                    "start_sec": start, "end_sec": end,
+                })
+                start = end
+            return [unit, *children]
+    return [unit]
+
+
+def _materialize_media(path: Path, source: Path, bundle: Path) -> Path:
+    """Expose original media to a coding-agent workdir without a fragile symlink."""
+    target = bundle / "media" / path.relative_to(source)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    if target.exists():
+        return target
+    try:
+        os.link(path, target)
+    except OSError:
+        shutil.copy2(path, target)
+    return target
+
+
+def _pdf_units(path: Path, unit: dict[str, Any], remaining: int) -> list[dict[str, Any]]:
+    try:
+        from pypdf import PdfReader
+        reader = PdfReader(str(path))
+        unit["pages"] = len(reader.pages)
+        children = []
+        for page_no, page in enumerate(reader.pages[:max(0, remaining - 1)], start=1):
+            text = page.extract_text() or ""
+            children.append({
+                "id": f"{unit['id']}:p{page_no}", "parent_id": unit["id"], "kind": "pdf_page",
+                **{k: v for k, v in unit.items() if k not in {"id", "parent_id", "kind"}},
+                "page": page_no, "text_excerpt": text[:1000],
+            })
+        return [unit, *children]
+    except Exception:
+        unit.update(pages=None, warning="PDF text extraction requires the optional pypdf dependency")
+        return [unit]
+
+
+def _ffprobe_metadata(path: Path) -> dict[str, Any]:
+    try:
+        proc = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", str(path)], capture_output=True, text=True, timeout=15, check=False)
+        data = json.loads(proc.stdout or "{}")
+        duration = (data.get("format") or {}).get("duration")
+        return {"duration_sec": float(duration) if duration is not None else None}
+    except Exception:
+        return {"duration_sec": None, "warning": "ffprobe metadata unavailable"}
+
+
+def _file_entry(path: Path, root: Path) -> dict[str, Any]:
+    suffix = path.suffix.lower()
+    return {"path": str(path.relative_to(root)), "suffix": suffix, "mime": mimetypes.guess_type(path.name)[0] or "application/octet-stream", "bytes": path.stat().st_size}
+
+
+def _media_kind(suffix: str) -> str:
+    if suffix in IMAGE_SUFFIXES:
+        return "image"
+    if suffix in PDF_SUFFIXES:
+        return "pdf"
+    if suffix in AUDIO_SUFFIXES:
+        return "audio"
+    return "video"
+
+
+def _is_junk(path: Path) -> bool:
+    return "__MACOSX" in path.parts or path.name in {".DS_Store", "Thumbs.db"}
+
+
+def _safe_name(value: str) -> str:
+    clean = "".join(c if c.isalnum() or c in "_-" else "_" for c in value).strip("_")
+    return clean or "dataset"
+
+
+def _append_jsonl(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(value, default=str) + "\n")
+        f.flush()
+        os.fsync(f.fileno())
+
+
+def _last_event_seq(path: Path) -> int:
+    if not path.exists():
+        return 0
+    try:
+        for line in reversed(path.read_text(encoding="utf-8").splitlines()):
+            if line.strip():
+                return int(json.loads(line).get("seq", 0))
+    except Exception:
+        pass
+    return 0

evalrx/analyzers/__init__.py

@@ -0,0 +1,42 @@
+"""Analyzers — functional taxonomy keyed by required capability (not black/white-box).
+
+Every analyzer declares ``requires`` (capabilities) + ``applies_to_modalities`` and
+self-registers on import.  Categories:
+
+    perturbation/   input masking (RISE; VL-SHAP/MM-SHAP)        — GENERATE/LOGPROBS
+    uncertainty/    entropy / self-consistency / verbalized conf  — LOGITS/GENERATE
+    hallucination/  POPE / CHAIR / OPERA / VCD                    — GENERATE/ATTENTION
+    attention/      summary / rollout / sink / relative-attn      — ATTENTION
+    attribution/    Grad-CAM / Chefer generic-attention           — GRADIENTS
+    lens/           logit-lens / tuned-lens                       — HIDDEN_STATES
+    patching/       causal tracing / activation patching          — HIDDEN_STATES (read+write)
+    geometry/       CKA / linear-probe                            — HIDDEN_STATES
+    reasoning/      extraction/termination hygiene; arithmetic,
+                    self-repair, step-value, knowledge split       — GENERATE
+    agent/          loop / ignored-obs / first-error / counterfactual — Trajectory
+
+Import is torch-tolerant: the ``attention`` subpackage loads torch at import, so on
+the light (pure-API) install it is skipped and its analyzers are simply not offered.
+"""
+
+from evalrx.analyzers import (  # noqa: F401  -- import to self-register
+    agent,
+    attribution,
+    geometry,
+    hallucination,
+    lens,
+    patching,
+    perturbation,
+    reasoning,
+    uncertainty,
+)
+from evalrx.analyzers.base import Analyzer, Result
+
+try:  # attention/summary imports torch+numpy at module load
+    from evalrx.analyzers import attention  # noqa: F401
+except ModuleNotFoundError as e:  # pragma: no cover - light install
+    if e.name not in {"torch", "numpy"}:
+        raise
+    attention = None  # type: ignore
+
+__all__ = ["Analyzer", "Result"]