auditkit 1.0.0__py3-none-any.whl

auditkit/report.py

@@ -0,0 +1,278 @@
+"""The result of a run: :class:`Prediction` rows and the :class:`RunResult`.
+
+Every run returns the same shape regardless of task or technique: aggregate
+:class:`~auditkit.score.Stat` per metric, a ``headline`` of means, and one
+:class:`Prediction` per sample — the answer browser. Reporting helpers
+(:meth:`RunResult.summary`, :meth:`RunResult.metric_table`,
+:meth:`RunResult.wrong_only`, :meth:`RunResult.save`) read off that.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from .score import Stat
+
+
+@dataclass
+class Prediction:
+    """One sample's record: what was asked, what came back, and how it scored."""
+
+    run_id: str
+    task: str
+    sample_id: str
+    prompt: str
+    raw_output: Optional[str]
+    parsed_answer: Optional[str]
+    expected: Optional[str]
+    correct: Optional[bool]
+    score: Optional[float]
+    choice_likelihoods: Optional[list[float]] = None
+    context: Any = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_doc(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "task": self.task,
+            "sample_id": self.sample_id,
+            "prompt": self.prompt,
+            "raw_output": self.raw_output,
+            "parsed_answer": self.parsed_answer,
+            "expected": self.expected,
+            "correct": self.correct,
+            "score": self.score,
+            "choice_likelihoods": self.choice_likelihoods,
+            "context": self.context,
+            "metadata": self.metadata,
+        }
+
+
+@dataclass
+class RunResult:
+    """What ``evaluate()`` returns: aggregates, per-sample rows, and a headline."""
+
+    run_id: str
+    fingerprint: str
+    stats: dict[str, Stat]
+    predictions: list[Prediction]
+    headline: dict[str, float]
+    config: Any = None
+    model_spec: Any = None
+    errors: list[dict] = field(default_factory=list)
+    failed_count: int = 0
+    experiment_name: str | None = None
+    tags: list[str] = field(default_factory=list)
+    # Efficiency/systems metrics — latency (ms) and throughput (req/s),
+    # measured by timing the run's own model calls (see Runner.run()).
+    # None unless RunConfig.track_performance=True was set on this run (the
+    # default is False -- most callers scoring a quick eval don't want this
+    # measured). When tracking IS on: {} on the reads_actual_output path (no
+    # model call happens there at all) or when a run made zero requests --
+    # that's a genuinely different state from "never asked to measure" (None).
+    perf: Optional[dict[str, Any]] = None
+    # Model size/identity — never caller-supplied. Introspected (real
+    # parameter count/sparsity/size) for local backends; identity-only
+    # ({"is_local": False, "model_name": ...}) for hosted-API backends,
+    # since there is no checkpoint to measure. See Model.model_info().
+    # None unless RunConfig.track_performance=True (see `perf` above).
+    model_size: Optional[dict[str, Any]] = None
+    # Real, provider-reported token counts (prompt_tokens/completion_tokens/
+    # total_tokens) — populated automatically by Runner from each backend's
+    # actual API response (see model/openai.py etc.), never estimated. All
+    # zero for local/callable backends that don't report usage. This is the
+    # only input .cost() needs from the run itself; $/token pricing is
+    # deliberately NOT bundled in the library (see .cost()'s docstring).
+    # None unless RunConfig.track_performance=True (see `perf` above).
+    token_usage: Optional[dict[str, int]] = None
+
+    def cost(self, pricing: dict[str, float]) -> Optional[float]:
+        """Dollar cost of this run, given *pricing* — never a library default.
+
+        There is no live pricing API and provider prices change over time, so
+        a hardcoded table would silently go stale; you must supply
+        ``{"input_per_1m": <$ per 1M prompt tokens>, "output_per_1m": <$ per
+        1M completion tokens>}`` yourself (check your provider's current
+        pricing page). Returns ``None`` if this run recorded no token usage
+        at all (a local/callable backend, nothing was ever generated, or
+        ``RunConfig.track_performance`` was never enabled on this run) —
+        there is nothing to price.
+        """
+        if not self.token_usage or not self.token_usage.get("total_tokens"):
+            return None
+        return (
+            self.token_usage["prompt_tokens"] / 1_000_000 * pricing["input_per_1m"]
+            + self.token_usage["completion_tokens"] / 1_000_000 * pricing["output_per_1m"]
+        )
+
+    @property
+    def samples(self) -> list[Prediction]:
+        return self.predictions
+
+    def metric_table(self) -> list[dict[str, Any]]:
+        """One row per metric: name, mean, stderr, and n."""
+        rows = []
+        for name, stat in self.stats.items():
+            rows.append({
+                "name": name,
+                "mean": stat.mean,
+                "stderr": stat.stderr,
+                "n": stat.count,
+            })
+        return rows
+
+    def summary(self) -> str:
+        """A compact human-readable metric table (mean ± stderr over n)."""
+        lines = [f"run {self.run_id} (fingerprint {self.fingerprint})"]
+        for row in self.metric_table():
+            lines.append(
+                f"  {row['name']}: {row['mean']:.4f} ± {row['stderr']:.4f}  (n={row['n']})"
+            )
+        perf_line = self._perf_line()
+        if perf_line:
+            lines.append(f"  {perf_line}")
+        size_line = self._model_size_line()
+        if size_line:
+            lines.append(f"  {size_line}")
+        if self.token_usage and self.token_usage.get("total_tokens"):
+            lines.append(
+                f"  tokens: {self.token_usage['prompt_tokens']:,} in + "
+                f"{self.token_usage['completion_tokens']:,} out = "
+                f"{self.token_usage['total_tokens']:,} total "
+                f"(pass pricing to .cost() for a $ figure)"
+            )
+        return "\n".join(lines)
+
+    def _perf_line(self) -> str:
+        """One-line latency/throughput summary, or '' if nothing was measured
+        (the reads_actual_output path makes no model call to time, or
+        RunConfig.track_performance was never enabled on this run)."""
+        if not self.perf:
+            return ""
+        lat = self.perf.get("latency_ms") or {}
+        thr = self.perf.get("throughput") or {}
+        if not lat.get("count"):
+            return ""
+        line = (
+            f"perf: latency={lat['mean']:.0f}ms "
+            f"(n={lat['count']} calls) | throughput={thr.get('rps', 0.0):.2f} req/s"
+        )
+        # Only shown when the backend actually reported token usage (hosted
+        # APIs) -- 0 tok/s for a local/callable backend would misleadingly
+        # read as "measured zero" rather than "not available here".
+        if thr.get("total_tokens_per_sec"):
+            line += f" | {thr['output_tokens_per_sec']:.1f} out-tok/s, {thr['total_tokens_per_sec']:.1f} total-tok/s"
+        return line
+
+    def _model_size_line(self) -> str:
+        """One-line model size summary, or '' if nothing was recorded at all."""
+        ms = self.model_size
+        if not ms:
+            return ""
+        if not ms.get("is_local"):
+            return f"model: {ms.get('model_name', '?')} (not a local model -- no measurable size)"
+        return (
+            f"model: {ms.get('model_name', '?')} -- "
+            f"{ms.get('total_params', 0):,} params "
+            f"({ms.get('sparsity', 0.0):.1%} sparse), {ms.get('size_mb', 0.0):.1f} MB"
+        )
+
+    def wrong_only(self) -> list[Prediction]:
+        """The predictions that were scored incorrect (the regression browser)."""
+        return [p for p in self.predictions if p.correct is False]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "fingerprint": self.fingerprint,
+            "headline": self.headline,
+            "stats": {name: stat.to_dict() for name, stat in self.stats.items()},
+            "predictions": [p.to_doc() for p in self.predictions],
+            "config": self.config.to_dict() if self.config else None,
+            "model_spec": self.model_spec,
+            "errors": self.errors,
+            "failed_count": self.failed_count,
+            "experiment_name": self.experiment_name,
+            "tags": self.tags,
+            "perf": self.perf,
+            "model_size": self.model_size,
+            "token_usage": self.token_usage,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RunResult:
+        from .score import Stat
+        from .runspec import RunConfig
+        stats = {}
+        for name, sdata in data.get("stats", {}).items():
+            stats[name] = Stat.from_dict({"name": name, **sdata})
+        predictions = [Prediction(**p) for p in data.get("predictions", [])]
+        return cls(
+            run_id=data["run_id"],
+            fingerprint=data["fingerprint"],
+            stats=stats,
+            predictions=predictions,
+            headline=data.get("headline", {}),
+            config=RunConfig.from_dict(data["config"]) if data.get("config") else None,
+            model_spec=data.get("model_spec"),
+            errors=data.get("errors", []),
+            failed_count=data.get("failed_count", 0),
+            experiment_name=data.get("experiment_name"),
+            tags=data.get("tags", []),
+            perf=data.get("perf", {}),
+            model_size=data.get("model_size", {}),
+            token_usage=data.get("token_usage", {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}),
+        )
+
+    @classmethod
+    def load(cls, path: str) -> RunResult:
+        import json
+        with open(path, "r", encoding="utf-8") as fh:
+            return cls.from_dict(json.load(fh))
+
+    def save(self, path: str, fmt: str = "json") -> None:
+        """Persist the run as ``json`` (full) or ``csv`` (per-prediction rows)."""
+        if fmt == "json":
+            with open(path, "w", encoding="utf-8") as fh:
+                json.dump(self.to_dict(), fh, indent=2, default=str)
+        elif fmt == "csv":
+            import csv as csv_mod
+            predictions = self.predictions
+            if not predictions:
+                with open(path, "w") as fh:
+                    pass
+                return
+            fields = list(predictions[0].to_doc().keys())
+            with open(path, "w", newline="") as fh:
+                writer = csv_mod.DictWriter(fh, fieldnames=fields)
+                writer.writeheader()
+                for p in predictions:
+                    writer.writerow(p.to_doc())
+        else:
+            raise ValueError(f"unknown format {fmt!r}; use 'json' or 'csv'")
+
+    def push_to_hub(
+        self,
+        repo_id: str,
+        private: bool = False,
+        token: str | None = None,
+    ) -> str:
+        """Push predictions to a Hub dataset and stamp the AuditKIT card."""
+        from .hf_publish import push_rows_to_hub
+
+        model = ""
+        if isinstance(self.model_spec, str):
+            model = self.model_spec
+        elif isinstance(self.model_spec, dict):
+            model = str(self.model_spec.get("name") or self.model_spec.get("model") or "")
+        return push_rows_to_hub(
+            [p.to_doc() for p in self.predictions],
+            repo_id,
+            private=private,
+            token=token,
+            kind="run",
+            method=self.experiment_name or "evaluate",
+            model=model,
+        )

auditkit/report_format.py

@@ -0,0 +1,52 @@
+"""Pretty-printed and markdown reports for RunResult."""
+from __future__ import annotations
+from typing import Any
+from .report import RunResult
+
+
+class Report:
+    def __init__(self, result: RunResult) -> None:
+        self.result = result
+
+    def metric_rows(self) -> list[dict[str, Any]]:
+        rows = []
+        for name, stat in self.result.stats.items():
+            rows.append({
+                "name": name,
+                "mean": stat.mean,
+                "stderr": stat.stderr,
+                "std": stat.std,
+                "n": stat.count,
+            })
+        return rows
+
+    def __str__(self) -> str:
+        lines = [f"Run {self.result.run_id}  (fingerprint {self.result.fingerprint})"]
+        lines.append("")
+        rows = self.metric_rows()
+        if rows:
+            lines.append(f"{'Metric':<30} {'Mean':>8} {'±Stderr':>8} {'Std':>8} {'N':>6}")
+            lines.append("-" * 62)
+            for r in rows:
+                lines.append(
+                    f"{r['name']:<30} {r['mean']:>8.4f} {r['stderr']:>8.4f} "
+                    f"{r['std']:>8.4f} {r['n']:>6}"
+                )
+            lines.append("")
+        lines.append(f"{len(self.result.predictions)} predictions")
+        wrong = len(self.result.wrong_only())
+        if wrong:
+            lines.append(f"{wrong} wrong / {len(self.result.predictions)} total")
+        return "\n".join(lines)
+
+    def markdown(self) -> str:
+        rows = self.metric_rows()
+        if not rows:
+            return ""
+        lines = ["| Metric | Mean | ±Stderr | Std | N |", "|---|---|---|---|---|"]
+        for r in rows:
+            lines.append(
+                f"| {r['name']} | {r['mean']:.4f} | {r['stderr']:.4f} "
+                f"| {r['std']:.4f} | {r['n']} |"
+            )
+        return "\n".join(lines)

auditkit/router.py

@@ -0,0 +1,54 @@
+"""Automatic adapter selection from the dataset's shape.
+
+Routing is a pure function of the **task** — what each sample carries — never of
+the model:
+
+- ``choices`` present            -> :class:`MCQAdapter`
+- ``retrieval_context`` present  -> :class:`RAGAdapter`
+- otherwise                      -> :class:`GenerationAdapter`
+
+Why not look at the model? Chat-vs-base formatting (an instruct model's chat
+template vs a base model's raw text) is applied by the **model backend** at
+generate-time, so any adapter's output is formatted correctly for whatever model
+runs it. That means routing never has to know the model, never has to download a
+tokenizer to probe it, and an MCQ/RAG/plain task is handled correctly on both
+base and instruct models without picking a different adapter.
+
+:class:`ChatAdapter` is intentionally **never** auto-selected: its only job now
+is to inject a specific system prompt, which is an explicit user choice — not
+something to guess (and auto-injecting a generic "You are a helpful assistant"
+would silently confound an eval).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Sequence, Union
+
+from .adapter import Adapter, GenerationAdapter, MCQAdapter, RAGAdapter
+from .sample import Sample
+
+
+def _first_sample(samples: Union[Sample, Sequence[Sample]]) -> Sample:
+    if isinstance(samples, Sample):
+        return samples
+    for s in samples:
+        return s
+    raise ValueError("route_adapter needs at least one sample to inspect")
+
+
+def route_adapter(samples: Union[Sample, Sequence[Sample]], model: Any = None,
+                  **_ignored: Any) -> Adapter:
+    """Pick an adapter for *samples* from their shape.
+
+    Routes on the field each adapter actually needs (``choices`` for MCQ,
+    ``retrieval_context`` for RAG), so a sample can never be routed to an
+    adapter that would then fail for lack of that field. ``model`` is accepted
+    for backward compatibility but ignored — routing is task-based; the model
+    backend handles model-specific formatting.
+    """
+    sample = _first_sample(samples)
+    if sample.choices:
+        return MCQAdapter()
+    if sample.retrieval_context:
+        return RAGAdapter()
+    return GenerationAdapter()