llm-search-eval 0.2.0__py3-none-any.whl

llm_search_eval/__init__.py

@@ -0,0 +1,33 @@
+"""Evaluate product search quality with IR metrics, LLM judges, and CI quality gates."""
+
+from llm_search_eval.evaluate import EvaluationConfig, EvaluationReport, evaluate
+from llm_search_eval.gate import GateResult, check_report
+from llm_search_eval.html_report import render_html
+from llm_search_eval.judges import Judge, Judgment, create_judge, register_judge
+from llm_search_eval.judges.panel import JudgePanel
+from llm_search_eval.models import Dataset, Product, Query, Run
+from llm_search_eval.usage import Budget, CostTracker, PriceTable
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "Budget",
+    "CostTracker",
+    "Dataset",
+    "EvaluationConfig",
+    "EvaluationReport",
+    "GateResult",
+    "Judge",
+    "JudgePanel",
+    "Judgment",
+    "PriceTable",
+    "Product",
+    "Query",
+    "Run",
+    "__version__",
+    "check_report",
+    "create_judge",
+    "evaluate",
+    "register_judge",
+    "render_html",
+]

llm_search_eval/__main__.py

@@ -0,0 +1,4 @@
+from llm_search_eval.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llm_search_eval/agreement.py

@@ -0,0 +1,261 @@
+"""Agreement statistics for graded relevance labels.
+
+Used to check whether several judges agree with each other (inter-rater
+reliability) and with human gold labels (calibration). All functions take plain
+integer grades, so they also work for labels that did not come from this package.
+
+Undefined statistics, for example kappa when every label is identical, are
+returned as ``None`` instead of an arbitrary number.
+"""
+
+from __future__ import annotations
+
+import itertools
+from collections import Counter
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from llm_search_eval.models import MAX_RELEVANCE, MIN_RELEVANCE
+
+DEFAULT_CATEGORIES: tuple[int, ...] = tuple(range(MIN_RELEVANCE, MAX_RELEVANCE + 1))
+WEIGHTS = (None, "linear", "quadratic")
+LEVELS = ("nominal", "ordinal", "interval")
+
+
+def _check_pairs(a: Sequence[int], b: Sequence[int]) -> None:
+    if len(a) != len(b):
+        raise ValueError("label sequences must have equal length")
+
+
+def percent_agreement(a: Sequence[int], b: Sequence[int]) -> float | None:
+    """Share of items where both raters chose the same grade."""
+    _check_pairs(a, b)
+    if not a:
+        return None
+    return sum(1 for x, y in zip(a, b, strict=True) if x == y) / len(a)
+
+
+def within_one(a: Sequence[int], b: Sequence[int]) -> float | None:
+    """Share of items where the grades differ by at most one step."""
+    _check_pairs(a, b)
+    if not a:
+        return None
+    return sum(1 for x, y in zip(a, b, strict=True) if abs(x - y) <= 1) / len(a)
+
+
+def confusion_matrix(
+    a: Sequence[int], b: Sequence[int], categories: Sequence[int] = DEFAULT_CATEGORIES
+) -> list[list[int]]:
+    """Counts with rows for ``a`` and columns for ``b``, in ``categories`` order."""
+    _check_pairs(a, b)
+    index = {c: i for i, c in enumerate(categories)}
+    matrix = [[0] * len(categories) for _ in categories]
+    for x, y in zip(a, b, strict=True):
+        if x not in index or y not in index:
+            raise ValueError(f"grade outside categories {list(categories)}: {x}, {y}")
+        matrix[index[x]][index[y]] += 1
+    return matrix
+
+
+def _weight(i: int, j: int, size: int, weights: str | None) -> float:
+    if weights is None:
+        return 0.0 if i == j else 1.0
+    span = max(1, size - 1)
+    if weights == "linear":
+        return abs(i - j) / span
+    if weights == "quadratic":
+        return ((i - j) / span) ** 2
+    raise ValueError(f"unknown weights {weights!r}; use None, 'linear', or 'quadratic'")
+
+
+def cohen_kappa(
+    a: Sequence[int],
+    b: Sequence[int],
+    weights: str | None = None,
+    categories: Sequence[int] = DEFAULT_CATEGORIES,
+) -> float | None:
+    """Cohen's kappa for two raters, optionally linear or quadratic weighted.
+
+    Quadratic weighting suits ordinal relevance grades because a 0 versus 3
+    disagreement is penalized more than 2 versus 3.
+    """
+    matrix = confusion_matrix(a, b, categories)
+    n = len(a)
+    if n == 0:
+        return None
+    size = len(categories)
+    rows = [sum(r) for r in matrix]
+    cols = [sum(matrix[i][j] for i in range(size)) for j in range(size)]
+    observed = expected = 0.0
+    for i in range(size):
+        for j in range(size):
+            w = _weight(i, j, size, weights)
+            observed += w * matrix[i][j] / n
+            expected += w * rows[i] * cols[j] / (n * n)
+    if expected == 0:
+        return None
+    return 1.0 - observed / expected
+
+
+def fleiss_kappa(
+    items: Sequence[Sequence[int]], categories: Sequence[int] = DEFAULT_CATEGORIES
+) -> float | None:
+    """Fleiss' kappa for any fixed number of raters per item.
+
+    ``items`` holds one sequence of grades per rated item. Every item needs the
+    same number of ratings (at least two); use :func:`krippendorff_alpha` when
+    some ratings are missing.
+    """
+    if not items:
+        return None
+    raters = len(items[0])
+    if raters < 2 or any(len(item) != raters for item in items):
+        raise ValueError("every item needs the same number of ratings, at least two")
+    index = {c: i for i, c in enumerate(categories)}
+    totals = [0] * len(categories)
+    agreement = 0.0
+    for item in items:
+        counts = [0] * len(categories)
+        for grade in item:
+            if grade not in index:
+                raise ValueError(f"grade {grade} outside categories {list(categories)}")
+            counts[index[grade]] += 1
+        for i, c in enumerate(counts):
+            totals[i] += c
+        agreement += (sum(c * c for c in counts) - raters) / (raters * (raters - 1))
+    p_bar = agreement / len(items)
+    shares = [t / (len(items) * raters) for t in totals]
+    p_e = sum(p * p for p in shares)
+    if p_e >= 1.0:
+        return None
+    return (p_bar - p_e) / (1.0 - p_e)
+
+
+def krippendorff_alpha(
+    units: Sequence[Sequence[int | None]], level: str = "ordinal"
+) -> float | None:
+    """Krippendorff's alpha with missing ratings allowed.
+
+    ``units`` holds one sequence per item with each rater's grade or ``None``.
+    Items with fewer than two grades are ignored. ``level`` is ``nominal``,
+    ``ordinal``, or ``interval``.
+    """
+    if level not in LEVELS:
+        raise ValueError(f"unknown level {level!r}; use one of: {', '.join(LEVELS)}")
+    coincidence: Counter[tuple[int, int]] = Counter()
+    pairable = 0
+    for unit in units:
+        values = [v for v in unit if v is not None]
+        m = len(values)
+        if m < 2:
+            continue
+        pairable += m
+        for i, j in itertools.permutations(range(m), 2):
+            coincidence[(values[i], values[j])] += 1.0 / (m - 1)  # type: ignore[assignment]
+    if pairable < 2:
+        return None
+    values_sorted = sorted({c for pair in coincidence for c in pair})
+    marginals = {c: sum(coincidence[(c, k)] for k in values_sorted) for c in values_sorted}
+    n = sum(marginals.values())
+
+    def delta(c: int, k: int) -> float:
+        if level == "nominal":
+            return 0.0 if c == k else 1.0
+        if level == "interval":
+            return float((c - k) ** 2)
+        low, high = min(c, k), max(c, k)
+        between = sum(marginals[g] for g in values_sorted if low <= g <= high)
+        return (between - (marginals[c] + marginals[k]) / 2) ** 2
+
+    observed = sum(coincidence[(c, k)] * delta(c, k) for c in values_sorted for k in values_sorted)
+    expected = sum(
+        marginals[c] * marginals[k] * delta(c, k) for c in values_sorted for k in values_sorted
+    )
+    if expected == 0:
+        return None
+    return 1.0 - (n - 1) * observed / expected
+
+
+def pairwise_agreement(
+    ratings: Mapping[str, Sequence[int | None]],
+) -> list[dict[str, Any]]:
+    """Cohen's kappa (plain and quadratic) for every pair of raters.
+
+    ``ratings`` maps rater name to grades aligned by item; ``None`` marks a
+    missing grade and excludes the item for that pair only.
+    """
+    out = []
+    for first, second in itertools.combinations(ratings, 2):
+        pairs = [
+            (x, y)
+            for x, y in zip(ratings[first], ratings[second], strict=True)
+            if x is not None and y is not None
+        ]
+        a = [x for x, _ in pairs]
+        b = [y for _, y in pairs]
+        out.append(
+            {
+                "a": first,
+                "b": second,
+                "n": len(pairs),
+                "percent_agreement": percent_agreement(a, b),
+                "cohen_kappa": cohen_kappa(a, b),
+                "weighted_kappa": cohen_kappa(a, b, weights="quadratic"),
+            }
+        )
+    return out
+
+
+def inter_judge_agreement(ratings: Mapping[str, Sequence[int | None]]) -> dict[str, Any]:
+    """Summary of agreement between several raters over the same items."""
+    names = list(ratings)
+    if len(names) < 2:
+        raise ValueError("agreement needs at least two raters")
+    length = len(ratings[names[0]])
+    if any(len(ratings[name]) != length for name in names):
+        raise ValueError("every rater needs one entry per item")
+    units = [[ratings[name][i] for name in names] for i in range(length)]
+    complete = [[v for v in unit if v is not None] for unit in units]
+    complete = [unit for unit in complete if len(unit) == len(names)]
+    unanimous = sum(1 for unit in complete if len(set(unit)) == 1)
+    return {
+        "raters": names,
+        "items": length,
+        "complete_items": len(complete),
+        "unanimous": None if not complete else unanimous / len(complete),
+        "fleiss_kappa": fleiss_kappa(complete) if complete else None,
+        "krippendorff_alpha": {
+            "nominal": krippendorff_alpha(units, "nominal"),
+            "ordinal": krippendorff_alpha(units, "ordinal"),
+            "interval": krippendorff_alpha(units, "interval"),
+        },
+        "pairwise": pairwise_agreement(ratings),
+    }
+
+
+def gold_agreement(
+    predicted: Sequence[int], gold: Sequence[int], threshold: int = 2
+) -> dict[str, Any]:
+    """Compare one rater with gold labels: exact, within one, binary, and kappa."""
+    _check_pairs(predicted, gold)
+    binary_p = [int(x >= threshold) for x in predicted]
+    binary_g = [int(x >= threshold) for x in gold]
+    n = len(gold)
+    return {
+        "n": n,
+        "exact": percent_agreement(predicted, gold),
+        "within_one": within_one(predicted, gold),
+        "binary_agreement": percent_agreement(binary_p, binary_g),
+        "cohen_kappa": cohen_kappa(predicted, gold),
+        "weighted_kappa": cohen_kappa(predicted, gold, weights="quadratic"),
+        "binary_kappa": cohen_kappa(binary_p, binary_g, categories=(0, 1)),
+        "mean_bias": (
+            None if not n else sum(p - g for p, g in zip(predicted, gold, strict=True)) / n
+        ),
+        "confusion": {
+            "rows": "gold",
+            "columns": "judge",
+            "categories": list(DEFAULT_CATEGORIES),
+            "matrix": confusion_matrix(gold, predicted),
+        },
+    }

llm_search_eval/assets/dashboard.css

@@ -0,0 +1,51 @@
+:root {
+  --bg: #f6f7f9; --card: #fff; --text: #1d2330; --muted: #5f6b7a; --line: #e2e6ec;
+  --accent: #2563eb; --good: #15803d; --bad: #b91c1c; --warn: #b45309;
+  --g0: #e5e7eb; --g1: #fde68a; --g2: #93c5fd; --g3: #86efac;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --bg: #11151c; --card: #1a2029; --text: #e6e9ef; --muted: #9aa5b4; --line: #2b3441;
+    --accent: #60a5fa; --good: #4ade80; --bad: #f87171; --warn: #fbbf24;
+    --g0: #374151; --g1: #854d0e; --g2: #1e40af; --g3: #166534;
+  }
+}
+* { box-sizing: border-box; }
+body { margin: 0; font: 14px/1.5 system-ui, -apple-system, "Segoe UI", Roboto, Vazirmatn, Tahoma, sans-serif; background: var(--bg); color: var(--text); }
+header { padding: 24px 32px 8px; }
+header h1 { margin: 0 0 4px; font-size: 22px; }
+header .meta { color: var(--muted); }
+nav { position: sticky; top: 0; z-index: 2; background: var(--bg); border-bottom: 1px solid var(--line); padding: 8px 32px; display: flex; gap: 16px; flex-wrap: wrap; }
+nav a { color: var(--accent); text-decoration: none; font-weight: 500; }
+main { padding: 16px 32px 48px; display: grid; gap: 20px; }
+section { background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: 18px 20px; }
+section h2 { margin: 0 0 12px; font-size: 17px; }
+.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 12px; }
+.card { border: 1px solid var(--line); border-radius: 8px; padding: 10px 12px; }
+.card .v { font-size: 20px; font-weight: 600; }
+.card .k { color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .04em; }
+table { border-collapse: collapse; width: 100%; }
+th, td { padding: 6px 8px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; }
+th { font-weight: 600; color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .03em; }
+td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; }
+.best { font-weight: 700; color: var(--accent); }
+.pos { color: var(--good); } .neg { color: var(--bad); } .muted { color: var(--muted); }
+.row { display: flex; gap: 12px; align-items: center; flex-wrap: wrap; margin-bottom: 10px; }
+select, input[type=search] { font: inherit; padding: 4px 8px; border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--text); }
+input[type=search] { min-width: 260px; }
+svg text { fill: var(--text); font-size: 11px; }
+svg .axis { stroke: var(--line); }
+.bar { fill: var(--accent); }
+.bar.pos { fill: var(--good); } .bar.neg { fill: var(--bad); }
+.split { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 20px; }
+.clickable tbody tr:hover, tr.selected { background: color-mix(in srgb, var(--accent) 10%, transparent); }
+.lists { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 14px; margin-top: 12px; }
+.list h3 { margin: 0 0 6px; font-size: 14px; }
+.list ol { margin: 0; padding-left: 26px; }
+.list li { margin: 0 0 6px; }
+.badge { display: inline-block; min-width: 20px; text-align: center; border-radius: 4px; padding: 0 5px; margin-right: 6px; font-weight: 600; font-size: 12px; color: #111; }
+.g0 { background: var(--g0); } .g1 { background: var(--g1); } .g2 { background: var(--g2); } .g3 { background: var(--g3); } .gx { background: transparent; border: 1px dashed var(--muted); color: var(--muted); }
+.src { color: var(--muted); font-size: 12px; }
+.reason { color: var(--muted); font-size: 12px; display: block; }
+.note { color: var(--muted); font-size: 13px; }
+footer { color: var(--muted); padding: 0 32px 32px; font-size: 12px; }

llm_search_eval/assets/dashboard.js

@@ -0,0 +1,286 @@
+(function () {
+  "use strict";
+  var DATA = JSON.parse(document.getElementById("report-data").textContent);
+  var SVG = "http://www.w3.org/2000/svg";
+  var cfg = DATA.config;
+  var primary = cfg.primary_metric;
+  var systems = DATA.systems;
+  var names = systems.map(function (s) { return s.name; });
+  var details = DATA.details || null;
+
+  function $(id) { return document.getElementById(id); }
+  function el(tag, attrs, children) {
+    var node = document.createElement(tag);
+    if (attrs) {
+      Object.keys(attrs).forEach(function (key) {
+        if (key === "text") node.textContent = attrs[key];
+        else if (key === "class") node.className = attrs[key];
+        else node.setAttribute(key, attrs[key]);
+      });
+    }
+    (children || []).forEach(function (child) { if (child) node.appendChild(child); });
+    return node;
+  }
+  function svg(tag, attrs) {
+    var node = document.createElementNS(SVG, tag);
+    Object.keys(attrs || {}).forEach(function (key) { node.setAttribute(key, attrs[key]); });
+    return node;
+  }
+  function fmt(v, d) { return v === null || v === undefined ? "n/a" : Number(v).toFixed(d === undefined ? 4 : d); }
+  function signed(v) { return v === null || v === undefined ? "n/a" : (v >= 0 ? "+" : "") + Number(v).toFixed(4); }
+  function pct(v) { return v === null || v === undefined ? "n/a" : (v >= 0 ? "+" : "") + (v * 100).toFixed(1) + "%"; }
+  function money(v) { return v === null || v === undefined ? "unknown" : "$" + Number(v).toFixed(4); }
+  function td(text, cls) { return el("td", { text: text, class: cls || "" }); }
+  function th(text, cls) { return el("th", { text: text, class: cls || "" }); }
+  function table(headers, rows, numeric) {
+    var head = el("tr", null, headers.map(function (h, i) { return th(h, i && numeric !== false ? "num" : ""); }));
+    var body = el("tbody", null, rows);
+    return el("table", null, [el("thead", null, [head]), body]);
+  }
+  function option(value, label, selected) {
+    var o = el("option", { value: value, text: label || value });
+    if (selected) o.selected = true;
+    return o;
+  }
+
+  function barChart(container, items, opts) {
+    opts = opts || {};
+    container.textContent = "";
+    var width = Math.max(320, container.clientWidth || 640);
+    var rowH = 22, labelW = opts.labelWidth || 140, valueW = 70;
+    var height = items.length * rowH + 10;
+    var chart = svg("svg", { width: width, height: height, role: "img", "aria-label": opts.label || "chart" });
+    var plotW = width - labelW - valueW;
+    var values = items.map(function (it) { return it.value || 0; });
+    var maxAbs = Math.max.apply(null, values.map(Math.abs).concat([opts.max || 0, 1e-9]));
+    var signedMode = values.some(function (v) { return v < 0; });
+    var zero = labelW + (signedMode ? plotW / 2 : 0);
+    var scale = (signedMode ? plotW / 2 : plotW) / maxAbs;
+    items.forEach(function (it, i) {
+      var y = 5 + i * rowH;
+      var v = it.value || 0;
+      var label = svg("text", { x: labelW - 6, y: y + 14, "text-anchor": "end" });
+      label.textContent = it.label.length > 22 ? it.label.slice(0, 21) + "\u2026" : it.label;
+      var title = svg("title", {});
+      title.textContent = it.label + ": " + (it.value === null ? "n/a" : fmt(it.value));
+      var w = Math.abs(v) * scale;
+      var bar = svg("rect", {
+        x: v >= 0 ? zero : zero - w, y: y + 3, width: Math.max(w, it.value === null ? 0 : 1), height: rowH - 8,
+        class: "bar" + (signedMode || opts.colorSign ? (v >= 0 ? " pos" : " neg") : ""), rx: 2
+      });
+      bar.appendChild(title);
+      var text = svg("text", { x: labelW + plotW + 6, y: y + 14 });
+      text.textContent = it.value === null ? "n/a" : (signedMode ? signed(v) : fmt(v));
+      chart.appendChild(label); chart.appendChild(bar); chart.appendChild(text);
+    });
+    if (signedMode) chart.appendChild(svg("line", { x1: zero, x2: zero, y1: 0, y2: height, class: "axis" }));
+    container.appendChild(chart);
+  }
+
+  function renderOverview() {
+    var ds = DATA.dataset, labels = DATA.labels, cost = DATA.cost || {};
+    var cards = [
+      ["Queries", ds.queries], ["Products", ds.products], ["Gold labels", ds.gold_labels],
+      ["Judged", labels.judged], ["From cache", labels.cached], ["Unlabeled", labels.unlabeled],
+      ["Tokens", cost.total_tokens === undefined ? "n/a" : cost.total_tokens.toLocaleString()],
+      ["Cost", cost.requests ? money(cost.cost_usd) : "$0"]
+    ];
+    var box = $("overview-cards");
+    cards.forEach(function (c) {
+      box.appendChild(el("div", { class: "card" }, [el("div", { class: "k", text: c[0] }), el("div", { class: "v", text: String(c[1]) })]));
+    });
+    $("meta").textContent = "Judge: " + (labels.judge || "none") + " \u00b7 depth " + cfg.depth +
+      " \u00b7 relevant when grade \u2265 " + cfg.threshold + " \u00b7 baseline " + cfg.baseline +
+      " \u00b7 generated " + DATA.generated_at + " by llm-search-eval " + DATA.tool.version;
+  }
+
+  function renderSummary() {
+    var metrics = cfg.metrics;
+    var best = {};
+    metrics.forEach(function (m) {
+      var vals = systems.map(function (s) { return s.metrics[m].mean; }).filter(function (v) { return v !== null; });
+      best[m] = vals.length ? Math.max.apply(null, vals) : null;
+    });
+    var rows = systems.map(function (s) {
+      return el("tr", null, [td(s.name)].concat(metrics.map(function (m) {
+        var v = s.metrics[m].mean;
+        var isBest = v !== null && systems.length > 1 && Math.abs(v - best[m]) < 1e-12;
+        return td(fmt(v), "num" + (isBest ? " best" : ""));
+      })));
+    });
+    $("summary-table").appendChild(table(["System"].concat(metrics), rows));
+    var select = $("metric-select");
+    metrics.forEach(function (m) { select.appendChild(option(m, m, m === primary)); });
+    function draw() {
+      var m = select.value;
+      barChart($("metric-chart"), systems.map(function (s) { return { label: s.name, value: s.metrics[m].mean }; }), { max: 1, label: m });
+    }
+    select.addEventListener("change", draw);
+    draw();
+  }
+
+  function renderComparison() {
+    var comp = DATA.comparison;
+    var section = $("comparison");
+    if (systems.length < 2) { section.hidden = true; return; }
+    var rows = comp.significance.map(function (t) {
+      var d = comp.deltas.filter(function (x) { return x.system === t.system && x.metric === t.metric; })[0] || {};
+      var cls = d.delta > 0 ? "num pos" : d.delta < 0 ? "num neg" : "num";
+      return el("tr", null, [td(t.system), td(signed(d.delta), cls), td(pct(d.relative), cls),
+        td(t.wins + " / " + t.ties + " / " + t.losses, "num"),
+        td(t.p_value === null ? "n/a" : t.p_value.toFixed(4) + (t.p_value < 0.05 ? " *" : ""), "num")]);
+    });
+    $("comparison-table").appendChild(table(["System", "\u0394 " + comp.primary_metric, "Relative", "Win / tie / loss", "p-value"], rows));
+    var select = $("compare-select");
+    names.filter(function (n) { return n !== comp.baseline; }).forEach(function (n, i) { select.appendChild(option(n, n, i === 0)); });
+    function draw() {
+      var a = systems.filter(function (s) { return s.name === select.value; })[0];
+      var b = systems.filter(function (s) { return s.name === comp.baseline; })[0];
+      var items = [];
+      Object.keys(b.per_query).forEach(function (q) {
+        var x = a.per_query[q][primary], y = b.per_query[q][primary];
+        if (x === null || y === null || Math.abs(x - y) < 1e-12) return;
+        var text = details && details.queries[q] ? q + " " + details.queries[q].text : q;
+        items.push({ label: text, value: x - y });
+      });
+      items.sort(function (p, q) { return q.value - p.value; });
+      $("delta-note").textContent = items.length ? "Per-query " + primary + " change of " + select.value + " versus " + comp.baseline + ", queries with no change omitted." : "No query changed between " + select.value + " and " + comp.baseline + ".";
+      barChart($("delta-chart"), items, { labelWidth: 220, label: "per-query delta" });
+    }
+    select.addEventListener("change", draw);
+    draw();
+  }
+
+  function renderLanguages() {
+    var by = DATA.by_language;
+    var langs = {};
+    Object.keys(by).forEach(function (s) { Object.keys(by[s]).forEach(function (l) { langs[l] = true; }); });
+    langs = Object.keys(langs).sort();
+    if (langs.length < 2) { $("languages").hidden = true; return; }
+    var rows = Object.keys(by).map(function (s) {
+      return el("tr", null, [td(s)].concat(langs.map(function (l) { return td(fmt(by[s][l].mean) + " (n=" + by[s][l].n + ")", "num"); })));
+    });
+    $("language-table").appendChild(table(["System"].concat(langs), rows));
+  }
+
+  function renderJudging() {
+    var j = DATA.judging;
+    if (!j) { $("judging").hidden = true; return; }
+    var dist = j.label_distribution;
+    barChart($("label-chart"), ["3", "2", "1", "0"].map(function (g) { return { label: "grade " + g, value: dist[g] || 0 }; }), { label: "label distribution" });
+    var info = $("judging-info");
+    info.appendChild(el("p", { text: "Judge: " + j.judge + ". " + j.pairs + " judged pair(s)." }));
+    if (j.agreement) {
+      var a = j.agreement;
+      var k = a.krippendorff_alpha;
+      info.appendChild(table(["Statistic", "Value"], [
+        el("tr", null, [td("Fleiss' kappa"), td(fmt(a.fleiss_kappa, 3), "num")]),
+        el("tr", null, [td("Krippendorff's alpha (ordinal)"), td(fmt(k.ordinal, 3), "num")]),
+        el("tr", null, [td("Krippendorff's alpha (nominal)"), td(fmt(k.nominal, 3), "num")]),
+        el("tr", null, [td("Unanimous items"), td(a.unanimous === null ? "n/a" : (a.unanimous * 100).toFixed(1) + "%", "num")])
+      ]));
+      info.appendChild(el("h3", { text: "Pairwise agreement" }));
+      info.appendChild(table(["Judges", "n", "Agreement", "Cohen's kappa", "Weighted kappa"], a.pairwise.map(function (p) {
+        return el("tr", null, [td(p.a + " / " + p.b), td(String(p.n), "num"), td(p.percent_agreement === null ? "n/a" : (p.percent_agreement * 100).toFixed(1) + "%", "num"), td(fmt(p.cohen_kappa, 3), "num"), td(fmt(p.weighted_kappa, 3), "num")]);
+      })));
+    }
+    var g = j.gold_agreement;
+    if (g) {
+      var box = $("gold-info");
+      box.appendChild(el("h3", { text: "Agreement with gold labels" }));
+      var raters = [[j.judge, g.judge]].concat(Object.keys(g.members).map(function (n) { return [n, g.members[n]]; }));
+      box.appendChild(table(["Rater", "n", "Exact", "Within one", "Binary", "Weighted kappa", "Bias"], raters.map(function (r) {
+        var s = r[1];
+        return el("tr", null, [td(r[0]), td(String(s.n), "num"), td((s.exact * 100).toFixed(1) + "%", "num"), td((s.within_one * 100).toFixed(1) + "%", "num"), td((s.binary_agreement * 100).toFixed(1) + "%", "num"), td(fmt(s.weighted_kappa, 3), "num"), td(signed(s.mean_bias), "num")]);
+      })));
+      var m = g.judge.confusion;
+      box.appendChild(el("p", { class: "note", text: "Confusion matrix: rows are gold grades, columns are judge grades." }));
+      box.appendChild(table(["gold \\ judge"].concat(m.categories.map(String)), m.matrix.map(function (row, i) {
+        return el("tr", null, [td(String(m.categories[i]))].concat(row.map(function (c) { return td(String(c), "num"); })));
+      })));
+    }
+  }
+
+  function renderCost() {
+    var c = DATA.cost;
+    if (!c || (!c.requests && !c.cache_hits && !c.budget.skipped_pairs)) { $("cost").hidden = true; return; }
+    var rows = Object.keys(c.models).map(function (name) {
+      var m = c.models[name];
+      return el("tr", null, [td(name), td(String(m.requests), "num"), td(m.prompt_tokens.toLocaleString(), "num"), td(m.completion_tokens.toLocaleString(), "num"), td(money(m.cost_usd), "num")]);
+    });
+    rows.push(el("tr", null, [td("Total"), td(String(c.requests), "num"), td(c.prompt_tokens.toLocaleString(), "num"), td(c.completion_tokens.toLocaleString(), "num"), td(money(c.cost_usd), "num best")]));
+    $("cost-table").appendChild(table(["Model", "Requests", "Input tokens", "Output tokens", "Cost"], rows));
+    var notes = [c.cache_hits + " judgment(s) served from cache at no cost."];
+    if (c.unpriced_models.length) notes.push("No price configured for: " + c.unpriced_models.join(", ") + ".");
+    if (c.budget.exhausted) notes.push("Budget reached (" + c.budget.reason + "); " + c.budget.skipped_pairs + " pair(s) were not judged.");
+    $("cost-note").textContent = notes.join(" ");
+  }
+
+  function renderQueries() {
+    if (!details) { $("queries").hidden = true; return; }
+    var qids = Object.keys(details.queries);
+    var body = el("tbody");
+    var rows = {};
+    qids.forEach(function (q) {
+      var info = details.queries[q];
+      var cells = [td(q), el("td", { dir: "auto", text: info.text }), td(info.lang)];
+      systems.forEach(function (s) { cells.push(td(fmt(s.per_query[q][primary], 3), "num")); });
+      var tr = el("tr", { "data-q": q }, cells);
+      tr.addEventListener("click", function () { showQuery(q); });
+      rows[q] = tr;
+      body.appendChild(tr);
+    });
+    var head = el("tr", null, [th("Id"), th("Query"), th("Lang")].concat(names.map(function (n) { return th(n + " " + primary, "num"); })));
+    $("query-table").appendChild(el("table", { class: "clickable" }, [el("thead", null, [head]), body]));
+    $("query-filter").addEventListener("input", function (e) {
+      var needle = e.target.value.toLowerCase();
+      qids.forEach(function (q) {
+        var hay = (q + " " + details.queries[q].text).toLowerCase();
+        rows[q].hidden = needle && hay.indexOf(needle) < 0;
+      });
+    });
+    function showQuery(q) {
+      Object.keys(rows).forEach(function (k) { rows[k].className = k === q ? "selected" : ""; });
+      var box = $("query-detail");
+      box.textContent = "";
+      var info = details.queries[q];
+      box.appendChild(el("h3", { dir: "auto", text: q + ": " + info.text }));
+      var labels = details.labels[q] || {};
+      var lists = el("div", { class: "lists" });
+      systems.forEach(function (s) {
+        var ranked = details.rankings[s.name][q] || [];
+        var ol = el("ol");
+        ranked.forEach(function (pid) {
+          var l = labels[pid];
+          var p = details.products[pid];
+          var grade = l ? l.grade : null;
+          var badge = el("span", { class: "badge " + (grade === null ? "gx" : "g" + grade), text: grade === null ? "?" : String(grade) });
+          var title = el("span", { dir: "auto", text: p ? p.title : pid + " (not in catalog)" });
+          var src = el("span", { class: "src", text: " " + pid + (l ? " \u00b7 " + l.source : " \u00b7 unlabeled") });
+          var parts = [badge, title, src];
+          if (l && l.reason) parts.push(el("span", { class: "reason", dir: "auto", text: l.reason }));
+          if (l && l.votes) parts.push(el("span", { class: "reason", text: "votes: " + Object.keys(l.votes).map(function (k) { return k + "=" + (l.votes[k] === null ? "x" : l.votes[k]); }).join(", ") }));
+          ol.appendChild(el("li", null, parts));
+        });
+        if (!ranked.length) ol.appendChild(el("li", { class: "muted", text: "no results" }));
+        var metrics = cfg.metrics.map(function (m) { return m + " " + fmt(s.per_query[q][m], 3); }).join(" \u00b7 ");
+        lists.appendChild(el("div", { class: "list" }, [el("h3", { text: s.name }), el("p", { class: "note", text: metrics }), ol]));
+      });
+      box.appendChild(lists);
+      var missing = Object.keys(labels).filter(function (pid) {
+        return labels[pid].grade >= cfg.threshold && !systems.some(function (s) { return (details.rankings[s.name][q] || []).indexOf(pid) >= 0; });
+      });
+      if (missing.length) box.appendChild(el("p", { class: "note", text: "Relevant products no system returned in the top " + cfg.depth + ": " + missing.join(", ") }));
+      box.scrollIntoView({ behavior: "smooth", block: "nearest" });
+    }
+    if (qids.length) showQuery(qids[0]);
+  }
+
+  renderOverview();
+  renderSummary();
+  renderComparison();
+  renderLanguages();
+  renderJudging();
+  renderCost();
+  renderQueries();
+})();