ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/report/plain_summary.py

@@ -0,0 +1,122 @@
+"""A plain-English summary of a run: what's wrong, in at most six lines.
+
+Shared by `ragaudit check` (printed in the terminal) and the top of the
+HTML run report, so both say the same thing.
+"""
+
+from dataclasses import dataclass
+
+from ragaudit.models.run import RunSummary
+from ragaudit.models.scoring import ScoredCase, Verdict
+
+HEALTHY_TRUE_PASS_RATE = 0.8
+"""A run is healthy when at least this share of answers is correct and backed by the right passage."""
+
+# Tie-break order when two failure classes are equally common: a retrieval
+# problem undermines everything downstream, and a lucky pass hides one.
+_FAILURE_ORDER = (Verdict.RETRIEVAL_FAILURE, Verdict.LUCKY_PASS, Verdict.GENERATION_FAILURE)
+
+
+@dataclass
+class PlainSummary:
+    status: str
+    """healthy, needs_work, or empty."""
+    verdict: str
+    retrieval: str
+    correctness: str
+    lucky: str
+    fix: str
+    dominant: Verdict | None
+
+    def lines(self) -> list[str]:
+        return [self.verdict, self.retrieval, self.correctness, self.lucky, f"Most useful fix: {self.fix}"]
+
+
+def _pct(rate: float) -> str:
+    return f"{rate:.0%}"
+
+
+def dominant_failure(summary: RunSummary) -> Verdict | None:
+    counts = {verdict: summary.verdict_counts.get(verdict, 0) for verdict in _FAILURE_ORDER}
+    worst = max(_FAILURE_ORDER, key=lambda verdict: (counts[verdict], -_FAILURE_ORDER.index(verdict)))
+    return worst if counts[worst] else None
+
+
+def _fix(dominant: Verdict | None, count: int, no_contexts: bool, match_by: str, no_id_hits: bool = False) -> str:
+    if no_id_hits:
+        return (
+            "no retrieved chunk id matched ragaudit's, which usually means your pipeline uses its own chunks. "
+            "Rerun with --match-by text, or return ragaudit's chunk ids in retrieved_chunk_ids."
+        )
+    if no_contexts:
+        return (
+            "your adapter returned no retrieved passages, so retrieval could not be scored. Return the text "
+            "of each retrieved passage in RAGResponse.contexts."
+        )
+    if dominant is None:
+        return "nothing stands out. Keep this check in CI to catch regressions."
+    if dominant == Verdict.RETRIEVAL_FAILURE:
+        hint = "retrieve more passages (raise top-k), add keyword/hybrid search, or try smaller chunks"
+        if match_by == "id":
+            hint += "; if your pipeline chunks the docs itself, rerun with --match-by text"
+        return f"improve retrieval. For {count} question(s) the right passage was not found and the answer was wrong: {hint}."
+    if dominant == Verdict.LUCKY_PASS:
+        return (
+            f"don't trust the correct answers yet. {count} were right without the right passage being retrieved, "
+            "so the model answered from memory. Fix retrieval first, and tell the model to answer only from the context."
+        )
+    return (
+        f"improve answer generation. For {count} question(s) the right passage was retrieved but the answer was "
+        "still wrong: tighten the prompt to answer from the context, show it fewer but better passages, or use a stronger model."
+    )
+
+
+def build_plain_summary(summary: RunSummary, scored_cases: list[ScoredCase]) -> PlainSummary:
+    total = summary.total_cases
+    if total == 0:
+        return PlainSummary(
+            status="empty",
+            verdict="Verdict: NO RESULTS. The run had no questions to score.",
+            retrieval="Retrieval hit rate: n/a",
+            correctness="Answer correctness: n/a",
+            lucky="Lucky passes: n/a",
+            fix="generate questions first (check that the docs folder has .txt or .md files).",
+            dominant=None,
+        )
+    counts = summary.verdict_counts
+    true_pass = counts.get(Verdict.TRUE_PASS, 0)
+    lucky = counts.get(Verdict.LUCKY_PASS, 0)
+    hits = sum(1 for case in scored_cases if case.retrieval.hit)
+    correct = sum(1 for case in scored_cases if case.correctness.is_correct)
+    match_by = next((case.retrieval.match_by for case in scored_cases), "id")
+    no_contexts = match_by == "text" and not any(case.response.contexts for case in scored_cases)
+    dominant = dominant_failure(summary)
+
+    if true_pass / total >= HEALTHY_TRUE_PASS_RATE and not no_contexts:
+        status = "healthy"
+        verdict = (
+            f"Verdict: HEALTHY. {true_pass} of {total} answers were correct and came from the right passage."
+        )
+    else:
+        status = "needs_work"
+        problem = {
+            Verdict.RETRIEVAL_FAILURE: "retrieval is missing the right passage",
+            Verdict.LUCKY_PASS: "correct answers are not coming from retrieval",
+            Verdict.GENERATION_FAILURE: "answers are wrong even when retrieval is right",
+            None: "too few answers are both correct and grounded",
+        }[None if no_contexts else dominant]
+        verdict = f"Verdict: NEEDS WORK. Only {true_pass} of {total} answers were correct and from the right passage; main problem: {problem}."
+
+    confirmed = summary.ablation_confirmed_count
+    lucky_line = f"Lucky passes: {lucky} (correct answer, but the right passage was not retrieved)"
+    if lucky and confirmed is not None:
+        lucky_line += f"; {confirmed} still correct with no context at all, so answered from memory"
+    return PlainSummary(
+        status=status,
+        verdict=verdict,
+        retrieval=f"Retrieval hit rate: {_pct(summary.retrieval_hit_rate)} ({hits} of {total} found the passage the question came from, matched by {match_by})",
+        correctness=f"Answer correctness: {_pct(summary.answer_correctness_rate)} ({correct} of {total} judged correct)",
+        lucky=lucky_line,
+        fix=_fix(dominant, counts.get(dominant, 0) if dominant else 0, no_contexts, match_by, match_by == "id" and hits == 0),
+        dominant=dominant,
+    )

ragaudit/report/templates/_v2_tooltip.jinja

@@ -0,0 +1,43 @@
+<script>
+(function () {
+  "use strict";
+  // One tooltip per chart figure. Each mark carries data-tip: the first
+  // line is the value (bold), the rest is context. textContent only.
+  Array.prototype.slice.call(document.querySelectorAll(".v2-figure")).forEach(function (figure) {
+    var tip = document.createElement("div");
+    tip.className = "v2-tooltip";
+    tip.hidden = true;
+    figure.appendChild(tip);
+
+    function show(mark, clientX, clientY) {
+      var lines = (mark.getAttribute("data-tip") || "").split("\n");
+      tip.textContent = "";
+      var head = document.createElement("strong");
+      head.textContent = lines[0];
+      tip.appendChild(head);
+      if (lines.length > 1) {
+        tip.appendChild(document.createTextNode(lines.slice(1).join("\n")));
+      }
+      tip.hidden = false;
+      var box = figure.getBoundingClientRect();
+      var x = clientX - box.left + 14;
+      var y = clientY - box.top + 14;
+      if (x + tip.offsetWidth > box.width) { x = Math.max(0, clientX - box.left - tip.offsetWidth - 14); }
+      tip.style.left = x + "px";
+      tip.style.top = y + "px";
+    }
+
+    function hide() { tip.hidden = true; }
+
+    Array.prototype.slice.call(figure.querySelectorAll("[data-tip]")).forEach(function (mark) {
+      mark.addEventListener("mousemove", function (event) { show(mark, event.clientX, event.clientY); });
+      mark.addEventListener("mouseleave", hide);
+      mark.addEventListener("focus", function () {
+        var r = mark.getBoundingClientRect();
+        show(mark, r.left + r.width / 2, r.top);
+      });
+      mark.addEventListener("blur", hide);
+    });
+  });
+})();
+</script>

ragaudit/report/templates/generate_report.html.jinja

@@ -0,0 +1,175 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>ragaudit generate report — {{ summary.run_id }}</title>
+<style>
+{{ style_css | safe }}
+{{ v2_css | safe }}
+</style>
+</head>
+<body>
+<div class="container">
+
+  <header class="report-header">
+    <h1>ragaudit generate report{% if is_mock %}<span class="v2-badge-mock">built with the mock LLM</span>{% endif %}</h1>
+    <div class="meta">
+      Run <code>{{ summary.run_id }}</code> ·
+      generator <code>{{ summary.generator_model }}</code> ·
+      verifier <code>{{ summary.verifier_model }}</code> ·
+      index <code>{{ (summary.index_version or "n/a")[:12] }}</code> ·
+      started {{ summary.created_at }}
+    </div>
+  </header>
+
+  <section class="headline">
+    <h2>The headline finding</h2>
+    <p class="headline-sentence">
+      {% if checked == 0 %}
+      Nothing new to generate in this run: every chunk already had questions or had been tried before.
+      {% else %}
+      Of the <strong>{{ checked }}</strong> candidate questions checked this run, <strong>{{ summary.generated }} ({{ pass_pct }}%)</strong>
+      passed every check and were saved.
+      {% if top_reason %}
+      The most common reason a question was thrown out was <strong>{{ top_reason[0] }}</strong> ({{ top_reason[1] }} of {{ summary.rejected }} rejected).
+      {% endif %}
+      {% endif %}
+    </p>
+    {% if is_mock %}
+    <div class="v2-callout warn">These questions were written and checked by ragaudit's offline mock LLM, which is deterministic and deliberately flawed so every rejection path gets exercised. Real runs use a real generator and a different verifier model.</div>
+    {% endif %}
+    <div class="stats-grid">
+      <div class="stat"><span class="value">{{ summary.generated }}</span><span class="label">Questions saved this run</span></div>
+      <div class="stat"><span class="value">{{ summary.rejected }}</span><span class="label">Rejected this run</span></div>
+      <div class="stat"><span class="value">{{ summary.split_counts.get("dev", 0) }} / {{ summary.split_counts.get("test", 0) }}</span><span class="label">New dev / new test</span></div>
+      <div class="stat"><span class="value">{{ summary.multi_hop_generated }}</span><span class="label">Multi-hop saved</span></div>
+    </div>
+    {% if summary.stopped_reason %}
+    <div class="v2-callout warn">Stopped early ({{ summary.stopped_reason }}): {{ summary.chunks_remaining }} chunk(s) and {{ summary.pairs_remaining }} chunk pair(s) are still pending. Run <code>ragaudit generate</code> again to continue where it stopped.</div>
+    {% endif %}
+  </section>
+
+  <section>
+    <h2>What's in the store now</h2>
+    <p>Every chunk is assigned to dev or test once, from a hash of its id, and its questions inherit that split, so no chunk feeds both. Editing a chunk never moves it. <code>tune</code> only ever sees the dev split; the test split stays locked until the final score.</p>
+    <div class="v2-table-wrap">
+      <table class="v2-table">
+        <thead><tr><th>Split</th><th class="num">Active</th><th class="num">Stale</th><th class="num">Rejected</th></tr></thead>
+        <tbody>
+          {% for split in ["dev", "test"] %}
+          <tr>
+            <td>{{ split }}{% if split == "test" %} (locked){% endif %}</td>
+            <td class="num">{{ totals[split]["active"] }}</td>
+            <td class="num">{{ totals[split]["stale"] }}</td>
+            <td class="num">{{ totals[split]["rejected"] }}</td>
+          </tr>
+          {% endfor %}
+          {% set unsplit = totals.get("none", {}).get("rejected", 0) %}{% if unsplit %}
+          <tr>
+            <td>neither (cross-split multi-hop)</td>
+            <td class="num">0</td>
+            <td class="num">0</td>
+            <td class="num">{{ unsplit }}</td>
+          </tr>
+          {% endif %}
+        </tbody>
+      </table>
+    </div>
+    {% if summary.corpus_diff %}
+    <p class="meta">Corpus this run: {{ summary.corpus_chunks }} chunks
+      ({% for key, value in summary.corpus_diff.items() %}{{ value }} {{ key }}{% if not loop.last %}, {% endif %}{% endfor %}).
+      Processed {{ summary.chunks_processed }} chunk(s), skipped {{ summary.chunks_skipped }} that already had questions,
+      {{ summary.pairs_processed }} multi-hop pair(s); {{ summary.duplicates }} duplicate(s); {{ summary.llm_calls }} LLM call(s).</p>
+    {% endif %}
+  </section>
+
+  <section>
+    <h2>Why questions were rejected</h2>
+    {% if reasons %}
+    <figure class="v2-figure">{{ reason_chart | safe }}</figure>
+    <div class="v2-table-wrap">
+      <table class="v2-table">
+        <thead><tr><th>Reason</th><th class="num">Rejected</th><th>Meaning</th></tr></thead>
+        <tbody>
+          {% for r in reasons %}
+          <tr><td><code>{{ r.reason }}</code></td><td class="num">{{ r.count }}</td><td>{{ r.label }}</td></tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
+    {% else %}
+    <p>No questions were rejected this run.</p>
+    {% endif %}
+  </section>
+
+  <section>
+    <h2>Lexical overlap</h2>
+    <p>
+      A question that copies its source passage's wording can be answered by keyword search alone, which makes
+      a retriever look better than it is. Each question is rewritten once; if more than
+      <strong>{{ (summary.lexical_threshold * 100) | round(0) | int }}%</strong> of its content words still appear in
+      the passage, it gets one more rewrite and is then rejected as <code>too_lexical</code>.
+      {% if overlap_median is not none %}Median overlap this run: <strong>{{ "%.2f" | format(overlap_median) }}</strong>.{% endif %}
+    </p>
+    {% if summary.lexical_overlaps %}
+    <figure class="v2-figure">{{ histogram | safe }}</figure>
+    <ul class="v2-legend">
+      <li><span class="v2-swatch" style="background: var(--accent)"></span>At or under the threshold: kept for verification</li>
+      <li><span class="v2-swatch" style="background: var(--v2-muted-mark)"></span>Over the threshold: rejected</li>
+    </ul>
+    <details class="v2-data">
+      <summary>Show as a table</summary>
+      <table class="v2-table">
+        <thead><tr><th>Overlap</th><th class="num">Questions</th><th>Outcome</th></tr></thead>
+        <tbody>
+          {% for row in histogram_rows %}
+          <tr><td>{{ row.range }}</td><td class="num">{{ row.count }}</td><td>{{ "kept" if row.kept else "rejected" }}</td></tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </details>
+    {% else %}
+    <p>No questions reached the overlap check this run.</p>
+    {% endif %}
+  </section>
+
+  <section>
+    <h2>Example questions (dev split)</h2>
+    <p>A sample of saved questions. Only dev-split questions are shown, so the report never leaks the locked test set.</p>
+    {% for q in accepted %}
+    <div class="v2-q">
+      <div><span class="v2-tag">{{ q.type }}</span><span class="v2-tag">overlap {{ q.overlap }}</span></div>
+      <div class="q">{{ q.question }}</div>
+      <div class="a">Answer: {{ q.answer }}</div>
+      <div class="src">Fact: {{ q.fact }} · from <code>{{ q.chunks }}</code></div>
+    </div>
+    {% else %}
+    <p>No dev-split questions yet.</p>
+    {% endfor %}
+  </section>
+
+  {% if rejected_by_reason %}
+  <section>
+    <h2>Example rejections</h2>
+    <p>Up to three per reason, from the dev split only, so the report never shows text from a test chunk.</p>
+    {% for reason, rows in rejected_by_reason.items() %}
+    <details>
+      <summary><code>{{ reason }}</code> ({{ rows | length }} shown)</summary>
+      {% for q in rows %}
+      <div class="v2-q">
+        <div class="q">{{ q.question }}</div>
+        {% if q.draft and q.draft != q.question %}<div class="a">First draft: {{ q.draft }}</div>{% endif %}
+        <div class="a">Answer: {{ q.answer }} · overlap {{ q.overlap }}</div>
+        <div class="src">Fact: {{ q.fact }} · from <code>{{ q.chunks }}</code></div>
+      </div>
+      {% endfor %}
+    </details>
+    {% endfor %}
+  </section>
+  {% endif %}
+
+</div>
+{% include "_v2_tooltip.jinja" %}
+</body>
+</html>

ragaudit/report/templates/report.html.jinja

@@ -0,0 +1,252 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>ragaudit report — {{ run.config.run_id }}</title>
+<style>
+{{ style_css | safe }}
+</style>
+</head>
+<body>
+<div class="container">
+
+  <header class="report-header">
+    <h1>ragaudit report</h1>
+    <div class="meta">
+      Run <code>{{ run.config.run_id }}</code> ·
+      test set <code>{{ run.config.testset_id }}</code> ·
+      adapter <code>{{ run.config.adapter_path }}</code> ·
+      started {{ run.config.started_at }}
+    </div>
+  </header>
+
+  <section class="summary-box summary-{{ plain_summary.status }}" aria-label="Summary">
+    <h2>In plain English</h2>
+    <ul class="summary-lines">
+      {% for label, text in summary_lines %}
+      <li><strong>{{ label }}:</strong> {{ text }}</li>
+      {% endfor %}
+    </ul>
+  </section>
+
+  <section class="headline">
+    <h2>The headline finding</h2>
+    <p class="headline-sentence">
+      {% if headline.kind == "lucky_pass" %}
+      Out of {{ total_cases }} test questions, <strong>{{ headline.count }} ({{ headline.pct }}%)</strong>
+      were answered correctly even though the pipeline never retrieved the passage the answer
+      actually came from. In other words, the model got these right despite retrieval &mdash;
+      not because of it.
+      {% elif headline.kind == "retrieval_failure" %}
+      The dominant story in this run is retrieval failure:
+      <strong>{{ headline.count }} of {{ total_cases }} questions ({{ headline.pct }}%)</strong>
+      were answered wrong because the pipeline never retrieved the passage that held the answer.
+      There were no lucky passes to mask it &mdash; when retrieval missed, the answer came back
+      wrong too.
+      {% elif headline.kind == "generation_failure" %}
+      The dominant story in this run is generation failure:
+      <strong>{{ headline.count }} of {{ total_cases }} questions ({{ headline.pct }}%)</strong>
+      were answered wrong even though the pipeline retrieved the right passage &mdash; the answer
+      was right there in the context and the model still got it wrong.
+      {% else %}
+      Every correct answer in this run (<strong>{{ headline.count }} of {{ total_cases }}</strong>)
+      was backed by a real, on-target retrieval, and there were no lucky passes and no failures
+      to report.
+      {% endif %}
+    </p>
+    <div class="stats-grid">
+      <div class="stat">
+        <span class="value">{{ total_cases }}</span>
+        <span class="label">Total questions</span>
+      </div>
+      <div class="stat">
+        <span class="value">{{ (summary.retrieval_hit_rate * 100) | round(1) }}%</span>
+        <span class="label">Retrieval hit rate</span>
+      </div>
+      <div class="stat">
+        <span class="value">{{ (summary.answer_correctness_rate * 100) | round(1) }}%</span>
+        <span class="label">Answer correctness rate</span>
+      </div>
+      <div class="stat">
+        <span class="value">
+          {% if summary.mean_groundedness is not none %}{{ (summary.mean_groundedness * 100) | round(1) }}%{% else %}n/a{% endif %}
+        </span>
+        <span class="label">Mean groundedness</span>
+      </div>
+    </div>
+  </section>
+
+  <section class="matrix">
+    <h2>Verdict breakdown</h2>
+    <table class="matrix-table">
+      <thead>
+        <tr>
+          <th></th>
+          <th>Retrieved</th>
+          <th>Not retrieved</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <th>Correct</th>
+          <td class="cell-true-pass">{{ verdict_matrix.true_pass }}<span class="cell-label">True pass</span></td>
+          <td class="cell-lucky-pass">{{ verdict_matrix.lucky_pass }}<span class="cell-label">Lucky pass</span></td>
+        </tr>
+        <tr>
+          <th>Wrong</th>
+          <td class="cell-generation-failure">{{ verdict_matrix.generation_failure }}<span class="cell-label">Generation failure</span></td>
+          <td class="cell-retrieval-failure">{{ verdict_matrix.retrieval_failure }}<span class="cell-label">Retrieval failure</span></td>
+        </tr>
+      </tbody>
+    </table>
+  </section>
+
+  <section class="ablation">
+    <h2>Ablation check</h2>
+    {% if lucky_pass_count == 0 %}
+    <p>No lucky-pass candidates were found in this run, so ablation was skipped &mdash; there was
+    nothing that qualified for it.</p>
+    {% elif not run.config.run_ablation %}
+    <p>Ablation was disabled for this run, so the {{ lucky_pass_count }} lucky-pass case(s) above are
+    unconfirmed candidates.</p>
+    {% else %}
+    <p>
+      Of the {{ lucky_pass_count }} lucky-pass case(s), <strong>{{ ablation_confirmed_count }}</strong>
+      stayed correct even with retrieval turned off entirely &mdash; confirming the model didn't
+      need retrieval to answer them.
+    </p>
+    {% endif %}
+  </section>
+
+  <section class="judge-failures">
+    <h2>Judge reliability</h2>
+    <p>
+      {% if judge_failure_count > 0 %}
+      <strong>{{ judge_failure_count }}</strong> of {{ total_cases }} case(s) had a judge response
+      that could not be parsed as valid JSON. Those cases were conservatively scored as
+      incorrect / not grounded rather than silently dropped &mdash; treat their verdicts with
+      extra caution.
+      {% else %}
+      All judge responses in this run parsed successfully &mdash; 0 parsing failures.
+      {% endif %}
+    </p>
+  </section>
+
+  <section class="cases">
+    <h2>Per-case results</h2>
+    <div class="filter-bar" role="group" aria-label="Filter by verdict">
+      <button type="button" class="filter-btn active" data-filter="all">All ({{ total_cases }})</button>
+      <button type="button" class="filter-btn" data-filter="true_pass">True pass ({{ verdict_matrix.true_pass }})</button>
+      <button type="button" class="filter-btn" data-filter="lucky_pass">Lucky pass ({{ verdict_matrix.lucky_pass }})</button>
+      <button type="button" class="filter-btn" data-filter="generation_failure">Generation failure ({{ verdict_matrix.generation_failure }})</button>
+      <button type="button" class="filter-btn" data-filter="retrieval_failure">Retrieval failure ({{ verdict_matrix.retrieval_failure }})</button>
+    </div>
+    <table id="case-table">
+      <thead>
+        <tr>
+          <th data-sort="question">Question</th>
+          <th data-sort="verdict">Verdict</th>
+          <th data-sort="rank">Retrieval rank</th>
+          <th>Details</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for row in rows %}
+        <tr class="case-row" data-verdict="{{ row.verdict }}" data-rank="{{ row.retrieval_rank if row.retrieval_rank is not none else -1 }}">
+          <td class="case-question">{{ row.question }}</td>
+          <td class="case-verdict"><span class="badge badge-{{ row.verdict }}">{{ row.verdict_label }}</span></td>
+          <td class="case-rank" data-label="Retrieval rank">{% if row.retrieval_rank is not none %}{{ row.retrieval_rank }}{% else %}not retrieved{% endif %}</td>
+          <td class="case-details">
+            <details>
+              <summary>Show</summary>
+              <dl>
+                <dt>Expected answer</dt>
+                <dd>{{ row.expected_answer }}</dd>
+                <dt>Model answer</dt>
+                <dd>{{ row.answer }}</dd>
+                <dt>Retrieved contexts</dt>
+                <dd class="contexts">
+                  {% if row.contexts %}
+                  <ol>
+                    {% for ctx in row.contexts %}
+                    <li>{{ ctx }}</li>
+                    {% endfor %}
+                  </ol>
+                  {% else %}
+                  <em>none</em>
+                  {% endif %}
+                </dd>
+                {% if row.judge_parse_failed %}
+                <dt>Judge warning</dt>
+                <dd class="warning">A judge response for this case could not be parsed; it was conservatively scored as incorrect / not grounded.</dd>
+                {% endif %}
+                {% if row.ablation %}
+                <dt>Ablation (no-context) answer</dt>
+                <dd>{{ row.ablation.answer }}</dd>
+                <dt>Ablation confirms lucky pass</dt>
+                <dd>{{ "Yes" if row.ablation.confirms_lucky_pass else "No" }}</dd>
+                {% endif %}
+              </dl>
+            </details>
+          </td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </section>
+
+</div>
+
+<script>
+(function () {
+  "use strict";
+
+  var filterButtons = Array.prototype.slice.call(document.querySelectorAll(".filter-btn"));
+  var rows = Array.prototype.slice.call(document.querySelectorAll("#case-table tbody tr"));
+
+  filterButtons.forEach(function (btn) {
+    btn.addEventListener("click", function () {
+      filterButtons.forEach(function (b) { b.classList.remove("active"); });
+      btn.classList.add("active");
+      var filter = btn.getAttribute("data-filter");
+      rows.forEach(function (row) {
+        var show = filter === "all" || row.getAttribute("data-verdict") === filter;
+        row.style.display = show ? "" : "none";
+      });
+    });
+  });
+
+  var sortState = {};
+  Array.prototype.slice.call(document.querySelectorAll("#case-table th[data-sort]")).forEach(function (th) {
+    th.addEventListener("click", function () {
+      var key = th.getAttribute("data-sort");
+      var ascending = !sortState[key];
+      sortState = {};
+      sortState[key] = ascending;
+
+      var tbody = document.querySelector("#case-table tbody");
+      var sorted = rows.slice().sort(function (a, b) {
+        var av, bv;
+        if (key === "rank") {
+          av = parseFloat(a.getAttribute("data-rank"));
+          bv = parseFloat(b.getAttribute("data-rank"));
+        } else if (key === "verdict") {
+          av = a.getAttribute("data-verdict");
+          bv = b.getAttribute("data-verdict");
+        } else {
+          av = a.cells[0].textContent.trim().toLowerCase();
+          bv = b.cells[0].textContent.trim().toLowerCase();
+        }
+        if (av < bv) { return ascending ? -1 : 1; }
+        if (av > bv) { return ascending ? 1 : -1; }
+        return 0;
+      });
+      sorted.forEach(function (row) { tbody.appendChild(row); });
+    });
+  });
+})();
+</script>
+
+</body>
+</html>