ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/report/templates/tune_report.html.jinja

@@ -0,0 +1,158 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="dark">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>ragaudit tune report — {{ result.tune_id }}</title>
+<style>
+{{ style_css | safe }}
+{{ v2_css | safe }}
+</style>
+</head>
+<body>
+<div class="container">
+
+  <header class="report-header">
+    <h1>ragaudit tune report{% if is_mock %}<span class="v2-badge-mock">built with the mock LLM</span>{% endif %}</h1>
+    <div class="meta">
+      Tune <code>{{ result.tune_id }}</code> ·
+      strategy <code>{{ result.strategy }}</code> ·
+      adapter <code>{{ result.adapter_path }}</code> ·
+      judge <code>{{ result.judge_model }}</code> ·
+      k={{ result.k }} ·
+      index <code>{{ result.index_version[:12] }}</code> ·
+      started {{ result.created_at }}
+    </div>
+  </header>
+
+  <section class="headline">
+    <h2>The headline finding</h2>
+    <p class="headline-sentence">
+      {% if best.index == baseline.index %}
+      No config beat the baseline on the dev set in {{ result.trials | length }} trial(s), so the baseline stays.
+      Its score on the locked test set is <strong>{{ "%.1f" | format(result.baseline_test.score) }}</strong>.
+      {% else %}
+      Tuning moved the honest score, measured once on the locked test set, from
+      <strong>{{ "%.1f" | format(result.baseline_test.score) }}</strong> to <strong>{{ "%.1f" | format(result.best_test.score) }}</strong>
+      ({{ "%+.1f" | format(improvement) }} points) after {{ result.trials | length }} trial(s).
+      The search only ever saw the dev set, where the same change was {{ "%+.1f" | format(dev_improvement) }} points.
+      {% endif %}
+    </p>
+    {% if is_mock %}
+    <div class="v2-callout warn">Built with ragaudit's offline mock LLM as both judge{% if result.strategy == "llm" %} and proposer{% endif %}. The numbers show how the loop behaves, not how good any real pipeline is.</div>
+    {% endif %}
+    <div class="stats-grid">
+      <div class="stat"><span class="value">{{ "%.1f" | format(result.baseline_test.score) }}</span><span class="label">Baseline, test</span></div>
+      <div class="stat"><span class="value">{{ "%.1f" | format(result.best_test.score) }}</span><span class="label">Best, test</span></div>
+      <div class="stat"><span class="value">{{ "%.1f" | format(best.dev.score) }}</span><span class="label">Best, dev</span></div>
+      <div class="stat"><span class="value">{{ "%+.1f" | format(result.overfit_gap) }}</span><span class="label">Dev minus test (best)</span></div>
+    </div>
+    <p class="meta">Score = share of questions where the gold passage was in the top {{ result.k }} <em>and</em> the answer was judged correct, ×100.
+      Dev: {{ best.dev.n_questions }} questions. Test: {{ result.best_test.n_questions }} questions.</p>
+  </section>
+
+  <section>
+    <h2>Overfitting check</h2>
+    {% if result.overfit_warning %}
+    <div class="v2-callout warn">
+      <strong>Likely overfit to the dev set.</strong> The best config scores {{ "%.1f" | format(best.dev.score) }} on dev but
+      {{ "%.1f" | format(result.best_test.score) }} on the locked test set, a gap of {{ "%.1f" | format(result.overfit_gap) }} points,
+      more than the {{ "%.1f" | format(result.overfit_threshold) }}-point threshold. Treat the dev score as optimistic,
+      and consider generating more questions before trusting this config.
+    </div>
+    {% else %}
+    <div class="v2-callout ok">
+      <strong>No sign of overfitting.</strong> The best config's dev score ({{ "%.1f" | format(best.dev.score) }}) is within
+      {{ "%.1f" | format(result.overfit_threshold) }} points of its locked test score ({{ "%.1f" | format(result.best_test.score) }});
+      the gap is {{ "%+.1f" | format(result.overfit_gap) }}.
+    </div>
+    {% endif %}
+    <figure class="v2-figure">{{ dev_test_chart | safe }}</figure>
+    <ul class="v2-legend">
+      <li><span class="v2-swatch" style="background: var(--v2-dev)"></span>Dev (searched on)</li>
+      <li><span class="v2-swatch" style="background: var(--v2-test)"></span>Locked test (scored once at the end)</li>
+    </ul>
+  </section>
+
+  <section>
+    <h2>Dev score per trial</h2>
+    <p>Each dot is one config evaluated on the dev set. The dashed line is the best dev score so far; the white dot is the config that was kept.
+      Trials by source: {% for source, count in sources.items() %}{{ count }} {{ source }}{% if not loop.last %}, {% endif %}{% endfor %}.
+      Stopped because: <code>{{ result.stopped_reason }}</code>. LLM calls used: {{ result.llm_calls }}.</p>
+    <figure class="v2-figure">{{ trial_chart | safe }}</figure>
+    <ul class="v2-legend">
+      <li><span class="v2-swatch dot" style="background: var(--v2-dev)"></span>Trial dev score</li>
+      <li><span class="v2-swatch dot" style="background: var(--text)"></span>Best (kept)</li>
+      <li><span class="v2-swatch line"></span>Best so far</li>
+    </ul>
+    <details class="v2-data">
+      <summary>Show every trial as a table</summary>
+      <div class="v2-table-wrap">
+        <table class="v2-table">
+          <thead><tr><th class="num">#</th><th>Source</th><th class="num">Dev score</th><th class="num">Pass</th><th class="num">Gen miss</th><th class="num">Ret miss</th><th>Changed from baseline</th><th>Rationale</th></tr></thead>
+          <tbody>
+            {% for t in trial_rows %}
+            <tr{% if t.is_best %} class="is-best"{% endif %}>
+              <td class="num">{{ t.index }}</td>
+              <td>{{ t.source }}{% if t.is_best %} (best){% endif %}</td>
+              <td class="num">{{ "%.1f" | format(t.score) }}</td>
+              <td class="num">{{ t.counts.get("pass", 0) }}</td>
+              <td class="num">{{ t.counts.get("generation_miss", 0) }}</td>
+              <td class="num">{{ t.counts.get("retrieval_miss", 0) }}</td>
+              <td><code>{{ t.changed }}</code></td>
+              <td>{{ t.rationale }}</td>
+            </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+      </div>
+    </details>
+  </section>
+
+  <section>
+    <h2>Failure classes before and after</h2>
+    <p><strong>Retrieval miss</strong>: the gold passage wasn't in the top {{ result.k }} (a correct answer without it counts here too, since it was luck).
+      <strong>Generation miss</strong>: the passage was retrieved but the answer was still wrong.</p>
+    <figure class="v2-figure">{{ failure_chart | safe }}</figure>
+    <ul class="v2-legend">
+      {% for key, name, cls in failure_classes %}
+      <li><span class="v2-swatch" style="background: var(--{{ {'pass': 'true-pass', 'generation_miss': 'generation-failure', 'retrieval_miss': 'retrieval-failure'}[key] }})"></span>{{ name }}</li>
+      {% endfor %}
+    </ul>
+    <details class="v2-data">
+      <summary>Show as a table</summary>
+      <table class="v2-table">
+        <thead><tr><th>Config, split</th>{% for key, name, cls in failure_classes %}<th class="num">{{ name }}</th>{% endfor %}</tr></thead>
+        <tbody>
+          {% for label, counts in stack_rows %}
+          <tr><td>{{ label }}</td>{% for key, name, cls in failure_classes %}<td class="num">{{ counts.get(key, 0) }}</td>{% endfor %}</tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </details>
+  </section>
+
+  <section>
+    <h2>Best config</h2>
+    {% if result.best_config_path %}<p>Written to <code>{{ result.best_config_path }}</code>.</p>{% endif %}
+    <div class="v2-table-wrap">
+      <table class="v2-table">
+        <thead><tr><th>Knob</th><th>Baseline</th><th>Best</th><th>Searched over</th></tr></thead>
+        <tbody>
+          {% for k in knobs %}
+          <tr>
+            <td><code>{{ k.knob }}</code></td>
+            <td>{{ k.baseline }}</td>
+            <td class="{{ 'v2-changed' if k.changed else 'v2-unchanged' }}">{{ k.best }}{% if k.changed %} (changed){% endif %}</td>
+            <td class="v2-unchanged">{{ result.search_space[k.knob] | join(", ") }}</td>
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
+  </section>
+
+</div>
+{% include "_v2_tooltip.jinja" %}
+</body>
+</html>

ragaudit/report/templates/v2.css

@@ -0,0 +1,286 @@
+/* v2-only additions for the generate and tune reports. Loaded after
+   style.css, which stays the single source for the shared look. */
+
+:root {
+  --v2-dev: #3987e5;
+  --v2-test: #d95926;
+  --v2-muted-mark: #4a5160;
+  --v2-grid: #262b35;
+}
+
+.v2-badge-mock {
+  display: inline-block;
+  margin-left: 0.5rem;
+  padding: 0.1rem 0.5rem;
+  border: 1px solid var(--lucky-pass);
+  border-radius: 999px;
+  color: var(--lucky-pass);
+  font-size: 0.75rem;
+  vertical-align: middle;
+}
+
+.v2-callout {
+  margin: 1rem 0;
+  padding: 0.75rem 1rem;
+  border-left: 3px solid var(--accent);
+  background: var(--surface-raised);
+  border-radius: 4px;
+}
+
+.v2-callout.warn {
+  border-left-color: var(--lucky-pass);
+}
+
+.v2-callout.ok {
+  border-left-color: var(--true-pass);
+}
+
+.v2-figure {
+  position: relative;
+  margin: 1rem 0 0.5rem;
+}
+
+.v2-chart {
+  display: block;
+  width: 100%;
+  height: auto;
+  overflow: visible;
+}
+
+.v2-grid {
+  stroke: var(--v2-grid);
+  stroke-width: 1;
+}
+
+.v2-tick,
+.v2-axis-title {
+  fill: var(--text-muted);
+  font-size: 12px;
+}
+
+.v2-label {
+  fill: var(--text);
+  font-size: 13px;
+}
+
+.v2-value {
+  fill: var(--text-muted);
+  font-size: 12px;
+  font-variant-numeric: tabular-nums;
+}
+
+.v2-inside {
+  fill: #0f1115;
+  font-size: 12px;
+  font-weight: 600;
+  pointer-events: none;
+}
+
+.v2-rule {
+  stroke: var(--text);
+  stroke-width: 1.5;
+  stroke-dasharray: 4 3;
+}
+
+.v2-mark {
+  outline: none;
+}
+
+.v2-mark:hover,
+.v2-mark:focus-visible {
+  filter: brightness(1.2);
+}
+
+.v2-fill-accent { fill: var(--accent); }
+.v2-fill-muted { fill: var(--v2-muted-mark); }
+.v2-fill-dev { fill: var(--v2-dev); }
+.v2-fill-test { fill: var(--v2-test); }
+.v2-fill-pass { fill: var(--true-pass); }
+.v2-fill-gen { fill: var(--generation-failure); }
+.v2-fill-ret { fill: var(--retrieval-failure); }
+
+.v2-dot {
+  fill: var(--v2-dev);
+  stroke: var(--surface);
+  stroke-width: 2;
+  pointer-events: none;
+}
+
+.v2-dot-best {
+  fill: var(--text);
+}
+
+.v2-hit {
+  fill: transparent;
+  outline: none;
+  cursor: default;
+}
+
+.v2-hit:focus-visible {
+  stroke: var(--text);
+  stroke-width: 1;
+}
+
+.v2-line-ref {
+  fill: none;
+  stroke: var(--text-muted);
+  stroke-width: 2;
+  stroke-dasharray: 2 4;
+}
+
+.v2-legend {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.25rem 1.25rem;
+  margin: 0.25rem 0 0;
+  padding: 0;
+  list-style: none;
+  font-size: 0.85rem;
+  color: var(--text-muted);
+}
+
+.v2-legend li {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.4rem;
+}
+
+.v2-swatch {
+  display: inline-block;
+  width: 12px;
+  height: 12px;
+  border-radius: 3px;
+}
+
+.v2-swatch.line {
+  width: 18px;
+  height: 0;
+  border-top: 2px dashed var(--text-muted);
+  border-radius: 0;
+}
+
+.v2-swatch.dot {
+  border-radius: 50%;
+}
+
+.v2-tooltip {
+  position: absolute;
+  z-index: 10;
+  max-width: 280px;
+  padding: 0.45rem 0.6rem;
+  background: var(--surface-raised);
+  border: 1px solid var(--border);
+  border-radius: 6px;
+  color: var(--text-muted);
+  font-size: 0.8rem;
+  line-height: 1.35;
+  pointer-events: none;
+  white-space: pre-line;
+  box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4);
+}
+
+.v2-tooltip strong {
+  display: block;
+  color: var(--text);
+  font-size: 0.95rem;
+}
+
+.v2-table-wrap {
+  overflow-x: auto;
+}
+
+table.v2-table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 0.9rem;
+}
+
+table.v2-table th,
+table.v2-table td {
+  padding: 0.45rem 0.6rem;
+  border-bottom: 1px solid var(--border);
+  text-align: left;
+  vertical-align: top;
+}
+
+table.v2-table th {
+  color: var(--text-muted);
+  font-weight: 600;
+}
+
+table.v2-table td.num,
+table.v2-table th.num {
+  text-align: right;
+  font-variant-numeric: tabular-nums;
+}
+
+table.v2-table tr.is-best td {
+  background: var(--surface-raised);
+}
+
+.v2-changed {
+  color: var(--text);
+  font-weight: 600;
+}
+
+.v2-unchanged {
+  color: var(--text-muted);
+}
+
+details.v2-data {
+  margin-top: 0.5rem;
+}
+
+details.v2-data summary {
+  color: var(--text-muted);
+  font-size: 0.85rem;
+}
+
+.v2-q {
+  margin: 0;
+  padding: 0.75rem 0;
+  border-bottom: 1px solid var(--border);
+}
+
+.v2-q:last-child {
+  border-bottom: none;
+}
+
+.v2-q .q {
+  color: var(--text);
+  font-weight: 600;
+}
+
+.v2-q .a,
+.v2-q .src {
+  color: var(--text-muted);
+  font-size: 0.875rem;
+}
+
+.v2-tag {
+  display: inline-block;
+  margin-right: 0.4rem;
+  padding: 0 0.4rem;
+  border: 1px solid var(--border);
+  border-radius: 4px;
+  color: var(--text-muted);
+  font-size: 0.75rem;
+}
+
+.meta code {
+  overflow-wrap: anywhere;
+}
+
+@media (max-width: 600px) {
+  .container {
+    padding: 1.25rem 1rem 3rem;
+  }
+
+  section {
+    padding: 1rem;
+  }
+
+  .stats-grid {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
+}