ragarena 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ragarena/__init__.py ADDED
@@ -0,0 +1,60 @@
1
+ """
2
+ ⚡ RagArena — evaluate & benchmark RAG strategies, LLMs and embedding models
3
+ across every popular provider with one unified API.
4
+
5
+ Quickstart::
6
+
7
+ from ragarena import evaluate, compare
8
+
9
+ report = evaluate(
10
+ questions=["What is RAG?"],
11
+ documents=[{"text": "RAG = retrieval-augmented generation..."}],
12
+ strategy="hybrid", # 13 strategies available
13
+ model="openai/gpt-4o-mini", # any provider/model
14
+ embedding_model="voyage/voyage-3",
15
+ metrics="quality",
16
+ )
17
+ report.print_summary()
18
+ """
19
+ from .catalog import (
20
+ ModelInfo, ProviderConfig,
21
+ CHAT_MODELS, EMBEDDING_MODELS, RERANK_MODELS, VECTOR_STORES,
22
+ PROVIDERS, EMBEDDING_PROVIDERS,
23
+ get_model, has_model, list_models, list_providers, estimate_cost,
24
+ )
25
+ from .router import completion, embedding, rerank, ModelResponse, EmbeddingResponse, Usage
26
+ from .strategies import (
27
+ Strategy, Chunk, StrategyResult, STRATEGIES, get_strategy,
28
+ NaiveRAG, HybridRAG, MultiQueryRAG, HyDERAG, RerankRAG, RAGFusion,
29
+ ContextualCompressionRAG, CRAGRAG, SelfRAG, QueryDecompositionRAG,
30
+ StepBackRAG, AgenticRAG, FLARERAG,
31
+ GraphLocalRAG, GraphGlobalRAG, GraphHybridRAG, GraphMixRAG, MultimodalRAG,
32
+ )
33
+ from .graph import GraphIndex
34
+ from .metrics import METRICS, DEFAULT_METRIC_SETS, MetricResult
35
+ from .index import VectorIndex, TextChunker, chunk_text, MultimodalDocument
36
+ from .engine import evaluate, compare, EvaluationReport, ComparisonResult
37
+
38
+ __version__ = "0.2.0"
39
+ __author__ = "RagArena contributors"
40
+
41
+ __all__ = [
42
+ # one-liner APIs
43
+ "completion", "embedding", "rerank", "evaluate", "compare",
44
+ # catalog
45
+ "list_models", "list_providers", "get_model", "has_model", "estimate_cost",
46
+ "CHAT_MODELS", "EMBEDDING_MODELS", "RERANK_MODELS", "VECTOR_STORES",
47
+ "PROVIDERS", "EMBEDDING_PROVIDERS", "ModelInfo",
48
+ # strategies
49
+ "STRATEGIES", "get_strategy", "Strategy", "Chunk", "StrategyResult",
50
+ "NaiveRAG", "HybridRAG", "MultiQueryRAG", "HyDERAG", "RerankRAG", "RAGFusion",
51
+ "ContextualCompressionRAG", "CRAGRAG", "SelfRAG", "QueryDecompositionRAG",
52
+ "StepBackRAG", "AgenticRAG", "FLARERAG",
53
+ "GraphLocalRAG", "GraphGlobalRAG", "GraphHybridRAG", "GraphMixRAG", "MultimodalRAG",
54
+ # graph / multimodal
55
+ "GraphIndex", "MultimodalDocument",
56
+ # metrics / index / engine
57
+ "METRICS", "DEFAULT_METRIC_SETS", "MetricResult",
58
+ "VectorIndex", "TextChunker", "chunk_text",
59
+ "EvaluationReport", "ComparisonResult",
60
+ ]
@@ -0,0 +1,4 @@
1
+ """FastAPI serving layer for RagArena — dashboard + REST API."""
2
+ from .server import app, start_server
3
+
4
+ __all__ = ["app", "start_server"]
@@ -0,0 +1,467 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>RagArena · RAG Strategy & Model Benchmarking</title>
7
+ <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
8
+ <style>
9
+ :root{
10
+ --bg:#0b0e14;--panel:#11151f;--panel2:#161b28;--border:#1f2635;
11
+ --text:#e6e9f0;--muted:#8a93a6;--accent:#6c5ce7;--accent2:#00d2ff;
12
+ --good:#2ecc71;--bad:#ff4757;--warn:#ffa502;--radius:12px;
13
+ }
14
+ *{margin:0;padding:0;box-sizing:border-box}
15
+ body{font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;
16
+ background:var(--bg);color:var(--text);font-size:14px}
17
+ .layout{display:flex;height:100vh}
18
+
19
+ /* ── sidebar ── */
20
+ .sidebar{width:220px;background:var(--panel);border-right:1px solid var(--border);
21
+ padding:20px 0;display:flex;flex-direction:column}
22
+ .logo{padding:0 20px 24px;font-size:19px;font-weight:800;letter-spacing:-.5px}
23
+ .logo span{background:linear-gradient(90deg,var(--accent),var(--accent2));
24
+ -webkit-background-clip:text;background-clip:text;color:transparent}
25
+ .nav{flex:1}
26
+ .nav button{display:flex;width:100%;gap:10px;align-items:center;padding:11px 22px;
27
+ background:none;border:none;color:var(--muted);cursor:pointer;font-size:13.5px;text-align:left}
28
+ .nav button:hover{color:var(--text);background:var(--panel2)}
29
+ .nav button.active{color:var(--text);background:var(--panel2);
30
+ border-left:3px solid var(--accent);padding-left:19px}
31
+ .footer-note{padding:16px 20px;color:var(--muted);font-size:11px;border-top:1px solid var(--border)}
32
+
33
+ /* ── main ── */
34
+ main{flex:1;overflow-y:auto;padding:28px 34px}
35
+ .page{display:none}.page.active{display:block}
36
+ h1{font-size:22px;margin-bottom:4px}
37
+ .sub{color:var(--muted);margin-bottom:24px}
38
+
39
+ /* cards */
40
+ .stat-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:14px;margin-bottom:22px}
41
+ .card{background:var(--panel);border:1px solid var(--border);border-radius:var(--radius);padding:18px}
42
+ .stat .v{font-size:26px;font-weight:700;margin-top:6px}
43
+ .stat .l{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.6px}
44
+ .row2{display:grid;grid-template-columns:1fr 1fr;gap:14px}
45
+
46
+ /* forms */
47
+ .form-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:14px}
48
+ .field label{display:block;color:var(--muted);font-size:12px;margin-bottom:6px}
49
+ input,select,textarea{width:100%;background:var(--panel2);border:1px solid var(--border);
50
+ border-radius:8px;padding:10px 12px;color:var(--text);font-size:13px;outline:none;font-family:inherit}
51
+ input:focus,select:focus,textarea:focus{border-color:var(--accent)}
52
+ textarea{resize:vertical;min-height:90px}
53
+ .btn{background:linear-gradient(90deg,var(--accent),#8b7cf7);color:#fff;border:none;
54
+ padding:11px 22px;border-radius:8px;font-weight:600;cursor:pointer;font-size:13.5px}
55
+ .btn:hover{opacity:.92}
56
+ .btn.ghost{background:var(--panel2);border:1px solid var(--border);color:var(--text)}
57
+ .badge{display:inline-block;padding:2px 9px;border-radius:99px;font-size:10.5px;font-weight:600}
58
+ .badge.b-openai{background:#10a37f22;color:#10a37f}
59
+ .badge.b-anthropic{background:#d9775722;color:#d97757}
60
+ .badge.b-google{background:#4285f422;color:#66a3ff}
61
+ .badge.b-groq{background:#f5503622;color:#ff7857}
62
+ .badge.b-mistral{background:#fa520f22;color:#ff7a3d}
63
+ .badge.b-cohere{background:#39594d22;color:#63d0a5}
64
+ .badge.b-voyage{background:#7f5af022;color:#9d85f5}
65
+ .badge.b-jina{background:#00b86b22;color:#00d68f}
66
+ .badge.b-huggingface,.badge.b-ollama{background:#ffb70322;color:#ffc94d}
67
+ .badge.b-deepseek{background:#4d6bfe22;color:#7590ff}
68
+ .badge.b-xai{background:#88888833;color:#ccc}
69
+ .badge.b-together,.badge.b-fireworks,.badge.b-perplexity,.badge.b-bedrock,
70
+ .badge.b-vertex,.badge.b-deepinfra,.badge.b-nvidia_nim{background:#6c5ce722;color:#a29bfe}
71
+
72
+ /* tables */
73
+ table{width:100%;border-collapse:collapse}
74
+ th,td{text-align:left;padding:11px 14px;border-bottom:1px solid var(--border)}
75
+ th{color:var(--muted);font-size:11.5px;text-transform:uppercase;letter-spacing:.5px;font-weight:600}
76
+ tr:hover td{background:var(--panel2)}
77
+ .num{font-variant-numeric:tabular-nums}
78
+ .good{color:var(--good)}.bad{color:var(--bad)}.warn{color:var(--warn)}
79
+
80
+ /* misc */
81
+ .chip-row{display:flex;flex-wrap:wrap;gap:8px;margin-top:8px}
82
+ .chip{background:var(--panel2);border:1px solid var(--border);border-radius:99px;
83
+ padding:5px 12px;font-size:12px;cursor:pointer}
84
+ .chip.sel{border-color:var(--accent);background:#6c5ce725}
85
+ .spinner{display:inline-block;width:15px;height:15px;border:2px solid #ffffff44;
86
+ border-top-color:#fff;border-radius:50%;animation:spin .7s linear infinite;vertical-align:-3px;margin-right:8px}
87
+ @keyframes spin{to{transform:rotate(360deg)}}
88
+ .hidden{display:none!important}
89
+ .searchbar{max-width:340px;margin-bottom:14px}
90
+ .mono{font-family:'SF Mono',Consolas,monospace;font-size:12px}
91
+ .answer-box{background:var(--panel2);border-radius:10px;padding:14px;line-height:1.55;margin-top:8px}
92
+ .kv{display:flex;gap:6px;align-items:center}
93
+ </style>
94
+ </head>
95
+ <body>
96
+ <div class="layout">
97
+
98
+ <div class="sidebar">
99
+ <div class="logo">⚡ RAG<span>Eval</span></div>
100
+ <div class="nav" id="nav">
101
+ <button data-page="dashboard" class="active">📊 Overview</button>
102
+ <button data-page="run">▶ New Evaluation</button>
103
+ <button data-page="compare">⚔️ Compare</button>
104
+ <button data-page="runs">🗂 Runs</button>
105
+ <button data-page="catalog">🧠 Model Catalog</button>
106
+ </div>
107
+ <div class="footer-note">13 strategies · any LLM · any embedding<br>unified provider/model routing</div>
108
+ </div>
109
+
110
+ <main>
111
+
112
+ <!-- ══════════ OVERVIEW ══════════ -->
113
+ <section class="page active" id="page-dashboard">
114
+ <h1>Overview</h1>
115
+ <p class="sub">Your RAG evaluation workspace</p>
116
+ <div class="stat-grid">
117
+ <div class="card stat"><div class="l">Runs stored</div><div class="v" id="st-runs">–</div></div>
118
+ <div class="card stat"><div class="l">Chat models</div><div class="v" id="st-chat">–</div></div>
119
+ <div class="card stat"><div class="l">Embedding models</div><div class="v" id="st-emb">–</div></div>
120
+ <div class="card stat"><div class="l">Strategies</div><div class="v" id="st-strat">–</div></div>
121
+ </div>
122
+ <div class="row2">
123
+ <div class="card"><h3 style="margin-bottom:14px">Recent runs</h3>
124
+ <table id="tbl-recent"><thead><tr><th>Run</th><th>Kind</th><th>Config</th></tr></thead><tbody></tbody></table>
125
+ </div>
126
+ <div class="card"><h3 style="margin-bottom:14px">Strategies available</h3>
127
+ <div class="chip-row" id="strat-chips"></div>
128
+ <h3 style="margin:18px 0 14px">Metric presets</h3>
129
+ <div class="chip-row" id="metric-chips"></div>
130
+ </div>
131
+ </div>
132
+ </section>
133
+
134
+ <!-- ══════════ NEW RUN ══════════ -->
135
+ <section class="page" id="page-run">
136
+ <h1>New evaluation</h1>
137
+ <p class="sub">Pick a strategy + models, paste your corpus & questions, get scored.</p>
138
+
139
+ <div class="form-grid card">
140
+ <div class="field"><label>Generator model (provider/name)</label>
141
+ <select id="r-model"></select></div>
142
+ <div class="field"><label>Embedding model</label>
143
+ <select id="r-embed"></select></div>
144
+ <div class="field"><label>Judge model (LLM-as-judge)</label>
145
+ <select id="r-judge"></select></div>
146
+ <div class="field"><label>Metric preset</label>
147
+ <select id="r-metrics">
148
+ <option value="production">production — faithfulness, relevance, latency, cost</option>
149
+ <option value="quality">quality — full retrieval+generation quality</option>
150
+ <option value="quick">quick — hit-rate + latency only</option>
151
+ <option value="full">full — everything incl. correctness</option>
152
+ </select></div>
153
+ </div>
154
+
155
+ <div class="card" style="margin-top:14px">
156
+ <label style="color:var(--muted);font-size:12px">Strategy</label>
157
+ <div class="chip-row" id="r-strategy"></div>
158
+ <div style="margin-top:10px" class="sub mono" id="r-strat-desc"></div>
159
+ </div>
160
+
161
+ <div class="row2" style="margin-top:14px">
162
+ <div class="field"><label>Documents (one per line)</label>
163
+ <textarea id="r-docs" rows="8" placeholder="Paris is the capital of France.&#10;The Eiffel Tower was built in 1889.&#10;..."></textarea></div>
164
+ <div class="field"><label>Questions (one per line) — prefix with 'Q::' to add reference answer</label>
165
+ <textarea id="r-questions" rows="8" placeholder="What is the capital of France?::Paris&#10;When was the Eiffel Tower built?::1889"></textarea></div>
166
+ </div>
167
+
168
+ <div style="margin-top:16px;display:flex;gap:10px;align-items:center">
169
+ <button class="btn" onclick="submitRun()"><span id="run-btn-txt">▶ Run evaluation</span></button>
170
+ <span class="sub" id="run-status"></span>
171
+ </div>
172
+
173
+ <div id="run-results" class="hidden" style="margin-top:22px">
174
+ <h3 style="margin-bottom:12px">Results</h3>
175
+ <div class="stat-grid" id="rr-stats"></div>
176
+ <div class="card"><table id="rr-table"><thead></thead><tbody></tbody></table></div>
177
+ </div>
178
+ </section>
179
+
180
+ <!-- ══════════ COMPARE ══════════ -->
181
+ <section class="page" id="page-compare">
182
+ <h1>Compare strategies × models</h1>
183
+ <p class="sub">Head-to-head leaderboard across multiple configurations on the same corpus.</p>
184
+
185
+ <div class="row2">
186
+ <div class="field"><label>Documents (one per line)</label>
187
+ <textarea id="c-docs" rows="7"></textarea></div>
188
+ <div class="field"><label>Questions (one per line, '::ref' optional)</label>
189
+ <textarea id="c-questions" rows="7"></textarea></div>
190
+ </div>
191
+
192
+ <div class="card" style="margin-top:14px">
193
+ <label style="color:var(--muted);font-size:12px">Configurations to compare</label>
194
+ <div id="c-configs" style="display:flex;flex-direction:column;gap:10px;margin-top:8px"></div>
195
+ <button class="btn ghost" style="margin-top:12px" onclick="addConfig()">+ Add config</button>
196
+ </div>
197
+
198
+ <div style="margin-top:16px;display:flex;gap:10px;align-items:center">
199
+ <button class="btn" onclick="submitCompare()">⚔️ Run comparison</button>
200
+ <span class="sub" id="cmp-status"></span>
201
+ </div>
202
+
203
+ <div id="cmp-results" class="hidden" style="margin-top:22px">
204
+ <h3 style="margin-bottom:12px">Leaderboard</h3>
205
+ <div class="card"><table id="cmp-table"><thead></thead><tbody></tbody></table></div>
206
+ <div class="row2" style="margin-top:14px">
207
+ <div class="card"><canvas id="cmp-chart-quality" height="230"></canvas></div>
208
+ <div class="card"><canvas id="cmp-chart-cost" height="230"></canvas></div>
209
+ </div>
210
+ </div>
211
+ </section>
212
+
213
+ <!-- ══════════ RUNS ══════════ -->
214
+ <section class="page" id="page-runs">
215
+ <h1>Evaluation runs</h1>
216
+ <p class="sub">Every run executed through this server</p>
217
+ <div class="card"><table id="tbl-runs"><thead>
218
+ <tr><th>ID</th><th>Kind</th><th>Config</th><th>Faithfulness</th><th>Relevance</th><th>Cost</th><th></th></tr>
219
+ </thead><tbody></tbody></table></div>
220
+ <div id="run-detail" class="hidden" style="margin-top:18px">
221
+ <div class="card"><h3 id="rd-title"></h3><div id="rd-body" style="margin-top:12px"></div></div>
222
+ </div>
223
+ </section>
224
+
225
+ <!-- ══════════ CATALOG ══════════ -->
226
+ <section class="page" id="page-catalog">
227
+ <h1>Model catalog</h1>
228
+ <p class="sub" id="cat-counts"></p>
229
+ <input class="searchbar" id="cat-search" placeholder="Search models…"
230
+ oninput="renderCatalog()">
231
+ <div class="chip-row" id="cat-filters" style="margin-bottom:14px"></div>
232
+ <div class="card"><table id="tbl-catalog"><thead>
233
+ <tr><th>Model ID</th><th>Type</th><th>Context</th><th>$ In/1M</th><th>$ Out/1M</th><th>Notes</th></tr>
234
+ </thead><tbody></tbody></table></div>
235
+ </section>
236
+
237
+ </main>
238
+ </div>
239
+
240
+ <script>
241
+ const $=id=>document.getElementById(id);
242
+ let CATALOG=[], STRATEGIES=[], METRICS_INFO={};
243
+ let catFilter='all';
244
+
245
+ /* ── nav ── */
246
+ $('nav').addEventListener('click',e=>{
247
+ const b=e.target.closest('button'); if(!b)return;
248
+ document.querySelectorAll('#nav button').forEach(x=>x.classList.remove('active'));
249
+ b.classList.add('active');
250
+ document.querySelectorAll('.page').forEach(p=>p.classList.remove('active'));
251
+ $('page-'+b.dataset.page).classList.add('active');
252
+ });
253
+
254
+ async function api(path){const r=await fetch(path);if(!r.ok)throw await r.text();return r.json();}
255
+
256
+ /* ── init ── */
257
+ (async function init(){
258
+ const [cat,strats,mets]=await Promise.all([
259
+ api('/api/catalog'),api('/api/strategies'),api('/api/metrics')]);
260
+ CATALOG=cat.models; STRATEGIES=strats; METRICS_INFO=mets;
261
+
262
+ $('st-chat').textContent=cat.counts.chat;
263
+ $('st-emb').textContent=cat.counts.embedding;
264
+ $('st-strat').textContent=strats.length;
265
+
266
+ // model selects
267
+ const chat=CATALOG.filter(m=>m.modality==='chat');
268
+ const emb=CATALOG.filter(m=>m.modality==='embedding');
269
+ fillSelect($('r-model'),chat,'openai/gpt-4o-mini');
270
+ fillSelect($('r-judge'),chat,'openai/gpt-4o-mini');
271
+ fillSelect($('r-emb'),emb,'openai/text-embedding-3-small');
272
+
273
+ // strategy chips
274
+ $('r-strategy').innerHTML=strats.map((s,i)=>
275
+ `<span class="chip ${i===0?'sel':''}" data-name="${s.name}">${s.name}</span>`).join('');
276
+ $('strat-chips').innerHTML=strats.map(s=>`<span class="chip">${s.name}</span>`).join('');
277
+ $('metric-chips').innerHTML=Object.keys(mets.presets).map(p=>`<span class="chip">${p}</span>`).join('');
278
+ $('r-strat-desc').textContent=strats[0].description;
279
+ $('r-strategy').addEventListener('click',e=>{
280
+ if(!e.target.classList.contains('chip'))return;
281
+ document.querySelectorAll('#r-strategy .chip').forEach(c=>c.classList.remove('sel'));
282
+ e.target.classList.add('sel');
283
+ $('r-strat-desc').textContent=strats.find(s=>s.name===e.target.dataset.name)?.description||'';
284
+ });
285
+
286
+ renderRecent(); renderCatalog();
287
+ })();
288
+
289
+ function fillSelect(sel,models,defaultVal){
290
+ sel.innerHTML=models.map(m=>
291
+ `<option value="${m.id}" ${m.id===defaultVal?'selected':''}>${m.id}</option>`).join('');
292
+ }
293
+
294
+ /* ── run eval ── */
295
+ async function submitRun(){
296
+ const btnTxt=$('run-btn-txt');btnTxt.innerHTML='<span class="spinner"></span>Running…';
297
+ $('run-status').textContent='retrieving → generating → judging…';
298
+ const qs=$('r-questions').value.split('\n').map(l=>l.trim()).filter(Boolean);
299
+ const questions=[],refs=[];
300
+ for(const l of qs){const [q,r]=l.split('::');questions.push(q.trim());refs.push(r?.trim()||null);}
301
+ const body={
302
+ questions,
303
+ documents:$('r-docs').value.split('\n').map(l=>l.trim()).filter(Boolean).map(t=>({text:t})),
304
+ reference_answers:refs.some(r=>r)?refs:null,
305
+ strategy:document.querySelector('#r-strategy .sel').dataset.name,
306
+ model:$('r-model').value,embedding_model:$('r-emb').value,
307
+ judge_model:$('r-judge').value,metrics:$('r-metrics').value};
308
+ try{
309
+ const res=await fetch('/api/evaluate',{method:'POST',
310
+ headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
311
+ if(!res.ok)throw new Error((await res.json()).detail||res.statusText);
312
+ const data=await res.json(); renderSingleResult(data);
313
+ $('run-status').textContent=`done in ${data.wall_time_s}s`;
314
+ }catch(err){$('run-status').innerHTML=`<span class="bad">${err.message||err}</span>`;}
315
+ btnTxt.textContent='▶ Run evaluation';
316
+ }
317
+
318
+ function renderSingleResult(d){
319
+ $('run-results').classList.remove('hidden');
320
+ const agg=d.aggregate;
321
+ $('rr-stats').innerHTML=['faithfulness','answer_relevance','context_precision','avg_latency_s','total_cost_usd']
322
+ .filter(k=>agg[k]!==undefined).map(k=>
323
+ `<div class="card stat"><div class="l">${k.replace(/_/g,' ')}</div>
324
+ <div class="v ${scoreClass(k,agg[k])}">${fmt(k,agg[k])}</div></div>`).join('');
325
+
326
+ const cols=['question','answer'];
327
+ const metricNames=[...new Set(d.samples.flatMap(s=>Object.keys(s.metrics)))];
328
+ $('rr-table thead').innerHTML='<tr>'+cols.concat(metricNames,['latency_s'])
329
+ .map(c=>`<th>${c.replace(/_/g,' ')}</th>`).join('')+'</tr>';
330
+ $('rr-table tbody').innerHTML=d.samples.map(s=>{
331
+ let row=cols.map(c=>`<td>${esc(String(s[c]??'')).slice(0,120)}</td>`).join('');
332
+ row+=metricNames.map(mn=>{
333
+ const m=s.metrics[mn];
334
+ return m?`<td class="num">${m.score}${m.details?.reasoning?` <span title="${esc(m.details.reasoning)}">ⓘ</span>`:''}</td>`
335
+ :'<td class="num">—</td>';}).join('');
336
+ return row+`<td class="num">${s.latency_s.toFixed(2)}s</td>`;
337
+ }).join('');
338
+ }
339
+
340
+ /* ── compare ── */
341
+ function addConfig(){
342
+ const div=document.createElement('div');
343
+ div.className='kv';div.style.gap='10px';
344
+ div.innerHTML=`<select style="flex:1">${STRATEGIES.map(s=>`<option>${s.name}</option>`).join('')}</select>
345
+ <select style="flex:2" class="cfg-model">${CATALOG.filter(m=>m.modality==='chat')
346
+ .map(m=>`<option>${m.id}</option>`).join('')}</select>
347
+ <button class="btn ghost" onclick="this.parentElement.remove()">✕</button>`;
348
+ $('c-configs').appendChild(div);
349
+ }
350
+ addConfig();
351
+
352
+ async function submitCompare(){
353
+ $('cmp-status').innerHTML='<span class="spinner"></span>battle in progress…';
354
+ const configs=[...document.querySelectorAll('#c-configs > div')].map(d=>({
355
+ strategy:d.children[0].value,model:d.children[1].value}));
356
+ const qs=$('c-questions').value.split('\n').map(l=>l.trim()).filter(Boolean);
357
+ const questions=[],refs=[];
358
+ for(const l of qs){const [q,r]=l.split('::');questions.push(q.trim());refs.push(r?.trim()||null);}
359
+ try{
360
+ const res=await fetch('/api/compare',{method:'POST',
361
+ headers:{'Content-Type':'application/json'},
362
+ body:JSON.stringify({
363
+ questions,configs,
364
+ documents:$('c-docs').value.split('\n').map(l=>l.trim()).filter(Boolean).map(t=>({text:t})),
365
+ reference_answers:refs.some(r=>r)?refs:null})});
366
+ if(!res.ok)throw new Error((await res.json()).detail||res.statusText);
367
+ const data=await res.json();renderCompare(data);
368
+ $('cmp-status').textContent='';
369
+ }catch(err){$('cmp-status').innerHTML=`<span class="bad">${err.message||err}</span>`;}
370
+ }
371
+
372
+ let charts={};
373
+ function renderCompare(data){
374
+ $('cmp-results').classList.remove('hidden');
375
+ const rows=Object.entries(data.matrix).map(([name,v])=>({config:name,...v}));
376
+ if(!rows.length)return;
377
+ const cols=Object.keys(rows[0]);
378
+ $('cmp-table thead').innerHTML='<tr>'+cols.map(c=>`<th>${c.replace(/_/g,' ')}</th>`).join('')+'</tr>';
379
+ $('cmp-table tbody').innerHTML=rows.map(r=>'<tr>'+
380
+ cols.map(c=>`<td class="num">${typeof r[c]==='number'?(c.includes('cost')?'$'+r[c].toFixed(5):r[c]):r[c]}</td>`).join('')+'</tr>').join('');
381
+
382
+ const labels=rows.map(r=>r.config);
383
+ const palette=['#6c5ce7','#00d2ff','#2ecc71','#ffa502','#ff4757','#a29bfe'];
384
+ const qMetrics=['faithfulness','answer_relevance','context_precision'].filter(m=>rows[0][m]!==undefined);
385
+ const mkChart=(id,datasets)=>{if(charts[id])charts[id].destroy();
386
+ charts[id]=new Chart($(id),{type:'bar',data:{labels,datasets},
387
+ options:{responsive:true,plugins:{legend:{labels:{color:'#8a93a6'}}},
388
+ scales:{x:{ticks:{color:'#8a93a6'}},y:{ticks:{color:'#8a93a6'},beginAtZero:true}}}})};
389
+ mkChart('cmp-chart-quality',qMetrics.map((m,i)=>({label:m,data:rows.map(r=>r[m]),
390
+ backgroundColor:palette[i%palette.length]})));
391
+ mkChart('cmp-chart-cost',[{label:'total cost ($)',data:rows.map(r=>r.total_cost_usd||0),
392
+ backgroundColor:'#ffa50288'},{label:'avg latency (s)',data:rows.map(r=>r.avg_latency_s||0),
393
+ backgroundColor:'#00d2ff66'}]);
394
+ }
395
+
396
+ /* ── runs list/detail ── */
397
+ async function renderRecent(){
398
+ try{
399
+ const runs=await api('/api/runs');
400
+ $('st-runs').textContent=runs.length;
401
+ $('tbl-recent tbody').innerHTML=runs.slice(-5).reverse().map(r=>
402
+ `<tr><td class="mono">${r.run_id}</td><td>${r.kind}</td><td>${r.strategy||''}</td></tr>`).join('')
403
+ ||'<tr><td colspan="3" style="color:var(--muted)">no runs yet — create one!</td></tr>';
404
+ $('tbl-runs tbody').innerHTML=runs.map(r=>{
405
+ const a=r.aggregate||{};
406
+ return `<tr><td class="mono">${r.run_id}</td><td>${r.kind}</td><td>${r.strategy||'comparison'}</td>
407
+ <td class="num">${a.faithfulness??'—'}</td><td class="num">${a.answer_relevance??'—'}</td>
408
+ <td class="num">${a.total_cost_usd!=null?'$'+Number(a.total_cost_usd).toFixed(5):'—'}</td>
409
+ <td><button class="btn ghost" onclick="viewRun('${r.run_id}')">view</button></td></tr>`;}).join('');
410
+ }catch(e){}
411
+ }
412
+
413
+ async function viewRun(id){
414
+ const d=await api('/api/runs/'+id);
415
+ $('run-detail').classList.remove('hidden');
416
+ $('rd-title').textContent=`Run ${id}`;
417
+ if(d.kind==='comparison'){
418
+ const rows=Object.entries(d.matrix).map(([n,v])=>({config:n,...v}));
419
+ const cols=Object.keys(rows[0]||{});
420
+ $('rd-body').innerHTML=`<table><thead><tr>${cols.map(c=>`<th>${c}</th>`).join('')}</tr></thead><tbody>${
421
+ rows.map(r=>`<tr>${cols.map(c=>`<td class="num">${r[c]}</td>`).join('')}</tr>`).join('')}</tbody></table>`;
422
+ }else{
423
+ $('rd-body').innerHTML=(d.samples||[]).map(s=>`
424
+ <div style="margin-bottom:16px"><b>${esc(s.question)}</b>
425
+ <div class="answer-box">${esc(s.answer)}</div>
426
+ <div style="margin-top:8px" class="mono">${
427
+ Object.entries(s.metrics).map(([k,m])=>`${k}=${m.score}`).join(' · ')} · ${s.latency_s.toFixed(2)}s</div></div>`).join('');
428
+ }
429
+ $('run-detail').scrollIntoView({behavior:'smooth'});
430
+ }
431
+
432
+ /* ── catalog ── */
433
+ const CAT_FILTERS=['all','chat','embedding','rerank'];
434
+ $('cat-filters').innerHTML=CAT_FILTERS.map(f=>
435
+ `<span class="chip ${f==='all'?'sel':''}" data-f="${f}">${f}</span>`).join('');
436
+ $('cat-filters').addEventListener('click',e=>{
437
+ if(!e.target.dataset.f)return;catFilter=e.target.dataset.f;
438
+ document.querySelectorAll('#cat-filters .chip').forEach(c=>c.classList.toggle('sel',c.dataset.f===catFilter));
439
+ renderCatalog();});
440
+
441
+ function renderCatalog(){
442
+ const q=($('cat-search').value||'').toLowerCase();
443
+ const provBadge=p=>`<span class="badge b-${p}">${p}</span>`;
444
+ const rows=CATALOG.filter(m=>(catFilter==='all'||m.modality===catFilter)&&
445
+ (!q||m.id.toLowerCase().includes(q)||m.description.toLowerCase().includes(q)));
446
+ $('cat-counts').textContent=`${CATALOG.length} models · ${
447
+ new Set(CATALOG.map(m=>m.provider)).size} providers`;
448
+ $('tbl-catalog tbody').innerHTML=rows.map(m=>`<tr>
449
+ <td>${provBadge(m.provider)} <span class="mono">${m.model_name}</span></td>
450
+ <td>${m.modality}</td>
451
+ <td class="num">${m.modality==='chat'?Math.round(m.context_window/1000)+'k':'—'}</td>
452
+ <td class="num">${m.input_cost?'$'+m.input_cost:'free'}</td>
453
+ <td class="num">${m.output_cost?'$'+m.output_cost:'—'}</td>
454
+ <td style="color:var(--muted)">${m.description||''}</td></tr>`).join('');
455
+ }
456
+
457
+ /* ── helpers ── */
458
+ function esc(s){return s.replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));}
459
+ function scoreClass(k,v){if(k.includes('latency'))return v<3?'good':v<8?'warn':'bad';
460
+ if(k.includes('cost'))return v<0.01?'good':v<0.05?'warn':'bad';
461
+ return v>=0.8?'good':v>=0.5?'warn':'bad';}
462
+ function fmt(k,v){if(k.includes('latency'))return v.toFixed(2)+'s';
463
+ if(k.includes('cost'))return '$'+Number(v).toFixed(5);
464
+ return typeof v==='number'&&k!=='total_tokens'?v.toFixed(3):v;}
465
+ </script>
466
+ </body>
467
+ </html>