quickthink 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. quickthink/__init__.py +6 -0
  2. quickthink/cli.py +187 -0
  3. quickthink/config.py +108 -0
  4. quickthink/engine.py +174 -0
  5. quickthink/inline_protocol.py +18 -0
  6. quickthink/ollama_client.py +38 -0
  7. quickthink/plan_grammar.py +24 -0
  8. quickthink/prompts.py +73 -0
  9. quickthink/routing.py +65 -0
  10. quickthink/ui_server.py +1465 -0
  11. quickthink-0.2.1.dist-info/METADATA +387 -0
  12. quickthink-0.2.1.dist-info/RECORD +16 -0
  13. quickthink-0.2.1.dist-info/WHEEL +5 -0
  14. quickthink-0.2.1.dist-info/entry_points.txt +2 -0
  15. quickthink-0.2.1.dist-info/licenses/LICENSE +176 -0
  16. quickthink-0.2.1.dist-info/top_level.txt +1 -0
quickthink/ui_server.py ADDED
@@ -0,0 +1,1465 @@
1
+ from __future__ import annotations
2
+
3
+ """
4
+ Local-only evaluation UI server.
5
+
6
+ Purpose:
7
+ - Fast browser workflow for running QuickThink prompts/evals without CLI.
8
+ - Mirrors runtime controls (`lite` / `two_pass`, routing visibility, latency metrics).
9
+
10
+ Status:
11
+ - Internal integration sandbox for team testing.
12
+ - Not part of the core publishable runtime path unless explicitly promoted.
13
+
14
+ Agent guidance:
15
+ - Keep core engine logic in `engine.py`; this file should stay a thin UI adapter.
16
+ - Prefer adding eval/review UX here over changing core behavior.
17
+ """
18
+
19
+ import json
20
+ import subprocess
21
+ import sys
22
+ import threading
23
+ import webbrowser
24
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
25
+ from pathlib import Path
26
+ from urllib.parse import parse_qs, urlsplit
27
+
28
+ from .config import MODEL_PROFILES, QuickThinkConfig
29
+ from .engine import QuickThinkEngine
30
+
31
+ HTML_PAGE = """<!doctype html>
32
+ <html lang="en">
33
+ <head>
34
+ <meta charset="utf-8" />
35
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
36
+ <title>QuickThink Local Eval Console</title>
37
+ <style>
38
+ :root {
39
+ --bg: #f6efe5;
40
+ --ink: #111111;
41
+ --panel: #fffaf3;
42
+ --accent: #d9480f;
43
+ --accent-2: #0b6e4f;
44
+ --muted: #5f5f5f;
45
+ --ring: #131313;
46
+ --border: #d7c9b6;
47
+ }
48
+ * { box-sizing: border-box; }
49
+ body {
50
+ margin: 0;
51
+ font-family: "Space Grotesk", "Avenir Next", "Helvetica Neue", sans-serif;
52
+ color: var(--ink);
53
+ background:
54
+ radial-gradient(1200px 500px at 10% -10%, #f3ccb1 0%, transparent 60%),
55
+ radial-gradient(800px 450px at 100% 0%, #cce6dd 0%, transparent 55%),
56
+ var(--bg);
57
+ min-height: 100vh;
58
+ padding: 1.25rem;
59
+ animation: fadeIn .3s ease-out;
60
+ }
61
+ @keyframes fadeIn {
62
+ from { opacity: 0; transform: translateY(4px); }
63
+ to { opacity: 1; transform: translateY(0); }
64
+ }
65
+ .wrap {
66
+ max-width: 1100px;
67
+ margin: 0 auto;
68
+ display: grid;
69
+ gap: 1rem;
70
+ grid-template-columns: 1fr;
71
+ }
72
+ .panel {
73
+ background: var(--panel);
74
+ border: 1px solid var(--border);
75
+ border-radius: 14px;
76
+ padding: 1rem;
77
+ box-shadow: 0 10px 20px rgba(0, 0, 0, 0.04);
78
+ }
79
+ h1 {
80
+ margin: 0;
81
+ letter-spacing: .4px;
82
+ font-size: 1.5rem;
83
+ }
84
+ h2 {
85
+ margin: 0;
86
+ letter-spacing: .2px;
87
+ font-size: 1.1rem;
88
+ }
89
+ p { margin: .4rem 0 0 0; color: var(--muted); }
90
+ .flow {
91
+ margin-top: .7rem;
92
+ display: flex;
93
+ flex-wrap: wrap;
94
+ gap: .45rem;
95
+ }
96
+ .chip {
97
+ border: 1px solid var(--border);
98
+ border-radius: 999px;
99
+ padding: .2rem .55rem;
100
+ background: #fff;
101
+ font-size: .78rem;
102
+ color: #333;
103
+ }
104
+ .chip.adv {
105
+ background: #f6fff8;
106
+ border-color: #b5d9c9;
107
+ color: #1b5f46;
108
+ }
109
+ .grid {
110
+ display: grid;
111
+ gap: .8rem;
112
+ grid-template-columns: repeat(2, minmax(0, 1fr));
113
+ }
114
+ .full { grid-column: 1 / -1; }
115
+ label {
116
+ font-size: .86rem;
117
+ font-weight: 600;
118
+ margin-bottom: .3rem;
119
+ display: block;
120
+ }
121
+ input, select, textarea, button {
122
+ width: 100%;
123
+ border-radius: 10px;
124
+ border: 1px solid var(--border);
125
+ background: #fffcf8;
126
+ color: var(--ink);
127
+ padding: .65rem .75rem;
128
+ font-size: .95rem;
129
+ font-family: inherit;
130
+ }
131
+ textarea {
132
+ resize: vertical;
133
+ min-height: 120px;
134
+ line-height: 1.35;
135
+ }
136
+ input:focus, select:focus, textarea:focus {
137
+ outline: 2px solid var(--ring);
138
+ outline-offset: 1px;
139
+ }
140
+ button:focus-visible {
141
+ outline: 2px solid var(--ring);
142
+ outline-offset: 2px;
143
+ }
144
+ .checks {
145
+ display: flex;
146
+ flex-wrap: wrap;
147
+ gap: 1rem;
148
+ margin-top: .3rem;
149
+ }
150
+ .checks label {
151
+ display: flex;
152
+ align-items: center;
153
+ gap: .4rem;
154
+ margin: 0;
155
+ font-weight: 500;
156
+ }
157
+ .checks input {
158
+ width: auto;
159
+ }
160
+ .actions {
161
+ display: flex;
162
+ gap: .6rem;
163
+ justify-content: flex-end;
164
+ margin-top: .6rem;
165
+ }
166
+ button {
167
+ width: auto;
168
+ cursor: pointer;
169
+ font-weight: 700;
170
+ border-color: #111111;
171
+ background: #ffffff;
172
+ }
173
+ .primary {
174
+ background: var(--accent);
175
+ color: #fff;
176
+ border-color: var(--accent);
177
+ }
178
+ .secondary {
179
+ background: var(--accent-2);
180
+ color: #fff;
181
+ border-color: var(--accent-2);
182
+ }
183
+ .status {
184
+ font-size: .9rem;
185
+ margin-top: .5rem;
186
+ color: var(--muted);
187
+ min-height: 1.4rem;
188
+ }
189
+ .status.is-success { color: #0b6e4f; }
190
+ .status.is-error { color: #b42318; }
191
+ .hint {
192
+ margin-top: .45rem;
193
+ color: var(--muted);
194
+ font-size: .82rem;
195
+ }
196
+ .mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
197
+ .result {
198
+ white-space: pre-wrap;
199
+ line-height: 1.45;
200
+ padding: .8rem;
201
+ border-radius: 10px;
202
+ border: 1px dashed var(--border);
203
+ background: #fff;
204
+ min-height: 80px;
205
+ margin-top: .4rem;
206
+ }
207
+ .metrics {
208
+ display: grid;
209
+ gap: .6rem;
210
+ grid-template-columns: repeat(3, minmax(0, 1fr));
211
+ }
212
+ .metric {
213
+ border: 1px solid var(--border);
214
+ border-radius: 10px;
215
+ padding: .6rem;
216
+ background: #fff;
217
+ }
218
+ .metric .k { color: var(--muted); font-size: .8rem; }
219
+ .metric .v { font-weight: 700; margin-top: .2rem; }
220
+ @media (max-width: 900px) {
221
+ .grid { grid-template-columns: 1fr; }
222
+ .metrics { grid-template-columns: 1fr; }
223
+ .actions { justify-content: stretch; }
224
+ button { flex: 1; }
225
+ }
226
+ </style>
227
+ </head>
228
+ <body>
229
+ <div class="wrap">
230
+ <section class="panel">
231
+ <h1>QuickThink Local Eval Console</h1>
232
+ <p>Run and evaluate local prompts with the same routing and validation gates used by CLI.</p>
233
+ <div class="flow">
234
+ <span class="chip">Recommended: 1) Preflight</span>
235
+ <span class="chip">2) Single Prompt</span>
236
+ <span class="chip">3) Batch Eval</span>
237
+ <span class="chip adv">Advanced: Ingest and Browsers</span>
238
+ </div>
239
+ </section>
240
+
241
+ <section class="panel">
242
+ <h2>1. Preflight Validation (Required)</h2>
243
+ <p>Required before eval runs. Uses <span class="mono">validate_prompt_set.py</span>.</p>
244
+ <div class="grid">
245
+ <div class="full">
246
+ <label for="promptSetPath">Prompt set path</label>
247
+ <input id="promptSetPath" value="docs/evals/prompt_set.jsonl" />
248
+ </div>
249
+ </div>
250
+ <div class="actions">
251
+ <button id="preflightBtn" class="secondary">Validate Prompt Set</button>
252
+ </div>
253
+ <div id="preflightStatus" class="status" role="status" aria-live="polite"></div>
254
+ <div class="result" id="preflightSha">Dataset SHA256: (not validated)</div>
255
+ <div class="result" id="preflightOutput">No preflight run yet. Validate to enable run actions.</div>
256
+ </section>
257
+
258
+ <section class="panel">
259
+ <h2>2. Run Single Prompt</h2>
260
+ <p>Canonical interactive flow for fast testing before batch runs.</p>
261
+ <div class="grid">
262
+ <div>
263
+ <label for="model">Model</label>
264
+ <select id="model"></select>
265
+ </div>
266
+ <div>
267
+ <label for="mode">Mode</label>
268
+ <select id="mode">
269
+ <option value="direct">direct (raw)</option>
270
+ <option value="lite">lite</option>
271
+ <option value="two_pass">two_pass</option>
272
+ </select>
273
+ </div>
274
+ <div>
275
+ <label for="lanePolicy">Lane policy</label>
276
+ <select id="lanePolicy">
277
+ <option value="default">default</option>
278
+ <option value="strict_safe">strict_safe</option>
279
+ </select>
280
+ </div>
281
+ <div class="full">
282
+ <label for="ollamaUrl">Ollama URL</label>
283
+ <input id="ollamaUrl" value="http://localhost:11434" />
284
+ </div>
285
+ <div class="full">
286
+ <label for="prompt">Prompt</label>
287
+ <textarea id="prompt" placeholder="Enter prompt for your eval case..."></textarea>
288
+ </div>
289
+ <div class="full">
290
+ <label for="continuity">Continuity hint (optional)</label>
291
+ <input id="continuity" placeholder="ctx:prior_goal,format_json" />
292
+ </div>
293
+ <div class="full">
294
+ <div class="checks">
295
+ <label title="Skip planner for very short prompts to reduce overhead."><input id="bypass" type="checkbox" checked /> bypass short prompts</label>
296
+ <label title="Display planner line in results."><input id="showPlan" type="checkbox" checked /> show plan</label>
297
+ <label title="Display routing metadata like score and plan budget."><input id="showRoute" type="checkbox" checked /> show route info</label>
298
+ </div>
299
+ <div class="hint">
300
+ Control help: bypass short prompts = skip planner for short inputs; show plan = reveal planner output;
301
+ show route info = reveal routing score and plan budget.
302
+ </div>
303
+ </div>
304
+ </div>
305
+ <div class="actions">
306
+ <button id="clearBtn">Clear</button>
307
+ <button id="runAllBtn" class="secondary" title="Requires typed prompt.">Compare 3 Modes</button>
308
+ <button id="runBtn" class="primary" title="Requires successful preflight.">Run Single Prompt</button>
309
+ </div>
310
+ <div id="runGateHint" class="hint">Preflight required to enable run actions.</div>
311
+ <div id="status" class="status" role="status" aria-live="polite">No run yet. Enter a prompt and run.</div>
312
+ </section>
313
+
314
+ <section class="panel">
315
+ <h2>3. Batch Eval Runner</h2>
316
+ <p>Run prompt_set.jsonl across direct/lite/two_pass with run manifest output.</p>
317
+ <div class="grid">
318
+ <div class="full">
319
+ <label for="evalPromptSetPath">Prompt set path</label>
320
+ <input id="evalPromptSetPath" value="docs/evals/prompt_set.jsonl" />
321
+ </div>
322
+ <div class="full">
323
+ <label for="evalOutPath">Results JSONL out</label>
324
+ <input id="evalOutPath" value="docs/evals/results/run_results.jsonl" />
325
+ </div>
326
+ <div class="full">
327
+ <label for="evalManifestPath">Manifest out</label>
328
+ <input id="evalManifestPath" value="docs/evals/results/run_manifest.json" />
329
+ </div>
330
+ <div>
331
+ <label for="evalRuns">Runs per prompt</label>
332
+ <input id="evalRuns" type="number" value="3" min="1" />
333
+ </div>
334
+ <div>
335
+ <label for="evalLimit">Prompt limit (0=all)</label>
336
+ <input id="evalLimit" type="number" value="0" min="0" />
337
+ </div>
338
+ <div class="full">
339
+ <label for="evalModels">Models (space-separated)</label>
340
+ <input id="evalModels" value="qwen2.5:1.5b mistral:7b gemma3:27b" />
341
+ </div>
342
+ </div>
343
+ <div class="actions">
344
+ <button id="runEvalSetBtn" class="secondary">Run Batch Eval</button>
345
+ </div>
346
+ <div id="evalSetStatus" class="status" role="status" aria-live="polite"></div>
347
+ <div class="result" id="evalSetOutput">No batch eval run yet.</div>
348
+ </section>
349
+
350
+ <section class="panel">
351
+ <h2>4. Ingest Validated Run File <span class="chip adv">Advanced</span></h2>
352
+ <p>Ingestion is blocked unless <span class="mono">validate_results.py</span> returns <span class="mono">status=OK</span>.</p>
353
+ <div class="grid">
354
+ <div class="full">
355
+ <label for="runFilePath">Run file path (JSONL)</label>
356
+ <input id="runFilePath" placeholder="docs/evals/results/run-YYYYMMDD-HHMM-batch.jsonl" />
357
+ </div>
358
+ <div>
359
+ <label for="expectedPrompts">Expected prompts</label>
360
+ <input id="expectedPrompts" type="number" value="120" min="0" />
361
+ </div>
362
+ <div>
363
+ <label for="expectedRuns">Expected runs</label>
364
+ <input id="expectedRuns" type="number" value="3" min="0" />
365
+ </div>
366
+ <div class="full">
367
+ <label for="models">Models (space-separated)</label>
368
+ <input id="models" value="qwen2.5:1.5b mistral:7b gemma3:27b" />
369
+ </div>
370
+ </div>
371
+ <div class="actions">
372
+ <button id="ingestBtn" class="secondary">Validate and Ingest</button>
373
+ </div>
374
+ <div id="ingestStatus" class="status" role="status" aria-live="polite"></div>
375
+ <div class="result" id="ingestOutput">No ingestion yet.</div>
376
+ </section>
377
+
378
+ <section class="panel">
379
+ <h2>Prompt Set Preview <span class="chip adv">Advanced</span></h2>
380
+ <div class="grid">
381
+ <div class="full">
382
+ <label for="browsePromptPath">Prompt set JSONL path</label>
383
+ <input id="browsePromptPath" value="docs/evals/prompt_set.jsonl" />
384
+ </div>
385
+ </div>
386
+ <div class="actions">
387
+ <button id="loadPromptsBtn">Preview Prompts</button>
388
+ </div>
389
+ <div id="promptBrowseStatus" class="status" role="status" aria-live="polite"></div>
390
+ <div class="result" id="promptBrowseOutput">No prompts loaded. Confirm prompt-set path, then select Preview Prompts.</div>
391
+ </section>
392
+
393
+ <section class="panel">
394
+ <h2>Result File Explorer <span class="chip adv">Advanced</span></h2>
395
+ <div class="grid">
396
+ <div class="full">
397
+ <label for="resultsFileSelect">Result file</label>
398
+ <select id="resultsFileSelect"></select>
399
+ </div>
400
+ </div>
401
+ <div class="actions">
402
+ <button id="refreshResultFilesBtn">Refresh Result Files</button>
403
+ <button id="loadResultRowsBtn">Open Rows</button>
404
+ </div>
405
+ <div id="resultsBrowseStatus" class="status" role="status" aria-live="polite"></div>
406
+ <div class="result" id="resultsBrowseOutput">No result files loaded yet.</div>
407
+ </section>
408
+
409
+ <section class="panel">
410
+ <h2>Run Output and Diagnostics</h2>
411
+ <p>Glossary: route score = routing complexity signal, plan budget = selected planning token budget.</p>
412
+ <div id="metrics" class="metrics"></div>
413
+ <div class="result" id="answer">No run yet. Run a prompt to view answer, plan, route, and latency metrics.</div>
414
+ <div class="result" id="plan">Plan hidden. Enable "show plan" to display planner output.</div>
415
+ <div class="result" id="route">Route details hidden. Enable "show route info" to inspect routing decisions.</div>
416
+ <div class="result" id="allModes">(no comparison output yet)</div>
417
+ </section>
418
+ </div>
419
+
420
+ <script>
421
+ const modelEl = document.getElementById("model");
422
+ const modeEl = document.getElementById("mode");
423
+ const lanePolicyEl = document.getElementById("lanePolicy");
424
+ const promptEl = document.getElementById("prompt");
425
+ const ollamaEl = document.getElementById("ollamaUrl");
426
+ const continuityEl = document.getElementById("continuity");
427
+ const bypassEl = document.getElementById("bypass");
428
+ const showPlanEl = document.getElementById("showPlan");
429
+ const showRouteEl = document.getElementById("showRoute");
430
+ const statusEl = document.getElementById("status");
431
+ const answerEl = document.getElementById("answer");
432
+ const planEl = document.getElementById("plan");
433
+ const routeEl = document.getElementById("route");
434
+ const allModesEl = document.getElementById("allModes");
435
+ const metricsEl = document.getElementById("metrics");
436
+ const runBtn = document.getElementById("runBtn");
437
+ const runAllBtn = document.getElementById("runAllBtn");
438
+ const runGateHintEl = document.getElementById("runGateHint");
439
+ const clearBtn = document.getElementById("clearBtn");
440
+ const preflightBtn = document.getElementById("preflightBtn");
441
+ const preflightStatusEl = document.getElementById("preflightStatus");
442
+ const preflightShaEl = document.getElementById("preflightSha");
443
+ const preflightOutputEl = document.getElementById("preflightOutput");
444
+ const promptSetPathEl = document.getElementById("promptSetPath");
445
+ const ingestBtn = document.getElementById("ingestBtn");
446
+ const runFilePathEl = document.getElementById("runFilePath");
447
+ const expectedPromptsEl = document.getElementById("expectedPrompts");
448
+ const expectedRunsEl = document.getElementById("expectedRuns");
449
+ const modelsEl = document.getElementById("models");
450
+ const ingestStatusEl = document.getElementById("ingestStatus");
451
+ const ingestOutputEl = document.getElementById("ingestOutput");
452
+ const evalPromptSetPathEl = document.getElementById("evalPromptSetPath");
453
+ const evalOutPathEl = document.getElementById("evalOutPath");
454
+ const evalManifestPathEl = document.getElementById("evalManifestPath");
455
+ const evalRunsEl = document.getElementById("evalRuns");
456
+ const evalLimitEl = document.getElementById("evalLimit");
457
+ const evalModelsEl = document.getElementById("evalModels");
458
+ const runEvalSetBtn = document.getElementById("runEvalSetBtn");
459
+ const evalSetStatusEl = document.getElementById("evalSetStatus");
460
+ const evalSetOutputEl = document.getElementById("evalSetOutput");
461
+ const browsePromptPathEl = document.getElementById("browsePromptPath");
462
+ const loadPromptsBtn = document.getElementById("loadPromptsBtn");
463
+ const promptBrowseStatusEl = document.getElementById("promptBrowseStatus");
464
+ const promptBrowseOutputEl = document.getElementById("promptBrowseOutput");
465
+ const resultsFileSelectEl = document.getElementById("resultsFileSelect");
466
+ const refreshResultFilesBtn = document.getElementById("refreshResultFilesBtn");
467
+ const loadResultRowsBtn = document.getElementById("loadResultRowsBtn");
468
+ const resultsBrowseStatusEl = document.getElementById("resultsBrowseStatus");
469
+ const resultsBrowseOutputEl = document.getElementById("resultsBrowseOutput");
470
+ let preflightOk = false;
471
+
472
+ function setStatusMessage(el, text, kind = "info") {
473
+ el.textContent = text;
474
+ el.classList.remove("is-info", "is-success", "is-error");
475
+ if (kind === "success") {
476
+ el.classList.add("is-success");
477
+ el.setAttribute("role", "status");
478
+ el.setAttribute("aria-live", "polite");
479
+ } else if (kind === "error") {
480
+ el.classList.add("is-error");
481
+ el.setAttribute("role", "alert");
482
+ el.setAttribute("aria-live", "assertive");
483
+ } else {
484
+ el.classList.add("is-info");
485
+ el.setAttribute("role", "status");
486
+ el.setAttribute("aria-live", "polite");
487
+ }
488
+ }
489
+
490
+ function setStatus(text, kind = "info") {
491
+ setStatusMessage(statusEl, text, kind);
492
+ }
493
+
494
+ function fmtMs(v) {
495
+ return Number(v || 0).toFixed(2) + " ms";
496
+ }
497
+
498
+ function setPreflightState(ok, sha) {
499
+ preflightOk = Boolean(ok);
500
+ runBtn.disabled = !ok;
501
+ runAllBtn.disabled = !ok;
502
+ runEvalSetBtn.disabled = !ok;
503
+ setStatusMessage(
504
+ preflightStatusEl,
505
+ ok
506
+ ? "Preflight passed. Single and batch runs are enabled."
507
+ : "Preflight required. Validate prompt set to enable run actions.",
508
+ ok ? "success" : "error"
509
+ );
510
+ runGateHintEl.textContent = ok
511
+ ? "Run actions enabled. You can execute single prompt or 3-mode comparison."
512
+ : "Preflight required to enable run actions.";
513
+ preflightShaEl.textContent = "Dataset SHA256: " + (sha || "(not available)");
514
+ }
515
+
516
+ function renderMetrics(data) {
517
+ const items = [
518
+ ["Plan latency", fmtMs(data.plan_latency_ms)],
519
+ ["Answer latency", fmtMs(data.answer_latency_ms)],
520
+ ["Total latency", fmtMs(data.total_latency_ms)],
521
+ ["Mode", data.mode],
522
+ ["Bypassed", String(data.bypassed)],
523
+ ["Plan repaired", String(data.plan_repaired)],
524
+ ];
525
+ metricsEl.innerHTML = "";
526
+ for (const [k, v] of items) {
527
+ const card = document.createElement("div");
528
+ card.className = "metric";
529
+ card.innerHTML = '<div class="k">' + k + '</div><div class="v">' + v + "</div>";
530
+ metricsEl.appendChild(card);
531
+ }
532
+ }
533
+
534
+ async function loadModels() {
535
+ const res = await fetch("/api/models");
536
+ const data = await res.json();
537
+ modelEl.innerHTML = "";
538
+ for (const m of data.models) {
539
+ const o = document.createElement("option");
540
+ o.value = m;
541
+ o.textContent = m;
542
+ modelEl.appendChild(o);
543
+ }
544
+ }
545
+
546
+ async function refreshGateState() {
547
+ const res = await fetch("/api/state");
548
+ const data = await res.json();
549
+ setPreflightState(Boolean(data.preflight_ok), data.dataset_sha256 || "");
550
+ if (data.preflight_output) {
551
+ preflightOutputEl.textContent = data.preflight_output;
552
+ }
553
+ if (data.last_ingestion && data.last_ingestion.output) {
554
+ ingestOutputEl.textContent = data.last_ingestion.output;
555
+ }
556
+ }
557
+
558
+ async function runPreflight() {
559
+ preflightBtn.disabled = true;
560
+ setStatusMessage(preflightStatusEl, "Running preflight validation...", "info");
561
+ try {
562
+ const res = await fetch("/api/preflight", {
563
+ method: "POST",
564
+ headers: { "Content-Type": "application/json" },
565
+ body: JSON.stringify({ path: promptSetPathEl.value.trim() || "docs/evals/prompt_set.jsonl" })
566
+ });
567
+ const data = await res.json();
568
+ preflightOutputEl.textContent = data.output || "";
569
+ setPreflightState(Boolean(data.preflight_ok), data.dataset_sha256 || "");
570
+ if (!res.ok) {
571
+ setStatusMessage(
572
+ preflightStatusEl,
573
+ "Preflight failed. Fix prompt-set validation errors shown below and run validation again.",
574
+ "error"
575
+ );
576
+ return;
577
+ }
578
+ setStatusMessage(preflightStatusEl, "Preflight passed. Run actions are enabled.", "success");
579
+ } catch (err) {
580
+ setStatusMessage(
581
+ preflightStatusEl,
582
+ "Preflight failed. Check prompt-set path and retry. Details: " + err.message,
583
+ "error"
584
+ );
585
+ } finally {
586
+ preflightBtn.disabled = false;
587
+ }
588
+ }
589
+
590
+ async function validateAndIngestRunFile() {
591
+ const path = runFilePathEl.value.trim();
592
+ if (!path) {
593
+ setStatusMessage(ingestStatusEl, "Run file path is required.", "error");
594
+ return;
595
+ }
596
+ ingestBtn.disabled = true;
597
+ setStatusMessage(ingestStatusEl, "Validating run file...", "info");
598
+ try {
599
+ const res = await fetch("/api/ingest-run", {
600
+ method: "POST",
601
+ headers: { "Content-Type": "application/json" },
602
+ body: JSON.stringify({
603
+ path,
604
+ expected_prompts: Number(expectedPromptsEl.value || "0"),
605
+ expected_runs: Number(expectedRunsEl.value || "0"),
606
+ models: (modelsEl.value || "").trim().split(/\\s+/).filter(Boolean)
607
+ })
608
+ });
609
+ const data = await res.json();
610
+ ingestOutputEl.textContent = data.output || "";
611
+ if (data.ingested) {
612
+ setStatusMessage(ingestStatusEl, "Ingestion complete. Validation status=OK.", "success");
613
+ } else {
614
+ setStatusMessage(
615
+ ingestStatusEl,
616
+ "Ingestion blocked: validator status is not OK. Resolve validator output and retry.",
617
+ "error"
618
+ );
619
+ }
620
+ } catch (err) {
621
+ setStatusMessage(
622
+ ingestStatusEl,
623
+ "Ingestion failed. Check run file path and validation settings, then retry. Details: " + err.message,
624
+ "error"
625
+ );
626
+ } finally {
627
+ ingestBtn.disabled = false;
628
+ }
629
+ }
630
+
631
+ async function runEvalSet() {
632
+ runEvalSetBtn.disabled = true;
633
+ setStatusMessage(evalSetStatusEl, "Running batch eval...", "info");
634
+ evalSetOutputEl.textContent = "";
635
+ try {
636
+ const res = await fetch("/api/run-eval-set", {
637
+ method: "POST",
638
+ headers: { "Content-Type": "application/json" },
639
+ body: JSON.stringify({
640
+ prompt_set: evalPromptSetPathEl.value.trim() || "docs/evals/prompt_set.jsonl",
641
+ out: evalOutPathEl.value.trim() || "docs/evals/results/run_results.jsonl",
642
+ manifest_out: evalManifestPathEl.value.trim() || "docs/evals/results/run_manifest.json",
643
+ runs: Number(evalRunsEl.value || "3"),
644
+ limit: Number(evalLimitEl.value || "0"),
645
+ models: (evalModelsEl.value || "").trim().split(/\\s+/).filter(Boolean),
646
+ ollama_url: ollamaEl.value.trim() || "http://localhost:11434",
647
+ continuity_hint: continuityEl.value.trim() || null
648
+ })
649
+ });
650
+ const data = await res.json();
651
+ evalSetOutputEl.textContent = data.output || "";
652
+ if (!res.ok) {
653
+ throw new Error(data.error || "Eval set run failed");
654
+ }
655
+ setStatusMessage(evalSetStatusEl, "Batch eval complete.", "success");
656
+ runFilePathEl.value = data.out_path || runFilePathEl.value;
657
+ } catch (err) {
658
+ setStatusMessage(
659
+ evalSetStatusEl,
660
+ "Batch eval failed. Check preflight state, model availability, and Ollama URL, then retry. Details: " + err.message,
661
+ "error"
662
+ );
663
+ } finally {
664
+ runEvalSetBtn.disabled = !preflightOk;
665
+ }
666
+ }
667
+
668
+ function toPrettyJsonLines(rows, maxRows) {
669
+ const slice = rows.slice(0, maxRows);
670
+ return slice.map((row) => JSON.stringify(row, null, 2)).join("\\n\\n");
671
+ }
672
+
673
+ async function loadPromptRows() {
674
+ const path = browsePromptPathEl.value.trim() || "docs/evals/prompt_set.jsonl";
675
+ loadPromptsBtn.disabled = true;
676
+ setStatusMessage(promptBrowseStatusEl, "Loading prompts...", "info");
677
+ try {
678
+ const res = await fetch("/api/prompts?path=" + encodeURIComponent(path) + "&offset=0&limit=20");
679
+ const data = await res.json();
680
+ if (!res.ok) {
681
+ throw new Error(data.error || "failed loading prompts");
682
+ }
683
+ setStatusMessage(
684
+ promptBrowseStatusEl,
685
+ "Loaded " + data.rows.length + " of " + data.total + " prompt rows.",
686
+ "success"
687
+ );
688
+ promptBrowseOutputEl.textContent = toPrettyJsonLines(data.rows, 20);
689
+ } catch (err) {
690
+ setStatusMessage(
691
+ promptBrowseStatusEl,
692
+ "Prompt load failed. Check prompt-set path and JSONL format, then retry. Details: " + err.message,
693
+ "error"
694
+ );
695
+ } finally {
696
+ loadPromptsBtn.disabled = false;
697
+ }
698
+ }
699
+
700
+ async function refreshResultFiles() {
701
+ refreshResultFilesBtn.disabled = true;
702
+ setStatusMessage(resultsBrowseStatusEl, "Refreshing result files...", "info");
703
+ try {
704
+ const res = await fetch("/api/results/files");
705
+ const data = await res.json();
706
+ if (!res.ok) {
707
+ throw new Error(data.error || "failed listing result files");
708
+ }
709
+ resultsFileSelectEl.innerHTML = "";
710
+ for (const file of data.files) {
711
+ const opt = document.createElement("option");
712
+ opt.value = file.path;
713
+ opt.textContent = file.path + " (" + file.size_bytes + " bytes)";
714
+ resultsFileSelectEl.appendChild(opt);
715
+ }
716
+ setStatusMessage(resultsBrowseStatusEl, "Found " + data.files.length + " result files.", "success");
717
+ if (data.files.length === 0) {
718
+ resultsBrowseOutputEl.textContent = "No result files found in docs/evals/results. Run batch eval first or check output path.";
719
+ }
720
+ } catch (err) {
721
+ setStatusMessage(
722
+ resultsBrowseStatusEl,
723
+ "Result file listing failed. Check docs/evals/results path and permissions. Details: " + err.message,
724
+ "error"
725
+ );
726
+ } finally {
727
+ refreshResultFilesBtn.disabled = false;
728
+ }
729
+ }
730
+
731
+ async function loadResultRows() {
732
+ const path = (resultsFileSelectEl.value || "").trim();
733
+ if (!path) {
734
+ setStatusMessage(resultsBrowseStatusEl, "Select a result file first.", "error");
735
+ return;
736
+ }
737
+ loadResultRowsBtn.disabled = true;
738
+ setStatusMessage(resultsBrowseStatusEl, "Loading result rows...", "info");
739
+ try {
740
+ const res = await fetch("/api/results/rows?path=" + encodeURIComponent(path) + "&offset=0&limit=20");
741
+ const data = await res.json();
742
+ if (!res.ok) {
743
+ throw new Error(data.error || "failed loading result rows");
744
+ }
745
+ setStatusMessage(
746
+ resultsBrowseStatusEl,
747
+ "Loaded " + data.rows.length + " of " + data.total + " rows from " + data.path,
748
+ "success"
749
+ );
750
+ resultsBrowseOutputEl.textContent = toPrettyJsonLines(data.rows, 20);
751
+ } catch (err) {
752
+ setStatusMessage(
753
+ resultsBrowseStatusEl,
754
+ "Result row load failed. Verify selected file path and JSONL validity. Details: " + err.message,
755
+ "error"
756
+ );
757
+ } finally {
758
+ loadResultRowsBtn.disabled = false;
759
+ }
760
+ }
761
+
762
+ function renderAllModes(results) {
763
+ const order = ["direct", "lite", "two_pass"];
764
+ const sections = [];
765
+ for (const mode of order) {
766
+ const data = results[mode];
767
+ if (!data) {
768
+ continue;
769
+ }
770
+ sections.push(
771
+ [
772
+ mode === "direct" ? "Mode: direct (raw)" : "Mode: " + mode,
773
+ "total_latency_ms=" + Number(data.total_latency_ms || 0).toFixed(2),
774
+ "plan_latency_ms=" + Number(data.plan_latency_ms || 0).toFixed(2),
775
+ "answer_latency_ms=" + Number(data.answer_latency_ms || 0).toFixed(2),
776
+ "bypassed=" + String(data.bypassed),
777
+ "route_score=" + String(data.route_score),
778
+ "selected_plan_budget=" + String(data.selected_plan_budget),
779
+ "",
780
+ "Plan:",
781
+ data.plan || "(none)",
782
+ "",
783
+ "Answer:",
784
+ data.answer || "(empty)",
785
+ ].join("\\n")
786
+ );
787
+ }
788
+ allModesEl.textContent = sections.join("\\n\\n----------------------------------------\\n\\n");
789
+ }
790
+
791
+ async function runAllModes() {
792
+ const prompt = promptEl.value.trim();
793
+ if (!prompt) {
794
+ setStatus("Prompt required for mode comparison. Use Run Batch Eval for prompt-set runs.", "error");
795
+ return;
796
+ }
797
+ runAllBtn.disabled = true;
798
+ runBtn.disabled = true;
799
+ setStatus("Running 3-mode comparison...", "info");
800
+ answerEl.textContent = "";
801
+ planEl.textContent = "";
802
+ routeEl.textContent = "";
803
+ allModesEl.textContent = "";
804
+ metricsEl.innerHTML = "";
805
+ try {
806
+ const res = await fetch("/api/ask-all", {
807
+ method: "POST",
808
+ headers: { "Content-Type": "application/json" },
809
+ body: JSON.stringify({
810
+ prompt,
811
+ model: modelEl.value,
812
+ ollama_url: ollamaEl.value.trim(),
813
+ lane_policy: lanePolicyEl.value,
814
+ bypass_short_prompts: bypassEl.checked,
815
+ continuity_hint: continuityEl.value.trim() || null
816
+ })
817
+ });
818
+ const data = await res.json();
819
+ if (!res.ok) {
820
+ throw new Error(data.error || "Request failed");
821
+ }
822
+ renderAllModes(data.results || {});
823
+ if (data.results && data.results.lite) {
824
+ renderMetrics(data.results.lite);
825
+ }
826
+ setStatus("3-mode comparison complete.", "success");
827
+ } catch (err) {
828
+ setStatus(
829
+ "Run failed. Check Ollama URL, selected model, and preflight state, then retry. Details: " + err.message,
830
+ "error"
831
+ );
832
+ } finally {
833
+ runAllBtn.disabled = !preflightOk;
834
+ runBtn.disabled = !preflightOk;
835
+ }
836
+ }
837
+
838
+ async function runPrompt() {
839
+ const prompt = promptEl.value.trim();
840
+ if (!prompt) {
841
+ setStatus("Enter a prompt to run a single request.", "error");
842
+ return;
843
+ }
844
+ runBtn.disabled = true;
845
+ runAllBtn.disabled = true;
846
+ setStatus("Running request...", "info");
847
+ answerEl.textContent = "";
848
+ planEl.textContent = "";
849
+ routeEl.textContent = "";
850
+ allModesEl.textContent = "";
851
+ metricsEl.innerHTML = "";
852
+
853
+ try {
854
+ const res = await fetch("/api/ask", {
855
+ method: "POST",
856
+ headers: { "Content-Type": "application/json" },
857
+ body: JSON.stringify({
858
+ prompt,
859
+ model: modelEl.value,
860
+ mode: modeEl.value,
861
+ ollama_url: ollamaEl.value.trim(),
862
+ lane_policy: lanePolicyEl.value,
863
+ bypass_short_prompts: bypassEl.checked,
864
+ continuity_hint: continuityEl.value.trim() || null
865
+ })
866
+ });
867
+ const data = await res.json();
868
+ if (!res.ok) {
869
+ throw new Error(data.error || "Request failed");
870
+ }
871
+ answerEl.textContent = "Answer\\n\\n" + (data.answer || "(empty)");
872
+ planEl.textContent = showPlanEl.checked
873
+ ? "Plan\\n\\n" + (data.plan || "(none)")
874
+ : "Plan hidden";
875
+ routeEl.textContent = showRouteEl.checked
876
+ ? "Route\\n\\n" + [
877
+ "mode=" + data.mode,
878
+ "bypassed=" + data.bypassed,
879
+ "score=" + data.route_score,
880
+ "plan_budget=" + data.selected_plan_budget
881
+ ].join(" ")
882
+ : "Route info hidden";
883
+ renderMetrics(data);
884
+ setStatus("Run complete.", "success");
885
+ } catch (err) {
886
+ setStatus(
887
+ "Run failed. Check Ollama URL, selected model, and server status, then retry. Details: " + err.message,
888
+ "error"
889
+ );
890
+ } finally {
891
+ runBtn.disabled = !preflightOk;
892
+ runAllBtn.disabled = !preflightOk;
893
+ }
894
+ }
895
+
896
+ function clearResults() {
897
+ answerEl.textContent = "No run yet. Run a prompt to view answer, plan, route, and latency metrics.";
898
+ planEl.textContent = "Plan hidden. Enable \"show plan\" to display planner output.";
899
+ routeEl.textContent = "Route details hidden. Enable \"show route info\" to inspect routing decisions.";
900
+ allModesEl.textContent = "(no comparison output yet)";
901
+ metricsEl.innerHTML = "";
902
+ setStatus("Output cleared.", "info");
903
+ }
904
+
905
+ runBtn.addEventListener("click", runPrompt);
906
+ runAllBtn.addEventListener("click", runAllModes);
907
+ clearBtn.addEventListener("click", clearResults);
908
+ preflightBtn.addEventListener("click", runPreflight);
909
+ ingestBtn.addEventListener("click", validateAndIngestRunFile);
910
+ runEvalSetBtn.addEventListener("click", runEvalSet);
911
+ loadPromptsBtn.addEventListener("click", loadPromptRows);
912
+ refreshResultFilesBtn.addEventListener("click", refreshResultFiles);
913
+ loadResultRowsBtn.addEventListener("click", loadResultRows);
914
+ loadModels().catch((err) =>
915
+ setStatus("Failed loading models. Check local server state and refresh. Details: " + err.message, "error")
916
+ );
917
+ refreshGateState().catch((err) =>
918
+ setStatusMessage(preflightStatusEl, "State load failed. Refresh and retry. Details: " + err.message, "error")
919
+ );
920
+ refreshResultFiles().catch((err) =>
921
+ setStatusMessage(resultsBrowseStatusEl, "Result file refresh failed. Details: " + err.message, "error")
922
+ );
923
+ loadPromptRows().catch((err) =>
924
+ setStatusMessage(promptBrowseStatusEl, "Prompt preview failed. Details: " + err.message, "error")
925
+ );
926
+ </script>
927
+ </body>
928
+ </html>
929
+ """
930
+
931
+ REPO_ROOT = Path(__file__).resolve().parents[2]
932
+ PROMPT_VALIDATOR = REPO_ROOT / "scripts" / "evals" / "validate_prompt_set.py"
933
+ RESULTS_VALIDATOR = REPO_ROOT / "scripts" / "evals" / "validate_results.py"
934
+ RUN_SUITE_SCRIPT = REPO_ROOT / "scripts" / "eval_harness" / "run_suite.py"
935
+ PROMPT_SET_DEFAULT = REPO_ROOT / "docs" / "evals" / "prompt_set.jsonl"
936
+ RESULTS_DIR_DEFAULT = REPO_ROOT / "docs" / "evals" / "results"
937
+ PROMPTS_DIR_DEFAULT = REPO_ROOT / "docs" / "evals"
938
+
939
+
940
+ def _extract_key(output: str, key: str) -> str | None:
941
+ prefix = f"{key}="
942
+ for line in output.splitlines():
943
+ line = line.strip()
944
+ if line.startswith(prefix):
945
+ return line[len(prefix) :].strip()
946
+ return None
947
+
948
+
949
+ def _validator_status(output: str, returncode: int) -> str:
950
+ status = _extract_key(output, "status")
951
+ if status:
952
+ return status
953
+ return "OK" if returncode == 0 else "FAILED"
954
+
955
+
956
+ def _run_validator(cmd: list[str]) -> dict[str, object]:
957
+ proc = subprocess.run(
958
+ cmd,
959
+ cwd=REPO_ROOT,
960
+ capture_output=True,
961
+ text=True,
962
+ check=False,
963
+ )
964
+ output = proc.stdout
965
+ if proc.stderr.strip():
966
+ output = f"{output}\n{proc.stderr}" if output else proc.stderr
967
+ status = _validator_status(output, proc.returncode)
968
+ return {
969
+ "status": status,
970
+ "ok": status == "OK",
971
+ "output": output.strip(),
972
+ "returncode": proc.returncode,
973
+ "sha256": _extract_key(output, "sha256"),
974
+ }
975
+
976
+
977
+ def _count_jsonl_rows(path: Path) -> int:
978
+ count = 0
979
+ with path.open("r", encoding="utf-8") as fh:
980
+ for line in fh:
981
+ if line.strip():
982
+ count += 1
983
+ return count
984
+
985
+
986
+ def _is_relative_to(path: Path, base: Path) -> bool:
987
+ try:
988
+ path.relative_to(base)
989
+ return True
990
+ except ValueError:
991
+ return False
992
+
993
+
994
+ def _resolve_repo_path(path_raw: str, *, allowed_root: Path | None = None) -> Path:
995
+ raw = path_raw.strip()
996
+ if not raw:
997
+ raise ValueError("path cannot be empty")
998
+
999
+ candidate = Path(raw)
1000
+ resolved = candidate.resolve() if candidate.is_absolute() else (REPO_ROOT / candidate).resolve()
1001
+
1002
+ if not _is_relative_to(resolved, REPO_ROOT.resolve()):
1003
+ raise ValueError("path must stay within repository root")
1004
+ if allowed_root is not None and not _is_relative_to(resolved, allowed_root.resolve()):
1005
+ rel = allowed_root.relative_to(REPO_ROOT)
1006
+ raise ValueError(f"path must stay within {rel}")
1007
+ return resolved
1008
+
1009
+
1010
+ def _read_jsonl_page(path: Path, offset: int, limit: int) -> dict[str, object]:
1011
+ if offset < 0:
1012
+ offset = 0
1013
+ if limit < 1:
1014
+ limit = 1
1015
+ if limit > 200:
1016
+ limit = 200
1017
+
1018
+ rows: list[dict[str, object]] = []
1019
+ total = 0
1020
+ with path.open("r", encoding="utf-8") as fh:
1021
+ for lineno, line in enumerate(fh, 1):
1022
+ line = line.strip()
1023
+ if not line:
1024
+ continue
1025
+ if total >= offset and len(rows) < limit:
1026
+ value = json.loads(line)
1027
+ if not isinstance(value, dict):
1028
+ raise ValueError(f"line {lineno}: top-level value must be an object")
1029
+ rows.append(value)
1030
+ total += 1
1031
+ return {"rows": rows, "total": total, "offset": offset, "limit": limit}
1032
+
1033
+
1034
+ def _list_result_jsonl_files() -> list[dict[str, object]]:
1035
+ if not RESULTS_DIR_DEFAULT.exists():
1036
+ return []
1037
+ files: list[dict[str, object]] = []
1038
+ for path in sorted(RESULTS_DIR_DEFAULT.glob("*.jsonl")):
1039
+ stat = path.stat()
1040
+ files.append(
1041
+ {
1042
+ "path": str(path.relative_to(REPO_ROOT)),
1043
+ "size_bytes": int(stat.st_size),
1044
+ "modified_epoch": float(stat.st_mtime),
1045
+ }
1046
+ )
1047
+ files.sort(key=lambda x: x["modified_epoch"], reverse=True)
1048
+ return files
1049
+
1050
+
1051
+ def _result_payload(result: object) -> dict[str, object]:
1052
+ return {
1053
+ "answer": result.answer,
1054
+ "plan": result.plan,
1055
+ "mode": result.mode,
1056
+ "bypassed": result.bypassed,
1057
+ "route_score": result.route_score,
1058
+ "selected_plan_budget": result.selected_plan_budget,
1059
+ "plan_repaired": result.plan_repaired,
1060
+ "plan_latency_ms": round(result.plan_latency_ms, 2),
1061
+ "answer_latency_ms": round(result.answer_latency_ms, 2),
1062
+ "total_latency_ms": round(result.total_latency_ms, 2),
1063
+ }
1064
+
1065
+
1066
+ def _run_eval_mode(
1067
+ *,
1068
+ prompt: str,
1069
+ model: str,
1070
+ mode: str,
1071
+ ollama_url: str,
1072
+ lane_policy: str,
1073
+ bypass_short_prompts: bool,
1074
+ continuity_hint: object,
1075
+ ) -> dict[str, object]:
1076
+ config = QuickThinkConfig.with_model_profile(model=model, ollama_url=ollama_url)
1077
+ config.continuity_hint = str(continuity_hint).strip() if continuity_hint else None
1078
+ config.lane_policy = lane_policy
1079
+
1080
+ if mode == "direct":
1081
+ config.mode = "direct"
1082
+ config.adaptive_routing = False
1083
+ config.bypass_short_prompts = True
1084
+ config.bypass_char_threshold = 10_000_000
1085
+ else:
1086
+ config.mode = mode
1087
+ config.bypass_short_prompts = bypass_short_prompts
1088
+
1089
+ result = QuickThinkEngine(config).run(prompt)
1090
+ return _result_payload(result)
1091
+
1092
+
1093
+ def serve_ui(host: str = "127.0.0.1", port: int = 7860, open_browser: bool = False) -> None:
1094
+ state_lock = threading.Lock()
1095
+ state: dict[str, object] = {
1096
+ "preflight_ok": False,
1097
+ "dataset_sha256": None,
1098
+ "preflight_output": "",
1099
+ "last_ingestion": None,
1100
+ }
1101
+
1102
+ class Handler(BaseHTTPRequestHandler):
1103
+ def _json(self, code: int, payload: dict[str, object]) -> None:
1104
+ body = json.dumps(payload).encode("utf-8")
1105
+ self.send_response(code)
1106
+ self.send_header("Content-Type", "application/json; charset=utf-8")
1107
+ self.send_header("Content-Length", str(len(body)))
1108
+ self.end_headers()
1109
+ self.wfile.write(body)
1110
+
1111
+ def _html(self, body: str) -> None:
1112
+ raw = body.encode("utf-8")
1113
+ self.send_response(200)
1114
+ self.send_header("Content-Type", "text/html; charset=utf-8")
1115
+ self.send_header("Content-Length", str(len(raw)))
1116
+ self.end_headers()
1117
+ self.wfile.write(raw)
1118
+
1119
+ def do_GET(self) -> None: # noqa: N802
1120
+ parsed = urlsplit(self.path)
1121
+ path = parsed.path
1122
+ query = parse_qs(parsed.query)
1123
+
1124
+ if path == "/":
1125
+ self._html(HTML_PAGE)
1126
+ return
1127
+ if path == "/api/models":
1128
+ self._json(200, {"models": list(MODEL_PROFILES.keys())})
1129
+ return
1130
+ if path == "/api/state":
1131
+ with state_lock:
1132
+ payload = dict(state)
1133
+ self._json(200, payload)
1134
+ return
1135
+ if path == "/api/prompts":
1136
+ prompt_set = str(query.get("path", [str(PROMPT_SET_DEFAULT.relative_to(REPO_ROOT))])[0])
1137
+ offset = int(query.get("offset", ["0"])[0])
1138
+ limit = int(query.get("limit", ["20"])[0])
1139
+ try:
1140
+ resolved = _resolve_repo_path(prompt_set, allowed_root=PROMPTS_DIR_DEFAULT)
1141
+ if not resolved.exists():
1142
+ self._json(404, {"error": f"file not found: {prompt_set}"})
1143
+ return
1144
+ page = _read_jsonl_page(resolved, offset=offset, limit=limit)
1145
+ except Exception as exc:
1146
+ self._json(400, {"error": str(exc)})
1147
+ return
1148
+ self._json(
1149
+ 200,
1150
+ {
1151
+ "path": prompt_set,
1152
+ "rows": page["rows"],
1153
+ "total": page["total"],
1154
+ "offset": page["offset"],
1155
+ "limit": page["limit"],
1156
+ },
1157
+ )
1158
+ return
1159
+ if path == "/api/results/files":
1160
+ self._json(200, {"files": _list_result_jsonl_files()})
1161
+ return
1162
+ if path == "/api/results/rows":
1163
+ result_path = str(query.get("path", [""])[0]).strip()
1164
+ if not result_path:
1165
+ self._json(400, {"error": "path query parameter is required"})
1166
+ return
1167
+ offset = int(query.get("offset", ["0"])[0])
1168
+ limit = int(query.get("limit", ["20"])[0])
1169
+ try:
1170
+ resolved = _resolve_repo_path(result_path, allowed_root=RESULTS_DIR_DEFAULT)
1171
+ if not resolved.exists():
1172
+ self._json(404, {"error": f"file not found: {result_path}"})
1173
+ return
1174
+ page = _read_jsonl_page(resolved, offset=offset, limit=limit)
1175
+ except Exception as exc:
1176
+ self._json(400, {"error": str(exc)})
1177
+ return
1178
+ self._json(
1179
+ 200,
1180
+ {
1181
+ "path": result_path,
1182
+ "rows": page["rows"],
1183
+ "total": page["total"],
1184
+ "offset": page["offset"],
1185
+ "limit": page["limit"],
1186
+ },
1187
+ )
1188
+ return
1189
+ self._json(404, {"error": "not found"})
1190
+
1191
+ def do_POST(self) -> None: # noqa: N802
1192
+ try:
1193
+ raw_len = int(self.headers.get("Content-Length", "0"))
1194
+ raw = self.rfile.read(raw_len)
1195
+ payload = json.loads(raw.decode("utf-8"))
1196
+
1197
+ if self.path == "/api/preflight":
1198
+ prompt_set_path = str(payload.get("path", "docs/evals/prompt_set.jsonl")).strip()
1199
+ try:
1200
+ resolved_prompt_set = _resolve_repo_path(
1201
+ prompt_set_path,
1202
+ allowed_root=PROMPTS_DIR_DEFAULT,
1203
+ )
1204
+ except Exception as exc:
1205
+ self._json(400, {"error": str(exc), "preflight_ok": False})
1206
+ return
1207
+ cmd = [sys.executable, str(PROMPT_VALIDATOR), "--path", str(resolved_prompt_set)]
1208
+ result = _run_validator(cmd)
1209
+ with state_lock:
1210
+ state["preflight_ok"] = result["ok"]
1211
+ state["dataset_sha256"] = result["sha256"]
1212
+ state["preflight_output"] = result["output"]
1213
+ code = 200 if result["ok"] else 400
1214
+ self._json(
1215
+ code,
1216
+ {
1217
+ "preflight_ok": result["ok"],
1218
+ "status": result["status"],
1219
+ "dataset_sha256": result["sha256"],
1220
+ "output": result["output"],
1221
+ },
1222
+ )
1223
+ return
1224
+
1225
+ if self.path == "/api/ingest-run":
1226
+ run_path_raw = str(payload.get("path", "")).strip()
1227
+ if not run_path_raw:
1228
+ self._json(400, {"error": "path is required", "ingested": False})
1229
+ return
1230
+ try:
1231
+ resolved_run_path = _resolve_repo_path(run_path_raw, allowed_root=RESULTS_DIR_DEFAULT)
1232
+ except Exception as exc:
1233
+ self._json(400, {"error": str(exc), "ingested": False})
1234
+ return
1235
+ expected_prompts = int(payload.get("expected_prompts", 0) or 0)
1236
+ expected_runs = int(payload.get("expected_runs", 0) or 0)
1237
+ models = payload.get("models", [])
1238
+ if isinstance(models, str):
1239
+ models_list = [m for m in models.split() if m]
1240
+ else:
1241
+ models_list = [str(m).strip() for m in models if str(m).strip()]
1242
+
1243
+ cmd = [
1244
+ sys.executable,
1245
+ str(RESULTS_VALIDATOR),
1246
+ "--path",
1247
+ str(resolved_run_path),
1248
+ "--expected-prompts",
1249
+ str(expected_prompts),
1250
+ "--expected-runs",
1251
+ str(expected_runs),
1252
+ ]
1253
+ if models_list:
1254
+ cmd.extend(["--models", *models_list])
1255
+ result = _run_validator(cmd)
1256
+
1257
+ if not result["ok"]:
1258
+ with state_lock:
1259
+ state["last_ingestion"] = {
1260
+ "ingested": False,
1261
+ "path": run_path_raw,
1262
+ "output": result["output"],
1263
+ "status": result["status"],
1264
+ }
1265
+ self._json(
1266
+ 400,
1267
+ {
1268
+ "ingested": False,
1269
+ "status": result["status"],
1270
+ "output": result["output"],
1271
+ },
1272
+ )
1273
+ return
1274
+
1275
+ rows = _count_jsonl_rows(resolved_run_path)
1276
+ ingestion_payload = {
1277
+ "ingested": True,
1278
+ "path": str(resolved_run_path.relative_to(REPO_ROOT)),
1279
+ "rows": rows,
1280
+ "status": result["status"],
1281
+ "output": result["output"],
1282
+ }
1283
+ with state_lock:
1284
+ state["last_ingestion"] = ingestion_payload
1285
+ self._json(200, ingestion_payload)
1286
+ return
1287
+
1288
+ if self.path == "/api/run-eval-set":
1289
+ with state_lock:
1290
+ preflight_ok = bool(state.get("preflight_ok"))
1291
+ dataset_sha256 = state.get("dataset_sha256")
1292
+ if not preflight_ok:
1293
+ self._json(
1294
+ 409,
1295
+ {
1296
+ "error": "Preflight required: run validate_prompt_set.py and obtain status=OK before eval runs.",
1297
+ "dataset_sha256": dataset_sha256,
1298
+ },
1299
+ )
1300
+ return
1301
+
1302
+ prompt_set = str(payload.get("prompt_set", "docs/evals/prompt_set.jsonl")).strip()
1303
+ out_path = str(payload.get("out", "docs/evals/results/run_results.jsonl")).strip()
1304
+ manifest_out = str(payload.get("manifest_out", "docs/evals/results/run_manifest.json")).strip()
1305
+ runs = int(payload.get("runs", 3) or 3)
1306
+ limit = int(payload.get("limit", 0) or 0)
1307
+ ollama_url = str(payload.get("ollama_url", "http://localhost:11434")).strip()
1308
+ continuity_hint = payload.get("continuity_hint", None)
1309
+ models = payload.get("models", [])
1310
+ if isinstance(models, str):
1311
+ models_list = [m for m in models.split() if m]
1312
+ else:
1313
+ models_list = [str(m).strip() for m in models if str(m).strip()]
1314
+ if runs < 1:
1315
+ self._json(400, {"error": "runs must be >= 1"})
1316
+ return
1317
+ try:
1318
+ resolved_prompt_set = _resolve_repo_path(prompt_set, allowed_root=PROMPTS_DIR_DEFAULT)
1319
+ resolved_out_path = _resolve_repo_path(out_path, allowed_root=RESULTS_DIR_DEFAULT)
1320
+ resolved_manifest_out = _resolve_repo_path(manifest_out, allowed_root=RESULTS_DIR_DEFAULT)
1321
+ except Exception as exc:
1322
+ self._json(400, {"error": str(exc)})
1323
+ return
1324
+
1325
+ cmd = [
1326
+ sys.executable,
1327
+ str(RUN_SUITE_SCRIPT),
1328
+ "--prompt-set",
1329
+ str(resolved_prompt_set),
1330
+ "--out",
1331
+ str(resolved_out_path),
1332
+ "--manifest-out",
1333
+ str(resolved_manifest_out),
1334
+ "--runs",
1335
+ str(runs),
1336
+ "--ollama-url",
1337
+ ollama_url,
1338
+ ]
1339
+ if models_list:
1340
+ cmd.extend(["--models", *models_list])
1341
+ if limit > 0:
1342
+ cmd.extend(["--limit", str(limit)])
1343
+ if continuity_hint:
1344
+ cmd.extend(["--continuity-hint", str(continuity_hint)])
1345
+
1346
+ proc = subprocess.run(
1347
+ cmd,
1348
+ cwd=REPO_ROOT,
1349
+ capture_output=True,
1350
+ text=True,
1351
+ check=False,
1352
+ )
1353
+ output = (proc.stdout or "").strip()
1354
+ if (proc.stderr or "").strip():
1355
+ output = f"{output}\n{proc.stderr.strip()}" if output else proc.stderr.strip()
1356
+ if proc.returncode != 0:
1357
+ self._json(
1358
+ 400,
1359
+ {
1360
+ "error": "eval harness run failed",
1361
+ "returncode": proc.returncode,
1362
+ "output": output,
1363
+ },
1364
+ )
1365
+ return
1366
+ self._json(
1367
+ 200,
1368
+ {
1369
+ "status": "OK",
1370
+ "out_path": str(resolved_out_path.relative_to(REPO_ROOT)),
1371
+ "manifest_out": str(resolved_manifest_out.relative_to(REPO_ROOT)),
1372
+ "output": output,
1373
+ },
1374
+ )
1375
+ return
1376
+
1377
+ if self.path not in {"/api/ask", "/api/ask-all"}:
1378
+ self._json(404, {"error": "not found"})
1379
+ return
1380
+
1381
+ with state_lock:
1382
+ preflight_ok = bool(state.get("preflight_ok"))
1383
+ dataset_sha256 = state.get("dataset_sha256")
1384
+ if not preflight_ok:
1385
+ self._json(
1386
+ 409,
1387
+ {
1388
+ "error": "Preflight required: run validate_prompt_set.py and obtain status=OK before eval runs.",
1389
+ "dataset_sha256": dataset_sha256,
1390
+ },
1391
+ )
1392
+ return
1393
+
1394
+ prompt = str(payload.get("prompt", "")).strip()
1395
+ model = str(payload.get("model", "qwen2.5:1.5b")).strip()
1396
+ mode = str(payload.get("mode", "lite")).strip()
1397
+ ollama_url = str(payload.get("ollama_url", "http://localhost:11434")).strip()
1398
+ lane_policy = str(payload.get("lane_policy", "default")).strip()
1399
+ bypass_short_prompts = bool(payload.get("bypass_short_prompts", True))
1400
+ continuity_hint = payload.get("continuity_hint", None)
1401
+
1402
+ if not prompt:
1403
+ self._json(400, {"error": "prompt is required"})
1404
+ return
1405
+ if mode not in {"direct", "lite", "two_pass"}:
1406
+ self._json(400, {"error": "mode must be direct, lite, or two_pass"})
1407
+ return
1408
+ if lane_policy not in {"default", "strict_safe"}:
1409
+ self._json(400, {"error": "lane_policy must be default or strict_safe"})
1410
+ return
1411
+ if self.path == "/api/ask-all":
1412
+ results = {
1413
+ "direct": _run_eval_mode(
1414
+ prompt=prompt,
1415
+ model=model,
1416
+ mode="direct",
1417
+ ollama_url=ollama_url,
1418
+ lane_policy=lane_policy,
1419
+ bypass_short_prompts=bypass_short_prompts,
1420
+ continuity_hint=continuity_hint,
1421
+ ),
1422
+ "lite": _run_eval_mode(
1423
+ prompt=prompt,
1424
+ model=model,
1425
+ mode="lite",
1426
+ ollama_url=ollama_url,
1427
+ lane_policy=lane_policy,
1428
+ bypass_short_prompts=bypass_short_prompts,
1429
+ continuity_hint=continuity_hint,
1430
+ ),
1431
+ "two_pass": _run_eval_mode(
1432
+ prompt=prompt,
1433
+ model=model,
1434
+ mode="two_pass",
1435
+ ollama_url=ollama_url,
1436
+ lane_policy=lane_policy,
1437
+ bypass_short_prompts=bypass_short_prompts,
1438
+ continuity_hint=continuity_hint,
1439
+ ),
1440
+ }
1441
+ self._json(200, {"results": results})
1442
+ return
1443
+
1444
+ payload_out = _run_eval_mode(
1445
+ prompt=prompt,
1446
+ model=model,
1447
+ mode=mode,
1448
+ ollama_url=ollama_url,
1449
+ lane_policy=lane_policy,
1450
+ bypass_short_prompts=bypass_short_prompts,
1451
+ continuity_hint=continuity_hint,
1452
+ )
1453
+ self._json(200, payload_out)
1454
+ except Exception as exc: # pragma: no cover - network/runtime dependent
1455
+ self._json(500, {"error": str(exc)})
1456
+
1457
+ def log_message(self, format: str, *args: object) -> None:
1458
+ return
1459
+
1460
+ server = ThreadingHTTPServer((host, port), Handler)
1461
+ url = f"http://{host}:{port}"
1462
+ if open_browser:
1463
+ webbrowser.open(url)
1464
+ print(f"quickthink UI listening on {url}")
1465
+ server.serve_forever()