warden-agent 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +166 -2
- package/package.json +47 -4
- package/src/agent/catalog.mjs +85 -0
- package/src/agent/config.mjs +44 -0
- package/src/agent/credentials.mjs +38 -0
- package/src/agent/doctor.mjs +75 -0
- package/src/agent/headless.mjs +35 -0
- package/src/agent/loop.mjs +105 -0
- package/src/agent/models.mjs +77 -0
- package/src/agent/prompt.mjs +53 -0
- package/src/agent/providers.mjs +244 -0
- package/src/agent/session.mjs +53 -0
- package/src/agent/texttools.mjs +85 -0
- package/src/agent/tools.mjs +226 -0
- package/src/agent/update.mjs +62 -0
- package/src/cli.mjs +71 -0
- package/src/supervisor/arena/arena.mjs +148 -0
- package/src/supervisor/arena/harnesses.mjs +112 -0
- package/src/supervisor/arena/judge.mjs +76 -0
- package/src/supervisor/arena/subjects.mjs +64 -0
- package/src/supervisor/capture.mjs +30 -0
- package/src/supervisor/cli.mjs +223 -0
- package/src/supervisor/config.mjs +33 -0
- package/src/supervisor/daemon.mjs +187 -0
- package/src/supervisor/dashboard/index.html +222 -0
- package/src/supervisor/db.mjs +86 -0
- package/src/supervisor/providers.mjs +120 -0
- package/src/supervisor/redact.mjs +37 -0
- package/src/supervisor/review.mjs +111 -0
- package/src/tui/app.mjs +500 -0
- package/src/tui/render.mjs +108 -0
- package/src/version.mjs +4 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// Blind judge panel. Candidates are shown as anonymous letters in a shuffled order; every judge
|
|
2
|
+
// runs twice (order and reversed order) to cancel position bias. Judges that are also contestants
|
|
3
|
+
// are excluded, and same-vendor overlap is reported so you know how independent the panel was.
|
|
4
|
+
import { getProvider } from "../providers.mjs";
|
|
5
|
+
import { redactDiff, redact } from "../redact.mjs";
|
|
6
|
+
|
|
7
|
+
const SYSTEM = `You are an impartial senior engineer judging several anonymous solutions to the same coding task.
|
|
8
|
+
Score each candidate 1-10 on: does it correctly and completely do what was asked, is it minimal (no unrequested changes), is the code clear and safe.
|
|
9
|
+
Objective check results are given as evidence: a candidate that fails the checks cannot score above 4. Do not guess which model wrote what.
|
|
10
|
+
Reply with ONLY JSON: {"scores":{"A":number,...},"notes":{"A":"one sentence",...}}`;
|
|
11
|
+
|
|
12
|
+
const vendorOf = (provider, model) => {
|
|
13
|
+
const s = `${provider} ${model || ""}`.toLowerCase();
|
|
14
|
+
if (/claude|anthropic|opus|sonnet|haiku/.test(s)) return "anthropic";
|
|
15
|
+
if (/gemini|google/.test(s)) return "google";
|
|
16
|
+
if (/gpt|openai|\bo[134]\b/.test(s)) return "openai";
|
|
17
|
+
return provider;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const LETTERS = "ABCDEFGHIJ";
|
|
21
|
+
|
|
22
|
+
function shuffle(arr, rng) {
|
|
23
|
+
const a = arr.slice();
|
|
24
|
+
for (let i = a.length - 1; i > 0; i--) { const j = Math.floor(rng() * (i + 1)); [a[i], a[j]] = [a[j], a[i]]; }
|
|
25
|
+
return a;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Decide who may judge. contestants: [{subject, provider?, model}]. judges: [{provider, model}]
|
|
29
|
+
export function pickJudges(judges, contestants) {
|
|
30
|
+
const warnings = [], used = [];
|
|
31
|
+
const cVendors = new Set(contestants.map((c) => vendorOf(c.provider || "", c.model)));
|
|
32
|
+
for (const j of judges) {
|
|
33
|
+
const same = contestants.some((c) => (c.model || "") === (j.model || "") && vendorOf(c.provider || "", c.model) === vendorOf(j.provider, j.model));
|
|
34
|
+
if (same) { warnings.push(`excluded judge ${j.provider}:${j.model ?? "default"} (same model as a contestant)`); continue; }
|
|
35
|
+
if (cVendors.has(vendorOf(j.provider, j.model))) warnings.push(`judge ${j.provider}:${j.model ?? "default"} shares a vendor with a contestant: possible self-preference bias`);
|
|
36
|
+
used.push(j);
|
|
37
|
+
}
|
|
38
|
+
return { used, warnings };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function judgePanel({ task, candidates, judges, cfg, rng = Math.random, maxDiffChars = 24000, provider = getProvider }) {
|
|
42
|
+
// candidates: [{subject, diff, checks:[{name,pass}], ok}]
|
|
43
|
+
if (!judges.length) return { scores: {}, detail: [], note: "no valid judges" };
|
|
44
|
+
const perDiff = Math.max(2000, Math.floor(maxDiffChars / Math.max(1, candidates.length)));
|
|
45
|
+
const baseOrder = shuffle(candidates.map((_, i) => i), rng);
|
|
46
|
+
const orders = [baseOrder, baseOrder.slice().reverse()];
|
|
47
|
+
const totals = candidates.map(() => []);
|
|
48
|
+
const detail = [];
|
|
49
|
+
|
|
50
|
+
await Promise.all(judges.flatMap((j) => orders.map(async (order, oi) => {
|
|
51
|
+
const letterOf = new Map(order.map((ci, pos) => [ci, LETTERS[pos]]));
|
|
52
|
+
const prompt = `## Task\n${redact(task.prompt)}\n\n` + order.map((ci, pos) => {
|
|
53
|
+
const c = candidates[ci];
|
|
54
|
+
const checks = c.checks.length ? c.checks.map((k) => `${k.name}: ${k.pass ? "PASS" : "FAIL"}`).join(", ") : "no checks";
|
|
55
|
+
let d = redactDiff(c.diff || "");
|
|
56
|
+
if (d.length > perDiff) d = d.slice(0, perDiff) + "\n[diff truncated]";
|
|
57
|
+
return `## Candidate ${LETTERS[pos]}\nChecks: ${checks}\n${c.ok ? "" : "(the agent errored or timed out)\n"}\`\`\`diff\n${d || "(no changes)"}\n\`\`\``;
|
|
58
|
+
}).join("\n\n");
|
|
59
|
+
try {
|
|
60
|
+
const r = await provider(j.provider, cfg).complete({ system: SYSTEM, prompt, model: j.model });
|
|
61
|
+
const m = r.text.match(/\{[\s\S]*\}/);
|
|
62
|
+
const out = JSON.parse(m[0]);
|
|
63
|
+
for (const [ci, letter] of letterOf) {
|
|
64
|
+
const s = Number(out.scores?.[letter]);
|
|
65
|
+
if (Number.isFinite(s) && s >= 1 && s <= 10) totals[ci].push(s);
|
|
66
|
+
}
|
|
67
|
+
detail.push({ judge: `${j.provider}:${j.model ?? "default"}`, order: oi, scores: Object.fromEntries([...letterOf].map(([ci, l]) => [candidates[ci].subject, out.scores?.[l] ?? null])), notes: Object.fromEntries([...letterOf].map(([ci, l]) => [candidates[ci].subject, out.notes?.[l] ?? ""])), costUsd: r.costUsd });
|
|
68
|
+
} catch (err) {
|
|
69
|
+
detail.push({ judge: `${j.provider}:${j.model ?? "default"}`, order: oi, error: err.message });
|
|
70
|
+
}
|
|
71
|
+
})));
|
|
72
|
+
|
|
73
|
+
const scores = {};
|
|
74
|
+
candidates.forEach((c, i) => { scores[c.subject] = totals[i].length ? +(totals[i].reduce((a, b) => a + b, 0) / totals[i].length).toFixed(2) : null; });
|
|
75
|
+
return { scores, detail };
|
|
76
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Subjects: the things being compared. Each runs a task inside an isolated worktree and
|
|
2
|
+
// returns { ok, error, durationMs, costUsd|null, tokens|null, turns|null }.
|
|
3
|
+
// claude-code Claude Code headless with --model (e.g. opus, sonnet, haiku, or a full model id)
|
|
4
|
+
// command any other harness/CLI: "cmd" template run in a shell inside the worktree.
|
|
5
|
+
// The prompt arrives on stdin AND in env WARDEN_PROMPT (never interpolated into the
|
|
6
|
+
// command line, so quoting can't break or inject). Example: "gemini -p \"$WARDEN_PROMPT\""
|
|
7
|
+
import { spawn } from "node:child_process";
|
|
8
|
+
import { harnessCommand, friendlyError } from "./harnesses.mjs";
|
|
9
|
+
|
|
10
|
+
function run(cmd, args, { cwd, input, timeoutMs, shell = false, env }) {
|
|
11
|
+
return new Promise((resolve) => {
|
|
12
|
+
const t0 = Date.now();
|
|
13
|
+
const p = spawn(cmd, args, { cwd, shell, windowsHide: true, env: { ...process.env, ...env } });
|
|
14
|
+
let out = "", err = "", killed = false;
|
|
15
|
+
const timer = setTimeout(() => { killed = true; p.kill(); }, timeoutMs);
|
|
16
|
+
p.stdout.on("data", (d) => (out += d));
|
|
17
|
+
p.stderr.on("data", (d) => (err += d));
|
|
18
|
+
p.on("error", (e) => { clearTimeout(timer); resolve({ code: -1, out, err: e.message, ms: Date.now() - t0, killed }); });
|
|
19
|
+
p.on("close", (code) => { clearTimeout(timer); resolve({ code, out, err, ms: Date.now() - t0, killed }); });
|
|
20
|
+
if (input != null) p.stdin.end(input); else p.stdin.end();
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export { run };
|
|
25
|
+
|
|
26
|
+
// Tools a subject may use. Bash is limited to the task's own check commands unless the task widens it.
|
|
27
|
+
export function defaultAllowedTools(task) {
|
|
28
|
+
const bash = (task.checks || []).map((c) => `Bash(${c.cmd}:*)`);
|
|
29
|
+
return task.allowedTools || ["Read", "Edit", "Write", "Glob", "Grep", ...bash];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export async function runSubject(subject, task, cwd, { timeoutMs = 600_000, maxUsd = 1 } = {}) {
|
|
33
|
+
if (subject.kind === "claude-code") {
|
|
34
|
+
const args = ["-p", "--output-format", "json", "--no-session-persistence", "--permission-mode", "acceptEdits",
|
|
35
|
+
"--max-budget-usd", String(subject.maxUsd ?? maxUsd), "--allowedTools", ...defaultAllowedTools(task)];
|
|
36
|
+
if (subject.model) args.push("--model", subject.model);
|
|
37
|
+
const r = await run("claude", args, { cwd, input: task.prompt, timeoutMs });
|
|
38
|
+
if (r.killed) return { ok: false, error: "timed out", durationMs: r.ms, costUsd: null, tokens: null, turns: null };
|
|
39
|
+
let j;
|
|
40
|
+
try { j = JSON.parse(r.out); } catch { return { ok: false, error: `bad output (exit ${r.code}): ${(r.err || r.out).slice(0, 200)}`, durationMs: r.ms, costUsd: null, tokens: null, turns: null }; }
|
|
41
|
+
const u = j.usage || {};
|
|
42
|
+
return {
|
|
43
|
+
ok: !j.is_error, error: j.is_error ? String(j.result || "error").slice(0, 300) : null,
|
|
44
|
+
durationMs: j.duration_ms ?? r.ms, costUsd: j.total_cost_usd ?? null,
|
|
45
|
+
tokens: (u.input_tokens ?? 0) + (u.output_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0),
|
|
46
|
+
turns: j.num_turns ?? null,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
if (subject.kind === "command") {
|
|
50
|
+
if (!subject.cmd) return { ok: false, error: "command subject needs `cmd`", durationMs: 0, costUsd: null, tokens: null, turns: null };
|
|
51
|
+
const r = await run(subject.cmd, [], { cwd, input: task.prompt, timeoutMs, shell: true, env: { WARDEN_PROMPT: task.prompt } });
|
|
52
|
+
return { ok: r.code === 0 && !r.killed, error: r.killed ? "timed out" : r.code === 0 ? null : `exit ${r.code}: ${r.err.slice(0, 200)}`, durationMs: r.ms, costUsd: null, tokens: null, turns: null };
|
|
53
|
+
}
|
|
54
|
+
const hc = harnessCommand(subject, task.prompt);
|
|
55
|
+
if (hc) {
|
|
56
|
+
// argv only, no shell: the prompt is one argument and cannot be interpreted as commands
|
|
57
|
+
const r = await run(hc.cmd, hc.args, { cwd, input: null, timeoutMs });
|
|
58
|
+
if (r.code === -1) return { ok: false, error: `could not start ${subject.kind}: ${r.err} (is it installed and on PATH?)`, durationMs: r.ms, costUsd: null, tokens: null, turns: null };
|
|
59
|
+
const parsed = hc.parse ? hc.parse(r.out) : {};
|
|
60
|
+
const fail = r.killed ? "timed out" : r.code !== 0 ? friendlyError(r.out, r.err, r.code) : parsed.error || null;
|
|
61
|
+
return { ok: !fail, error: fail, durationMs: r.ms, costUsd: parsed.costUsd ?? null, tokens: parsed.tokens ?? null, turns: parsed.turns ?? null };
|
|
62
|
+
}
|
|
63
|
+
return { ok: false, error: `unknown subject kind "${subject.kind}"`, durationMs: 0, costUsd: null, tokens: null, turns: null };
|
|
64
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Repo + transcript capture, run when a session stops.
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
|
|
5
|
+
const MAX_DIFF = 2_000_000; // bytes, cap so a huge diff can't bloat the DB
|
|
6
|
+
const MAX_TRANSCRIPT = 20_000_000;
|
|
7
|
+
|
|
8
|
+
function git(cwd, args) {
|
|
9
|
+
return new Promise((resolve) => {
|
|
10
|
+
execFile("git", ["-C", cwd, ...args], { maxBuffer: 50_000_000, windowsHide: true }, (err, stdout) =>
|
|
11
|
+
resolve(err ? null : stdout)
|
|
12
|
+
);
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function captureSnapshot(cwd, transcriptPath) {
|
|
17
|
+
let git_status = null, git_diff = null, transcript_copy = null;
|
|
18
|
+
if (cwd && fs.existsSync(cwd)) {
|
|
19
|
+
git_status = await git(cwd, ["status", "--porcelain"]);
|
|
20
|
+
// tracked changes vs HEAD; fall back to plain diff if the repo has no commits
|
|
21
|
+
git_diff = (await git(cwd, ["diff", "HEAD"])) ?? (await git(cwd, ["diff"]));
|
|
22
|
+
if (git_diff && git_diff.length > MAX_DIFF) git_diff = git_diff.slice(0, MAX_DIFF) + "\n[truncated]";
|
|
23
|
+
}
|
|
24
|
+
try {
|
|
25
|
+
if (transcriptPath && fs.statSync(transcriptPath).size <= MAX_TRANSCRIPT) {
|
|
26
|
+
transcript_copy = fs.readFileSync(transcriptPath, "utf8");
|
|
27
|
+
}
|
|
28
|
+
} catch {}
|
|
29
|
+
return { git_status, git_diff, transcript_copy };
|
|
30
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// warden CLI: daemon | watch | sessions | replay <id> | diff <id>
|
|
3
|
+
import readline from "node:readline/promises";
|
|
4
|
+
import { startDaemon, PORT } from "./daemon.mjs";
|
|
5
|
+
import { loadConfig } from "./config.mjs";
|
|
6
|
+
import { ROLES, buildContext, estimate, planReviewers, runReviewers } from "./review.mjs";
|
|
7
|
+
import { resolveSpec } from "./providers.mjs";
|
|
8
|
+
|
|
9
|
+
import { openDb } from "./db.mjs";
|
|
10
|
+
import { loadTask, runArena, rank, leaderboard } from "./arena/arena.mjs";
|
|
11
|
+
import { pickJudges } from "./arena/judge.mjs";
|
|
12
|
+
import { defaultAllowedTools } from "./arena/subjects.mjs";
|
|
13
|
+
import { HARNESSES, describeHarness, resolveBin } from "./arena/harnesses.mjs";
|
|
14
|
+
import { execFileSync } from "node:child_process";
|
|
15
|
+
|
|
16
|
+
const s_tools = (task) => defaultAllowedTools(task).join(", ") + " (claude-code subjects only)";
|
|
17
|
+
const fromRow = (r) => ({ subject: r.subject, model: r.model, ok: !!r.ok, error: r.error, checks: JSON.parse(r.checks || "[]"), checksPassed: !!r.checks_passed, judgeAvg: r.judge_avg, costUsd: r.cost_usd, durationMs: r.duration_ms, tokens: r.tokens, turns: r.turns, filesChanged: r.files_changed, linesAdded: r.lines_added, linesRemoved: r.lines_removed, human: r.human });
|
|
18
|
+
|
|
19
|
+
function printArena(runId, ranked, task) {
|
|
20
|
+
const ms = (x) => (x == null ? "-" : x >= 60000 ? `${(x / 60000).toFixed(1)}m` : `${(x / 1000).toFixed(0)}s`);
|
|
21
|
+
console.log(`\n${c.dim}run ${runId} ${String(task.prompt || "").slice(0, 80)}${c.reset}`);
|
|
22
|
+
console.log(`${"#".padEnd(3)}${"subject".padEnd(16)} ${"checks".padEnd(8)} ${"judge".padStart(6)} ${"cost".padStart(8)} ${"time".padStart(6)} ${"tokens".padStart(8)} ${"turns".padStart(5)} ${"files +/-".padStart(12)}`);
|
|
23
|
+
ranked.forEach((r, i) => {
|
|
24
|
+
if (!r.ok) r._unranked = true;
|
|
25
|
+
const chk = !r.ok ? c.red + "ERROR" + c.reset : r.checksPassed ? c.green + "PASS" + c.reset : c.red + "FAIL" + c.reset;
|
|
26
|
+
const pad = (s, n) => s + " ".repeat(Math.max(0, n - s.replace(/\x1b\[[0-9;]*m/g, "").length));
|
|
27
|
+
console.log(`${(r._unranked ? "-" : String(i + 1)).padEnd(3)}${r.subject.padEnd(16)} ${pad(chk, 8)} ${(r.judgeAvg == null ? "-" : r.judgeAvg.toFixed(1)).padStart(6)} ${(r.costUsd == null ? "?" : "$" + r.costUsd.toFixed(3)).padStart(8)} ${ms(r.durationMs).padStart(6)} ${String(r.tokens ?? "-").padStart(8)} ${String(r.turns ?? "-").padStart(5)} ${(`${r.filesChanged ?? "-"} +${r.linesAdded ?? 0}/-${r.linesRemoved ?? 0}`).padStart(12)}`);
|
|
28
|
+
if (!r.ok) console.log(` ${c.dim}${r.error}${c.reset}`);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function printReport(r) {
|
|
33
|
+
const vc = { pass: c.green, concerns: c.yellow, fail: c.red, error: c.red }[r.verdict] || "";
|
|
34
|
+
console.log(`\n${vc}VERDICT: ${String(r.verdict).toUpperCase()}${c.reset} ${c.dim}cost $${(r.cost_usd ?? 0).toFixed(3)}${r.cost_unknown_for?.length ? " + unpriced: " + r.cost_unknown_for.join(", ") : ""}${c.reset}`);
|
|
35
|
+
for (const x of r.roles) console.log(` ${x.ok ? "•" : "x"} ${x.role.padEnd(12)} ${x.ok ? x.verdict : "ERROR " + x.error}${x.summary ? c.dim + " " + x.summary + c.reset : ""}`);
|
|
36
|
+
for (const f of r.findings) {
|
|
37
|
+
const sc = { high: c.red, medium: c.yellow, low: c.dim }[f.severity] || "";
|
|
38
|
+
console.log(`\n ${sc}[${f.severity}]${c.reset} ${f.file ?? ""}${f.line ? ":" + f.line : ""} ${c.dim}(${f.roles.join(", ")})${c.reset}\n ${f.message}`);
|
|
39
|
+
}
|
|
40
|
+
if (!r.findings.length) console.log("\n no findings");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const BASE = `http://127.0.0.1:${PORT}`;
|
|
44
|
+
const c = { dim: "\x1b[2m", cyan: "\x1b[36m", green: "\x1b[32m", yellow: "\x1b[33m", magenta: "\x1b[35m", red: "\x1b[31m", reset: "\x1b[0m" };
|
|
45
|
+
const col = { PreToolUse: c.cyan, PostToolUse: c.green, UserPromptSubmit: c.magenta, Stop: c.yellow, SubagentStop: c.yellow, Snapshot: c.dim };
|
|
46
|
+
const line = (e) =>
|
|
47
|
+
`${c.dim}${new Date(e.ts).toLocaleTimeString()} ${String(e.session_id || "").slice(0, 8)}${c.reset} ` +
|
|
48
|
+
`${col[e.type] || ""}${e.type.padEnd(16)}${c.reset} ${e.tool_name ? e.tool_name + " " : ""}${e.summary || ""}` +
|
|
49
|
+
(e.duration_ms != null ? ` ${c.dim}${e.duration_ms}ms${c.reset}` : "");
|
|
50
|
+
|
|
51
|
+
async function get(path) {
|
|
52
|
+
try { const r = await fetch(BASE + path); if (!r.ok) throw new Error(`${r.status}`); return await r.json(); }
|
|
53
|
+
catch (err) { console.error(`${c.red}cannot reach daemon at ${BASE}${c.reset} (${err.message}). Start it: warden daemon`); process.exit(1); }
|
|
54
|
+
}
|
|
55
|
+
// resolve a session id prefix
|
|
56
|
+
async function resolveId(prefix) {
|
|
57
|
+
const all = await get("/sessions");
|
|
58
|
+
const hit = all.filter((s) => s.id.startsWith(prefix));
|
|
59
|
+
if (hit.length !== 1) { console.error(hit.length ? "ambiguous id prefix" : "no such session"); process.exit(1); }
|
|
60
|
+
return hit[0].id;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Wrapped in a function (no top-level await) so the standalone build can bundle it.
|
|
64
|
+
async function main() {
|
|
65
|
+
const [cmd, arg] = process.argv.slice(2);
|
|
66
|
+
switch (cmd) {
|
|
67
|
+
case "daemon":
|
|
68
|
+
await startDaemon();
|
|
69
|
+
break;
|
|
70
|
+
case "watch": {
|
|
71
|
+
const res = await fetch(BASE + "/stream").catch(() => null);
|
|
72
|
+
if (!res) { console.error("daemon not running. Start it: warden daemon"); process.exit(1); }
|
|
73
|
+
console.log(`${c.dim}watching ${BASE}/stream (ctrl+c to quit)${c.reset}`);
|
|
74
|
+
const dec = new TextDecoder(); let buf = "";
|
|
75
|
+
for await (const chunk of res.body) {
|
|
76
|
+
buf += dec.decode(chunk, { stream: true });
|
|
77
|
+
let i;
|
|
78
|
+
while ((i = buf.indexOf("\n\n")) >= 0) {
|
|
79
|
+
const msg = buf.slice(0, i); buf = buf.slice(i + 2);
|
|
80
|
+
if (msg.startsWith("data: ")) console.log(line(JSON.parse(msg.slice(6))));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
case "sessions": {
|
|
86
|
+
for (const s of await get("/sessions")) {
|
|
87
|
+
console.log(`${s.id.slice(0, 8)} ${s.status.padEnd(8)} ${String(s.event_count).padStart(4)} events ${new Date(s.started_at).toLocaleString()} ${c.dim}${s.cwd}${c.reset}\n ${c.dim}${(s.first_prompt || "").slice(0, 90)}${c.reset}`);
|
|
88
|
+
}
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
case "replay": {
|
|
92
|
+
if (!arg) { console.error("usage: warden replay <session-id-prefix>"); process.exit(1); }
|
|
93
|
+
const id = await resolveId(arg);
|
|
94
|
+
for (const e of await get(`/sessions/${id}/events`)) console.log(line({ ...e, session_id: id }));
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
case "diff": {
|
|
98
|
+
if (!arg) { console.error("usage: warden diff <session-id-prefix>"); process.exit(1); }
|
|
99
|
+
const snap = await get(`/sessions/${await resolveId(arg)}/snapshot`);
|
|
100
|
+
console.log(`${c.dim}git status:${c.reset}\n${snap.git_status || "(clean or not a repo)"}\n${c.dim}git diff:${c.reset}\n${snap.git_diff || "(none)"}`);
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "review": {
|
|
104
|
+
if (!arg) { console.error("usage: warden review <id> [--yes] [--roles a,b] [--with provider:model,provider:model]"); process.exit(1); }
|
|
105
|
+
const flags = process.argv.slice(4);
|
|
106
|
+
const flagVal = (n) => flags.find((f) => f.startsWith(`--${n}=`))?.slice(n.length + 3) ?? (flags.includes(`--${n}`) ? flags[flags.indexOf(`--${n}`) + 1] : null);
|
|
107
|
+
const cfg = loadConfig();
|
|
108
|
+
const rolesArg = flagVal("roles"), withArg = flagVal("with");
|
|
109
|
+
const specs = planReviewers(cfg, { roles: rolesArg ? rolesArg.split(",") : null, withModels: withArg ? withArg.split(",") : null });
|
|
110
|
+
const bad = specs.filter((s) => !ROLES[s.role]);
|
|
111
|
+
if (bad.length) { console.error(`unknown role(s): ${bad.map((s) => s.role).join(", ")}. Available: ${Object.keys(ROLES).join(", ")}`); process.exit(1); }
|
|
112
|
+
try { specs.forEach((s) => s.provider === "claude-cli" || resolveSpec(cfg, s.provider)); } catch (e) { console.error(e.message); process.exit(1); }
|
|
113
|
+
if (cfg.mode === "observe") { console.error("mode is 'observe': reviews are disabled. Change mode in ~/.warden/config.json"); process.exit(1); }
|
|
114
|
+
|
|
115
|
+
const id = await resolveId(arg);
|
|
116
|
+
const [session, events, snap] = await Promise.all([get("/sessions").then((a) => a.find((s) => s.id === id)), get(`/sessions/${id}/events`), get(`/sessions/${id}/snapshot`)]);
|
|
117
|
+
const ctx = buildContext(session, events, snap, cfg);
|
|
118
|
+
const est = estimate(ctx, specs, cfg);
|
|
119
|
+
const spent = (await get("/spend/today")).usd;
|
|
120
|
+
// where the (redacted) context will actually be sent
|
|
121
|
+
const dest = (p) => { if (p === "claude-cli") return "Anthropic via your Claude Code login"; try { return new URL(resolveSpec(cfg, p).baseUrl).host; } catch { return p; } };
|
|
122
|
+
|
|
123
|
+
console.log(`\nReview proposal for session ${id.slice(0, 8)} ${c.dim}${(session.first_prompt || "").slice(0, 70)}${c.reset}`);
|
|
124
|
+
console.log(` reviewers:`);
|
|
125
|
+
for (const s of specs) console.log(` ${s.label.padEnd(42)} ${c.dim}-> ${dest(s.provider)}${c.reset}`);
|
|
126
|
+
console.log(` sends: ${ctx.length} chars (secrets redacted) to each destination above`);
|
|
127
|
+
console.log(` est. cost: ~$${est.usd}${est.unknown.length ? ` + UNKNOWN for ${est.unknown.length} reviewer(s) (no price configured)` : ""} (today's spend $${spent.toFixed(3)} of $${cfg.maxUsdPerDay}; output capped at ${cfg.maxOutputTokens} tokens)`);
|
|
128
|
+
if (spent + est.usd > cfg.maxUsdPerDay) { console.error(`${c.red}Refusing: would exceed the daily cap of $${cfg.maxUsdPerDay}.${c.reset}`); process.exit(1); }
|
|
129
|
+
|
|
130
|
+
if (!flags.includes("--yes")) {
|
|
131
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
132
|
+
const ans = await rl.question("Approve and run? [y/N] "); rl.close();
|
|
133
|
+
if (!/^y(es)?$/i.test(ans.trim())) { console.log("Cancelled. Nothing was sent."); break; }
|
|
134
|
+
}
|
|
135
|
+
const report = await runReviewers({ context: ctx, cfg, specs, onProgress: (r, s) => console.log(`${c.dim} ${r}: ${s}${c.reset}`) });
|
|
136
|
+
await fetch(BASE + "/reviews", { method: "POST", body: JSON.stringify({ session_id: id, ...report }) }).catch(() => {});
|
|
137
|
+
printReport(report);
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
case "reviews": {
|
|
141
|
+
if (!arg) { console.error("usage: warden reviews <id>"); process.exit(1); }
|
|
142
|
+
for (const r of await get(`/sessions/${await resolveId(arg)}/reviews`)) { console.log(`\n${c.dim}${new Date(r.ts).toLocaleString()}${c.reset}`); printReport(JSON.parse(r.report)); }
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
case "harnesses": {
|
|
146
|
+
// which agent CLIs can the Arena use on this machine?
|
|
147
|
+
for (const [kind, h] of [["claude-code", { bin: "claude", evidence: "used by Warden itself" }], ...Object.entries(HARNESSES)]) {
|
|
148
|
+
const { cmd, pre } = resolveBin(h.bin);
|
|
149
|
+
let ver = null;
|
|
150
|
+
try { ver = execFileSync(cmd, [...pre, "--version"], { encoding: "utf8", timeout: 15000, windowsHide: true, stdio: ["ignore", "pipe", "ignore"] }).trim().split("\n")[0]; } catch {}
|
|
151
|
+
console.log(`${(ver ? c.green + "found " : c.red + "missing ") + c.reset}${kind.padEnd(12)} ${ver ? ver.padEnd(14) : "".padEnd(14)} ${c.dim}${h.evidence}${c.reset}`);
|
|
152
|
+
}
|
|
153
|
+
console.log(`${c.dim}gemini: no adapter (docs say Gemini CLI was replaced by Antigravity CLI; use a generic "command" subject).${c.reset}`);
|
|
154
|
+
break;
|
|
155
|
+
}
|
|
156
|
+
case "arena": {
|
|
157
|
+
const [, , , sub, a1, a2] = process.argv; // warden arena <sub> ...
|
|
158
|
+
const flags = process.argv.slice(3);
|
|
159
|
+
const db = openDb();
|
|
160
|
+
const fmtMs = (ms) => (ms == null ? "-" : ms >= 60000 ? `${(ms / 60000).toFixed(1)}m` : `${(ms / 1000).toFixed(0)}s`);
|
|
161
|
+
const fmtUsd = (u) => (u == null ? "?" : `$${u.toFixed(3)}`);
|
|
162
|
+
if (sub === "run") {
|
|
163
|
+
if (!a1) { console.error("usage: warden arena run <task.json> [--yes] [--keep]"); process.exit(1); }
|
|
164
|
+
const cfg = loadConfig();
|
|
165
|
+
const task = loadTask(a1);
|
|
166
|
+
const judgeCfg = (task.judges || cfg.judges || []).map((j) => (typeof j === "string" ? { provider: j.split(":")[0], model: j.split(":").slice(1).join(":") || null } : j));
|
|
167
|
+
const { used, warnings } = pickJudges(judgeCfg, task.subjects.map((s) => ({ subject: s.name, provider: s.kind === "claude-code" ? "claude-cli" : "", model: s.model })));
|
|
168
|
+
const cap = task.maxUsdPerSubject ?? 1;
|
|
169
|
+
console.log(`\nArena proposal: ${c.dim}${task.id}${c.reset}\n task: ${task.prompt.slice(0, 120)}`);
|
|
170
|
+
console.log(` repo: ${task.repo} (base ${task.base || "HEAD"}; each subject gets its own git worktree, your checkout is not touched)`);
|
|
171
|
+
console.log(` subjects:`);
|
|
172
|
+
for (const s of task.subjects) {
|
|
173
|
+
console.log(` ${s.name.padEnd(16)} ${s.kind}${s.model ? " / " + s.model : ""}${s.cmd ? " cmd: " + s.cmd : ""}${s.extraArgs?.length ? " extra args: " + s.extraArgs.join(" ") : ""}`);
|
|
174
|
+
const hd = describeHarness(s.kind);
|
|
175
|
+
if (hd) console.log(` ${c.yellow}${hd.risk}${c.reset}\n ${c.dim}adapter status: ${hd.evidence}${c.reset}`);
|
|
176
|
+
}
|
|
177
|
+
console.log(` tools: ${s_tools(task)}`);
|
|
178
|
+
console.log(` checks run in each worktree after the subject finishes (commands from the task file):`);
|
|
179
|
+
for (const k of task.checks) console.log(` ${k.name || ""}: ${k.cmd}`);
|
|
180
|
+
console.log(` judges: ${used.length ? used.map((j) => `${j.provider}:${j.model ?? "default"}`).join(", ") : "none"} (blind, each run twice with the order swapped)`);
|
|
181
|
+
for (const w of warnings) console.log(` ${c.yellow}note: ${w}${c.reset}`);
|
|
182
|
+
console.log(` max spend: up to $${(cap * task.subjects.length).toFixed(2)} for subjects (${cap}/subject cap) + judging`);
|
|
183
|
+
console.log(` subjects run ${task.subjects.some((s) => s.kind === "command") ? c.yellow + "arbitrary commands from the task file (command subjects) " + c.reset : ""}with ${task.subjects.length} parallel agents, each editing files in its worktree.`);
|
|
184
|
+
if (!flags.includes("--yes")) {
|
|
185
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
186
|
+
const ans = await rl.question("Approve and run? [y/N] "); rl.close();
|
|
187
|
+
if (!/^y(es)?$/i.test(ans.trim())) { console.log("Cancelled. Nothing was run."); break; }
|
|
188
|
+
}
|
|
189
|
+
const out = await runArena({ task, db, cfg, keep: flags.includes("--keep"), onProgress: (n, s) => console.log(`${c.dim} ${n}: ${s}${c.reset}`) });
|
|
190
|
+
printArena(out.runId, out.ranked, task);
|
|
191
|
+
for (const e of out.judgeErrors) console.log(`${c.yellow}judge error (${e.judge}): ${e.error}${c.reset}`);
|
|
192
|
+
console.log(`\n${c.dim}saved as run ${out.runId}. Rate: warden arena rate ${out.runId} <subject> up|down${c.reset}`);
|
|
193
|
+
} else if (sub === "board") {
|
|
194
|
+
const rows = leaderboard(db, a1);
|
|
195
|
+
if (!rows.length) { console.log("no arena runs yet"); break; }
|
|
196
|
+
console.log(`\n${"subject".padEnd(16)} ${"runs".padStart(4)} ${"wins".padStart(5)} ${"checks".padStart(7)} ${"judge".padStart(6)} ${"avg cost".padStart(9)} ${"avg time".padStart(9)} ${"human".padStart(6)}`);
|
|
197
|
+
for (const r of rows) console.log(`${r.subject.padEnd(16)} ${String(r.runs).padStart(4)} ${(r.winRate * 100).toFixed(0).padStart(4)}% ${(r.passRate * 100).toFixed(0).padStart(6)}% ${(r.judgeAvg == null ? "-" : r.judgeAvg.toFixed(1)).padStart(6)} ${fmtUsd(r.costAvg).padStart(9)} ${fmtMs(r.msAvg).padStart(9)} ${(r.human >= 0 ? "+" : "") + r.human}`.replace("NaN", "-"));
|
|
198
|
+
console.log(`${c.dim}${rows[0].runs < 5 ? "Few runs so far: treat rankings as anecdotes, not evidence." : ""}${c.reset}`);
|
|
199
|
+
} else if (sub === "show") {
|
|
200
|
+
const run = db.prepare("SELECT * FROM arena_runs WHERE id LIKE ?").get(`${a1 || ""}%`);
|
|
201
|
+
if (!run) { console.error("no such run"); process.exit(1); }
|
|
202
|
+
const rows = db.prepare("SELECT * FROM arena_results WHERE run_id=?").all(run.id);
|
|
203
|
+
printArena(run.id, rank(rows.map(fromRow)), { prompt: run.prompt });
|
|
204
|
+
} else if (sub === "diff") {
|
|
205
|
+
const row = db.prepare("SELECT diff FROM arena_results WHERE run_id LIKE ? AND subject=?").get(`${a1 || ""}%`, a2);
|
|
206
|
+
console.log(row?.diff || "(no diff)");
|
|
207
|
+
} else if (sub === "rate") {
|
|
208
|
+
const val = { up: 1, down: -1 }[flags[3]];
|
|
209
|
+
if (!a1 || !a2 || val == null) { console.error("usage: warden arena rate <runId> <subject> up|down"); process.exit(1); }
|
|
210
|
+
const r = db.prepare("UPDATE arena_results SET human=? WHERE run_id LIKE ? AND subject=?").run(val, `${a1}%`, a2);
|
|
211
|
+
console.log(r.changes ? "saved" : "no matching run/subject");
|
|
212
|
+
} else {
|
|
213
|
+
console.log("usage: warden arena <run <task.json> | board [taskId] | show <runId> | diff <runId> <subject> | rate <runId> <subject> up|down>");
|
|
214
|
+
}
|
|
215
|
+
db.close();
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
default:
|
|
219
|
+
console.log("usage: warden <daemon | watch | sessions | replay <id> | diff <id> | review <id> | reviews <id> | arena ...>");
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
main().catch((e) => { console.error(`warden: ${e?.stack || e}`); process.exitCode = 1; });
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// ~/.warden/config.json (created with safe defaults on first read)
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { DATA_DIR } from "./db.mjs";
|
|
5
|
+
|
|
6
|
+
export const DEFAULTS = {
|
|
7
|
+
mode: "ask", // observe | ask. ("auto-on-stop" comes later.)
|
|
8
|
+
provider: "claude-cli",
|
|
9
|
+
model: null, // null = provider default
|
|
10
|
+
roles: ["correctness", "test-gap", "security"],
|
|
11
|
+
maxUsdPerRun: 0.5, // hard cap per reviewer call
|
|
12
|
+
maxUsdPerDay: 3, // refuse new reviews once today's spend reaches this
|
|
13
|
+
maxDiffChars: 60000,
|
|
14
|
+
maxOutputTokens: 2000, // bounds spend for API providers
|
|
15
|
+
// Optional. Run specific roles on specific models, e.g.
|
|
16
|
+
// "reviewers": [{"role":"correctness","provider":"gemini","model":"<model>"},
|
|
17
|
+
// {"role":"security","provider":"openai","model":"<model>"}]
|
|
18
|
+
reviewers: null,
|
|
19
|
+
// Arena judges, e.g. ["gemini:<model>", "openai:<model>"]. Judges that are also contestants are excluded.
|
|
20
|
+
judges: ["claude-cli"],
|
|
21
|
+
// Optional USD per 1M tokens, needed to cost API calls: "prices": {"gemini/<model>": {"in": 0, "out": 0}}
|
|
22
|
+
prices: {},
|
|
23
|
+
// Optional custom OpenAI-compatible servers: "providers": {"lmstudio": {"kind":"openai","baseUrl":"http://127.0.0.1:1234/v1","free":true}}
|
|
24
|
+
providers: {},
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const FILE = path.join(DATA_DIR, "config.json");
|
|
28
|
+
|
|
29
|
+
export function loadConfig() {
|
|
30
|
+
let user = {};
|
|
31
|
+
try { user = JSON.parse(fs.readFileSync(FILE, "utf8")); } catch {}
|
|
32
|
+
return { ...DEFAULTS, ...user };
|
|
33
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// wardend: records Claude Code hook events, snapshots repo state on Stop,
|
|
2
|
+
// serves a small local HTTP API + SSE stream.
|
|
3
|
+
import http from "node:http";
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import crypto from "node:crypto";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { getAsset } from "node:sea";
|
|
9
|
+
import { leaderboard } from "./arena/arena.mjs";
|
|
10
|
+
import { openDb } from "./db.mjs";
|
|
11
|
+
import { loadConfig } from "./config.mjs";
|
|
12
|
+
import { captureSnapshot } from "./capture.mjs";
|
|
13
|
+
|
|
14
|
+
export const PORT = Number(process.env.WARDEN_PORT || 7777);
|
|
15
|
+
// In the standalone build the page is embedded as an asset; otherwise it sits next to this file.
|
|
16
|
+
const DASHBOARD = typeof __WARDEN_SEA__ !== "undefined" ? null : path.join(path.dirname(fileURLToPath(import.meta.url)), "dashboard", "index.html");
|
|
17
|
+
function readDashboard(cb) {
|
|
18
|
+
if (DASHBOARD) return fs.readFile(DASHBOARD, cb);
|
|
19
|
+
try { cb(null, getAsset("dashboard.html", "utf8")); } catch (e) { cb(e); }
|
|
20
|
+
}
|
|
21
|
+
const HOST = "127.0.0.1";
|
|
22
|
+
|
|
23
|
+
export function summarize(e) {
|
|
24
|
+
const t = e.tool_input || {};
|
|
25
|
+
switch (e.hook_event_name) {
|
|
26
|
+
case "UserPromptSubmit": return String(e.prompt || "").slice(0, 200);
|
|
27
|
+
case "PreToolUse":
|
|
28
|
+
case "PostToolUse":
|
|
29
|
+
return `${t.file_path || t.command || t.pattern || t.url || t.description || ""}`.slice(0, 200);
|
|
30
|
+
case "Stop": return String(e.last_assistant_message || "").slice(0, 200);
|
|
31
|
+
default: return "";
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function startDaemon({ dbFile, port = PORT, quiet = false } = {}) {
|
|
36
|
+
const db = openDb(dbFile);
|
|
37
|
+
const q = {
|
|
38
|
+
upsertSession: db.prepare(`
|
|
39
|
+
INSERT INTO sessions (id, cwd, transcript_path, started_at, last_event_at, first_prompt, event_count)
|
|
40
|
+
VALUES (@id, @cwd, @tp, @ts, @ts, @prompt, 1)
|
|
41
|
+
ON CONFLICT(id) DO UPDATE SET last_event_at=@ts, event_count=event_count+1,
|
|
42
|
+
status=CASE WHEN @type='Stop' THEN 'stopped' ELSE 'running' END,
|
|
43
|
+
first_prompt=COALESCE(first_prompt, @prompt)`),
|
|
44
|
+
insertEvent: db.prepare(`INSERT INTO events (session_id, ts, type, tool_name, summary, duration_ms, payload)
|
|
45
|
+
VALUES (?,?,?,?,?,?,?)`),
|
|
46
|
+
insertSnap: db.prepare(`INSERT INTO snapshots (session_id, ts, git_status, git_diff, transcript_copy)
|
|
47
|
+
VALUES (?,?,?,?,?)`),
|
|
48
|
+
};
|
|
49
|
+
const clients = new Set(); // SSE subscribers
|
|
50
|
+
const broadcast = (obj) => { const line = `data: ${JSON.stringify(obj)}\n\n`; for (const r of clients) r.write(line); };
|
|
51
|
+
|
|
52
|
+
function ingest(e) {
|
|
53
|
+
const ts = Date.now();
|
|
54
|
+
const type = e.hook_event_name || "Unknown";
|
|
55
|
+
const sid = e.session_id || "unknown";
|
|
56
|
+
q.upsertSession.run({
|
|
57
|
+
id: sid, cwd: e.cwd ?? null, tp: e.transcript_path ?? null, ts, type,
|
|
58
|
+
prompt: type === "UserPromptSubmit" ? String(e.prompt || "").slice(0, 300) : null,
|
|
59
|
+
});
|
|
60
|
+
delete e._sentAt;
|
|
61
|
+
const summary = summarize(e);
|
|
62
|
+
q.insertEvent.run(sid, ts, type, e.tool_name ?? null, summary, e.duration_ms ?? null, JSON.stringify(e));
|
|
63
|
+
broadcast({ session_id: sid, ts, type, tool_name: e.tool_name ?? null, summary, duration_ms: e.duration_ms ?? null });
|
|
64
|
+
if (type === "Stop" || type === "SubagentStop") {
|
|
65
|
+
// snapshot off the hot path; never blocks the hook response
|
|
66
|
+
captureSnapshot(e.cwd, e.transcript_path).then((s) => {
|
|
67
|
+
q.insertSnap.run(sid, Date.now(), s.git_status, s.git_diff, s.transcript_copy);
|
|
68
|
+
broadcast({ session_id: sid, ts: Date.now(), type: "Snapshot", summary: `diff ${s.git_diff?.length ?? 0}B` });
|
|
69
|
+
// Approval-gated: only PROPOSE a review. Nothing runs, nothing is spent, until the user approves.
|
|
70
|
+
if (type === "Stop" && loadConfig().mode === "ask" && s.git_diff) {
|
|
71
|
+
broadcast({ session_id: sid, ts: Date.now(), type: "ReviewProposal", summary: `review available: warden review ${sid.slice(0, 8)}` });
|
|
72
|
+
}
|
|
73
|
+
}).catch((err) => console.error("snapshot failed:", err.message));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const json = (res, code, obj) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(obj)); };
|
|
78
|
+
|
|
79
|
+
const server = http.createServer((req, res) => {
|
|
80
|
+
// Browser safety. The daemon holds your code and transcripts, so:
|
|
81
|
+
// - Host must be loopback (blocks DNS-rebinding: a web page can't make the daemon look like evil.com)
|
|
82
|
+
// - a browser-sent Origin must be the daemon itself (blocks other websites from reading or POSTing)
|
|
83
|
+
// curl/hooks/CLI send no Origin and are unaffected.
|
|
84
|
+
const hostHdr = req.headers.host || "";
|
|
85
|
+
if (!/^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/.test(hostHdr)) return res.writeHead(403).end("forbidden host");
|
|
86
|
+
if (req.headers.origin && req.headers.origin !== `http://${hostHdr}`) return res.writeHead(403).end("forbidden origin");
|
|
87
|
+
|
|
88
|
+
const url = new URL(req.url, `http://${HOST}`);
|
|
89
|
+
if (req.method === "GET" && (url.pathname === "/" || url.pathname === "/index.html")) {
|
|
90
|
+
return readDashboard((err, html) => {
|
|
91
|
+
if (err) return res.writeHead(500).end("dashboard missing");
|
|
92
|
+
// per-request nonce: only our own script/style run, so even if escaping ever failed, injected markup can't execute
|
|
93
|
+
const nonce = crypto.randomBytes(16).toString("base64");
|
|
94
|
+
res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store",
|
|
95
|
+
"content-security-policy": `default-src 'none'; style-src 'nonce-${nonce}'; script-src 'nonce-${nonce}'; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'` });
|
|
96
|
+
res.end(String(html).replaceAll("__NONCE__", nonce));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
if (req.method === "POST" && url.pathname === "/arena/rate") {
|
|
100
|
+
let body = "";
|
|
101
|
+
req.on("data", (d) => { body += d; });
|
|
102
|
+
req.on("end", () => {
|
|
103
|
+
try {
|
|
104
|
+
const { run_id, subject, value } = JSON.parse(body);
|
|
105
|
+
if (![1, -1, 0].includes(value)) throw new Error("value must be 1, -1 or 0");
|
|
106
|
+
const r = db.prepare("UPDATE arena_results SET human=? WHERE run_id=? AND subject=?").run(value === 0 ? null : value, run_id, subject);
|
|
107
|
+
json(res, r.changes ? 200 : 404, { ok: !!r.changes });
|
|
108
|
+
} catch (err) { json(res, 400, { error: err.message }); }
|
|
109
|
+
});
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
if (req.method === "POST" && url.pathname === "/event") {
|
|
113
|
+
let body = "";
|
|
114
|
+
req.on("data", (d) => { body += d; if (body.length > 10_000_000) req.destroy(); });
|
|
115
|
+
req.on("end", () => {
|
|
116
|
+
res.writeHead(204).end(); // answer first so the hook never waits
|
|
117
|
+
try { ingest(JSON.parse(body)); } catch (err) { if (!quiet) console.error("bad event:", err.message); }
|
|
118
|
+
});
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
if (req.method === "POST" && url.pathname === "/reviews") {
|
|
122
|
+
let body = "";
|
|
123
|
+
req.on("data", (d) => { body += d; });
|
|
124
|
+
req.on("end", () => {
|
|
125
|
+
try {
|
|
126
|
+
const r = JSON.parse(body);
|
|
127
|
+
db.prepare("INSERT INTO reviews (session_id, ts, verdict, cost_usd, report) VALUES (?,?,?,?,?)")
|
|
128
|
+
.run(r.session_id, Date.now(), r.verdict ?? null, r.cost_usd ?? 0, JSON.stringify(r));
|
|
129
|
+
broadcast({ session_id: r.session_id, ts: Date.now(), type: "ReviewDone", summary: `${r.verdict} ($${(r.cost_usd ?? 0).toFixed(3)})` });
|
|
130
|
+
json(res, 201, { ok: true });
|
|
131
|
+
} catch (err) { json(res, 400, { error: err.message }); }
|
|
132
|
+
});
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
if (req.method !== "GET") return res.writeHead(404).end();
|
|
136
|
+
if (url.pathname === "/spend/today") {
|
|
137
|
+
const start = new Date(); start.setHours(0, 0, 0, 0);
|
|
138
|
+
return json(res, 200, db.prepare("SELECT COALESCE(SUM(cost_usd),0) AS usd FROM reviews WHERE ts>=?").get(start.getTime()));
|
|
139
|
+
}
|
|
140
|
+
{
|
|
141
|
+
const rm = url.pathname.match(/^\/sessions\/([^/]+)\/reviews$/);
|
|
142
|
+
if (rm) return json(res, 200, db.prepare("SELECT id,ts,verdict,cost_usd,report FROM reviews WHERE session_id=? ORDER BY id DESC").all(decodeURIComponent(rm[1])));
|
|
143
|
+
}
|
|
144
|
+
if (url.pathname === "/health") return res.writeHead(200).end("ok");
|
|
145
|
+
if (url.pathname === "/stream") {
|
|
146
|
+
res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" });
|
|
147
|
+
res.write(": connected\n\n");
|
|
148
|
+
clients.add(res); req.on("close", () => clients.delete(res));
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
if (url.pathname === "/sessions") {
|
|
152
|
+
return json(res, 200, db.prepare(`SELECT s.*, (SELECT verdict FROM reviews r WHERE r.session_id=s.id ORDER BY r.id DESC LIMIT 1) AS review_verdict
|
|
153
|
+
FROM sessions s ORDER BY last_event_at DESC LIMIT 100`).all());
|
|
154
|
+
}
|
|
155
|
+
if (url.pathname === "/arena/runs") {
|
|
156
|
+
return json(res, 200, db.prepare(`SELECT r.id, r.ts, r.task_id, r.prompt, r.judges,
|
|
157
|
+
(SELECT COUNT(*) FROM arena_results x WHERE x.run_id=r.id) AS subjects FROM arena_runs r ORDER BY r.ts DESC LIMIT 100`).all());
|
|
158
|
+
}
|
|
159
|
+
if (url.pathname === "/arena/board") return json(res, 200, leaderboard(db, url.searchParams.get("task") || undefined));
|
|
160
|
+
{
|
|
161
|
+
const am = url.pathname.match(/^\/arena\/runs\/([^/]+)$/);
|
|
162
|
+
if (am) {
|
|
163
|
+
const run = db.prepare("SELECT * FROM arena_runs WHERE id=?").get(decodeURIComponent(am[1]));
|
|
164
|
+
if (!run) return json(res, 404, { error: "no such run" });
|
|
165
|
+
const results = db.prepare("SELECT * FROM arena_results WHERE run_id=?").all(run.id)
|
|
166
|
+
.map((r) => ({ ...r, ok: !!r.ok, checks_passed: !!r.checks_passed, checks: JSON.parse(r.checks || "[]"), judge_detail: JSON.parse(r.judge_detail || "[]"), judges: undefined }));
|
|
167
|
+
return json(res, 200, { run: { ...run, judges: JSON.parse(run.judges || "{}") }, results });
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
const m = url.pathname.match(/^\/sessions\/([^/]+)\/(events|snapshot)$/);
|
|
171
|
+
if (m) {
|
|
172
|
+
const id = decodeURIComponent(m[1]);
|
|
173
|
+
if (m[2] === "events") return json(res, 200, db.prepare("SELECT id,ts,type,tool_name,summary,duration_ms FROM events WHERE session_id=? ORDER BY id").all(id));
|
|
174
|
+
const snap = db.prepare("SELECT * FROM snapshots WHERE session_id=? ORDER BY id DESC LIMIT 1").get(id);
|
|
175
|
+
if (!snap) return json(res, 404, { error: "no snapshot" });
|
|
176
|
+
// the transcript copy can be hundreds of KB; only send it when asked
|
|
177
|
+
const { transcript_copy, ...lite } = snap;
|
|
178
|
+
return json(res, 200, url.searchParams.get("full") ? snap : { ...lite, transcript_bytes: transcript_copy?.length ?? 0 });
|
|
179
|
+
}
|
|
180
|
+
res.writeHead(404).end();
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
return new Promise((resolve) => server.listen(port, HOST, () => {
|
|
184
|
+
if (!quiet) console.log(`warden daemon on http://${HOST}:${port}`);
|
|
185
|
+
resolve({ server, db, close: () => { for (const r of clients) r.end(); server.close(); db.close(); } });
|
|
186
|
+
}));
|
|
187
|
+
}
|