speedpatrol 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +366 -80
- package/package.json +1 -1
- package/src/ui/app.js +1086 -257
- package/src/ui/index.html +24 -8
- package/src/ui/style.css +618 -158
package/dist/bin.js
CHANGED
|
@@ -332,6 +332,9 @@ async function loadConfig(cwd) {
|
|
|
332
332
|
const dir = findConfigDir(cwd);
|
|
333
333
|
if (!dir)
|
|
334
334
|
throw new ConfigError(`No ${CONFIG_DIR}/${CONFIG_FILE} here or above. Run \`speedpatrol init\`.`);
|
|
335
|
+
return loadConfigDir(dir);
|
|
336
|
+
}
|
|
337
|
+
async function loadConfigDir(dir) {
|
|
335
338
|
const file = join(dir, CONFIG_FILE);
|
|
336
339
|
let raw;
|
|
337
340
|
try {
|
|
@@ -996,22 +999,22 @@ function compareSamples(base, head, better, thresholds) {
|
|
|
996
999
|
const excludesZero = ci[0] > 0 || ci[1] < 0;
|
|
997
1000
|
const significant = p < thresholds.alpha && excludesZero;
|
|
998
1001
|
const large = Math.abs(deltaPct) >= thresholds.relative && Math.abs(deltaAbs) >= thresholds.absolute;
|
|
999
|
-
let
|
|
1002
|
+
let verdict2;
|
|
1000
1003
|
if (significant && large) {
|
|
1001
1004
|
const worse = better === "lower" ? deltaAbs > 0 : deltaAbs < 0;
|
|
1002
|
-
|
|
1005
|
+
verdict2 = worse ? "regression" : "improvement";
|
|
1003
1006
|
} else if (!significant && Math.max(bs.cv, hs.cv) > thresholds.maxCv && large) {
|
|
1004
|
-
|
|
1007
|
+
verdict2 = "noisy";
|
|
1005
1008
|
} else {
|
|
1006
|
-
|
|
1009
|
+
verdict2 = "unchanged";
|
|
1007
1010
|
}
|
|
1008
|
-
return { deltaPct, deltaAbs, ci, pValue: p, verdict, ...significant && !large ? { underThreshold: true } : {} };
|
|
1011
|
+
return { deltaPct, deltaAbs, ci, pValue: p, verdict: verdict2, ...significant && !large ? { underThreshold: true } : {} };
|
|
1009
1012
|
}
|
|
1010
1013
|
var SEVERITY = { regression: 4, noisy: 3, improvement: 2, unchanged: 1, missing: 0 };
|
|
1011
1014
|
function worstVerdict(verdicts) {
|
|
1012
1015
|
if (verdicts.length === 0)
|
|
1013
1016
|
return "missing";
|
|
1014
|
-
return verdicts.reduce((worst,
|
|
1017
|
+
return verdicts.reduce((worst, verdict2) => SEVERITY[verdict2] > SEVERITY[worst] ? verdict2 : worst);
|
|
1015
1018
|
}
|
|
1016
1019
|
function poolResults(runs) {
|
|
1017
1020
|
const pooled = /* @__PURE__ */ new Map();
|
|
@@ -1131,8 +1134,8 @@ var VERDICT_ICON = {
|
|
|
1131
1134
|
missing: "\u26AB"
|
|
1132
1135
|
};
|
|
1133
1136
|
function row(metric) {
|
|
1134
|
-
const
|
|
1135
|
-
return `| ${VERDICT_ICON[metric.verdict]} ${
|
|
1137
|
+
const verdict2 = metric.underThreshold ? `${metric.verdict} (under threshold)` : metric.verdict;
|
|
1138
|
+
return `| ${VERDICT_ICON[metric.verdict]} ${verdict2} | \`${metric.benchmark}\` | ${metric.metric} | ${formatValue(metric.base?.median, metric.unit)} | ${formatValue(metric.head?.median, metric.unit)} | ${formatPct(metric.deltaPct)} | ${formatPct(metric.ci[0])} to ${formatPct(metric.ci[1])} | ${metric.pValue < 1e-3 ? "<0.001" : metric.pValue.toFixed(3)} |`;
|
|
1136
1139
|
}
|
|
1137
1140
|
function comparisonMarkdown(comparison, options = {}) {
|
|
1138
1141
|
const shown = comparison.metrics.filter((metric) => options.all || metric.primary || metric.verdict === "regression" || metric.verdict === "improvement");
|
|
@@ -2314,12 +2317,13 @@ async function callTool(tools, name, input, budget) {
|
|
|
2314
2317
|
async function serveTools(tools, options = {}) {
|
|
2315
2318
|
const budget = { calls: 0, max: options.maxCalls ?? Number.POSITIVE_INFINITY };
|
|
2316
2319
|
const running = /* @__PURE__ */ new Map();
|
|
2320
|
+
const lane = { tail: Promise.resolve() };
|
|
2317
2321
|
const http = createServer((request, response) => {
|
|
2318
2322
|
if (request.url !== "/mcp") {
|
|
2319
2323
|
response.writeHead(404).end();
|
|
2320
2324
|
return;
|
|
2321
2325
|
}
|
|
2322
|
-
void handle(request, response, tools, options, budget, running).catch((error) => {
|
|
2326
|
+
void handle(request, response, tools, options, budget, running, lane).catch((error) => {
|
|
2323
2327
|
if (!response.headersSent)
|
|
2324
2328
|
response.writeHead(500).end(String(error));
|
|
2325
2329
|
});
|
|
@@ -2342,7 +2346,7 @@ async function serveTools(tools, options = {}) {
|
|
|
2342
2346
|
}
|
|
2343
2347
|
};
|
|
2344
2348
|
}
|
|
2345
|
-
async function handle(request, response, tools, options, budget, running) {
|
|
2349
|
+
async function handle(request, response, tools, options, budget, running, lane) {
|
|
2346
2350
|
const server = new Server({ name: "speedpatrol", version: "0.0.0" }, { capabilities: { tools: {} } });
|
|
2347
2351
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
2348
2352
|
tools: tools.map((tool2) => ({
|
|
@@ -2360,7 +2364,10 @@ async function handle(request, response, tools, options, budget, running) {
|
|
|
2360
2364
|
const result2 = await joined;
|
|
2361
2365
|
return { content: [{ type: "text", text: result2.text }], isError: result2.isError ?? false };
|
|
2362
2366
|
}
|
|
2363
|
-
const
|
|
2367
|
+
const exclusive = tools.find((item) => item.name === call.params.name)?.exclusive ?? false;
|
|
2368
|
+
const pending = exclusive ? lane.tail.then(() => callTool(tools, call.params.name, input, budget)) : callTool(tools, call.params.name, input, budget);
|
|
2369
|
+
if (exclusive)
|
|
2370
|
+
lane.tail = pending;
|
|
2364
2371
|
running.set(key, pending);
|
|
2365
2372
|
const result = await pending.finally(() => running.delete(key));
|
|
2366
2373
|
options.onCall?.(call.params.name, input, result, Date.now() - started);
|
|
@@ -2459,8 +2466,13 @@ ${task.prompt}`;
|
|
|
2459
2466
|
const child = spawn2("/bin/sh", ["-c", command], {
|
|
2460
2467
|
cwd: task.workdir,
|
|
2461
2468
|
// measure_change runs for minutes; Claude Code's default MCP tool
|
|
2462
|
-
// timeout abandons it and the agent calls it again over the first.
|
|
2463
|
-
|
|
2469
|
+
// timeout abandons it and the agent calls it again over the first. It also
|
|
2470
|
+
// aborts any call silent for 300 s (an A/B, or a call queued behind one).
|
|
2471
|
+
env: {
|
|
2472
|
+
MCP_TOOL_TIMEOUT: String(task.timeoutMs),
|
|
2473
|
+
CLAUDE_CODE_MCP_TOOL_IDLE_TIMEOUT: String(task.timeoutMs),
|
|
2474
|
+
...process.env
|
|
2475
|
+
},
|
|
2464
2476
|
detached: true,
|
|
2465
2477
|
stdio: ["pipe", "pipe", "pipe"]
|
|
2466
2478
|
});
|
|
@@ -2708,12 +2720,13 @@ import { readFile as readFile4, writeFile as writeFile5 } from "node:fs/promises
|
|
|
2708
2720
|
import { join as join8 } from "node:path";
|
|
2709
2721
|
|
|
2710
2722
|
// ../agents/dist/tools/schema.js
|
|
2711
|
-
function tool(name, description, fields, required, run) {
|
|
2723
|
+
function tool(name, description, fields, required, run, options = {}) {
|
|
2712
2724
|
return {
|
|
2713
2725
|
name,
|
|
2714
2726
|
description,
|
|
2715
2727
|
inputSchema: { type: "object", properties: fields, required, additionalProperties: false },
|
|
2716
|
-
run
|
|
2728
|
+
run,
|
|
2729
|
+
...options
|
|
2717
2730
|
};
|
|
2718
2731
|
}
|
|
2719
2732
|
function str(input, key) {
|
|
@@ -2906,7 +2919,7 @@ function measureCommitTool(loaded, store, log2) {
|
|
|
2906
2919
|
text: comparisonMarkdown(comparison),
|
|
2907
2920
|
summary: `${head.sha.slice(0, 8)} vs ${base.sha.slice(0, 8)}: ${comparison.verdict}.`
|
|
2908
2921
|
};
|
|
2909
|
-
});
|
|
2922
|
+
}, { exclusive: true });
|
|
2910
2923
|
}
|
|
2911
2924
|
|
|
2912
2925
|
// ../agents/dist/tools/repo.js
|
|
@@ -3138,7 +3151,7 @@ async function runOptimizer(options) {
|
|
|
3138
3151
|
log: log2
|
|
3139
3152
|
});
|
|
3140
3153
|
return { text: comparisonMarkdown(comparison), summary: `Measured: ${comparison.verdict}.` };
|
|
3141
|
-
}),
|
|
3154
|
+
}, { exclusive: true }),
|
|
3142
3155
|
tool("git_diff", "Your changes so far.", {}, [], async () => ({
|
|
3143
3156
|
text: await tryGit(worktree, ["diff", "HEAD"]) || "No changes yet."
|
|
3144
3157
|
})),
|
|
@@ -3310,8 +3323,8 @@ async function markFixed(store, run, loaded) {
|
|
|
3310
3323
|
const now = run.results.find((r) => r.benchmark === regression.benchmark)?.metrics[regression.metric];
|
|
3311
3324
|
if (!before || !now)
|
|
3312
3325
|
continue;
|
|
3313
|
-
const { verdict } = compareSamples(before.values, now.values, now.better, loaded.config.thresholds);
|
|
3314
|
-
if (
|
|
3326
|
+
const { verdict: verdict2 } = compareSamples(before.values, now.values, now.better, loaded.config.thresholds);
|
|
3327
|
+
if (verdict2 === "regression" || verdict2 === "noisy")
|
|
3315
3328
|
continue;
|
|
3316
3329
|
const updated = {
|
|
3317
3330
|
...regression,
|
|
@@ -3510,7 +3523,227 @@ import { fileURLToPath } from "node:url";
|
|
|
3510
3523
|
import { existsSync as existsSync8 } from "node:fs";
|
|
3511
3524
|
import { readFile as readFile7 } from "node:fs/promises";
|
|
3512
3525
|
import { join as join9 } from "node:path";
|
|
3526
|
+
|
|
3527
|
+
// ../dashboard/dist/summary.js
|
|
3528
|
+
var OPEN = /* @__PURE__ */ new Set(["open", "filed"]);
|
|
3529
|
+
var isOpen = (regression) => OPEN.has(regression.status);
|
|
3530
|
+
var plural = (count, word, many = `${word}s`) => `${count} ${count === 1 ? word : many}`;
|
|
3531
|
+
var byStart = (a, b) => a.startedAt.localeCompare(b.startedAt);
|
|
3532
|
+
function projectSummary(input) {
|
|
3533
|
+
const { config, runs, baselines, comparisons, regressions, experiments } = input;
|
|
3534
|
+
const configError = "error" in config ? config.error : void 0;
|
|
3535
|
+
const defined = "error" in config ? void 0 : config.benchmarks;
|
|
3536
|
+
const newest = baselines.at(-1);
|
|
3537
|
+
const measuredIds = new Set(newest?.results.map((result) => result.benchmark) ?? []);
|
|
3538
|
+
const unmeasured = (defined ?? []).filter((benchmark) => !measuredIds.has(benchmark.id)).map((item) => item.id);
|
|
3539
|
+
const withErrors = (newest?.results ?? []).filter((result) => result.errors.length).map((item) => item.benchmark);
|
|
3540
|
+
const critical = defined ? defined.filter((benchmark) => benchmark.critical).length : newest?.results.filter((result) => result.critical).length ?? 0;
|
|
3541
|
+
const open = regressions.filter(isOpen);
|
|
3542
|
+
const byStatus = { open: 0, filed: 0, dismissed: 0, fixed: 0 };
|
|
3543
|
+
for (const regression of regressions)
|
|
3544
|
+
byStatus[regression.status]++;
|
|
3545
|
+
const experimentCount = (...statuses) => experiments.filter((experiment) => statuses.includes(experiment.status)).length;
|
|
3546
|
+
const waiting = experiments.filter((experiment) => experiment.status === "improved");
|
|
3547
|
+
const lastRun = pickLastRun(runs, comparisons);
|
|
3548
|
+
const machine = newest?.machine ?? runs.find((run) => run.id === lastRun?.id)?.machine;
|
|
3549
|
+
const attention = [];
|
|
3550
|
+
if (configError) {
|
|
3551
|
+
attention.push({
|
|
3552
|
+
tone: "fail",
|
|
3553
|
+
title: "The config does not load",
|
|
3554
|
+
detail: configError,
|
|
3555
|
+
command: "npx speedpatrol doctor"
|
|
3556
|
+
});
|
|
3557
|
+
}
|
|
3558
|
+
if (lastRun?.failed) {
|
|
3559
|
+
attention.push({
|
|
3560
|
+
tone: "fail",
|
|
3561
|
+
title: "The last run failed",
|
|
3562
|
+
detail: lastRun.failed,
|
|
3563
|
+
href: "#/runs",
|
|
3564
|
+
command: "npx speedpatrol run"
|
|
3565
|
+
});
|
|
3566
|
+
}
|
|
3567
|
+
for (const regression of open)
|
|
3568
|
+
attention.push(regressionItem(regression));
|
|
3569
|
+
if (unmeasured.length) {
|
|
3570
|
+
attention.push({
|
|
3571
|
+
tone: "warn",
|
|
3572
|
+
title: `${plural(unmeasured.length, "benchmark has", "benchmarks have")} no baseline`,
|
|
3573
|
+
detail: `The newest baseline does not measure ${unmeasured.join(", ")}.`,
|
|
3574
|
+
href: "#/benchmarks",
|
|
3575
|
+
command: "npx speedpatrol run"
|
|
3576
|
+
});
|
|
3577
|
+
}
|
|
3578
|
+
if (withErrors.length) {
|
|
3579
|
+
attention.push({
|
|
3580
|
+
tone: "warn",
|
|
3581
|
+
title: `${plural(withErrors.length, "benchmark")} had errors in the last baseline`,
|
|
3582
|
+
detail: `Some iterations of ${withErrors.join(", ")} failed. Their medians use fewer samples.`,
|
|
3583
|
+
href: "#/benchmarks"
|
|
3584
|
+
});
|
|
3585
|
+
}
|
|
3586
|
+
for (const experiment of waiting) {
|
|
3587
|
+
attention.push({
|
|
3588
|
+
tone: "warn",
|
|
3589
|
+
title: `A speed-up of ${experiment.target.benchmark} waits to be published`,
|
|
3590
|
+
detail: experiment.summary ?? experiment.hypothesis ?? `The A/B run confirmed it on ${experiment.branch}.`,
|
|
3591
|
+
href: `#/experiments/${encodeURIComponent(experiment.id)}`,
|
|
3592
|
+
command: "npx speedpatrol publish"
|
|
3593
|
+
});
|
|
3594
|
+
}
|
|
3595
|
+
for (const role of input.stoppedAgents) {
|
|
3596
|
+
attention.push({
|
|
3597
|
+
tone: "warn",
|
|
3598
|
+
title: `The ${role} stopped`,
|
|
3599
|
+
detail: "Its status says that it works, but its process is gone. Start the patrol again.",
|
|
3600
|
+
href: "#/activity",
|
|
3601
|
+
command: "npx speedpatrol"
|
|
3602
|
+
});
|
|
3603
|
+
}
|
|
3604
|
+
return {
|
|
3605
|
+
name: input.name,
|
|
3606
|
+
...machine ? { machine } : {},
|
|
3607
|
+
...configError ? { configError } : {},
|
|
3608
|
+
verdict: verdict({
|
|
3609
|
+
configError,
|
|
3610
|
+
lastRun,
|
|
3611
|
+
open,
|
|
3612
|
+
baselines: baselines.length,
|
|
3613
|
+
comparisons: comparisons.length,
|
|
3614
|
+
warnings: attention.filter((item) => item.tone === "warn"),
|
|
3615
|
+
measured: measuredIds.size
|
|
3616
|
+
}),
|
|
3617
|
+
benchmarks: {
|
|
3618
|
+
...defined ? { defined: defined.length } : {},
|
|
3619
|
+
measured: measuredIds.size,
|
|
3620
|
+
critical,
|
|
3621
|
+
regressed: new Set(open.map((regression) => regression.benchmark)).size,
|
|
3622
|
+
withErrors,
|
|
3623
|
+
unmeasured
|
|
3624
|
+
},
|
|
3625
|
+
regressions: { open: open.length, byStatus, total: regressions.length },
|
|
3626
|
+
experiments: {
|
|
3627
|
+
active: experimentCount("proposed", "measuring"),
|
|
3628
|
+
waiting: waiting.length,
|
|
3629
|
+
published: experimentCount("published"),
|
|
3630
|
+
rejected: experimentCount("rejected", "failed"),
|
|
3631
|
+
total: experiments.length
|
|
3632
|
+
},
|
|
3633
|
+
runs: {
|
|
3634
|
+
total: runs.length,
|
|
3635
|
+
baselines: baselines.length,
|
|
3636
|
+
failed: runs.filter((run) => run.failed).length,
|
|
3637
|
+
comparisons: comparisons.length
|
|
3638
|
+
},
|
|
3639
|
+
...lastRun ? { lastRun } : {},
|
|
3640
|
+
attention,
|
|
3641
|
+
...baselines.length ? {} : { next: nextStep(defined?.length ?? 0) }
|
|
3642
|
+
};
|
|
3643
|
+
}
|
|
3644
|
+
function pickLastRun(runs, comparisons) {
|
|
3645
|
+
const sorted = [...runs].sort(byStart);
|
|
3646
|
+
const run = sorted.filter((item) => item.purpose === "baseline").at(-1) ?? sorted.at(-1);
|
|
3647
|
+
if (!run)
|
|
3648
|
+
return void 0;
|
|
3649
|
+
const comparison = [...comparisons].sort((a, b) => b.createdAt.localeCompare(a.createdAt)).find((item) => item.head.runs.includes(run.id));
|
|
3650
|
+
return {
|
|
3651
|
+
id: run.id,
|
|
3652
|
+
purpose: run.purpose,
|
|
3653
|
+
sha: run.source.sha,
|
|
3654
|
+
...run.source.subject ? { subject: run.source.subject } : {},
|
|
3655
|
+
...run.source.branch ? { branch: run.source.branch } : {},
|
|
3656
|
+
startedAt: run.startedAt,
|
|
3657
|
+
finishedAt: run.finishedAt,
|
|
3658
|
+
...run.failed ? { failed: run.failed } : {},
|
|
3659
|
+
benchmarks: run.results.length,
|
|
3660
|
+
...comparison ? { comparison: { id: comparison.id, verdict: comparison.verdict } } : {}
|
|
3661
|
+
};
|
|
3662
|
+
}
|
|
3663
|
+
function nextStep(defined) {
|
|
3664
|
+
if (!defined) {
|
|
3665
|
+
return {
|
|
3666
|
+
why: "The project has no benchmarks yet. The scout finds the critical paths and writes them.",
|
|
3667
|
+
command: "npx speedpatrol scout"
|
|
3668
|
+
};
|
|
3669
|
+
}
|
|
3670
|
+
return {
|
|
3671
|
+
why: `The config has ${plural(defined, "benchmark")}, but no baseline. A run on a clean commit records one.`,
|
|
3672
|
+
command: "npx speedpatrol run"
|
|
3673
|
+
};
|
|
3674
|
+
}
|
|
3675
|
+
function regressionItem(regression) {
|
|
3676
|
+
const href = `#/regressions/${encodeURIComponent(regression.id)}`;
|
|
3677
|
+
const title = regression.title || `${regression.benchmark} ${regression.metric} got slower`;
|
|
3678
|
+
if (regression.status === "open" && !regression.analysis) {
|
|
3679
|
+
return {
|
|
3680
|
+
tone: "fail",
|
|
3681
|
+
title,
|
|
3682
|
+
detail: "Confirmed. The analyst did not explain it yet.",
|
|
3683
|
+
href,
|
|
3684
|
+
command: `npx speedpatrol analyze ${regression.id}`
|
|
3685
|
+
};
|
|
3686
|
+
}
|
|
3687
|
+
if (!regression.issueUrl) {
|
|
3688
|
+
return {
|
|
3689
|
+
tone: "fail",
|
|
3690
|
+
title,
|
|
3691
|
+
detail: "Explained. No issue is open for it yet.",
|
|
3692
|
+
href,
|
|
3693
|
+
command: "npx speedpatrol publish"
|
|
3694
|
+
};
|
|
3695
|
+
}
|
|
3696
|
+
return {
|
|
3697
|
+
tone: "fail",
|
|
3698
|
+
title,
|
|
3699
|
+
detail: "Filed. The patrol closes it when a baseline is back within the thresholds.",
|
|
3700
|
+
href
|
|
3701
|
+
};
|
|
3702
|
+
}
|
|
3703
|
+
function verdict(facts) {
|
|
3704
|
+
if (facts.configError)
|
|
3705
|
+
return { tone: "fail", text: "Config error", detail: "Speedpatrol cannot read the config." };
|
|
3706
|
+
if (!facts.lastRun) {
|
|
3707
|
+
return { tone: "info", text: "Not measured yet", detail: "No run is recorded for this project." };
|
|
3708
|
+
}
|
|
3709
|
+
if (facts.lastRun.failed) {
|
|
3710
|
+
return { tone: "fail", text: "Last run failed", detail: "The newest numbers are older than the last run." };
|
|
3711
|
+
}
|
|
3712
|
+
if (facts.open.length) {
|
|
3713
|
+
return {
|
|
3714
|
+
tone: "fail",
|
|
3715
|
+
text: facts.open.length === 1 ? "1 open regression" : `${facts.open.length} open regressions`,
|
|
3716
|
+
detail: `Slower: ${[...new Set(facts.open.map((item) => item.benchmark))].join(", ")}.`
|
|
3717
|
+
};
|
|
3718
|
+
}
|
|
3719
|
+
if (!facts.baselines) {
|
|
3720
|
+
return { tone: "info", text: "No baseline yet", detail: "A run on a clean commit records the first baseline." };
|
|
3721
|
+
}
|
|
3722
|
+
const [warning] = facts.warnings;
|
|
3723
|
+
if (warning) {
|
|
3724
|
+
return {
|
|
3725
|
+
tone: "warn",
|
|
3726
|
+
text: facts.warnings.length === 1 ? "One item needs you" : `${facts.warnings.length} items need you`,
|
|
3727
|
+
detail: warning.title
|
|
3728
|
+
};
|
|
3729
|
+
}
|
|
3730
|
+
if (facts.baselines <= 1 && !facts.comparisons) {
|
|
3731
|
+
return {
|
|
3732
|
+
tone: "info",
|
|
3733
|
+
text: "First baseline recorded",
|
|
3734
|
+
detail: "Speedpatrol compares the next commit with it."
|
|
3735
|
+
};
|
|
3736
|
+
}
|
|
3737
|
+
return {
|
|
3738
|
+
tone: "pass",
|
|
3739
|
+
text: "No open regressions",
|
|
3740
|
+
detail: `${plural(facts.measured, "benchmark")} measured on the last baseline.`
|
|
3741
|
+
};
|
|
3742
|
+
}
|
|
3743
|
+
|
|
3744
|
+
// ../dashboard/dist/data.js
|
|
3513
3745
|
var ROLES = ["scout", "analyst", "optimizer", "runner", "patrol"];
|
|
3746
|
+
var PROBLEM_ORDER = ["regression", "errors", "unmeasured", "stale", void 0];
|
|
3514
3747
|
function pidAlive(pid) {
|
|
3515
3748
|
try {
|
|
3516
3749
|
process.kill(pid, 0);
|
|
@@ -3585,40 +3818,40 @@ var DashboardData = class {
|
|
|
3585
3818
|
const runs = await Promise.all(ordered.map((entry) => this.store.getRun(entry.runId)));
|
|
3586
3819
|
return { machine, runs: runs.filter((run) => run !== void 0 && !run.failed) };
|
|
3587
3820
|
}
|
|
3588
|
-
|
|
3589
|
-
|
|
3821
|
+
/** The benchmarks of the config, or the error that stops the load. */
|
|
3822
|
+
async config() {
|
|
3823
|
+
try {
|
|
3824
|
+
const { benchmarks } = await loadConfigDir(this.configDir);
|
|
3825
|
+
return { benchmarks: benchmarks.map(({ id: id2, critical, kind }) => ({ id: id2, critical, kind })) };
|
|
3826
|
+
} catch (error) {
|
|
3827
|
+
return { error: error instanceof Error ? error.message : String(error) };
|
|
3828
|
+
}
|
|
3829
|
+
}
|
|
3830
|
+
async summary() {
|
|
3831
|
+
const [name, config, agents, runs, baselines, comparisons, regressions, experiments] = await Promise.all([
|
|
3590
3832
|
this.projectName(),
|
|
3833
|
+
this.config(),
|
|
3591
3834
|
this.agents(),
|
|
3592
3835
|
this.store.listRuns(),
|
|
3593
3836
|
this.baselineRuns(),
|
|
3594
|
-
this.store.listRegressions(),
|
|
3595
|
-
this.store.listExperiments(),
|
|
3596
3837
|
this.store.listComparisons(),
|
|
3597
|
-
this.store.
|
|
3838
|
+
this.store.listRegressions(),
|
|
3839
|
+
this.store.listExperiments()
|
|
3598
3840
|
]);
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
runs: runs.length,
|
|
3614
|
-
baselines: baselines.runs.length,
|
|
3615
|
-
comparisons: comparisons.length,
|
|
3616
|
-
openRegressions: regressions.filter((item) => item.status === "open" || item.status === "filed").length,
|
|
3617
|
-
improvedExperiments: experiments.filter((item) => item.status === "improved" || item.status === "published").length
|
|
3618
|
-
},
|
|
3619
|
-
agents,
|
|
3620
|
-
patrol: patrol2
|
|
3621
|
-
};
|
|
3841
|
+
return projectSummary({
|
|
3842
|
+
name,
|
|
3843
|
+
config,
|
|
3844
|
+
runs,
|
|
3845
|
+
baselines: baselines.runs,
|
|
3846
|
+
comparisons,
|
|
3847
|
+
regressions,
|
|
3848
|
+
experiments,
|
|
3849
|
+
stoppedAgents: agents.filter((agent) => agent.state === "stopped").map((agent) => agent.role)
|
|
3850
|
+
});
|
|
3851
|
+
}
|
|
3852
|
+
async agentsAndPatrol() {
|
|
3853
|
+
const [agents, patrol2] = await Promise.all([this.agents(), this.store.patrolState()]);
|
|
3854
|
+
return { agents, patrol: patrol2 };
|
|
3622
3855
|
}
|
|
3623
3856
|
/** The comparison of exactly these two commits, newest first. */
|
|
3624
3857
|
stepComparison(comparisons, base, head) {
|
|
@@ -3627,7 +3860,12 @@ var DashboardData = class {
|
|
|
3627
3860
|
return newestFirst(comparisons).find((comparison) => comparison.base.sha === base.source.sha && comparison.head.sha === head.source.sha);
|
|
3628
3861
|
}
|
|
3629
3862
|
async benchmarks() {
|
|
3630
|
-
const [{ machine, runs }, comparisons] = await Promise.all([
|
|
3863
|
+
const [{ machine, runs }, comparisons, regressions, config] = await Promise.all([
|
|
3864
|
+
this.baselineRuns(),
|
|
3865
|
+
this.store.listComparisons(),
|
|
3866
|
+
this.store.listRegressions(),
|
|
3867
|
+
this.config()
|
|
3868
|
+
]);
|
|
3631
3869
|
const latestRun = runs.at(-1);
|
|
3632
3870
|
const step = this.stepComparison(comparisons, runs.at(-2), latestRun);
|
|
3633
3871
|
const rows = /* @__PURE__ */ new Map();
|
|
@@ -3641,9 +3879,10 @@ var DashboardData = class {
|
|
|
3641
3879
|
primary: result.primary,
|
|
3642
3880
|
unit: series?.unit ?? "ms",
|
|
3643
3881
|
better: series?.better ?? "lower",
|
|
3882
|
+
errors: 0,
|
|
3644
3883
|
series: []
|
|
3645
3884
|
};
|
|
3646
|
-
Object.assign(row2, { kind: result.kind, critical: result.critical });
|
|
3885
|
+
Object.assign(row2, { kind: result.kind, critical: result.critical, errors: result.errors.length });
|
|
3647
3886
|
if (series && result.primary === row2.primary)
|
|
3648
3887
|
row2.series.push(point(run, series.summary));
|
|
3649
3888
|
if (series && result.primary !== row2.primary) {
|
|
@@ -3653,15 +3892,44 @@ var DashboardData = class {
|
|
|
3653
3892
|
rows.set(result.benchmark, row2);
|
|
3654
3893
|
}
|
|
3655
3894
|
}
|
|
3895
|
+
for (const defined of "benchmarks" in config ? config.benchmarks : []) {
|
|
3896
|
+
if (rows.has(defined.id))
|
|
3897
|
+
continue;
|
|
3898
|
+
rows.set(defined.id, {
|
|
3899
|
+
id: defined.id,
|
|
3900
|
+
kind: defined.kind,
|
|
3901
|
+
critical: defined.critical,
|
|
3902
|
+
primary: "",
|
|
3903
|
+
unit: "ms",
|
|
3904
|
+
better: "lower",
|
|
3905
|
+
errors: 0,
|
|
3906
|
+
series: []
|
|
3907
|
+
});
|
|
3908
|
+
}
|
|
3909
|
+
const open = newestFirst(regressions.filter(isOpen));
|
|
3656
3910
|
for (const row2 of rows.values()) {
|
|
3657
3911
|
const latest = row2.series.at(-1);
|
|
3658
3912
|
const previous = row2.series.at(-2);
|
|
3913
|
+
const inLatest = latestRun?.results.some((result) => result.benchmark === row2.id) ?? false;
|
|
3659
3914
|
row2.latest = latest?.runId === latestRun?.id ? latest : void 0;
|
|
3660
3915
|
if (row2.latest && previous)
|
|
3661
3916
|
row2.deltaPct = row2.latest.median / previous.median - 1;
|
|
3662
3917
|
row2.verdict = step?.metrics.find((metric) => metric.benchmark === row2.id && metric.metric === row2.primary)?.verdict;
|
|
3918
|
+
if (!inLatest)
|
|
3919
|
+
row2.errors = 0;
|
|
3920
|
+
const regression = open.find((item) => item.benchmark === row2.id);
|
|
3921
|
+
if (regression) {
|
|
3922
|
+
row2.regression = {
|
|
3923
|
+
id: regression.id,
|
|
3924
|
+
status: regression.status,
|
|
3925
|
+
deltaPct: regression.deltaPct,
|
|
3926
|
+
metric: regression.metric,
|
|
3927
|
+
badSha: regression.badSha
|
|
3928
|
+
};
|
|
3929
|
+
}
|
|
3930
|
+
row2.problem = regression ? "regression" : row2.errors ? "errors" : !row2.series.length ? "unmeasured" : !inLatest ? "stale" : void 0;
|
|
3663
3931
|
}
|
|
3664
|
-
const benchmarks = [...rows.values()].sort((a, b) => Number(b.critical) - Number(a.critical) || a.id.localeCompare(b.id));
|
|
3932
|
+
const benchmarks = [...rows.values()].sort((a, b) => PROBLEM_ORDER.indexOf(a.problem) - PROBLEM_ORDER.indexOf(b.problem) || Number(b.critical) - Number(a.critical) || a.id.localeCompare(b.id));
|
|
3665
3933
|
return { machine, benchmarks };
|
|
3666
3934
|
}
|
|
3667
3935
|
async benchmark(id2) {
|
|
@@ -3713,20 +3981,37 @@ var DashboardData = class {
|
|
|
3713
3981
|
};
|
|
3714
3982
|
}
|
|
3715
3983
|
async runs() {
|
|
3716
|
-
const runs = await this.store.listRuns();
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
purpose
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
3984
|
+
const [runs, comparisons] = await Promise.all([this.store.listRuns(), this.store.listComparisons()]);
|
|
3985
|
+
const first = /* @__PURE__ */ new Map();
|
|
3986
|
+
for (const run of [...runs].sort((a, b) => a.startedAt.localeCompare(b.startedAt))) {
|
|
3987
|
+
if (run.purpose === "baseline" && !run.failed && !first.has(run.machine.fingerprint)) {
|
|
3988
|
+
first.set(run.machine.fingerprint, run.id);
|
|
3989
|
+
}
|
|
3990
|
+
}
|
|
3991
|
+
const byHead = /* @__PURE__ */ new Map();
|
|
3992
|
+
for (const comparison of newestFirst(comparisons).reverse()) {
|
|
3993
|
+
for (const runId of comparison.head.runs)
|
|
3994
|
+
byHead.set(runId, comparison);
|
|
3995
|
+
}
|
|
3996
|
+
return runs.sort((a, b) => b.startedAt.localeCompare(a.startedAt)).map((run) => {
|
|
3997
|
+
const comparison = byHead.get(run.id);
|
|
3998
|
+
return {
|
|
3999
|
+
id: run.id,
|
|
4000
|
+
purpose: run.purpose,
|
|
4001
|
+
startedAt: run.startedAt,
|
|
4002
|
+
finishedAt: run.finishedAt,
|
|
4003
|
+
sha: run.source.sha,
|
|
4004
|
+
subject: run.source.subject,
|
|
4005
|
+
branch: run.source.branch,
|
|
4006
|
+
dirty: run.source.dirty,
|
|
4007
|
+
machine: run.machine.fingerprint,
|
|
4008
|
+
benchmarks: run.results.length,
|
|
4009
|
+
errors: run.results.filter((result) => result.errors.length).length,
|
|
4010
|
+
failed: run.failed,
|
|
4011
|
+
first: first.get(run.machine.fingerprint) === run.id,
|
|
4012
|
+
comparison: comparison ? { id: comparison.id, verdict: comparison.verdict, mode: comparison.mode } : void 0
|
|
4013
|
+
};
|
|
4014
|
+
});
|
|
3730
4015
|
}
|
|
3731
4016
|
async comparisons() {
|
|
3732
4017
|
const comparisons = newestFirst(await this.store.listComparisons());
|
|
@@ -3753,16 +4038,14 @@ var DashboardData = class {
|
|
|
3753
4038
|
return newestFirst(await this.store.listExperiments());
|
|
3754
4039
|
}
|
|
3755
4040
|
async sessions() {
|
|
3756
|
-
const ids = await this.store.sessions();
|
|
4041
|
+
const [ids, agents] = await Promise.all([this.store.sessions(), this.agents()]);
|
|
4042
|
+
const live = new Set(agents.filter((agent) => agent.state === "running").map((agent) => agent.sessionId));
|
|
3757
4043
|
const views = await Promise.all(ids.map(async (id2) => sessionView(id2, await this.store.events(id2))));
|
|
4044
|
+
for (const view of views)
|
|
4045
|
+
if (view.state === "running" && !live.has(view.id))
|
|
4046
|
+
view.state = "stopped";
|
|
3758
4047
|
return views.sort((a, b) => (b.lastAt ?? "").localeCompare(a.lastAt ?? ""));
|
|
3759
4048
|
}
|
|
3760
|
-
/** The newest events across the most recent sessions, newest first. */
|
|
3761
|
-
async activity(limit) {
|
|
3762
|
-
const sessions = (await this.sessions()).slice(0, 10);
|
|
3763
|
-
const events = (await Promise.all(sessions.map((session) => this.store.events(session.id)))).flat();
|
|
3764
|
-
return events.sort((a, b) => b.at.localeCompare(a.at)).slice(0, limit);
|
|
3765
|
-
}
|
|
3766
4049
|
};
|
|
3767
4050
|
function hotspotsOf(run, result) {
|
|
3768
4051
|
return {
|
|
@@ -3776,13 +4059,19 @@ function sessionView(id2, events) {
|
|
|
3776
4059
|
const first = events[0];
|
|
3777
4060
|
const last = events.at(-1);
|
|
3778
4061
|
if (!first || !last)
|
|
3779
|
-
return { id: id2, events: 0, state: "empty" };
|
|
4062
|
+
return { id: id2, events: 0, toolCalls: 0, errors: 0, state: "empty" };
|
|
4063
|
+
const cost = events.reduce((sum, event) => sum + (event.costUsd ?? 0), 0);
|
|
3780
4064
|
return {
|
|
3781
4065
|
id: id2,
|
|
3782
4066
|
role: first.role,
|
|
4067
|
+
goal: events.find((event) => event.kind === "start")?.summary,
|
|
3783
4068
|
startedAt: first.at,
|
|
3784
4069
|
lastAt: last.at,
|
|
4070
|
+
durationMs: Date.parse(last.at) - Date.parse(first.at),
|
|
3785
4071
|
events: events.length,
|
|
4072
|
+
toolCalls: events.filter((event) => event.kind === "tool-call").length,
|
|
4073
|
+
errors: events.filter((event) => event.kind === "error").length,
|
|
4074
|
+
...cost ? { costUsd: cost } : {},
|
|
3786
4075
|
last: last.summary,
|
|
3787
4076
|
state: last.kind === "error" ? "failed" : last.kind === "finish" ? "done" : "running"
|
|
3788
4077
|
};
|
|
@@ -3844,7 +4133,6 @@ async function handle2(req, res, data) {
|
|
|
3844
4133
|
if (req.method !== "GET" && req.method !== "HEAD")
|
|
3845
4134
|
throw new HttpError(405, "read-only");
|
|
3846
4135
|
const raw = (req.url ?? "/").split("?")[0];
|
|
3847
|
-
const query = new URL(req.url ?? "/", "http://localhost").searchParams;
|
|
3848
4136
|
const parts = raw.split("/").slice(1);
|
|
3849
4137
|
if (parts[0] !== "api")
|
|
3850
4138
|
return serveUi(res, raw);
|
|
@@ -3852,8 +4140,10 @@ async function handle2(req, res, data) {
|
|
|
3852
4140
|
if (rest.length)
|
|
3853
4141
|
throw new HttpError(404, "not found");
|
|
3854
4142
|
switch (resource) {
|
|
3855
|
-
case "
|
|
3856
|
-
return json(res, await data.
|
|
4143
|
+
case "summary":
|
|
4144
|
+
return json(res, await data.summary());
|
|
4145
|
+
case "agents":
|
|
4146
|
+
return json(res, await data.agentsAndPatrol());
|
|
3857
4147
|
case "benchmarks":
|
|
3858
4148
|
return json(res, key ? found(await data.benchmark(id(key)), "benchmark") : await data.benchmarks());
|
|
3859
4149
|
case "runs":
|
|
@@ -3870,10 +4160,6 @@ async function handle2(req, res, data) {
|
|
|
3870
4160
|
if (sub !== "events")
|
|
3871
4161
|
throw new HttpError(404, "not found");
|
|
3872
4162
|
return json(res, await data.store.events(id(key)));
|
|
3873
|
-
case "activity": {
|
|
3874
|
-
const limit = Number(query.get("limit") ?? 60);
|
|
3875
|
-
return json(res, await data.activity(Number.isFinite(limit) ? Math.max(1, Math.min(500, limit)) : 60));
|
|
3876
|
-
}
|
|
3877
4163
|
default:
|
|
3878
4164
|
throw new HttpError(404, "not found");
|
|
3879
4165
|
}
|
|
@@ -4239,7 +4525,7 @@ Docs: https://github.com/agent-labs-dev/speedpatrol`;
|
|
|
4239
4525
|
// ../cli/dist/bin.js
|
|
4240
4526
|
function version2() {
|
|
4241
4527
|
if (true)
|
|
4242
|
-
return "0.1.
|
|
4528
|
+
return "0.1.2";
|
|
4243
4529
|
try {
|
|
4244
4530
|
const here = dirname3(fileURLToPath2(import.meta.url));
|
|
4245
4531
|
return JSON.parse(readFileSync2(join11(here, "../../speedpatrol/package.json"), "utf8")).version;
|
package/package.json
CHANGED