pi-warden 0.71.0 → 0.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/dist/adaptive.d.ts +3 -2
- package/dist/adaptive.js +4 -3
- package/dist/backend.d.ts +33 -14
- package/dist/backend.js +27 -14
- package/dist/config.d.ts +13 -8
- package/dist/config.js +20 -20
- package/dist/conscience.d.ts +1 -1
- package/dist/extension.d.ts +14 -3
- package/dist/extension.js +410 -64
- package/dist/guard.d.ts +2 -1
- package/dist/guard.js +4 -4
- package/dist/holds.d.ts +1 -1
- package/dist/holds.js +3 -2
- package/dist/host-compat.d.ts +2 -0
- package/dist/host-compat.js +132 -0
- package/dist/host-dirs.d.ts +10 -0
- package/dist/host-dirs.js +21 -0
- package/dist/index-cmd.d.ts +6 -5
- package/dist/index-cmd.js +16 -24
- package/dist/index.d.ts +12 -4
- package/dist/index.js +6 -2
- package/dist/init.js +31 -3
- package/dist/learning.d.ts +12 -11
- package/dist/learning.js +60 -38
- package/dist/loops.d.ts +2 -1
- package/dist/loops.js +3 -2
- package/dist/panel.d.ts +4 -1
- package/dist/panel.js +11 -6
- package/dist/prefs.d.ts +1 -1
- package/dist/prefs.js +3 -2
- package/dist/rules-audit.d.ts +208 -0
- package/dist/rules-audit.js +417 -0
- package/dist/rules-calibrate.d.ts +196 -0
- package/dist/rules-calibrate.js +474 -0
- package/dist/rules-lint.d.ts +2 -0
- package/dist/rules-lint.js +7 -2
- package/dist/rules-log.d.ts +6 -1
- package/dist/rules-log.js +9 -3
- package/dist/rules-report.d.ts +3 -0
- package/dist/rules-report.js +5 -0
- package/dist/rules.d.ts +72 -5
- package/dist/rules.js +116 -54
- package/dist/shape.js +8 -2
- package/dist/turn-rules.d.ts +86 -0
- package/dist/turn-rules.js +232 -0
- package/dist/widget-render.d.ts +23 -0
- package/dist/widget-render.js +82 -0
- package/dist/widget.d.ts +5 -22
- package/dist/widget.js +3 -80
- package/package.json +5 -3
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { discoverSourceFiles } from "./discover.js";
|
|
4
|
+
import { evaluateRules, gitIgnored, projectPath, skipReason } from "./rules.js";
|
|
5
|
+
/**
|
|
6
|
+
* `/warden rules audit` and `/warden bench`: two plain functions over the rules guard.
|
|
7
|
+
*
|
|
8
|
+
* The audit judges existing files against the project rules as if each had just been written, one `write` request per
|
|
9
|
+
* file. Nothing leaves the machine without the confirm dialog (or `--yes` in a headless run), the results never reach
|
|
10
|
+
* the rules log, and the only file written is the Markdown copy under `.pi-warden/`. The bench measures what one of
|
|
11
|
+
* those requests costs on this machine with a fixed built-in sample, so no project content is sent for it at all.
|
|
12
|
+
*/
|
|
13
|
+
/** Files the audit judges unless `--max` says otherwise. */
|
|
14
|
+
export const AUDIT_DEFAULT_MAX = 50;
|
|
15
|
+
/** Requests in flight at once; small so a big audit cannot burst the backend. */
|
|
16
|
+
export const AUDIT_CONCURRENCY = 4;
|
|
17
|
+
/** Source files the discovery walk collects before it stops; the cap counts what it found. */
|
|
18
|
+
export const AUDIT_DISCOVERY_MAX = 5000;
|
|
19
|
+
export const AUDIT_REPORT_FILE = ".pi-warden/rules-audit.md";
|
|
20
|
+
export const RULES_AUDIT_USAGE = "Usage: /warden rules audit [paths...] [--max N] [--yes], where N is a positive integer (default 50) and paths default to the project.";
|
|
21
|
+
export const BENCH_USAGE = "Usage: /warden bench [--runs N], where N is a positive integer (default 10).";
|
|
22
|
+
export function parseRulesAuditArgs(tokens) {
|
|
23
|
+
const paths = [];
|
|
24
|
+
let max = AUDIT_DEFAULT_MAX;
|
|
25
|
+
let yes = false;
|
|
26
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
27
|
+
const token = tokens[index];
|
|
28
|
+
if (token === "--yes") {
|
|
29
|
+
yes = true;
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (token === "--max") {
|
|
33
|
+
const value = tokens[++index];
|
|
34
|
+
const count = Number(value);
|
|
35
|
+
if (value === undefined || !Number.isInteger(count) || count < 1)
|
|
36
|
+
return { paths, max, yes, error: `--max needs a positive integer. ${RULES_AUDIT_USAGE}` };
|
|
37
|
+
max = count;
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
if (token.startsWith("-"))
|
|
41
|
+
return { paths, max, yes, error: `Unknown option ${token}. ${RULES_AUDIT_USAGE}` };
|
|
42
|
+
paths.push(token);
|
|
43
|
+
}
|
|
44
|
+
return { paths: paths.length ? paths : ["."], max, yes };
|
|
45
|
+
}
|
|
46
|
+
export function parseBenchArgs(tokens) {
|
|
47
|
+
let runs = 10;
|
|
48
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
49
|
+
const token = tokens[index];
|
|
50
|
+
if (token === "--runs") {
|
|
51
|
+
const value = tokens[++index];
|
|
52
|
+
const count = Number(value);
|
|
53
|
+
if (value === undefined || !Number.isInteger(count) || count < 1)
|
|
54
|
+
return { runs, error: `--runs needs a positive integer. ${BENCH_USAGE}` };
|
|
55
|
+
runs = count;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
return { runs, error: `Unknown option ${token}. ${BENCH_USAGE}` };
|
|
59
|
+
}
|
|
60
|
+
return { runs };
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* The source files under `paths` the audit will judge: found by `discoverSourceFiles`, inside the project, not ignored
|
|
64
|
+
* by git, not kept out by `rules.exclude` or `rules.skip`, and with at least one rule that applies to them. Capped in
|
|
65
|
+
* sorted order so the same command judges the same files twice.
|
|
66
|
+
*/
|
|
67
|
+
export function collectAuditFiles(cwd, paths, set, config, max) {
|
|
68
|
+
const seen = new Set();
|
|
69
|
+
const candidates = [];
|
|
70
|
+
const missing = [];
|
|
71
|
+
for (const raw of paths) {
|
|
72
|
+
const abs = resolve(cwd, raw);
|
|
73
|
+
let stat;
|
|
74
|
+
try {
|
|
75
|
+
stat = statSync(abs);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
stat = undefined;
|
|
79
|
+
}
|
|
80
|
+
const rel = stat ? relative(resolve(cwd), abs).split(sep).join("/") : undefined;
|
|
81
|
+
// The project root is the one path `projectPath` rejects: it is not inside itself.
|
|
82
|
+
if (!stat || rel === undefined || rel.startsWith("..") || isAbsolute(rel)) {
|
|
83
|
+
missing.push(raw);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const found = stat.isDirectory()
|
|
87
|
+
? discoverSourceFiles(abs, AUDIT_DISCOVERY_MAX).map(name => projectPath(join(abs, name), cwd)).filter((path) => path !== undefined)
|
|
88
|
+
: rel ? [rel] : [];
|
|
89
|
+
if (!stat.isDirectory() && !stat.isFile())
|
|
90
|
+
missing.push(raw);
|
|
91
|
+
for (const candidate of found) {
|
|
92
|
+
if (seen.has(candidate))
|
|
93
|
+
continue;
|
|
94
|
+
seen.add(candidate);
|
|
95
|
+
if (gitIgnored(candidate, cwd))
|
|
96
|
+
continue;
|
|
97
|
+
if (skipReason({ tool: "write", path: candidate }, set, config))
|
|
98
|
+
continue;
|
|
99
|
+
candidates.push(candidate);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
candidates.sort();
|
|
103
|
+
return { files: candidates.slice(0, max), matched: candidates.length, leftOut: Math.max(0, candidates.length - max), missing };
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* One `write` request per selected file, at most AUDIT_CONCURRENCY at a time, after the confirm dialog (or `--yes`).
|
|
107
|
+
* With no judge nothing is sent and no dialog is shown. The Markdown copy is written to `AUDIT_REPORT_FILE`; the rules
|
|
108
|
+
* log is never touched.
|
|
109
|
+
*/
|
|
110
|
+
export async function runRulesAudit(options) {
|
|
111
|
+
if (!options.judge)
|
|
112
|
+
return { status: "no-judge" };
|
|
113
|
+
const { set } = options;
|
|
114
|
+
if (!set)
|
|
115
|
+
return { status: "no-files", reason: "no rules file resolved, so there is nothing to judge the files against" };
|
|
116
|
+
if (set.proseOnly)
|
|
117
|
+
return { status: "no-files", reason: `no rules found in ${set.sources.join(", ")} (prose only)` };
|
|
118
|
+
const plan = collectAuditFiles(options.cwd, options.paths, set, options.config, options.max);
|
|
119
|
+
if (!plan.files.length)
|
|
120
|
+
return { status: "no-files", reason: `no source file under ${options.paths.join(", ")} has a rule that applies to it` };
|
|
121
|
+
if (!options.yes) {
|
|
122
|
+
if (!options.confirm)
|
|
123
|
+
return { status: "needs-yes", files: plan.files.length, leftOut: plan.leftOut };
|
|
124
|
+
const title = `Send ${plan.files.length} file${plan.files.length === 1 ? "" : "s"} to ${options.destination} for a rules audit?`;
|
|
125
|
+
const body = [
|
|
126
|
+
`Each file is judged against the project rules as if it had just been written.`,
|
|
127
|
+
`What leaves the machine: a redacted sample of each file (up to 6000 characters per file) and the rule text, sent to ${options.destination}.`,
|
|
128
|
+
plan.leftOut ? `${plan.leftOut} file${plan.leftOut === 1 ? "" : "s"} left out at the --max ${options.max} cap.` : "",
|
|
129
|
+
"Nothing is written to the rules log.",
|
|
130
|
+
].filter(Boolean).join("\n");
|
|
131
|
+
if (!await options.confirm(title, body))
|
|
132
|
+
return { status: "cancelled" };
|
|
133
|
+
}
|
|
134
|
+
const queue = plan.files.map((path, index) => ({ path, index }));
|
|
135
|
+
const results = Array.from({ length: plan.files.length });
|
|
136
|
+
const unreadable = [];
|
|
137
|
+
const errors = [];
|
|
138
|
+
const started = Date.now();
|
|
139
|
+
const worker = async () => {
|
|
140
|
+
for (;;) {
|
|
141
|
+
const next = queue.shift();
|
|
142
|
+
if (!next)
|
|
143
|
+
return;
|
|
144
|
+
let content;
|
|
145
|
+
try {
|
|
146
|
+
content = readFileSync(join(options.cwd, next.path), "utf8");
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
unreadable.push(next.path);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (!content.trim()) {
|
|
153
|
+
unreadable.push(next.path);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
const verdict = await evaluateRules("write", { path: next.path, content }, {
|
|
157
|
+
cwd: options.cwd, config: options.config, set, judge: options.judge, timeoutMs: options.timeoutMs, ...(options.signal ? { signal: options.signal } : {}),
|
|
158
|
+
});
|
|
159
|
+
if (verdict.source === "error")
|
|
160
|
+
errors.push(next.path);
|
|
161
|
+
results[next.index] = { path: next.path, verdict };
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
await Promise.all(Array.from({ length: AUDIT_CONCURRENCY }, worker));
|
|
165
|
+
const judged = results.filter((entry) => entry !== undefined);
|
|
166
|
+
const rows = ruleAuditRows(judged);
|
|
167
|
+
const outcome = {
|
|
168
|
+
paths: [...options.paths],
|
|
169
|
+
max: options.max,
|
|
170
|
+
selected: plan.files.length,
|
|
171
|
+
judged: judged.length,
|
|
172
|
+
leftOut: plan.leftOut,
|
|
173
|
+
missing: plan.missing,
|
|
174
|
+
unreadable,
|
|
175
|
+
errors,
|
|
176
|
+
results: judged,
|
|
177
|
+
rows,
|
|
178
|
+
flagged: judged.filter(entry => entry.verdict.findings.length > 0).length,
|
|
179
|
+
sources: set.sources,
|
|
180
|
+
rules: rows.length,
|
|
181
|
+
aggregate: set.aggregate !== undefined && !set.rules.length,
|
|
182
|
+
reportPath: AUDIT_REPORT_FILE,
|
|
183
|
+
elapsedMs: Date.now() - started,
|
|
184
|
+
};
|
|
185
|
+
writeRulesAuditReport(options.cwd, rulesAuditMarkdown(outcome));
|
|
186
|
+
return { status: "done", outcome };
|
|
187
|
+
}
|
|
188
|
+
// ---------------------------------------------------------------------------
|
|
189
|
+
// The Markdown copy
|
|
190
|
+
function usd(value) {
|
|
191
|
+
return `$${value.toFixed(6)}`;
|
|
192
|
+
}
|
|
193
|
+
function ruleCell(id) {
|
|
194
|
+
return id.length <= 44 ? id : `${id.slice(0, 43)}…`;
|
|
195
|
+
}
|
|
196
|
+
/** The per-rule table body, flagged rules first. */
|
|
197
|
+
export function formatRulesAuditRows(rows) {
|
|
198
|
+
const lines = [`${"rule".padEnd(44)}${"judged".padStart(6)}${"flagged".padStart(8)}${"mean".padStart(7)}`];
|
|
199
|
+
for (const row of rows) {
|
|
200
|
+
lines.push(`${ruleCell(row.id).padEnd(44)}${String(row.judged).padStart(6)}${String(row.flagged).padStart(8)}${row.mean.toFixed(2).padStart(7)}`);
|
|
201
|
+
}
|
|
202
|
+
return lines.join("\n");
|
|
203
|
+
}
|
|
204
|
+
/** Files with findings, worst first (most findings, then the strongest score). */
|
|
205
|
+
export function flaggedAuditFiles(results) {
|
|
206
|
+
return results
|
|
207
|
+
.map(entry => ({
|
|
208
|
+
path: entry.path,
|
|
209
|
+
findings: entry.verdict.findings.map(finding => ({ name: finding.name, score: finding.violation })),
|
|
210
|
+
soft: (entry.verdict.softFindings ?? []).map(finding => ({ name: finding.name, score: finding.violation })),
|
|
211
|
+
}))
|
|
212
|
+
.filter(entry => entry.findings.length > 0 || entry.soft.length > 0)
|
|
213
|
+
.sort((a, b) => (b.findings.length - a.findings.length)
|
|
214
|
+
|| (Math.max(0, ...b.findings.map(finding => finding.score)) - Math.max(0, ...a.findings.map(finding => finding.score)))
|
|
215
|
+
|| a.path.localeCompare(b.path));
|
|
216
|
+
}
|
|
217
|
+
/** One line per file, rule names with scores indented under the path. */
|
|
218
|
+
export function formatRulesAuditFiles(results) {
|
|
219
|
+
const flagged = flaggedAuditFiles(results);
|
|
220
|
+
if (!flagged.length)
|
|
221
|
+
return "no file flagged";
|
|
222
|
+
const lines = [];
|
|
223
|
+
for (const entry of flagged) {
|
|
224
|
+
lines.push(entry.path);
|
|
225
|
+
for (const finding of entry.findings)
|
|
226
|
+
lines.push(` ${finding.name} ${finding.score.toFixed(2)}`);
|
|
227
|
+
for (const finding of entry.soft)
|
|
228
|
+
lines.push(` ${finding.name} ${finding.score.toFixed(2)} (double-check)`);
|
|
229
|
+
}
|
|
230
|
+
return lines.join("\n");
|
|
231
|
+
}
|
|
232
|
+
/** The terminal output: summary, the table by rule, the file list, and where the Markdown copy went. */
|
|
233
|
+
export function formatRulesAudit(outcome) {
|
|
234
|
+
const lines = [
|
|
235
|
+
`Rules audit: ${outcome.judged} file${outcome.judged === 1 ? "" : "s"} judged as new writes against ${outcome.sources.join(", ")} (${outcome.rules} rule${outcome.rules === 1 ? "" : "s"} in play); ${outcome.leftOut} left out at the --max ${outcome.max} cap; ${outcome.flagged} of ${outcome.judged} flagged.`,
|
|
236
|
+
];
|
|
237
|
+
if (outcome.aggregate)
|
|
238
|
+
lines.push("The rules document has no rule headings, so each file is judged against it as one document.");
|
|
239
|
+
lines.push(formatRulesAuditRows(outcome.rows));
|
|
240
|
+
lines.push("flagged files, worst first:");
|
|
241
|
+
lines.push(formatRulesAuditFiles(outcome.results));
|
|
242
|
+
if (outcome.missing.length)
|
|
243
|
+
lines.push(`not found in the project: ${outcome.missing.join("; ")}.`);
|
|
244
|
+
if (outcome.unreadable.length)
|
|
245
|
+
lines.push(`not judged (unreadable or empty): ${outcome.unreadable.join("; ")}.`);
|
|
246
|
+
if (outcome.errors.length)
|
|
247
|
+
lines.push(`judgment failed for ${outcome.errors.length} file${outcome.errors.length === 1 ? "" : "s"}: ${outcome.errors.join("; ")}.`);
|
|
248
|
+
lines.push(`Markdown copy: ${outcome.reportPath}. Nothing was recorded in the rules log.`);
|
|
249
|
+
return lines.join("\n");
|
|
250
|
+
}
|
|
251
|
+
/** The Markdown copy of one audit run. */
|
|
252
|
+
export function rulesAuditMarkdown(outcome, date = new Date()) {
|
|
253
|
+
const lines = [
|
|
254
|
+
"# Rules audit",
|
|
255
|
+
"",
|
|
256
|
+
`- Date: ${date.toISOString()}`,
|
|
257
|
+
`- Paths: ${outcome.paths.join(", ")}`,
|
|
258
|
+
`- Judged: ${outcome.judged} of ${outcome.selected} selected files as new writes (${outcome.leftOut} left out at the --max ${outcome.max} cap)`,
|
|
259
|
+
`- Rules in force: ${outcome.sources.join(", ")} (${outcome.rules} rule${outcome.rules === 1 ? "" : "s"} in play${outcome.aggregate ? ", judged as one document" : ""})`,
|
|
260
|
+
`- Flagged: ${outcome.flagged} of ${outcome.judged} files`,
|
|
261
|
+
"",
|
|
262
|
+
"## By rule",
|
|
263
|
+
"",
|
|
264
|
+
"| rule | judged | flagged | mean |",
|
|
265
|
+
"| --- | ---: | ---: | ---: |",
|
|
266
|
+
...outcome.rows.map(row => `| \`${row.id}\` | ${row.judged} | ${row.flagged} | ${row.mean.toFixed(2)} |`),
|
|
267
|
+
"",
|
|
268
|
+
"## Flagged files, worst first",
|
|
269
|
+
"",
|
|
270
|
+
...(flaggedAuditFiles(outcome.results).length
|
|
271
|
+
? flaggedAuditFiles(outcome.results).map(entry => `- \`${entry.path}\` — ${[
|
|
272
|
+
...entry.findings.map(finding => `${finding.name} ${finding.score.toFixed(2)}`),
|
|
273
|
+
...entry.soft.map(finding => `${finding.name} ${finding.score.toFixed(2)} (double-check)`),
|
|
274
|
+
].join("; ")}`)
|
|
275
|
+
: ["- no file flagged"]),
|
|
276
|
+
"",
|
|
277
|
+
];
|
|
278
|
+
if (outcome.missing.length)
|
|
279
|
+
lines.push(`Not found in the project: ${outcome.missing.join("; ")}.`, "");
|
|
280
|
+
if (outcome.unreadable.length)
|
|
281
|
+
lines.push(`Not judged (unreadable or empty): ${outcome.unreadable.join("; ")}.`, "");
|
|
282
|
+
if (outcome.errors.length)
|
|
283
|
+
lines.push(`Judgment failed for ${outcome.errors.length} file${outcome.errors.length === 1 ? "" : "s"}: ${outcome.errors.join("; ")}.`, "");
|
|
284
|
+
lines.push("Nothing was recorded in the rules log.");
|
|
285
|
+
return `${lines.join("\n")}\n`;
|
|
286
|
+
}
|
|
287
|
+
/** Write the Markdown copy and return the path it went to. */
|
|
288
|
+
export function writeRulesAuditReport(cwd, markdown) {
|
|
289
|
+
const path = join(cwd, ...AUDIT_REPORT_FILE.split("/"));
|
|
290
|
+
mkdirSync(join(cwd, ".pi-warden"), { recursive: true });
|
|
291
|
+
writeFileSync(path, markdown);
|
|
292
|
+
return path;
|
|
293
|
+
}
|
|
294
|
+
// ---------------------------------------------------------------------------
|
|
295
|
+
// Per-rule rows
|
|
296
|
+
/** One row per rule asked anywhere in the audit: files judged, files flagged, mean score; flagged first. */
|
|
297
|
+
export function ruleAuditRows(results) {
|
|
298
|
+
const rows = new Map();
|
|
299
|
+
for (const entry of results) {
|
|
300
|
+
for (const score of entry.verdict.scores ?? []) {
|
|
301
|
+
const row = rows.get(score.id) ?? { id: score.id, name: score.name, judged: 0, flagged: 0, mean: 0, sum: 0 };
|
|
302
|
+
row.judged += 1;
|
|
303
|
+
row.sum += score.violation;
|
|
304
|
+
if (entry.verdict.findings.some(finding => finding.id === score.id))
|
|
305
|
+
row.flagged += 1;
|
|
306
|
+
rows.set(score.id, row);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return [...rows.values()]
|
|
310
|
+
.map(row => ({ id: row.id, name: row.name, judged: row.judged, flagged: row.flagged, mean: row.judged ? row.sum / row.judged : 0 }))
|
|
311
|
+
.sort((a, b) => (b.flagged - a.flagged) || (b.mean - a.mean) || a.id.localeCompare(b.id));
|
|
312
|
+
}
|
|
313
|
+
// ---------------------------------------------------------------------------
|
|
314
|
+
// Bench
|
|
315
|
+
/**
|
|
316
|
+
* The one fixed sample the bench judges. It is built in, so no project content is sent for a bench run; only the
|
|
317
|
+
* active rule text travels with it.
|
|
318
|
+
*/
|
|
319
|
+
export const BENCH_SAMPLE = {
|
|
320
|
+
path: "src/warden-bench-sample.ts",
|
|
321
|
+
content: [
|
|
322
|
+
"// Built-in so /warden bench measures a rules check without sending project content.",
|
|
323
|
+
"export interface Sample { id: string; label: string }",
|
|
324
|
+
"",
|
|
325
|
+
"export function groupByLabel(items: readonly Sample[]): Map<string, Sample[]> {",
|
|
326
|
+
" const groups = new Map<string, Sample[]>();",
|
|
327
|
+
" for (const item of items) {",
|
|
328
|
+
" const bucket = groups.get(item.label);",
|
|
329
|
+
" if (bucket) bucket.push(item);",
|
|
330
|
+
" else groups.set(item.label, [item]);",
|
|
331
|
+
" }",
|
|
332
|
+
" return groups;",
|
|
333
|
+
"}",
|
|
334
|
+
"",
|
|
335
|
+
"export function labels(items: readonly Sample[]): string[] {",
|
|
336
|
+
" return [...new Set(items.map((item) => item.label))].sort();",
|
|
337
|
+
"}",
|
|
338
|
+
"",
|
|
339
|
+
].join("\n"),
|
|
340
|
+
};
|
|
341
|
+
/** Nearest-rank percentile of a sorted, non-empty list of numbers. */
|
|
342
|
+
export function percentile(sorted, p) {
|
|
343
|
+
if (!sorted.length)
|
|
344
|
+
return 0;
|
|
345
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
|
346
|
+
return sorted[index];
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Judge the built-in sample `runs` times against the active rules, one request at a time so the latency numbers mean
|
|
350
|
+
* something, and report what the checks cost on this machine. With no judge nothing is sent; a sample the rules keep
|
|
351
|
+
* out (`rules.exclude`, `rules.skip`) is reported and never sent either.
|
|
352
|
+
*/
|
|
353
|
+
export async function runBench(options) {
|
|
354
|
+
if (!options.judge)
|
|
355
|
+
return { status: "no-judge" };
|
|
356
|
+
const target = { tool: "write", path: BENCH_SAMPLE.path };
|
|
357
|
+
const reason = skipReason(target, options.set, options.config);
|
|
358
|
+
if (reason)
|
|
359
|
+
return { status: "skipped", reason };
|
|
360
|
+
const now = options.now ?? (() => Date.now());
|
|
361
|
+
const before = options.getUsage?.();
|
|
362
|
+
const timings = [];
|
|
363
|
+
let flagged = 0;
|
|
364
|
+
let errors = 0;
|
|
365
|
+
let asked = 0;
|
|
366
|
+
let aggregate = false;
|
|
367
|
+
for (let run = 0; run < options.runs; run++) {
|
|
368
|
+
const started = now();
|
|
369
|
+
const verdict = await evaluateRules("write", { path: BENCH_SAMPLE.path, content: BENCH_SAMPLE.content }, {
|
|
370
|
+
cwd: options.cwd, config: options.config, set: options.set, judge: options.judge, timeoutMs: options.timeoutMs, ...(options.signal ? { signal: options.signal } : {}),
|
|
371
|
+
});
|
|
372
|
+
timings.push(now() - started);
|
|
373
|
+
if (verdict.source === "error")
|
|
374
|
+
errors++;
|
|
375
|
+
if (verdict.findings.length)
|
|
376
|
+
flagged++;
|
|
377
|
+
asked = verdict.asked;
|
|
378
|
+
aggregate = verdict.aggregate;
|
|
379
|
+
}
|
|
380
|
+
const sorted = [...timings].sort((a, b) => a - b);
|
|
381
|
+
const after = options.getUsage?.();
|
|
382
|
+
const usage = before && after ? {
|
|
383
|
+
requests: after.requestsStarted - before.requestsStarted,
|
|
384
|
+
inputTokens: (after.inputTokens - before.inputTokens) / options.runs,
|
|
385
|
+
usd: (after.estimatedUsd - before.estimatedUsd) / options.runs,
|
|
386
|
+
} : undefined;
|
|
387
|
+
return {
|
|
388
|
+
status: "done",
|
|
389
|
+
runs: options.runs,
|
|
390
|
+
asked,
|
|
391
|
+
aggregate,
|
|
392
|
+
flagged,
|
|
393
|
+
errors,
|
|
394
|
+
requests: usage?.requests ?? options.runs,
|
|
395
|
+
p50: Math.round(percentile(sorted, 50)),
|
|
396
|
+
p95: Math.round(percentile(sorted, 95)),
|
|
397
|
+
...(usage ? { meanInputTokens: usage.inputTokens, costPerCheck: usage.usd, costPer100: usage.usd * 100 } : {}),
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
/** The bench output: the numbers, and the line that says no project content was sent. */
|
|
401
|
+
export function formatBench(result) {
|
|
402
|
+
const lines = [
|
|
403
|
+
`Bench: ${result.runs} check${result.runs === 1 ? "" : "s"} of one built-in sample file against the active rules (${result.asked} rule${result.asked === 1 ? "" : "s"} per check${result.aggregate ? ", document judged as one rule" : ""}).`,
|
|
404
|
+
"The sample is built in and no project content is sent, so no confirmation was needed.",
|
|
405
|
+
`Latency: p50 ${result.p50} ms, p95 ${result.p95} ms; requests ${result.requests}${result.errors ? ` (${result.errors} failed)` : ""}.`,
|
|
406
|
+
];
|
|
407
|
+
if (result.flagged)
|
|
408
|
+
lines.push(`Findings in ${result.flagged} of ${result.runs} checks (the sample is judged as written code, not as a fixture).`);
|
|
409
|
+
lines.push(result.meanInputTokens === undefined
|
|
410
|
+
? "Input tokens per check: unknown (no usage counters)."
|
|
411
|
+
: `Input tokens per check: ${Math.round(result.meanInputTokens)} (mean).`);
|
|
412
|
+
const { costPerCheck, costPer100 } = result;
|
|
413
|
+
lines.push(costPerCheck === undefined || costPer100 === undefined
|
|
414
|
+
? "Estimated cost per check: unknown (no usage counters)."
|
|
415
|
+
: `Estimated cost per check: ${usd(costPerCheck)}; per 100 edits: ${usd(costPer100)}.`);
|
|
416
|
+
return lines.join("\n");
|
|
417
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import type { Judge } from "pi-typesafe";
|
|
2
|
+
import type { RulesConfig } from "./config.js";
|
|
3
|
+
import type { RulesCheckResult } from "./rules-lint.js";
|
|
4
|
+
import type { RuleRecord } from "./rules-log.js";
|
|
5
|
+
import type { RuleFlag } from "./rules-report.js";
|
|
6
|
+
import type { EditView, Rule, RuleSet, RulesTarget } from "./rules.js";
|
|
7
|
+
/**
|
|
8
|
+
* `/warden rules calibrate`: replay recent git history through the project rules and find the rules that fire on
|
|
9
|
+
* everything or cannot decide. A rule the sample never caught shows `no violation in sample` and is not flagged: the
|
|
10
|
+
* replayed commits are mostly reviewed, compliant code, so a rule that never fires there is often a rule people follow. The core is a plain function of the history and the rule set: the history in, per-rule
|
|
11
|
+
* results out. The caller collects the history (`collectHistory`, read-only `git log -p`), shows the confirm dialog, and
|
|
12
|
+
* writes the scores to the local rules log; nothing here sends anything on its own and nothing here writes a file.
|
|
13
|
+
*
|
|
14
|
+
* Each changed file of each commit becomes one `edit` input: the hunk's removed lines are `oldText`, its added lines are
|
|
15
|
+
* `newText`, and the file after the commit supplies the surrounding context (`before`/`after` around the change). Binary,
|
|
16
|
+
* generated, gitignored files and anything under `rules.exclude` or `rules.skip` are skipped, and the requests are capped
|
|
17
|
+
* at `--max`. `/warden rules tune` reads the flagged rules from the result (and from `/warden rules check`) and asks the
|
|
18
|
+
* session's agent to rewrite them; that prompt is the only thing tune sends.
|
|
19
|
+
*/
|
|
20
|
+
/** Non-merge commits read when `--commits` is absent. */
|
|
21
|
+
export declare const CALIBRATE_DEFAULT_COMMITS = 20;
|
|
22
|
+
/** Requests sent when `--max` is absent. */
|
|
23
|
+
export declare const CALIBRATE_DEFAULT_MAX = 40;
|
|
24
|
+
export interface HistoryHunk {
|
|
25
|
+
/** Removed lines of the hunk, newline-joined: the edit's `oldText`. */
|
|
26
|
+
oldText: string;
|
|
27
|
+
/** Added lines of the hunk, newline-joined: the edit's `newText`. */
|
|
28
|
+
newText: string;
|
|
29
|
+
/** The hunk as the diff shows it: context plus removed lines, the file before the change. */
|
|
30
|
+
before: string;
|
|
31
|
+
/** The hunk as the diff shows it: context plus added lines, the file after the change. */
|
|
32
|
+
after: string;
|
|
33
|
+
/** 1-based first line of the hunk on the file's after side. */
|
|
34
|
+
start: number;
|
|
35
|
+
/** Lines the hunk has on the after side (context plus added). */
|
|
36
|
+
count: number;
|
|
37
|
+
}
|
|
38
|
+
export interface HistoryFile {
|
|
39
|
+
/** Project-relative path with forward slashes, as the commit changed it. */
|
|
40
|
+
path: string;
|
|
41
|
+
/** A binary patch carries no lines to judge. */
|
|
42
|
+
binary: boolean;
|
|
43
|
+
hunks: HistoryHunk[];
|
|
44
|
+
/** The file's text after the commit, when it still exists there; the edit's context. */
|
|
45
|
+
after?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface HistoryCommit {
|
|
48
|
+
hash: string;
|
|
49
|
+
/** ISO timestamp of the commit. */
|
|
50
|
+
at: string;
|
|
51
|
+
files: HistoryFile[];
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The changed files of each commit in one `git log -p` output: one record per commit, one per changed file, and one
|
|
55
|
+
* `oldText`/`newText` pair per hunk. Pure: the caller runs git and passes the text.
|
|
56
|
+
*/
|
|
57
|
+
export declare function parseHistory(log: string): HistoryCommit[];
|
|
58
|
+
/**
|
|
59
|
+
* The last `commits` non-merge commits with their patches, and the file after each commit as the edit's context. Read-only:
|
|
60
|
+
* `git log -p` and `git show`, nothing else. Throws when git cannot read the history (no commits yet, not a repository).
|
|
61
|
+
*/
|
|
62
|
+
export declare function collectHistory(cwd: string, commits: number): HistoryCommit[];
|
|
63
|
+
/** A file the history replay skips, with the same reason wording the live guard reports. */
|
|
64
|
+
export interface CalibrateSkip {
|
|
65
|
+
path: string;
|
|
66
|
+
commit: string;
|
|
67
|
+
reason: string;
|
|
68
|
+
}
|
|
69
|
+
/** Why a changed file is not replayed, or undefined when it is judged. Binary and generated are decided here. */
|
|
70
|
+
export declare function changeSkipReason(file: HistoryFile): string | undefined;
|
|
71
|
+
/** The hunk's `oldText`/`newText` pairs as one `edit` input: the file after the commit around each change is the context. */
|
|
72
|
+
export declare function editViews(file: HistoryFile, contextLines?: number, maxEdits?: number): {
|
|
73
|
+
edits: EditView[];
|
|
74
|
+
moreEdits: number;
|
|
75
|
+
};
|
|
76
|
+
export interface CalibrateSample {
|
|
77
|
+
commit: string;
|
|
78
|
+
/** ISO timestamp of the commit the change came from. */
|
|
79
|
+
at: string;
|
|
80
|
+
path: string;
|
|
81
|
+
target: RulesTarget;
|
|
82
|
+
}
|
|
83
|
+
export interface CalibratePlan {
|
|
84
|
+
/** One `edit` input per judged change, in history order (newest first), at most `maxRequests`. */
|
|
85
|
+
samples: CalibrateSample[];
|
|
86
|
+
/** Samples past `maxRequests`, not sent. */
|
|
87
|
+
dropped: CalibrateSample[];
|
|
88
|
+
/** Requests the run would send: `samples.length`. */
|
|
89
|
+
requests: number;
|
|
90
|
+
skipped: CalibrateSkip[];
|
|
91
|
+
}
|
|
92
|
+
export interface PlanOptions {
|
|
93
|
+
set: RuleSet;
|
|
94
|
+
config: RulesConfig;
|
|
95
|
+
/** The project root the paths are relative to; also decides `gitignore` skips. */
|
|
96
|
+
cwd: string;
|
|
97
|
+
maxRequests: number;
|
|
98
|
+
}
|
|
99
|
+
/** Which changed files of the history are judged, in what shape, and which are skipped and why. Pure apart from `git check-ignore`. */
|
|
100
|
+
export declare function planCalibration(history: readonly HistoryCommit[], options: PlanOptions): CalibratePlan;
|
|
101
|
+
export interface RuleCalibration {
|
|
102
|
+
id: string;
|
|
103
|
+
name: string;
|
|
104
|
+
/** How often the rule was asked: its `paths` matched a judged file. */
|
|
105
|
+
applied: number;
|
|
106
|
+
fired: number;
|
|
107
|
+
/** fired / applied. */
|
|
108
|
+
firedRate: number;
|
|
109
|
+
meanViolation: number;
|
|
110
|
+
/**
|
|
111
|
+
* True when the sample held no violation of this rule. Not a flag: replayed commits are mostly reviewed, compliant
|
|
112
|
+
* code, so a rule that never fires there is often a rule people follow. The line says `no violation in sample`.
|
|
113
|
+
*/
|
|
114
|
+
unfired: boolean;
|
|
115
|
+
/** The `/warden report` flags with the same thresholds, without `never fires`: see `unfired`. */
|
|
116
|
+
flags: RuleFlag[];
|
|
117
|
+
}
|
|
118
|
+
export interface Calibration {
|
|
119
|
+
rows: RuleCalibration[];
|
|
120
|
+
/** Rules in the current rule set that no judged file reached. */
|
|
121
|
+
neverApplied: Array<{
|
|
122
|
+
id: string;
|
|
123
|
+
name: string;
|
|
124
|
+
}>;
|
|
125
|
+
/** Requests sent. */
|
|
126
|
+
requests: number;
|
|
127
|
+
/** Requests that came back without an answer; their files are not scored. */
|
|
128
|
+
failed: number;
|
|
129
|
+
skipped: CalibrateSkip[];
|
|
130
|
+
/** Judged changes past the cap, not sent. */
|
|
131
|
+
dropped: number;
|
|
132
|
+
/** One record per scored rule, `source: "calibrate"`, for the local rules log. */
|
|
133
|
+
records: RuleRecord[];
|
|
134
|
+
commits: number;
|
|
135
|
+
}
|
|
136
|
+
export interface CalibrateOptions extends PlanOptions {
|
|
137
|
+
judge?: Judge | undefined;
|
|
138
|
+
timeoutMs: number;
|
|
139
|
+
/** Session id for the log records; replays default to `calibrate`. */
|
|
140
|
+
session?: string;
|
|
141
|
+
signal?: AbortSignal | undefined;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Replay the history through every rule of the set and return per-rule results, worst first: how often the rule applied,
|
|
145
|
+
* fired, its mean score, and the flags `/warden report` uses. The rules the sample never reached are listed apart. Each
|
|
146
|
+
* score also comes back as a `RuleRecord` with `source: "calibrate"` for the local rules log. Never throws; a request
|
|
147
|
+
* that fails is counted in `failed` and leaves its file unscored. With no judge nothing is sent and nothing is scored.
|
|
148
|
+
*/
|
|
149
|
+
export declare function calibrate(history: readonly HistoryCommit[], options: CalibrateOptions): Promise<Calibration>;
|
|
150
|
+
/** One line per rule, worst first: how often it applied and fired, the mean score, and the flags; then the unheard rules. */
|
|
151
|
+
export declare function formatCalibration(result: Calibration): string;
|
|
152
|
+
export type CalibrateGate = {
|
|
153
|
+
kind: "send";
|
|
154
|
+
} | {
|
|
155
|
+
kind: "confirm";
|
|
156
|
+
} | {
|
|
157
|
+
kind: "refuse";
|
|
158
|
+
reason: string;
|
|
159
|
+
};
|
|
160
|
+
/**
|
|
161
|
+
* Nothing leaves the machine before the user sees what it is: a UI shows the confirm dialog, and a headless run needs the
|
|
162
|
+
* explicit `--yes`. `--yes` stands in for the dialog in either case.
|
|
163
|
+
*/
|
|
164
|
+
export declare function calibrateGate(options: {
|
|
165
|
+
hasUI: boolean;
|
|
166
|
+
yes: boolean;
|
|
167
|
+
}): CalibrateGate;
|
|
168
|
+
/** What the confirm dialog shows: how many requests go out and what leaves the machine, as redacted diff samples. */
|
|
169
|
+
export declare function calibrationNotice(plan: CalibratePlan): string;
|
|
170
|
+
export interface TuneRule {
|
|
171
|
+
id: string;
|
|
172
|
+
name: string;
|
|
173
|
+
/** The rule's current text as the agent sees it in the rules file. */
|
|
174
|
+
body: string;
|
|
175
|
+
/** One line per flag: why this rule is up for a rewrite. */
|
|
176
|
+
reasons: string[];
|
|
177
|
+
}
|
|
178
|
+
/** The rules the latest calibrate flagged, plus the ones `/warden rules check` flagged this session, with the reasons. */
|
|
179
|
+
export declare function tuneTargets(options: {
|
|
180
|
+
calibration?: Calibration | undefined;
|
|
181
|
+
check?: RulesCheckResult | undefined;
|
|
182
|
+
rules: readonly Rule[];
|
|
183
|
+
}): TuneRule[];
|
|
184
|
+
/** One prompt for the session's agent: each flagged rule, why it was flagged, and the ask for a concrete rewrite. */
|
|
185
|
+
export declare function buildTunePrompt(targets: readonly TuneRule[]): string;
|
|
186
|
+
export type TuneRequest = {
|
|
187
|
+
prompt: string;
|
|
188
|
+
} | {
|
|
189
|
+
reason: string;
|
|
190
|
+
};
|
|
191
|
+
/** The prompt to send, or the reason to say so and send nothing when no rule is flagged. */
|
|
192
|
+
export declare function tuneRequest(options: {
|
|
193
|
+
calibration?: Calibration | undefined;
|
|
194
|
+
check?: RulesCheckResult | undefined;
|
|
195
|
+
rules: readonly Rule[];
|
|
196
|
+
}): TuneRequest;
|