simt-dev 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +239 -0
- package/bin/simt-dev.mjs +21 -2
- package/docs/decisions/ADR-064-the-context-meter-counts-what-the-tool-wrote.md +107 -0
- package/docs/decisions/ADR-065-a-pack-is-one-shared-rule-set-pinned-by-digest.md +97 -0
- package/docs/decisions/ADR-066-a-monorepo-projects-once-per-package.md +115 -0
- package/docs/decisions/ADR-067-which-harnesses-read-agents-md-is-a-column.md +110 -0
- package/docs/decisions/ADR-068-the-mcp-revision-list-cites-what-was-read.md +111 -0
- package/docs/decisions/ADR-069-the-conformance-lint-checks-the-published-specification.md +90 -0
- package/docs/decisions/ADR-070-the-json-payload-carries-its-own-version.md +91 -0
- package/docs/decisions/ADR-071-a-rule-that-can-be-narrowed-is-written-narrow.md +106 -0
- package/docs/decisions/ADR-072-the-budget-is-a-declaration-beside-the-gates.md +104 -0
- package/docs/decisions/ADR-073-a-kernel-is-a-smaller-projection-the-repository-asks-for.md +92 -0
- package/docs/decisions/ADR-074-a-rule-and-a-decision-are-readable-by-uri.md +99 -0
- package/docs/decisions/ADR-075-nothing-leaves-the-machine.md +56 -0
- package/docs/decisions/ADR-076-a-change-to-what-agents-read-is-measured.md +70 -0
- package/docs/decisions/ADR-077-the-command-card-leads-the-region-and-reads-committed-evidence.md +91 -0
- package/docs/decisions/ADR-078-a-line-the-repository-already-proves-does-not-ship.md +117 -0
- package/docs/decisions/ADR-079-the-rules-are-enforced-at-the-tool-call-from-one-policy-table.md +162 -0
- package/docs/decisions/ADR-080-the-tools-own-names-are-not-repositories-and-neither-reader-decides-by-shape.md +99 -0
- package/docs/decisions/ADR-081-a-tracker-is-read-through-the-repositorys-template.md +36 -0
- package/docs/decisions/ADR-082-the-tracker-override-is-a-manifest-key.md +35 -0
- package/package.json +2 -1
- package/rules/core.md +1 -0
- package/rules/stacks/android.md +2 -0
- package/rules/stacks/cloudflare-workers.md +2 -0
- package/rules/stacks/connectiq.md +1 -0
- package/rules/stacks/container.md +2 -0
- package/rules/stacks/dart.md +2 -0
- package/rules/stacks/deno.md +2 -0
- package/rules/stacks/dotnet.md +2 -0
- package/rules/stacks/elixir.md +2 -0
- package/rules/stacks/go.md +2 -0
- package/rules/stacks/jvm.md +2 -0
- package/rules/stacks/node.md +2 -0
- package/rules/stacks/python.md +2 -0
- package/rules/stacks/react-native.md +2 -0
- package/rules/stacks/react-web.md +2 -0
- package/rules/stacks/ruby.md +2 -0
- package/rules/stacks/rust.md +2 -0
- package/rules/stacks/swift.md +2 -0
- package/rules/stacks/typescript.md +2 -0
- package/rules/stacks/uv.md +2 -0
- package/src/agenthooks/decide.mjs +123 -0
- package/src/agenthooks/policy.mjs +78 -0
- package/src/agenthooks/render.mjs +240 -0
- package/src/argv.mjs +55 -1
- package/src/bench/ablate.mjs +630 -0
- package/src/bench/adapters.json +149 -0
- package/src/bench/grade.mjs +229 -0
- package/src/bench/mine.mjs +811 -0
- package/src/bench/probes.mjs +1350 -0
- package/src/bench/report.mjs +899 -0
- package/src/bench/run.mjs +1002 -0
- package/src/commands/agenthook.mjs +282 -0
- package/src/commands/bench.mjs +359 -0
- package/src/commands/compare.mjs +15 -0
- package/src/commands/graph.mjs +45 -21
- package/src/commands/harness.mjs +301 -0
- package/src/commands/help.mjs +73 -2
- package/src/commands/index.mjs +67 -25
- package/src/commands/init.mjs +12 -8
- package/src/commands/mcp.mjs +20 -5
- package/src/commands/plan.mjs +99 -0
- package/src/commands/rules.mjs +42 -57
- package/src/commands/skills.mjs +67 -92
- package/src/commands/skillsinstall.mjs +17 -2
- package/src/commands/sync.mjs +62 -6
- package/src/commands/tokens.mjs +169 -0
- package/src/compare/row.mjs +73 -2
- package/src/decisions/index.mjs +23 -5
- package/src/graph/spec.mjs +227 -0
- package/src/graph/speccheck.mjs +68 -0
- package/src/hooks/dispatcher.mjs +35 -3
- package/src/kernel/card.mjs +254 -0
- package/src/kernel/inferability.mjs +188 -0
- package/src/mcp/resources.mjs +157 -0
- package/src/mcp/schema.mjs +139 -0
- package/src/mcp/server.mjs +228 -20
- package/src/mcp/tools.mjs +372 -1
- package/src/measure/context.mjs +542 -0
- package/src/measure/loaderView.mjs +435 -0
- package/src/measure/session.mjs +267 -0
- package/src/monorepo.mjs +339 -0
- package/src/output/index.mjs +78 -2
- package/src/packs.mjs +182 -0
- package/src/plan/index.mjs +650 -0
- package/src/plan/template.mjs +331 -0
- package/src/projection/loaders.mjs +689 -0
- package/src/projection/manifest.mjs +114 -3
- package/src/projection/project.mjs +140 -9
- package/src/projection/rules.mjs +365 -22
- package/src/projection/surfaces.mjs +155 -23
- package/src/projection/targets.mjs +37 -0
- package/src/toolwords.mjs +69 -0
- package/templates/plan/tracker.json +36 -0
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
// simt-dev agent-hook <event> | agent-hooks install | agent-hooks status
|
|
2
|
+
//
|
|
3
|
+
// Three commands, one subject: the rules the tool projects, enforced while the agent is
|
|
4
|
+
// working rather than when git is committing. `agent-hook` is the entry point a harness
|
|
5
|
+
// calls; it reads one JSON payload on stdin and answers in that harness's dialect.
|
|
6
|
+
// `install` writes the entry into each harness's own config, and `status` reports what
|
|
7
|
+
// each harness will do and what still has to be trusted by hand.
|
|
8
|
+
//
|
|
9
|
+
// The entry point is on a hot path — it runs before every shell command an agent wants
|
|
10
|
+
// to run — so it imports the policy module, the decision, the renderer and nothing that
|
|
11
|
+
// walks the repository. A hook that measured the tree before answering would be exactly
|
|
12
|
+
// the latency the plan's acceptance criterion exists to prevent.
|
|
13
|
+
//
|
|
14
|
+
// A denial exits 2, which is what every harness in the table reads as "block". A mistake
|
|
15
|
+
// in the hook's own invocation must NOT exit 2: a typo in a config would then block every
|
|
16
|
+
// tool call. Failures of this command exit 1, which a harness reports as a failed hook,
|
|
17
|
+
// and that is the right blast radius for a hook that is broken rather than refusing.
|
|
18
|
+
|
|
19
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
20
|
+
import { dirname, join } from 'node:path';
|
|
21
|
+
|
|
22
|
+
import { EXIT, emit, emitJson, fail, isJson, note, writeMode } from '../output/index.mjs';
|
|
23
|
+
import { CLI } from '../name.mjs';
|
|
24
|
+
import { ALLOW_ENV, EVENTS, POLICY_IDS } from '../agenthooks/policy.mjs';
|
|
25
|
+
import { decide, factsFor } from '../agenthooks/decide.mjs';
|
|
26
|
+
import {
|
|
27
|
+
adapter,
|
|
28
|
+
adapters,
|
|
29
|
+
decisionJson,
|
|
30
|
+
factsFrom,
|
|
31
|
+
hookPath,
|
|
32
|
+
hookRows,
|
|
33
|
+
installedIn,
|
|
34
|
+
rendered,
|
|
35
|
+
} from '../agenthooks/render.mjs';
|
|
36
|
+
|
|
37
|
+
export async function agentHookCommand(args, flags) {
|
|
38
|
+
const event = args[0];
|
|
39
|
+
if (!EVENTS.includes(event)) {
|
|
40
|
+
return fail(`unknown agent event \`${event ?? ''}\``, EXIT.FAIL, `known events: ${EVENTS.join(', ')}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const id = flags.harness ?? args[1];
|
|
44
|
+
const row = id ? adapter(id) : null;
|
|
45
|
+
if (row === null) {
|
|
46
|
+
return fail(
|
|
47
|
+
id === undefined ? `agent-hook needs --harness` : `unknown harness \`${id}\``,
|
|
48
|
+
EXIT.FAIL,
|
|
49
|
+
`--harness <id>, one of: ${adapters().map((a) => a.id).join(', ')}`,
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const { payload, empty, error } = await readPayload();
|
|
54
|
+
if (empty || error) {
|
|
55
|
+
// Nothing to check is not a violation. A harness that calls the hook with nothing on
|
|
56
|
+
// stdin, or with a payload we cannot parse, has not told us that anything is wrong;
|
|
57
|
+
// blocking there would stop real work on no evidence, which is how a hook gets
|
|
58
|
+
// uninstalled the same afternoon it was installed.
|
|
59
|
+
note(`note ${id}: ${empty ? 'no payload on stdin' : `payload is not JSON (${error})`}; nothing to check`);
|
|
60
|
+
return EXIT.OK;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const { root, command, writes } = factsFrom(row, payload);
|
|
64
|
+
const facts = await factsFor(root, { command, writes });
|
|
65
|
+
const decision = decide({ event, facts, env: process.env });
|
|
66
|
+
|
|
67
|
+
// Both notes are about a conclusion the author is about to draw: a lift that did
|
|
68
|
+
// nothing because of a typo, and a lift that is the reason the command they expected to
|
|
69
|
+
// be stopped was not.
|
|
70
|
+
if (decision.unknown.length > 0) {
|
|
71
|
+
note(`note ${ALLOW_ENV} names no policy: ${decision.unknown.join(', ')}; known: ${POLICY_IDS.join(', ')}`);
|
|
72
|
+
}
|
|
73
|
+
if (decision.lifted.length > 0) {
|
|
74
|
+
note(`note allowed by ${ALLOW_ENV}: ${decision.lifted.join(', ')}`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const json = decisionJson(row, decision);
|
|
78
|
+
if (json === null) return EXIT.OK;
|
|
79
|
+
|
|
80
|
+
emit(JSON.stringify(json));
|
|
81
|
+
for (const message of decision.messages) note(`blocked ${message}`);
|
|
82
|
+
return row.denyExit;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export async function agentHooksCommand(sub, args, flags) {
|
|
86
|
+
const root = flags.root ?? args[0] ?? process.cwd();
|
|
87
|
+
if (sub === 'install') return installHooks(root, flags);
|
|
88
|
+
if (sub === 'status') return hooksStatus(root, flags);
|
|
89
|
+
return fail(`unknown agent-hooks subcommand ${sub}`, EXIT.USAGE, `run: ${CLI} agent-hooks status`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function installHooks(root, flags) {
|
|
93
|
+
const mode = writeMode(flags);
|
|
94
|
+
const plan = [];
|
|
95
|
+
for (const row of adapters()) {
|
|
96
|
+
const path = hookPath(row.id);
|
|
97
|
+
const existing = await readIfExists(join(root, path));
|
|
98
|
+
const result = rendered(row, existing);
|
|
99
|
+
plan.push({ id: row.id, path, action: result.error ? 'refused' : result.action, error: result.error, text: result.text });
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// The readers this tool does not write an entry for, reported rather than skipped in
|
|
103
|
+
// silence: a command that writes four files and says nothing about the fifth harness
|
|
104
|
+
// in the repository reads as "that one is covered too".
|
|
105
|
+
const others = [];
|
|
106
|
+
for (const r of hookRows()) {
|
|
107
|
+
if (r.path === null) others.push({ id: r.id, path: null, state: 'none', note: r.note });
|
|
108
|
+
else if (adapter(r.id) === null) others.push({ id: r.id, path: r.path, state: 'other', note: r.note });
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const refused = plan.filter((p) => p.error);
|
|
112
|
+
|
|
113
|
+
if (!mode.apply) {
|
|
114
|
+
if (isJson(flags)) {
|
|
115
|
+
emitJson({
|
|
116
|
+
ok: refused.length === 0,
|
|
117
|
+
command: 'agent-hooks install',
|
|
118
|
+
root,
|
|
119
|
+
dryRun: true,
|
|
120
|
+
plan: plan.map(({ text, ...rest }) => rest),
|
|
121
|
+
notWritten: others,
|
|
122
|
+
filesWritten: false,
|
|
123
|
+
});
|
|
124
|
+
return refused.length > 0 ? EXIT.FAIL : EXIT.OK;
|
|
125
|
+
}
|
|
126
|
+
emit(
|
|
127
|
+
[
|
|
128
|
+
`repo ${root}`,
|
|
129
|
+
...plan.map((p) => `${p.action.toUpperCase().padEnd(9)} ${p.id.padEnd(12)} ${p.path}${p.error ? ` (${p.error})` : ''}`),
|
|
130
|
+
...others.map((o) => `${o.state === 'none' ? 'NO FILE ' : 'NOT OURS'} ${o.id.padEnd(12)} ${o.path ?? '(no hooks file is documented)'}`),
|
|
131
|
+
'',
|
|
132
|
+
refused.length > 0
|
|
133
|
+
? `REFUSED ${refused.length} config file(s) could not be read; nothing written`
|
|
134
|
+
: `PLAN nothing written; apply with: ${CLI} agent-hooks install --yes`,
|
|
135
|
+
].join('\n'),
|
|
136
|
+
{ maxBytes: flags.maxBytes ?? 0 },
|
|
137
|
+
);
|
|
138
|
+
return refused.length > 0 ? EXIT.FAIL : EXIT.OK;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// Refused beats partial: a config this tool cannot parse is one it must not rewrite,
|
|
142
|
+
// and writing the other three while one is unreadable leaves the repository in a state
|
|
143
|
+
// where the report it just printed is no longer true.
|
|
144
|
+
if (refused.length > 0) {
|
|
145
|
+
return fail(`refused to write ${refused.map((p) => `${p.path} (${p.error})`).join('; ')}`, EXIT.FAIL);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const written = [];
|
|
149
|
+
for (const p of plan.filter((x) => x.action !== 'unchanged')) {
|
|
150
|
+
await mkdir(join(root, dirname(p.path)), { recursive: true });
|
|
151
|
+
await writeFile(join(root, p.path), p.text);
|
|
152
|
+
written.push(p);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const notes = [];
|
|
156
|
+
for (const p of written) {
|
|
157
|
+
const trust = adapter(p.id).trust;
|
|
158
|
+
if (trust) notes.push(`${p.id}: ${trust}`);
|
|
159
|
+
}
|
|
160
|
+
if (written.length === 0) notes.push('every documented hooks file already carried this tool’s entry');
|
|
161
|
+
|
|
162
|
+
if (isJson(flags)) {
|
|
163
|
+
emitJson({
|
|
164
|
+
ok: true,
|
|
165
|
+
command: 'agent-hooks install',
|
|
166
|
+
root,
|
|
167
|
+
plan: plan.map(({ text, ...rest }) => rest),
|
|
168
|
+
notWritten: others,
|
|
169
|
+
written: written.map((p) => p.path),
|
|
170
|
+
filesWritten: true,
|
|
171
|
+
notes,
|
|
172
|
+
});
|
|
173
|
+
return EXIT.OK;
|
|
174
|
+
}
|
|
175
|
+
emit(
|
|
176
|
+
[
|
|
177
|
+
`repo ${root}`,
|
|
178
|
+
...plan.map((p) => `${p.action.toUpperCase().padEnd(9)} ${p.id.padEnd(12)} ${p.path}`),
|
|
179
|
+
...others.map((o) => `${o.state === 'none' ? 'NO FILE ' : 'NOT OURS'} ${o.id.padEnd(12)} ${o.path ?? '(no hooks file is documented)'}`),
|
|
180
|
+
'',
|
|
181
|
+
`WROTE ${written.length} hooks file(s); each one runs ${CLI} agent-hook before the tool call`,
|
|
182
|
+
].join('\n'),
|
|
183
|
+
{ maxBytes: flags.maxBytes ?? 0 },
|
|
184
|
+
);
|
|
185
|
+
for (const o of others) note(`note ${o.id}: not written - ${o.note}`);
|
|
186
|
+
for (const n of notes) note(`note ${n}`);
|
|
187
|
+
note(`note never committed or pushed (ADR-001); review, then commit yourself`);
|
|
188
|
+
return EXIT.OK;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async function hooksStatus(root, flags) {
|
|
192
|
+
const rows = [];
|
|
193
|
+
const problems = [];
|
|
194
|
+
|
|
195
|
+
for (const row of adapters()) {
|
|
196
|
+
const path = hookPath(row.id);
|
|
197
|
+
const existing = await readIfExists(join(root, path));
|
|
198
|
+
if (existing === null) {
|
|
199
|
+
rows.push({ id: row.id, path, state: 'absent', entries: 0, note: null });
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
let config;
|
|
203
|
+
try {
|
|
204
|
+
config = JSON.parse(existing);
|
|
205
|
+
} catch (e) {
|
|
206
|
+
rows.push({ id: row.id, path, state: 'unreadable', entries: 0, note: e.message });
|
|
207
|
+
problems.push(`${path} is not valid JSON: ${e.message}`);
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
// A file that parses but is not an object cannot hold a hooks table, and `install`
|
|
211
|
+
// refuses to write into one. Status has to agree with it, or one question gets two
|
|
212
|
+
// answers and the quiet one is the one a reader acts on.
|
|
213
|
+
if (config === null || typeof config !== 'object' || Array.isArray(config)) {
|
|
214
|
+
rows.push({ id: row.id, path, state: 'unreadable', entries: 0, note: 'not a JSON object' });
|
|
215
|
+
problems.push(`${path} is not a JSON object, so this tool will not write into it`);
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
const found = installedIn(row, config).length;
|
|
219
|
+
rows.push({
|
|
220
|
+
id: row.id,
|
|
221
|
+
path,
|
|
222
|
+
state: found > 0 ? 'present' : 'absent',
|
|
223
|
+
entries: found,
|
|
224
|
+
note: found > 0 ? (row.trust ?? null) : null,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// The readers whose hooks are not a file this tool can write — plugins, or a vendor
|
|
229
|
+
// page that documents none — are reported from the same table that documents them,
|
|
230
|
+
// rather than from a second list of the harnesses this command happens to know.
|
|
231
|
+
for (const r of hookRows()) {
|
|
232
|
+
if (r.path === null) rows.push({ id: r.id, path: null, state: 'none', entries: 0, note: r.note });
|
|
233
|
+
else if (adapter(r.id) === null) rows.push({ id: r.id, path: r.path, state: 'other', entries: 0, note: r.note });
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (isJson(flags)) {
|
|
237
|
+
emitJson({ ok: problems.length === 0, command: 'agent-hooks status', root, harnesses: rows, problems });
|
|
238
|
+
return problems.length === 0 ? EXIT.OK : EXIT.FAIL;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const lines = [`repo ${root}`];
|
|
242
|
+
for (const r of rows) {
|
|
243
|
+
const state = { present: 'PRESENT ', absent: 'ABSENT ', unreadable: 'INVALID ', none: 'NO FILE ', other: 'NOT OURS' }[r.state];
|
|
244
|
+
lines.push(`${state} ${r.id.padEnd(12)} ${r.path ?? '(no hooks file is documented)'}`);
|
|
245
|
+
if (r.state === 'absent') lines.push(` ${' '.repeat(12)} install: ${CLI} agent-hooks install --yes`);
|
|
246
|
+
if (r.note) lines.push(` ${' '.repeat(12)} ${r.note}`);
|
|
247
|
+
}
|
|
248
|
+
if (problems.length > 0) {
|
|
249
|
+
lines.push('', 'problems:');
|
|
250
|
+
for (const p of problems) lines.push(` ${p}`);
|
|
251
|
+
}
|
|
252
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
253
|
+
return problems.length === 0 ? EXIT.OK : EXIT.FAIL;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function readIfExists(path) {
|
|
257
|
+
try {
|
|
258
|
+
return await readFile(path, 'utf8');
|
|
259
|
+
} catch {
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Read the one payload the harness has already written to stdin.
|
|
266
|
+
*
|
|
267
|
+
* A terminal on stdin means a person ran this by hand, and reading would hang until they
|
|
268
|
+
* pressed Ctrl-D — so the isTTY check is the difference between a diagnostic and a
|
|
269
|
+
* command that looks like it has frozen.
|
|
270
|
+
*/
|
|
271
|
+
async function readPayload() {
|
|
272
|
+
if (process.stdin.isTTY) return { payload: {}, empty: true };
|
|
273
|
+
const chunks = [];
|
|
274
|
+
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
275
|
+
const text = Buffer.concat(chunks).toString('utf8').trim();
|
|
276
|
+
if (text === '') return { payload: {}, empty: true };
|
|
277
|
+
try {
|
|
278
|
+
return { payload: JSON.parse(text) };
|
|
279
|
+
} catch (e) {
|
|
280
|
+
return { payload: {}, error: e.message };
|
|
281
|
+
}
|
|
282
|
+
}
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
// `simt bench` - benchmark task mining (`bench mine`, R01), runner (`bench run`, R02)
|
|
2
|
+
// and statistical report (`bench report`, R03).
|
|
3
|
+
//
|
|
4
|
+
// Like every writer in this CLI, `bench mine` and `bench run` are previews by
|
|
5
|
+
// default and write artifacts to `.simt-dev/bench/` only when authorised with
|
|
6
|
+
// `--yes` (and not `--dry-run`). `bench report` is read-only.
|
|
7
|
+
|
|
8
|
+
import { resolve } from 'node:path';
|
|
9
|
+
|
|
10
|
+
import { CLI } from '../name.mjs';
|
|
11
|
+
import { EXIT, emit, emitJson, fail, isJson, writeMode } from '../output/index.mjs';
|
|
12
|
+
import { DEFAULT_MINE_TARGET, mineTasks } from '../bench/mine.mjs';
|
|
13
|
+
import { regradeRun, runBench } from '../bench/run.mjs';
|
|
14
|
+
import { buildBenchReport } from '../bench/report.mjs';
|
|
15
|
+
import { ablateBench } from '../bench/ablate.mjs';
|
|
16
|
+
import { runBenchProbes } from '../bench/probes.mjs';
|
|
17
|
+
import { listFlag } from './skills.mjs';
|
|
18
|
+
|
|
19
|
+
export async function benchMineCommand(positional, flags) {
|
|
20
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
21
|
+
const { dryRun, apply } = writeMode(flags);
|
|
22
|
+
|
|
23
|
+
const result = await mineTasks(root, {
|
|
24
|
+
apply,
|
|
25
|
+
target: DEFAULT_MINE_TARGET,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
if (!result.ok) {
|
|
29
|
+
return fail(
|
|
30
|
+
`${root}: ${result.error}`,
|
|
31
|
+
EXIT.CONFIG,
|
|
32
|
+
`point --root at a git repository and rerun: ${CLI} bench mine --root <path> --yes`,
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (isJson(flags)) {
|
|
37
|
+
emitJson({
|
|
38
|
+
ok: true,
|
|
39
|
+
command: 'bench mine',
|
|
40
|
+
root: result.root,
|
|
41
|
+
dryRun,
|
|
42
|
+
applied: result.applied,
|
|
43
|
+
candidates: result.candidatesCount,
|
|
44
|
+
tasksDir: result.tasksDir,
|
|
45
|
+
vaultRoot: result.vaultRoot,
|
|
46
|
+
tasks: result.tasks,
|
|
47
|
+
});
|
|
48
|
+
return EXIT.OK;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const lines = [
|
|
52
|
+
`bench mine ${result.root}`,
|
|
53
|
+
`mode ${result.applied ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview (pass --yes to validate and write)'}`,
|
|
54
|
+
`commits ${result.candidatesCount} candidate(s) touch both test and implementation paths`,
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
if (result.applied) {
|
|
58
|
+
lines.push(`mined ${result.tasks.length} verified task(s)`);
|
|
59
|
+
lines.push(`tasks ${result.tasksDir}`);
|
|
60
|
+
lines.push(`vault ${result.vaultRoot}`);
|
|
61
|
+
for (const task of result.tasks) {
|
|
62
|
+
const subject = task.prompt.split('\n')[0];
|
|
63
|
+
lines.push(
|
|
64
|
+
`task ${task.id} f2p=${task.f2p.length} p2p=${task.f2p.length} files=${task.gold.files} +${task.gold.linesAdded}/-${task.gold.linesRemoved} ${subject}`,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
} else {
|
|
68
|
+
lines.push(`preview ${result.tasks.length} candidate(s) queued for RED/GOLD worktree validation`);
|
|
69
|
+
for (const item of result.tasks) {
|
|
70
|
+
lines.push(`queue ${item.id} tests=${item.testFiles.length} impl=${item.implFiles.length} ${item.subject}`);
|
|
71
|
+
}
|
|
72
|
+
lines.push(`remedy run: ${CLI} bench mine --yes`);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
76
|
+
return EXIT.OK;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function benchRunCommand(positional, flags) {
|
|
80
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
81
|
+
const { dryRun, apply } = writeMode(flags);
|
|
82
|
+
|
|
83
|
+
if (flags.regrade !== undefined) {
|
|
84
|
+
let result;
|
|
85
|
+
try {
|
|
86
|
+
result = await regradeRun(root, flags.regrade, { apply });
|
|
87
|
+
} catch (error) {
|
|
88
|
+
return fail(
|
|
89
|
+
`${root}: ${error.message ?? error}`,
|
|
90
|
+
EXIT.CONFIG,
|
|
91
|
+
`pass a stored run name under .simt-dev/bench/runs/: ${CLI} bench run --regrade <run>`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const code = result.allReproduced ? EXIT.OK : EXIT.DRIFT;
|
|
96
|
+
if (isJson(flags)) {
|
|
97
|
+
emitJson({
|
|
98
|
+
ok: code === EXIT.OK,
|
|
99
|
+
command: 'bench run',
|
|
100
|
+
mode: 'regrade',
|
|
101
|
+
root: result.root,
|
|
102
|
+
target: result.target,
|
|
103
|
+
dryRun,
|
|
104
|
+
applied: result.applied,
|
|
105
|
+
regraded: result.regraded,
|
|
106
|
+
reproducedCount: result.reproducedCount,
|
|
107
|
+
allReproduced: result.allReproduced,
|
|
108
|
+
runs: result.runs,
|
|
109
|
+
});
|
|
110
|
+
return code;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const lines = [
|
|
114
|
+
`bench regrade ${result.target}`,
|
|
115
|
+
`mode ${result.applied ? 'applied' : 'verify (pass --yes to rewrite stored grades)'}`,
|
|
116
|
+
`summary ${result.reproducedCount}/${result.regraded} reproduced exactly`,
|
|
117
|
+
];
|
|
118
|
+
for (const run of result.runs) {
|
|
119
|
+
lines.push(
|
|
120
|
+
`run ${run.runId}/${run.task} arm=${run.arm} resolved=${run.resolved} gates=${run.gatesPassed} reproduced=${run.reproduced}`,
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
124
|
+
return code;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const arms = listFlag(flags.arm);
|
|
128
|
+
const tasks = listFlag(flags.task);
|
|
129
|
+
const harness = flags.harness ?? 'gold';
|
|
130
|
+
const model = flags.model;
|
|
131
|
+
const reps = flags.reps ?? 3;
|
|
132
|
+
|
|
133
|
+
let result;
|
|
134
|
+
try {
|
|
135
|
+
result = await runBench(root, {
|
|
136
|
+
apply,
|
|
137
|
+
arms: arms.length > 0 ? arms : ['current'],
|
|
138
|
+
harness,
|
|
139
|
+
model,
|
|
140
|
+
reps,
|
|
141
|
+
tasks,
|
|
142
|
+
});
|
|
143
|
+
} catch (error) {
|
|
144
|
+
return fail(
|
|
145
|
+
`${root}: ${error.message ?? error}`,
|
|
146
|
+
EXIT.CONFIG,
|
|
147
|
+
`mine tasks first with \`${CLI} bench mine --yes\` or check --arm / --harness flags`,
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (isJson(flags)) {
|
|
152
|
+
emitJson({
|
|
153
|
+
ok: true,
|
|
154
|
+
command: 'bench run',
|
|
155
|
+
root: result.root,
|
|
156
|
+
dryRun,
|
|
157
|
+
applied: result.applied,
|
|
158
|
+
harness: result.harness,
|
|
159
|
+
model: result.model,
|
|
160
|
+
arms: result.arms,
|
|
161
|
+
reps: result.reps,
|
|
162
|
+
taskCount: result.taskCount,
|
|
163
|
+
runs: result.runs,
|
|
164
|
+
});
|
|
165
|
+
return EXIT.OK;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const lines = [
|
|
169
|
+
`bench run ${result.root}`,
|
|
170
|
+
`mode ${result.applied ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview (pass --yes to execute and write runs)'}`,
|
|
171
|
+
`harness ${result.harness} (model: ${result.model})`,
|
|
172
|
+
`arms ${result.arms.join(', ')} (reps: ${result.reps}, tasks: ${result.taskCount})`,
|
|
173
|
+
];
|
|
174
|
+
|
|
175
|
+
if (result.applied) {
|
|
176
|
+
const resolvedCount = result.runs.filter((r) => r.resolved).length;
|
|
177
|
+
const gatesCount = result.runs.filter((r) => r.gatesPassed).length;
|
|
178
|
+
lines.push(`summary resolved=${resolvedCount}/${result.runs.length} gates=${gatesCount}/${result.runs.length}`);
|
|
179
|
+
for (const run of result.runs) {
|
|
180
|
+
lines.push(
|
|
181
|
+
`run ${run.runId}/${run.task} arm=${run.arm} rep=${run.rep} resolved=${run.resolved} gates=${run.gatesPassed} files=${run.diff.files} outside=${run.diff.outsideBlastRadius}`,
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
} else {
|
|
185
|
+
lines.push(`planned ${result.runs.length} isolated task run(s)`);
|
|
186
|
+
lines.push(`remedy run: ${CLI} bench run --yes`);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
190
|
+
return EXIT.OK;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export async function benchReportCommand(positional, flags) {
|
|
194
|
+
const compareItems = listFlag(flags.compare);
|
|
195
|
+
const pos = [...positional];
|
|
196
|
+
if (compareItems.length === 1 && pos.length >= 1) {
|
|
197
|
+
compareItems.push(pos.shift());
|
|
198
|
+
}
|
|
199
|
+
if (flags.compare !== undefined && compareItems.length !== 2) {
|
|
200
|
+
return fail(
|
|
201
|
+
'--compare requires two arm names: --compare <arm-a> <arm-b>',
|
|
202
|
+
EXIT.USAGE,
|
|
203
|
+
`pass two arms to compare: ${CLI} bench report --compare bare current`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const root = resolve(flags.root ?? pos[0] ?? process.cwd());
|
|
208
|
+
const arms = listFlag(flags.arm);
|
|
209
|
+
|
|
210
|
+
let report;
|
|
211
|
+
try {
|
|
212
|
+
report = await buildBenchReport(root, {
|
|
213
|
+
arms,
|
|
214
|
+
harness: flags.harness,
|
|
215
|
+
model: flags.model,
|
|
216
|
+
compare: compareItems.length === 2 ? compareItems : null,
|
|
217
|
+
});
|
|
218
|
+
} catch (error) {
|
|
219
|
+
return fail(
|
|
220
|
+
`${root}: ${error.message ?? error}`,
|
|
221
|
+
EXIT.CONFIG,
|
|
222
|
+
`run benchmark arms first with \`${CLI} bench run --yes\` or fix .simt-dev/bench/prices.json`,
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (isJson(flags)) {
|
|
227
|
+
emitJson({
|
|
228
|
+
ok: true,
|
|
229
|
+
command: 'bench report',
|
|
230
|
+
root: report.root,
|
|
231
|
+
pricesLoaded: report.pricesLoaded,
|
|
232
|
+
runCount: report.runCount,
|
|
233
|
+
arms: report.arms,
|
|
234
|
+
comparison: report.comparison,
|
|
235
|
+
});
|
|
236
|
+
return EXIT.OK;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
emit(report.markdown, { maxBytes: flags.maxBytes ?? 0 });
|
|
240
|
+
return EXIT.OK;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export async function benchAblateCommand(positional, flags) {
|
|
244
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
245
|
+
const { dryRun, apply } = writeMode(flags);
|
|
246
|
+
const harness = flags.harness ?? 'gold';
|
|
247
|
+
const model = flags.model;
|
|
248
|
+
const reps = flags.reps ?? 1;
|
|
249
|
+
|
|
250
|
+
let result;
|
|
251
|
+
try {
|
|
252
|
+
result = await ablateBench(root, {
|
|
253
|
+
apply,
|
|
254
|
+
harness,
|
|
255
|
+
model,
|
|
256
|
+
reps,
|
|
257
|
+
includeAll: flags.all === true,
|
|
258
|
+
});
|
|
259
|
+
} catch (error) {
|
|
260
|
+
return fail(
|
|
261
|
+
`${root}: ${error.message ?? error}`,
|
|
262
|
+
EXIT.CONFIG,
|
|
263
|
+
`mine tasks first with \`${CLI} bench mine --yes\` and rerun \`${CLI} bench ablate --yes\``,
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (isJson(flags)) {
|
|
268
|
+
emitJson({
|
|
269
|
+
ok: true,
|
|
270
|
+
command: 'bench ablate',
|
|
271
|
+
root: result.root,
|
|
272
|
+
dryRun,
|
|
273
|
+
applied: result.applied,
|
|
274
|
+
pricesLoaded: result.pricesLoaded,
|
|
275
|
+
canarySize: result.canarySize,
|
|
276
|
+
canaryTasks: result.canaryTasks,
|
|
277
|
+
totalTasks: result.totalTasks,
|
|
278
|
+
sections: result.sections,
|
|
279
|
+
ranking: result.ranking,
|
|
280
|
+
deletionCandidates: result.deletionCandidates,
|
|
281
|
+
});
|
|
282
|
+
return EXIT.OK;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
const lines = [
|
|
286
|
+
`bench ablate ${result.root}`,
|
|
287
|
+
`mode ${result.applied ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview / stored runs (pass --yes to execute canary and escalate)'}`,
|
|
288
|
+
`sections ${result.sections.length} (canary: ${result.canaryTasks}/${result.totalTasks} tasks)`,
|
|
289
|
+
`delete ${result.deletionCandidates.length > 0 ? result.deletionCandidates.join(', ') : '(none)'}`,
|
|
290
|
+
'',
|
|
291
|
+
result.markdown,
|
|
292
|
+
];
|
|
293
|
+
if (!result.applied) {
|
|
294
|
+
lines.push('');
|
|
295
|
+
lines.push(`remedy run: ${CLI} bench ablate --yes`);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
299
|
+
return EXIT.OK;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export async function benchProbesCommand(positional, flags) {
|
|
303
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
304
|
+
const { dryRun, apply } = writeMode(flags);
|
|
305
|
+
const arms = listFlag(flags.arm);
|
|
306
|
+
const probes = listFlag(flags.task);
|
|
307
|
+
const harness = flags.harness ?? 'gold';
|
|
308
|
+
const reps = flags.reps ?? 1;
|
|
309
|
+
|
|
310
|
+
let result;
|
|
311
|
+
try {
|
|
312
|
+
result = await runBenchProbes(root, {
|
|
313
|
+
apply,
|
|
314
|
+
arms: arms.length > 0 ? arms : ['bare', 'current'],
|
|
315
|
+
probes,
|
|
316
|
+
harness,
|
|
317
|
+
reps,
|
|
318
|
+
});
|
|
319
|
+
} catch (error) {
|
|
320
|
+
return fail(
|
|
321
|
+
`${root}: ${error.message ?? error}`,
|
|
322
|
+
EXIT.CONFIG,
|
|
323
|
+
`check --arm and --harness flags and rerun \`${CLI} bench probes --yes\``,
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if (isJson(flags)) {
|
|
328
|
+
emitJson({
|
|
329
|
+
ok: true,
|
|
330
|
+
command: 'bench probes',
|
|
331
|
+
root: result.root,
|
|
332
|
+
dryRun,
|
|
333
|
+
applied: result.applied,
|
|
334
|
+
probeCount: result.probeCount,
|
|
335
|
+
probes: result.probes,
|
|
336
|
+
arms: result.arms,
|
|
337
|
+
runs: result.runs,
|
|
338
|
+
});
|
|
339
|
+
return EXIT.OK;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const lines = [
|
|
343
|
+
`bench probes ${result.root}`,
|
|
344
|
+
`mode ${result.applied ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview / stored runs (pass --yes to execute probes)'}`,
|
|
345
|
+
`probes ${result.probeCount}`,
|
|
346
|
+
'',
|
|
347
|
+
result.markdown,
|
|
348
|
+
];
|
|
349
|
+
if (!result.applied) {
|
|
350
|
+
lines.push('');
|
|
351
|
+
lines.push(`remedy run: ${CLI} bench probes --yes`);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
355
|
+
return EXIT.OK;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
|
package/src/commands/compare.mjs
CHANGED
|
@@ -98,6 +98,8 @@ export async function compareCommand(positional, flags) {
|
|
|
98
98
|
noRegion: summary.noRegion,
|
|
99
99
|
malformed: summary.malformed,
|
|
100
100
|
staleRuleFiles: summary.staleRuleFiles,
|
|
101
|
+
packageRegions: summary.packageRegions,
|
|
102
|
+
stalePackages: summary.stalePackages,
|
|
101
103
|
hooksRouted: summary.hooksRouted,
|
|
102
104
|
withProblems: summary.withProblems,
|
|
103
105
|
withOwnHooks: summary.withOwnHooks,
|
|
@@ -155,6 +157,14 @@ function render(rows, summary, found) {
|
|
|
155
157
|
`repos ${summary.repos} (${summary.adopted} adopted, ${summary.unmanaged} unmanaged${summary.broken > 0 ? `, ${summary.broken} broken` : ''})`,
|
|
156
158
|
`region ${summary.current} current, ${summary.drifted} drifted, ${summary.noRegion} with no region, ${summary.malformed} malformed`,
|
|
157
159
|
`rules ${summary.staleRuleFiles === 0 ? 'every rule file current' : `${summary.staleRuleFiles} rule file${summary.staleRuleFiles === 1 ? '' : 's'} not current`}`,
|
|
160
|
+
// One line, and only when there is a package region to report. A workspace of
|
|
161
|
+
// single-repository members is the common case and must not grow a line that
|
|
162
|
+
// always reads "none".
|
|
163
|
+
...(summary.packageRegions > 0
|
|
164
|
+
? [
|
|
165
|
+
`packages ${summary.stalePackages === 0 ? `every package region current (${summary.packageRegions})` : `${summary.stalePackages} of ${summary.packageRegions} package region${summary.packageRegions === 1 ? '' : 's'} not current`}`,
|
|
166
|
+
]
|
|
167
|
+
: []),
|
|
158
168
|
`hooks ${summary.hooksRouted}/${summary.adopted} adopted repos route the shared dispatcher`,
|
|
159
169
|
` ${summary.withOwnHooks} repo${summary.withOwnHooks === 1 ? '' : 's'} run their own hooks; adopting replaces them (a note, not a problem)`,
|
|
160
170
|
`problems ${summary.withProblems} repo${summary.withProblems === 1 ? '' : 's'}`,
|
|
@@ -167,6 +177,11 @@ function render(rows, summary, found) {
|
|
|
167
177
|
lines.push(` adoption ${r.adoption}${r.id && r.id !== r.name ? ` (id ${r.id})` : ''}`);
|
|
168
178
|
lines.push(` stack ${r.stack.length > 0 ? r.stack.join(', ') : '(none)'}${r.declaredStack.length > 0 ? ` (manifest declares ${r.declaredStack.join(', ')})` : ''}`);
|
|
169
179
|
if (r.facts.length > 0) lines.push(` facts ${r.facts.join(', ')}`);
|
|
180
|
+
// Per package, and only for a repository that has packages: a single-repository
|
|
181
|
+
// member must not print an empty list of packages on every line of every run.
|
|
182
|
+
for (const p of r.packages) {
|
|
183
|
+
lines.push(` package ${p.path} ${p.status}${p.profiles.length > 0 ? ` (${p.profiles.join(', ')})` : ''}`);
|
|
184
|
+
}
|
|
170
185
|
lines.push(` region ${r.region}${r.regionDetail ? ` (${r.regionDetail})` : ''}`);
|
|
171
186
|
// The rule files are the answer to "does every editor see the same rule?".
|
|
172
187
|
lines.push(
|