simt-dev 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -872
- package/README.md +19 -133
- package/bin/simt.mjs +19 -55
- package/docs/decisions/ADR-014-workspace-is-configuration.md +1 -1
- package/docs/decisions/ADR-099-the-release-verifies-served-bytes-and-the-formula-mirrors-them.md +1 -1
- package/docs/decisions/ADR-117-the-darwin-release-builds-on-supported-macos-15-runners-and-a-re-run-is-idempotent.md +81 -0
- package/docs/decisions/ADR-118-the-bench-corpus-is-keyed-on-history-is-append-only-and-is-portable-as-a-sealed-document.md +91 -0
- package/docs/decisions/ADR-119-the-environment-variable-points-into-a-workspace-the-repo-declared.md +90 -0
- package/docs/decisions/ADR-120-a-pre-push-whose-tree-is-already-proven-green-skips-the-test-gate-on-that-evidence.md +96 -0
- package/docs/decisions/ADR-122-the-shims-move-into-the-state-directory-and-the-legacy-directory-removes-itself.md +94 -0
- package/docs/decisions/ADR-123-a-surplus-positional-is-not-refused-because-arity-is-not-in-the-source.md +57 -0
- package/docs/decisions/ADR-124-the-manifest-has-a-schema-in-the-tool-and-its-unknown-keys-stay-the-repositorys.md +64 -0
- package/docs/decisions/ADR-125-a-shared-context-belongs-to-a-read-and-a-writer-reads-directly.md +65 -0
- package/package.json +10 -2
- package/rules/core.md +7 -0
- package/rules/mdc/slice-commit.mdc +57 -4
- package/src/agenthooks/prompt.mjs +5 -2
- package/src/agenthooks/shim.mjs +1 -1
- package/src/api.mjs +29 -0
- package/src/argv.mjs +29 -3
- package/src/bench/adapters.json +16 -0
- package/src/bench/corpus.mjs +150 -0
- package/src/bench/grade.mjs +23 -49
- package/src/bench/mine.mjs +199 -35
- package/src/bench/probes.mjs +99 -62
- package/src/bench/process.mjs +128 -0
- package/src/bench/report.mjs +17 -4
- package/src/bench/run.mjs +242 -142
- package/src/brief/assemble.mjs +38 -3
- package/src/ci/workflow.mjs +30 -7
- package/src/cli.mjs +45 -0
- package/src/commands/agent.mjs +3 -4
- package/src/commands/agenthook.mjs +4 -5
- package/src/commands/bench.mjs +190 -17
- package/src/commands/brief.mjs +5 -4
- package/src/commands/ci.mjs +2 -1
- package/src/commands/compare.mjs +42 -42
- package/src/commands/completions.mjs +2 -6
- package/src/commands/decide.mjs +4 -4
- package/src/commands/distill.mjs +4 -3
- package/src/commands/doctor.mjs +11 -7
- package/src/commands/evidence.mjs +6 -9
- package/src/commands/explain.mjs +0 -1
- package/src/commands/gate.mjs +16 -12
- package/src/commands/graph.mjs +2 -3
- package/src/commands/harness.mjs +3 -2
- package/src/commands/help.mjs +125 -17
- package/src/commands/hooks.mjs +100 -30
- package/src/commands/ignore.mjs +2 -1
- package/src/commands/impact.mjs +4 -7
- package/src/commands/index.mjs +273 -188
- package/src/commands/init.mjs +2 -1
- package/src/commands/lease.mjs +6 -7
- package/src/commands/lessons.mjs +10 -9
- package/src/commands/local.mjs +3 -3
- package/src/commands/map.mjs +14 -9
- package/src/commands/mcp.mjs +0 -1
- package/src/commands/pack.mjs +9 -6
- package/src/commands/plan.mjs +6 -5
- package/src/commands/plugin.mjs +2 -2
- package/src/commands/prbody.mjs +3 -3
- package/src/commands/repoindex.mjs +4 -3
- package/src/commands/review.mjs +2 -2
- package/src/commands/rules.mjs +33 -16
- package/src/commands/shared.mjs +2 -3
- package/src/commands/skills.mjs +28 -26
- package/src/commands/skillscompile.mjs +4 -5
- package/src/commands/skillsinstall.mjs +2 -2
- package/src/commands/slice.mjs +21 -10
- package/src/commands/spec.mjs +151 -0
- package/src/commands/status.mjs +4 -7
- package/src/commands/sync.mjs +13 -11
- package/src/commands/tests.mjs +6 -9
- package/src/commands/tokens.mjs +2 -2
- package/src/commands/undo.mjs +5 -15
- package/src/commands/upgrade.mjs +7 -3
- package/src/commands/why.mjs +9 -8
- package/src/compare/row.mjs +0 -2
- package/src/context.mjs +64 -0
- package/src/decisions/index-doc.mjs +46 -0
- package/src/diff.mjs +1 -1
- package/src/distill/distill.mjs +14 -1
- package/src/distill/engine.mjs +5 -1
- package/src/graph/load.mjs +4 -1
- package/src/hooks/capability.mjs +3 -3
- package/src/hooks/dispatcher.mjs +198 -43
- package/src/hooks/gates.mjs +0 -43
- package/src/hooks/resolve.mjs +2 -7
- package/src/hooks/shim.mjs +5 -2
- package/src/identity.mjs +29 -1
- package/src/journal.mjs +78 -9
- package/src/ledger/index.mjs +30 -1
- package/src/local/client.mjs +4 -6
- package/src/locate.mjs +33 -9
- package/src/manifest/schema.mjs +213 -0
- package/src/mcp/resources.mjs +7 -10
- package/src/mcp/server.mjs +70 -16
- package/src/mcp/tools.mjs +45 -7
- package/src/measure/session.mjs +3 -2
- package/src/node-version.mjs +37 -0
- package/src/output/index.mjs +111 -3
- package/src/packs/manage.mjs +17 -3
- package/src/packs.mjs +20 -2
- package/src/projection/lock.mjs +11 -1
- package/src/projection/manifest.mjs +20 -34
- package/src/projection/rules.mjs +4 -3
- package/src/repo-root.mjs +19 -0
- package/src/resolve/fingerprint.mjs +19 -15
- package/src/review/critic.mjs +2 -1
- package/src/toolwords.mjs +3 -0
- package/src/workspace.mjs +55 -10
- package/docs/decisions/README.md +0 -600
package/src/ci/workflow.mjs
CHANGED
|
@@ -167,7 +167,7 @@ function checkoutFull() {
|
|
|
167
167
|
* to. The own gate also varies the platforms and the Node versions, because this
|
|
168
168
|
* package is the one whose floor and whose ADR-012 claim have to hold on more than
|
|
169
169
|
* the minimum. */
|
|
170
|
-
function skeleton(name, steps, { matrix = null } = {}) {
|
|
170
|
+
function skeleton(name, steps, { matrix = null, timeoutMinutes = 5, gateMain = false } = {}) {
|
|
171
171
|
const axes = matrix
|
|
172
172
|
? Object.entries(matrix)
|
|
173
173
|
.map(([axis, values]) => ` ${axis}: [${values.map((v) => `'${v}'`).join(', ')}]`)
|
|
@@ -182,9 +182,22 @@ function skeleton(name, steps, { matrix = null } = {}) {
|
|
|
182
182
|
// it is written escaped, the same way the `concurrency` line below spells its own two.
|
|
183
183
|
const runner = matrix?.os ? '\${{ matrix.os }}' : 'ubuntu-latest';
|
|
184
184
|
const job = matrix?.node ? '\${{ matrix.os }} node \${{ matrix.node }}' : name;
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
185
|
+
// Two trigger blocks, because the two copies answer different questions. An
|
|
186
|
+
// adopting repository merges through pull requests, which is where its gate runs.
|
|
187
|
+
// This repository merges locally, so a merge never opens a pull request: without a
|
|
188
|
+
// push trigger the commit that lands on `main` is the one no gate ever ran on
|
|
189
|
+
// (review 0003 Q57). Each block ships only in the copy whose claim it states.
|
|
190
|
+
const trigger = gateMain
|
|
191
|
+
? `# Pull requests ask whether a change may merge; a push to \`main\` asks whether
|
|
192
|
+
# what already merged is sound. This repository merges locally, so a merge never
|
|
193
|
+
# opens a pull request — without the second trigger the commit that lands on
|
|
194
|
+
# \`main\` is the one no gate ran on.
|
|
195
|
+
on:
|
|
196
|
+
pull_request:
|
|
197
|
+
push:
|
|
198
|
+
branches: [main]
|
|
199
|
+
`
|
|
200
|
+
: `# Pull requests only, and that is a decision rather than a default.
|
|
188
201
|
#
|
|
189
202
|
# A gate that runs on every push re-runs itself for every commit that pushes a
|
|
190
203
|
# branch forward, including the ones nobody asked it about, and its verdict has to
|
|
@@ -194,7 +207,10 @@ function skeleton(name, steps, { matrix = null } = {}) {
|
|
|
194
207
|
# on \`main\` is whatever CI the repository already has there.
|
|
195
208
|
on:
|
|
196
209
|
pull_request:
|
|
210
|
+
`;
|
|
211
|
+
return `name: ${name}
|
|
197
212
|
|
|
213
|
+
${trigger}
|
|
198
214
|
# Read-only is enough to check a repo, and a gate job does not need to write to it.
|
|
199
215
|
permissions:
|
|
200
216
|
contents: read
|
|
@@ -207,7 +223,7 @@ jobs:
|
|
|
207
223
|
check:
|
|
208
224
|
name: ${job}
|
|
209
225
|
runs-on: ${runner}
|
|
210
|
-
${strategy} timeout-minutes:
|
|
226
|
+
${strategy} timeout-minutes: ${timeoutMinutes}
|
|
211
227
|
steps:
|
|
212
228
|
${steps}`;
|
|
213
229
|
}
|
|
@@ -268,7 +284,7 @@ export function renderWorkflow({ own = false, npm = true, manager = null, versio
|
|
|
268
284
|
' - name: tests',
|
|
269
285
|
' run: npm test',
|
|
270
286
|
' - name: skill registry lint',
|
|
271
|
-
' run:
|
|
287
|
+
' run: npm run lint',
|
|
272
288
|
' - name: the gate copy in this repo is current',
|
|
273
289
|
// The drift this exception exists to prevent, checked where it would be caught.
|
|
274
290
|
// Without it a template change lands with the copy still describing the old
|
|
@@ -282,7 +298,14 @@ export function renderWorkflow({ own = false, npm = true, manager = null, versio
|
|
|
282
298
|
// sentence was false in the one repository that could have made it true, and no
|
|
283
299
|
// check read it - the package declares `os` for npm to enforce, and nothing here
|
|
284
300
|
// ever ran a test on a second platform.
|
|
285
|
-
|
|
301
|
+
// The slowest leg (macOS, full suite plus the registry lint) ran 4 min 28 s
|
|
302
|
+
// against a 5-minute timeout, so the timeout that was meant to bound a hang
|
|
303
|
+
// was about to fail a passing run instead. 15 leaves the hang bounded and the
|
|
304
|
+
// run alone; the sharding that would make it short again is Q92, not this row.
|
|
305
|
+
//
|
|
306
|
+
// `gateMain` because this repository merges locally: `pull_request` alone left
|
|
307
|
+
// the one commit that reached `main` ungated (review 0003 Q57).
|
|
308
|
+
{ matrix: { os: ['ubuntu-latest', 'macos-latest'], node: [floor, '24', '26'] }, timeoutMinutes: 15, gateMain: true },
|
|
286
309
|
)}\n`;
|
|
287
310
|
}
|
|
288
311
|
|
package/src/cli.mjs
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// The CLI proper, imported by `bin/simt.mjs` after the runtime floor is checked.
|
|
2
|
+
//
|
|
3
|
+
// It lives here rather than in `bin/` because a static import in the entry point is
|
|
4
|
+
// evaluated before the entry point's own code: the version check could not run first,
|
|
5
|
+
// and an old Node failed inside the module graph instead of saying which version it
|
|
6
|
+
// needed (review 0003 Q54).
|
|
7
|
+
|
|
8
|
+
import { parse, rawJsonFlag } from './argv.mjs';
|
|
9
|
+
import { EXIT, fail, jsonFailMode, jsonIndentDefault } from './output/index.mjs';
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
const argv = process.argv.slice(2);
|
|
13
|
+
const parsed = parse(argv);
|
|
14
|
+
if (!parsed.ok) {
|
|
15
|
+
// The flags do not exist yet, so the JSON mode is read off the raw words:
|
|
16
|
+
// a parse failure under `--json` is a JSON document on stdout, not text on
|
|
17
|
+
// stderr with an empty stdout (Q11, ADR-070 ruling 4).
|
|
18
|
+
jsonFailMode(rawJsonFlag(argv));
|
|
19
|
+
process.exit(fail(parsed.error.message, parsed.error.code));
|
|
20
|
+
}
|
|
21
|
+
const { flags, positional } = parsed.value;
|
|
22
|
+
|
|
23
|
+
// Set before anything can fail: `no command given` under `--json` is a JSON
|
|
24
|
+
// document too, and a failure that prints as text with an empty stdout breaks
|
|
25
|
+
// the same ruling the parse failure above honours (Q11, ADR-070 ruling 4).
|
|
26
|
+
jsonIndentDefault(flags.pretty === true);
|
|
27
|
+
jsonFailMode(flags.json === true);
|
|
28
|
+
|
|
29
|
+
// No arguments at all is a usage error, not a silent help dump: an agent that
|
|
30
|
+
// ran the wrong thing should be told so rather than handed 40 lines of prose.
|
|
31
|
+
if (positional.length === 0 && !flags.help && !flags.version) {
|
|
32
|
+
process.exit(fail('no command given', EXIT.USAGE));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const { dispatch } = await import('./commands/index.mjs');
|
|
36
|
+
const code = (await dispatch(argv)) ?? EXIT.OK;
|
|
37
|
+
// Assigned, not forced: a large document on stdout (a `--json --diff` over a
|
|
38
|
+
// skill-sized body exceeds the 64 KiB pipe buffer) is still draining when the
|
|
39
|
+
// command resolves, and `process.exit()` cuts it — exit 0 with half a JSON
|
|
40
|
+
// document on stdout. Letting the loop empty first makes the exit honest
|
|
41
|
+
// (plan 0003 R50; the S10 contract asserts the document parses).
|
|
42
|
+
process.exitCode = code;
|
|
43
|
+
} catch (error) {
|
|
44
|
+
process.exit(fail(error?.message ?? String(error), EXIT.FAIL));
|
|
45
|
+
}
|
package/src/commands/agent.mjs
CHANGED
|
@@ -27,9 +27,10 @@ import {
|
|
|
27
27
|
rendered,
|
|
28
28
|
writeAgentPlan,
|
|
29
29
|
} from '../agents/render.mjs';
|
|
30
|
+
import { commandRoot } from '../repo-root.mjs';
|
|
30
31
|
|
|
31
32
|
export async function agentsCommand(sub, args, flags) {
|
|
32
|
-
const root = flags
|
|
33
|
+
const root = commandRoot(flags, args);
|
|
33
34
|
if (sub === 'install') return installAgents(root, flags);
|
|
34
35
|
if (sub === 'status') return agentsStatus(root, flags);
|
|
35
36
|
return fail(`unknown agents subcommand ${sub}`, EXIT.USAGE, `run: ${CLI} agents status`);
|
|
@@ -102,7 +103,6 @@ async function installAgents(root, flags) {
|
|
|
102
103
|
'',
|
|
103
104
|
`PLAN nothing written; apply with: ${CLI} agents install --yes`,
|
|
104
105
|
].join('\n'),
|
|
105
|
-
{ maxBytes: flags.maxBytes ?? 0 },
|
|
106
106
|
);
|
|
107
107
|
// After the report: the report says what, the diff shows it (R50).
|
|
108
108
|
emitDiffs(diffs, flags);
|
|
@@ -135,7 +135,6 @@ async function installAgents(root, flags) {
|
|
|
135
135
|
'',
|
|
136
136
|
`WROTE ${written.length} agent file(s)`,
|
|
137
137
|
].join('\n'),
|
|
138
|
-
{ maxBytes: flags.maxBytes ?? 0 },
|
|
139
138
|
);
|
|
140
139
|
// After the report: the report says what, the diff shows it (R50).
|
|
141
140
|
emitDiffs(diffs, flags);
|
|
@@ -193,7 +192,7 @@ async function agentsStatus(root, flags) {
|
|
|
193
192
|
lines.push('', 'problems:');
|
|
194
193
|
for (const p of problems) lines.push(` ${p}`);
|
|
195
194
|
}
|
|
196
|
-
emit(lines.join('\n')
|
|
195
|
+
emit(lines.join('\n'));
|
|
197
196
|
return problems.length === 0 ? EXIT.OK : EXIT.FAIL;
|
|
198
197
|
}
|
|
199
198
|
|
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
installedIn,
|
|
45
45
|
rendered,
|
|
46
46
|
} from '../agenthooks/render.mjs';
|
|
47
|
+
import { commandRoot } from '../repo-root.mjs';
|
|
47
48
|
|
|
48
49
|
export async function agentHookCommand(args, flags) {
|
|
49
50
|
const event = args[0];
|
|
@@ -77,7 +78,7 @@ export async function agentHookCommand(args, flags) {
|
|
|
77
78
|
// dynamic import that only this event opens.
|
|
78
79
|
if (event === 'prompt-submit') {
|
|
79
80
|
const { deliverPrompt } = await import('../agenthooks/prompt.mjs');
|
|
80
|
-
const delivered = await deliverPrompt(row, payload);
|
|
81
|
+
const delivered = await deliverPrompt(row, payload, { root: flags.root });
|
|
81
82
|
if (delivered.skipped) {
|
|
82
83
|
note(`note ${id}: no packet delivered (${delivered.skipped})`);
|
|
83
84
|
return EXIT.OK;
|
|
@@ -167,7 +168,7 @@ export async function agentHookCommand(args, flags) {
|
|
|
167
168
|
}
|
|
168
169
|
|
|
169
170
|
export async function agentHooksCommand(sub, args, flags) {
|
|
170
|
-
const root = flags
|
|
171
|
+
const root = commandRoot(flags, args);
|
|
171
172
|
if (sub === 'install') return installHooks(root, flags);
|
|
172
173
|
if (sub === 'status') return hooksStatus(root, flags);
|
|
173
174
|
return fail(`unknown agent-hooks subcommand ${sub}`, EXIT.USAGE, `run: ${CLI} agent-hooks status`);
|
|
@@ -286,7 +287,6 @@ async function installHooks(root, flags) {
|
|
|
286
287
|
? `REFUSED ${refused.length} config file(s) could not be read; nothing written`
|
|
287
288
|
: `PLAN nothing written; apply with: ${CLI} agent-hooks install --yes`,
|
|
288
289
|
].join('\n'),
|
|
289
|
-
{ maxBytes: flags.maxBytes ?? 0 },
|
|
290
290
|
);
|
|
291
291
|
// After the report: the report says what, the diff shows it (R50).
|
|
292
292
|
emitDiffs(diffs, flags);
|
|
@@ -350,7 +350,6 @@ async function installHooks(root, flags) {
|
|
|
350
350
|
'',
|
|
351
351
|
`WROTE ${written.length} hooks file(s)${shimWritten ? ' and the shim' : ''}; each one runs the ${AGENT_HOOK_SHIM} shim before the tool call`,
|
|
352
352
|
].join('\n'),
|
|
353
|
-
{ maxBytes: flags.maxBytes ?? 0 },
|
|
354
353
|
);
|
|
355
354
|
// After the report: the report says what, the diff shows it (R50).
|
|
356
355
|
emitDiffs(diffs, flags);
|
|
@@ -448,7 +447,7 @@ async function hooksStatus(root, flags) {
|
|
|
448
447
|
lines.push('', 'problems:');
|
|
449
448
|
for (const p of problems) lines.push(` ${p}`);
|
|
450
449
|
}
|
|
451
|
-
emit(lines.join('\n')
|
|
450
|
+
emit(lines.join('\n'));
|
|
452
451
|
return problems.length === 0 ? EXIT.OK : EXIT.FAIL;
|
|
453
452
|
}
|
|
454
453
|
|
package/src/commands/bench.mjs
CHANGED
|
@@ -5,11 +5,13 @@
|
|
|
5
5
|
// default and write artifacts to `.simt/bench/` only when authorised with
|
|
6
6
|
// `--yes` (and not `--dry-run`). `bench report` is read-only.
|
|
7
7
|
|
|
8
|
-
import { resolve } from 'node:path';
|
|
8
|
+
import { dirname, resolve } from 'node:path';
|
|
9
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
9
10
|
|
|
10
11
|
import { CLI } from '../name.mjs';
|
|
11
12
|
import { EXIT, emit, emitJson, fail, isJson, writeMode } from '../output/index.mjs';
|
|
12
13
|
import { DEFAULT_MINE_TARGET, mineTasks } from '../bench/mine.mjs';
|
|
14
|
+
import { buildCorpusDocument, verifyCorpusDocument, writeCorpusDocument } from '../bench/corpus.mjs';
|
|
13
15
|
import { regradeRun, runBench } from '../bench/run.mjs';
|
|
14
16
|
import { buildBenchReport } from '../bench/report.mjs';
|
|
15
17
|
import { formatRecall, measureRecall } from '../index/map.mjs';
|
|
@@ -18,14 +20,33 @@ import { runBenchProbes } from '../bench/probes.mjs';
|
|
|
18
20
|
import { selectRuleVariants } from '../bench/variants.mjs';
|
|
19
21
|
import { criticBench } from '../review/critic.mjs';
|
|
20
22
|
import { listFlag } from './skills.mjs';
|
|
23
|
+
import { commandRoot } from '../repo-root.mjs';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Refuse an authorised bench run that names no harness.
|
|
27
|
+
*
|
|
28
|
+
* The default harness is `gold`, which applies the reference patch instead of asking
|
|
29
|
+
* an agent: a `--yes` run with no harness measured the grader and stored the result as
|
|
30
|
+
* an arm's evidence (review 0003 Q49). A default cannot be mistaken for a measurement
|
|
31
|
+
* if there is no default.
|
|
32
|
+
*/
|
|
33
|
+
function requireHarness(flags, command) {
|
|
34
|
+
if (typeof flags.harness === 'string' && flags.harness.length > 0) return null;
|
|
35
|
+
return fail(
|
|
36
|
+
`${command} --yes needs an explicit --harness`,
|
|
37
|
+
EXIT.USAGE,
|
|
38
|
+
`pass --harness <name>: an agent adapter such as \`--harness claude\`, or a calibration harness (\`gold\`, \`noop\`)`,
|
|
39
|
+
);
|
|
40
|
+
}
|
|
21
41
|
|
|
22
42
|
export async function benchMineCommand(positional, flags) {
|
|
23
|
-
const root = resolve(flags
|
|
43
|
+
const root = resolve(commandRoot(flags, positional));
|
|
24
44
|
const { dryRun, apply } = writeMode(flags);
|
|
25
45
|
|
|
26
46
|
const result = await mineTasks(root, {
|
|
27
47
|
apply,
|
|
28
48
|
target: DEFAULT_MINE_TARGET,
|
|
49
|
+
replace: flags.replace === true,
|
|
29
50
|
});
|
|
30
51
|
|
|
31
52
|
if (!result.ok) {
|
|
@@ -64,7 +85,7 @@ export async function benchMineCommand(positional, flags) {
|
|
|
64
85
|
for (const task of result.tasks) {
|
|
65
86
|
const subject = task.prompt.split('\n')[0];
|
|
66
87
|
lines.push(
|
|
67
|
-
`task ${task.id} f2p=${task.f2p.length} p2p=${task.
|
|
88
|
+
`task ${task.id} f2p=${task.f2p.length} p2p=${task.p2p.length} files=${task.gold.files} +${task.gold.linesAdded}/-${task.gold.linesRemoved} ${subject}`,
|
|
68
89
|
);
|
|
69
90
|
}
|
|
70
91
|
} else {
|
|
@@ -75,12 +96,12 @@ export async function benchMineCommand(positional, flags) {
|
|
|
75
96
|
lines.push(`remedy run: ${CLI} bench mine --yes`);
|
|
76
97
|
}
|
|
77
98
|
|
|
78
|
-
emit(lines.join('\n')
|
|
99
|
+
emit(lines.join('\n'));
|
|
79
100
|
return EXIT.OK;
|
|
80
101
|
}
|
|
81
102
|
|
|
82
103
|
export async function benchRunCommand(positional, flags) {
|
|
83
|
-
const root = resolve(flags
|
|
104
|
+
const root = resolve(commandRoot(flags, positional));
|
|
84
105
|
const { dryRun, apply } = writeMode(flags);
|
|
85
106
|
|
|
86
107
|
if (flags.regrade !== undefined) {
|
|
@@ -123,10 +144,15 @@ export async function benchRunCommand(positional, flags) {
|
|
|
123
144
|
`run ${run.runId}/${run.task} arm=${run.arm} resolved=${run.resolved} gates=${run.gatesPassed} reproduced=${run.reproduced}`,
|
|
124
145
|
);
|
|
125
146
|
}
|
|
126
|
-
emit(lines.join('\n')
|
|
147
|
+
emit(lines.join('\n'));
|
|
127
148
|
return code;
|
|
128
149
|
}
|
|
129
150
|
|
|
151
|
+
if (apply) {
|
|
152
|
+
const missing = requireHarness(flags, 'bench run');
|
|
153
|
+
if (missing !== null) return missing;
|
|
154
|
+
}
|
|
155
|
+
|
|
130
156
|
const arms = listFlag(flags.arm);
|
|
131
157
|
const tasks = listFlag(flags.task);
|
|
132
158
|
const harness = flags.harness ?? 'gold';
|
|
@@ -189,7 +215,7 @@ export async function benchRunCommand(positional, flags) {
|
|
|
189
215
|
lines.push(`remedy run: ${CLI} bench run --yes`);
|
|
190
216
|
}
|
|
191
217
|
|
|
192
|
-
emit(lines.join('\n')
|
|
218
|
+
emit(lines.join('\n'));
|
|
193
219
|
return EXIT.OK;
|
|
194
220
|
}
|
|
195
221
|
|
|
@@ -243,7 +269,7 @@ export async function benchReportCommand(positional, flags) {
|
|
|
243
269
|
return EXIT.OK;
|
|
244
270
|
}
|
|
245
271
|
|
|
246
|
-
emit(`${report.markdown}\n\n### Map\n\n${formatRecall(mapRecall)}
|
|
272
|
+
emit(`${report.markdown}\n\n### Map\n\n${formatRecall(mapRecall)}`);
|
|
247
273
|
return EXIT.OK;
|
|
248
274
|
}
|
|
249
275
|
|
|
@@ -256,8 +282,12 @@ async function recallOrUnmeasured(root) {
|
|
|
256
282
|
}
|
|
257
283
|
|
|
258
284
|
export async function benchAblateCommand(positional, flags) {
|
|
259
|
-
const root = resolve(flags
|
|
285
|
+
const root = resolve(commandRoot(flags, positional));
|
|
260
286
|
const { dryRun, apply } = writeMode(flags);
|
|
287
|
+
if (apply) {
|
|
288
|
+
const missing = requireHarness(flags, 'bench ablate');
|
|
289
|
+
if (missing !== null) return missing;
|
|
290
|
+
}
|
|
261
291
|
const harness = flags.harness ?? 'gold';
|
|
262
292
|
const model = flags.model;
|
|
263
293
|
const reps = flags.reps ?? 1;
|
|
@@ -302,13 +332,18 @@ export async function benchAblateCommand(positional, flags) {
|
|
|
302
332
|
lines.push(`remedy run: ${CLI} bench ablate --yes`);
|
|
303
333
|
}
|
|
304
334
|
|
|
305
|
-
emit(lines.join('\n')
|
|
335
|
+
emit(lines.join('\n'));
|
|
306
336
|
return EXIT.OK;
|
|
307
337
|
}
|
|
308
338
|
|
|
309
339
|
export async function benchProbesCommand(positional, flags) {
|
|
310
|
-
const root = resolve(flags
|
|
340
|
+
const root = resolve(commandRoot(flags, positional));
|
|
311
341
|
const { dryRun, apply } = writeMode(flags);
|
|
342
|
+
if (apply) {
|
|
343
|
+
const missing = requireHarness(flags, 'bench probes');
|
|
344
|
+
if (missing !== null) return missing;
|
|
345
|
+
}
|
|
346
|
+
|
|
312
347
|
const arms = listFlag(flags.arm);
|
|
313
348
|
const probes = listFlag(flags.task);
|
|
314
349
|
const harness = flags.harness ?? 'gold';
|
|
@@ -353,13 +388,17 @@ export async function benchProbesCommand(positional, flags) {
|
|
|
353
388
|
lines.push(`remedy run: ${CLI} bench probes --yes`);
|
|
354
389
|
}
|
|
355
390
|
|
|
356
|
-
emit(lines.join('\n')
|
|
391
|
+
emit(lines.join('\n'));
|
|
357
392
|
return EXIT.OK;
|
|
358
393
|
}
|
|
359
394
|
|
|
360
395
|
export async function benchVariantsCommand(positional, flags) {
|
|
361
|
-
const root = resolve(flags
|
|
396
|
+
const root = resolve(commandRoot(flags, positional));
|
|
362
397
|
const { dryRun, apply } = writeMode(flags);
|
|
398
|
+
if (apply) {
|
|
399
|
+
const missing = requireHarness(flags, 'bench variants');
|
|
400
|
+
if (missing !== null) return missing;
|
|
401
|
+
}
|
|
363
402
|
let result;
|
|
364
403
|
try {
|
|
365
404
|
result = await selectRuleVariants(root, {
|
|
@@ -390,7 +429,7 @@ export async function benchVariantsCommand(positional, flags) {
|
|
|
390
429
|
),
|
|
391
430
|
];
|
|
392
431
|
if (!result.applied) lines.push(`remedy run: ${CLI} bench variants --yes`);
|
|
393
|
-
emit(lines.join('\n')
|
|
432
|
+
emit(lines.join('\n'));
|
|
394
433
|
return EXIT.OK;
|
|
395
434
|
}
|
|
396
435
|
|
|
@@ -405,13 +444,13 @@ export async function benchVariantsCommand(positional, flags) {
|
|
|
405
444
|
// the row says precision >= 0.8 on >= 30 labelled diffs promotes it to a gate,
|
|
406
445
|
// and until a bench here shows that, every report calls the critic advisory.
|
|
407
446
|
export async function benchCriticCommand(positional, flags, commands = []) {
|
|
408
|
-
const root = resolve(flags
|
|
447
|
+
const root = resolve(commandRoot(flags, positional));
|
|
409
448
|
const labelsPath = flags.labels ?? '.simt/local/critic-labels.jsonl';
|
|
410
449
|
const result = await criticBench(root, { labelsPath, flags, commands });
|
|
411
450
|
if (typeof result === 'number') return result;
|
|
412
451
|
|
|
413
452
|
if (isJson(flags)) {
|
|
414
|
-
emitJson(result
|
|
453
|
+
emitJson(result);
|
|
415
454
|
return EXIT.OK;
|
|
416
455
|
}
|
|
417
456
|
const lines = [
|
|
@@ -424,6 +463,140 @@ export async function benchCriticCommand(positional, flags, commands = []) {
|
|
|
424
463
|
];
|
|
425
464
|
if (result.labelsNote) lines.push(`labels ${result.labelsNote}`);
|
|
426
465
|
for (const error of result.errors) lines.push(`error ${error}`);
|
|
427
|
-
emit(lines.join('\n')
|
|
466
|
+
emit(lines.join('\n'));
|
|
467
|
+
return EXIT.OK;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// `simt bench export` / `simt bench import` (review 0003 Q48).
|
|
471
|
+
//
|
|
472
|
+
// The vault is keyed on the repository's root commit and lives outside every
|
|
473
|
+
// checkout. That is what makes it safe as a hidden verifier, and it is also
|
|
474
|
+
// why a clone cannot read the corpus mined on another machine: the committed
|
|
475
|
+
// tasks failed with ENOENT from every other checkout. A sealed document
|
|
476
|
+
// carries the descriptors and both patches for every task in one file, so an
|
|
477
|
+
// export on one machine and an import on another reproduce the corpus
|
|
478
|
+
// exactly, and the seal is checked before a single byte is trusted.
|
|
479
|
+
|
|
480
|
+
export async function benchExportCommand(positional, flags) {
|
|
481
|
+
const out = positional[0];
|
|
482
|
+
if (!out) {
|
|
483
|
+
return fail(
|
|
484
|
+
'bench export needs a target file: bench export <corpus.json>',
|
|
485
|
+
EXIT.USAGE,
|
|
486
|
+
`export the sealed corpus: ${CLI} bench export corpus.json --yes`,
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
const root = resolve(commandRoot(flags));
|
|
490
|
+
const { dryRun, apply } = writeMode(flags);
|
|
491
|
+
|
|
492
|
+
let doc;
|
|
493
|
+
try {
|
|
494
|
+
doc = await buildCorpusDocument(root);
|
|
495
|
+
} catch (error) {
|
|
496
|
+
return fail(
|
|
497
|
+
`${root}: ${error.message ?? error}`,
|
|
498
|
+
EXIT.CONFIG,
|
|
499
|
+
`mine tasks first: ${CLI} bench mine --yes`,
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
if (doc.taskCount === 0) {
|
|
504
|
+
return fail(
|
|
505
|
+
`${root}: no mined tasks to export`,
|
|
506
|
+
EXIT.USAGE,
|
|
507
|
+
`mine tasks first: ${CLI} bench mine --yes`,
|
|
508
|
+
);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
if (apply) {
|
|
512
|
+
const absOut = resolve(out);
|
|
513
|
+
await mkdir(dirname(absOut), { recursive: true });
|
|
514
|
+
await writeFile(absOut, `${JSON.stringify(doc, null, 2)}\n`, 'utf8');
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
if (isJson(flags)) {
|
|
518
|
+
emitJson({
|
|
519
|
+
ok: true,
|
|
520
|
+
command: 'bench export',
|
|
521
|
+
root,
|
|
522
|
+
dryRun,
|
|
523
|
+
applied: apply,
|
|
524
|
+
out: resolve(out),
|
|
525
|
+
taskCount: doc.taskCount,
|
|
526
|
+
seal: doc.seal,
|
|
527
|
+
});
|
|
528
|
+
return EXIT.OK;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
const lines = [
|
|
532
|
+
`bench export ${resolve(out)}`,
|
|
533
|
+
`mode ${apply ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview (pass --yes to write)'}`,
|
|
534
|
+
`tasks ${doc.taskCount}`,
|
|
535
|
+
`seal ${doc.seal}`,
|
|
536
|
+
];
|
|
537
|
+
if (!apply) lines.push(`remedy run: ${CLI} bench export ${out} --yes`);
|
|
538
|
+
emit(lines.join('\n'));
|
|
539
|
+
return EXIT.OK;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
export async function benchImportCommand(positional, flags) {
|
|
543
|
+
const file = positional[0];
|
|
544
|
+
if (!file) {
|
|
545
|
+
return fail(
|
|
546
|
+
'bench import needs a corpus file: bench import <corpus.json>',
|
|
547
|
+
EXIT.USAGE,
|
|
548
|
+
`import a sealed corpus: ${CLI} bench import corpus.json --yes`,
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
const root = resolve(commandRoot(flags));
|
|
552
|
+
const { dryRun, apply } = writeMode(flags);
|
|
553
|
+
|
|
554
|
+
let doc;
|
|
555
|
+
try {
|
|
556
|
+
doc = JSON.parse(await readFile(file, 'utf8'));
|
|
557
|
+
} catch (error) {
|
|
558
|
+
return fail(
|
|
559
|
+
`${file}: ${error.message ?? error}`,
|
|
560
|
+
EXIT.USAGE,
|
|
561
|
+
`export one with: ${CLI} bench export ${file} --yes`,
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
const problem = verifyCorpusDocument(doc);
|
|
566
|
+
if (problem !== null) {
|
|
567
|
+
return fail(
|
|
568
|
+
`${file}: ${problem}`,
|
|
569
|
+
EXIT.CONFIG,
|
|
570
|
+
`export a fresh corpus: ${CLI} bench export ${file} --yes`,
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
const result = await writeCorpusDocument(root, doc, { apply });
|
|
575
|
+
|
|
576
|
+
if (isJson(flags)) {
|
|
577
|
+
emitJson({
|
|
578
|
+
ok: true,
|
|
579
|
+
command: 'bench import',
|
|
580
|
+
root,
|
|
581
|
+
file: resolve(file),
|
|
582
|
+
dryRun,
|
|
583
|
+
applied: result.applied,
|
|
584
|
+
taskCount: result.tasks,
|
|
585
|
+
tasksDir: result.tasksDir,
|
|
586
|
+
vaultRoot: result.vaultRoot,
|
|
587
|
+
seal: doc.seal,
|
|
588
|
+
});
|
|
589
|
+
return EXIT.OK;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
const lines = [
|
|
593
|
+
`bench import ${resolve(file)}`,
|
|
594
|
+
`mode ${apply ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview (pass --yes to write)'}`,
|
|
595
|
+
`imported ${result.tasks} task(s) seal=${doc.seal}`,
|
|
596
|
+
`tasks ${result.tasksDir}`,
|
|
597
|
+
`vault ${result.vaultRoot}`,
|
|
598
|
+
];
|
|
599
|
+
if (!apply) lines.push(`remedy run: ${CLI} bench import ${file} --yes`);
|
|
600
|
+
emit(lines.join('\n'));
|
|
428
601
|
return EXIT.OK;
|
|
429
602
|
}
|
package/src/commands/brief.mjs
CHANGED
|
@@ -19,8 +19,9 @@ import { CLI } from '../name.mjs';
|
|
|
19
19
|
import { EXIT, emit, emitJson, fail, isJson } from '../output/index.mjs';
|
|
20
20
|
import { DEFAULT_BRIEF_BYTES, explainPacket } from '../brief/packet.mjs';
|
|
21
21
|
import { assembleBrief } from '../brief/assemble.mjs';
|
|
22
|
+
import { commandRoot } from '../repo-root.mjs';
|
|
22
23
|
|
|
23
|
-
export async function briefCommand(args, flags) {
|
|
24
|
+
export async function briefCommand(args, flags, commands = []) {
|
|
24
25
|
if (args.length > 1) {
|
|
25
26
|
return fail(`brief takes one task, not ${args.length}`, EXIT.USAGE, `run: ${CLI} brief "<task>"`);
|
|
26
27
|
}
|
|
@@ -33,11 +34,12 @@ export async function briefCommand(args, flags) {
|
|
|
33
34
|
}
|
|
34
35
|
|
|
35
36
|
const result = await assembleBrief({
|
|
36
|
-
root: flags
|
|
37
|
+
root: commandRoot(flags),
|
|
37
38
|
task: String(args[0] ?? flags.task ?? ''),
|
|
38
39
|
paths: flags.paths === undefined ? null : flags.paths,
|
|
39
40
|
budget,
|
|
40
41
|
flags,
|
|
42
|
+
commands,
|
|
41
43
|
});
|
|
42
44
|
if (result.error) return result.error;
|
|
43
45
|
// Unreachable from here — only a delivered brief asks for a cold index to be reported
|
|
@@ -67,12 +69,11 @@ export async function briefCommand(args, flags) {
|
|
|
67
69
|
explain: flags.explain === true ? explainPacket(items, { cut }) : undefined,
|
|
68
70
|
text: packed.text,
|
|
69
71
|
},
|
|
70
|
-
{ pretty: flags.pretty === true, maxBytes: Number(flags.maxBytes ?? 0) || 0 },
|
|
71
72
|
);
|
|
72
73
|
return EXIT.OK;
|
|
73
74
|
}
|
|
74
75
|
|
|
75
76
|
const body = flags.explain === true ? [packed.text, '', 'EXPLAIN', ...explainPacket(items, { cut })] : [packed.text];
|
|
76
|
-
emit(body.join('\n')
|
|
77
|
+
emit(body.join('\n'));
|
|
77
78
|
return EXIT.OK;
|
|
78
79
|
}
|
package/src/commands/ci.mjs
CHANGED
|
@@ -27,6 +27,7 @@ import { EXIT, emit, emitJson, isJson, note, writeMode } from '../output/index.m
|
|
|
27
27
|
import { planWorkflow, renderWorkflow, WORKFLOW_FILE } from '../ci/workflow.mjs';
|
|
28
28
|
import { planDependabot, renderDependabot, DEPENDABOT_FILE } from '../ci/dependabot.mjs';
|
|
29
29
|
import { CLI } from '../name.mjs';
|
|
30
|
+
import { commandRoot } from '../repo-root.mjs';
|
|
30
31
|
|
|
31
32
|
/**
|
|
32
33
|
* The two plans, with the render and the plan-time data each needs kept attached.
|
|
@@ -43,7 +44,7 @@ function targets() {
|
|
|
43
44
|
}
|
|
44
45
|
|
|
45
46
|
export async function ciCommand(positional, flags) {
|
|
46
|
-
const root = flags
|
|
47
|
+
const root = commandRoot(flags, positional);
|
|
47
48
|
const mode = writeMode(flags);
|
|
48
49
|
const checkOnly = flags.check === true;
|
|
49
50
|
|