simt-dev 0.6.3 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +162 -841
- package/README.md +21 -135
- package/agents/simt-agent.md +84 -0
- package/bin/simt.mjs +19 -55
- package/docs/decisions/ADR-009-skill-layout.md +1 -1
- package/docs/decisions/ADR-014-workspace-is-configuration.md +1 -1
- package/docs/decisions/ADR-086-the-stable-prefix-comes-first-and-the-lock-records-both-digests.md +1 -1
- package/docs/decisions/ADR-099-the-release-verifies-served-bytes-and-the-formula-mirrors-them.md +1 -1
- package/docs/decisions/ADR-106-the-harness-hook-runs-through-a-repo-shim.md +46 -0
- package/docs/decisions/ADR-107-an-install-can-name-one-harness-and-the-skipped-ones-are-named.md +51 -0
- package/docs/decisions/ADR-108-an-adoption-builds-the-index-and-a-command-refreshes-it-on-read.md +59 -0
- package/docs/decisions/ADR-109-simt-dev-is-its-own-consumer-and-the-guards-that-kept-it-from-being-one-are-retired.md +47 -0
- package/docs/decisions/ADR-110-the-lock-file-follows-the-manifest-name-and-the-old-name-is-read-until-the-next-write.md +43 -0
- package/docs/decisions/ADR-111-the-tool-ships-one-agent-rendered-per-harness-and-installs-it-with-init-and-a-command.md +75 -0
- package/docs/decisions/ADR-112-the-registry-is-what-simt-dev-ships-and-a-repo-owns-its-own-skills.md +81 -0
- package/docs/decisions/ADR-113-a-pre-push-that-carries-only-documentation-skips-the-test-gate-and-says-so.md +87 -0
- package/docs/decisions/ADR-114-the-bench-pairs-at-the-task-level-a-missing-cost-is-not-zero-and-an-unasked-cross-group-compare-is-refused.md +74 -0
- package/docs/decisions/ADR-115-the-index-stamps-its-schema-version-migrates-older-databases-in-place-and-serves-newer-ones-from-a-sidecar.md +69 -0
- package/docs/decisions/ADR-116-rust-support-reaches-the-example-path-the-indent-and-the-bench-runner.md +100 -0
- package/docs/decisions/ADR-117-the-darwin-release-builds-on-supported-macos-15-runners-and-a-re-run-is-idempotent.md +81 -0
- package/docs/decisions/ADR-118-the-bench-corpus-is-keyed-on-history-is-append-only-and-is-portable-as-a-sealed-document.md +91 -0
- package/docs/decisions/ADR-119-the-environment-variable-points-into-a-workspace-the-repo-declared.md +90 -0
- package/docs/decisions/ADR-120-a-pre-push-whose-tree-is-already-proven-green-skips-the-test-gate-on-that-evidence.md +96 -0
- package/docs/decisions/ADR-122-the-shims-move-into-the-state-directory-and-the-legacy-directory-removes-itself.md +94 -0
- package/docs/decisions/ADR-123-a-surplus-positional-is-not-refused-because-arity-is-not-in-the-source.md +57 -0
- package/docs/decisions/ADR-124-the-manifest-has-a-schema-in-the-tool-and-its-unknown-keys-stay-the-repositorys.md +64 -0
- package/docs/decisions/ADR-125-a-shared-context-belongs-to-a-read-and-a-writer-reads-directly.md +65 -0
- package/package.json +13 -3
- package/rules/core.md +9 -2
- package/rules/mdc/slice-commit.mdc +57 -4
- package/rules/mdc/test-gate.mdc +2 -0
- package/rules/stacks/android.md +3 -3
- package/rules/stacks/connectiq.md +2 -2
- package/rules/stacks/dart.md +1 -1
- package/rules/stacks/deno.md +1 -1
- package/rules/stacks/dotnet.md +1 -1
- package/rules/stacks/elixir.md +1 -1
- package/rules/stacks/go.md +1 -1
- package/rules/stacks/jvm.md +3 -2
- package/rules/stacks/node.md +2 -2
- package/rules/stacks/python.md +7 -6
- package/rules/stacks/ruby.md +1 -1
- package/rules/stacks/rust.md +1 -1
- package/rules/stacks/swift.md +1 -1
- package/rules/stacks/typescript.md +2 -2
- package/rules/stacks/uv.md +3 -2
- package/skills/simt-commands/SKILL.md +70 -0
- package/src/advice/advice.mjs +7 -3
- package/src/agenthooks/decide.mjs +141 -9
- package/src/agenthooks/opencode.mjs +72 -0
- package/src/agenthooks/prompt.mjs +10 -3
- package/src/agenthooks/render.mjs +186 -19
- package/src/agenthooks/shim.mjs +55 -0
- package/src/agents/render.mjs +277 -0
- package/src/api.mjs +29 -0
- package/src/argv.mjs +37 -3
- package/src/bench/ablate.mjs +31 -14
- package/src/bench/adapters.json +16 -0
- package/src/bench/corpus.mjs +150 -0
- package/src/bench/grade.mjs +23 -49
- package/src/bench/mine.mjs +247 -36
- package/src/bench/probes.mjs +104 -68
- package/src/bench/process.mjs +128 -0
- package/src/bench/report.mjs +260 -84
- package/src/bench/run.mjs +251 -150
- package/src/bench/variants.mjs +112 -40
- package/src/brief/assemble.mjs +44 -3
- package/src/ci/workflow.mjs +46 -14
- package/src/cli.mjs +45 -0
- package/src/commands/agent.mjs +205 -0
- package/src/commands/agenthook.mjs +113 -20
- package/src/commands/bench.mjs +192 -18
- package/src/commands/brief.mjs +5 -4
- package/src/commands/ci.mjs +2 -1
- package/src/commands/compare.mjs +50 -49
- package/src/commands/completions.mjs +2 -6
- package/src/commands/decide.mjs +10 -9
- package/src/commands/distill.mjs +4 -3
- package/src/commands/doctor.mjs +11 -7
- package/src/commands/evidence.mjs +19 -12
- package/src/commands/explain.mjs +0 -1
- package/src/commands/gate.mjs +98 -28
- package/src/commands/graph.mjs +20 -8
- package/src/commands/harness.mjs +3 -2
- package/src/commands/help.mjs +160 -31
- package/src/commands/hooks.mjs +163 -50
- package/src/commands/ignore.mjs +2 -1
- package/src/commands/impact.mjs +8 -9
- package/src/commands/index.mjs +278 -180
- package/src/commands/init.mjs +171 -38
- package/src/commands/lease.mjs +206 -39
- package/src/commands/lessons.mjs +72 -15
- package/src/commands/local.mjs +3 -3
- package/src/commands/map.mjs +47 -27
- package/src/commands/mcp.mjs +0 -1
- package/src/commands/pack.mjs +9 -6
- package/src/commands/plan.mjs +6 -5
- package/src/commands/plugin.mjs +3 -3
- package/src/commands/prbody.mjs +3 -3
- package/src/commands/repoindex.mjs +5 -3
- package/src/commands/review.mjs +2 -2
- package/src/commands/rules.mjs +124 -33
- package/src/commands/shared.mjs +2 -3
- package/src/commands/skills.mjs +82 -45
- package/src/commands/skillscompile.mjs +12 -12
- package/src/commands/skillsinstall.mjs +14 -2
- package/src/commands/slice.mjs +396 -26
- package/src/commands/spec.mjs +151 -0
- package/src/commands/status.mjs +4 -7
- package/src/commands/sync.mjs +50 -33
- package/src/commands/tests.mjs +11 -11
- package/src/commands/tokens.mjs +2 -2
- package/src/commands/undo.mjs +29 -31
- package/src/commands/upgrade.mjs +40 -4
- package/src/commands/why.mjs +19 -12
- package/src/compare/hooksstate.mjs +23 -1
- package/src/compare/learn.mjs +82 -57
- package/src/compare/parallel.mjs +50 -0
- package/src/compare/row.mjs +26 -3
- package/src/completions.mjs +5 -0
- package/src/context.mjs +64 -0
- package/src/decisions/dir.mjs +84 -0
- package/src/decisions/index-doc.mjs +46 -0
- package/src/decisions/index.mjs +109 -11
- package/src/diff.mjs +1 -1
- package/src/distill/distill.mjs +17 -2
- package/src/distill/engine.mjs +5 -1
- package/src/ecosystem/node-runner.mjs +144 -0
- package/src/explain.mjs +18 -6
- package/src/graph/load.mjs +42 -20
- package/src/hooks/capability.mjs +17 -4
- package/src/hooks/dispatcher.mjs +513 -36
- package/src/hooks/gates.mjs +0 -43
- package/src/hooks/resolve.mjs +11 -10
- package/src/hooks/shim.mjs +17 -9
- package/src/identity.mjs +96 -15
- package/src/index/extract.mjs +192 -11
- package/src/index/persist.mjs +307 -47
- package/src/index/store.mjs +263 -60
- package/src/index/tests.mjs +6 -35
- package/src/journal.mjs +372 -84
- package/src/kernel/card.mjs +37 -12
- package/src/kernel/checks.mjs +44 -9
- package/src/kernel/variants.mjs +27 -16
- package/src/lease.mjs +97 -15
- package/src/ledger/index.mjs +120 -25
- package/src/local/client.mjs +4 -6
- package/src/locate.mjs +33 -9
- package/src/manifest/schema.mjs +213 -0
- package/src/mcp/resources.mjs +12 -13
- package/src/mcp/server.mjs +151 -38
- package/src/mcp/tools.mjs +90 -16
- package/src/measure/session.mjs +3 -2
- package/src/memory/decide.mjs +98 -43
- package/src/memory/lessons.mjs +168 -33
- package/src/name.mjs +9 -0
- package/src/node-version.mjs +37 -0
- package/src/output/index.mjs +111 -3
- package/src/packs/manage.mjs +17 -3
- package/src/packs.mjs +20 -2
- package/src/plan/index.mjs +31 -8
- package/src/plan/template.mjs +27 -5
- package/src/plugins.mjs +6 -2
- package/src/prbody.mjs +7 -5
- package/src/projection/baselines.mjs +1 -1
- package/src/projection/disclosure.mjs +2 -2
- package/src/projection/loaders.mjs +41 -1
- package/src/projection/lock.mjs +31 -14
- package/src/projection/manifest.mjs +89 -53
- package/src/projection/markers.mjs +43 -3
- package/src/projection/mdc.mjs +34 -7
- package/src/projection/project.mjs +162 -18
- package/src/projection/rules.mjs +58 -6
- package/src/projection/surfaces.mjs +9 -1
- package/src/repo-root.mjs +88 -0
- package/src/resolve/body.mjs +4 -0
- package/src/resolve/fingerprint.mjs +24 -15
- package/src/resolve/local.mjs +58 -0
- package/src/review/critic.mjs +6 -3
- package/src/secrets.mjs +165 -22
- package/src/state.mjs +76 -0
- package/src/status.mjs +9 -4
- package/src/toolwords.mjs +8 -0
- package/src/tsconfig.mjs +39 -99
- package/src/workspace.mjs +55 -10
- package/templates/plan/tracker.json +1 -1
- package/docs/decisions/README.md +0 -591
package/src/bench/variants.mjs
CHANGED
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
// Discovers `simt-variants` slots across applicable rules, runs candidate
|
|
4
4
|
// variants on the 10-task canary suite (escalating when significant), records
|
|
5
5
|
// the winning variant with `{winner, variant, model, harness, dR, dC, n, p}`
|
|
6
|
-
// in `simt
|
|
6
|
+
// in `simt.lock.json`, and re-opens the experiment whenever the target
|
|
7
7
|
// model or harness changes.
|
|
8
8
|
|
|
9
9
|
import { mkdir, readdir, readFile, unlink, writeFile } from 'node:fs/promises';
|
|
10
10
|
import { dirname, join, resolve } from 'node:path';
|
|
11
11
|
|
|
12
12
|
import { extractVariantsFromRule, readLockedVariants } from '../kernel/variants.mjs';
|
|
13
|
-
import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
13
|
+
import { LOCK_FILE, MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
14
14
|
import { writeLockVariants } from '../projection/lock.mjs';
|
|
15
15
|
import { readManifest } from '../projection/manifest.mjs';
|
|
16
16
|
import { loadMdcRules } from '../projection/mdc.mjs';
|
|
@@ -21,12 +21,22 @@ import { CANARY_ALPHA, DEFAULT_CANARY_SIZE, DEL_R_TOLERANCE, shouldEscalateCanar
|
|
|
21
21
|
import {
|
|
22
22
|
DEFAULT_BOOTSTRAP_SAMPLES,
|
|
23
23
|
DEFAULT_BOOTSTRAP_SEED,
|
|
24
|
+
collapseArmByTask,
|
|
24
25
|
compareArms,
|
|
25
26
|
evaluateRunRecord,
|
|
27
|
+
holmAdjust,
|
|
26
28
|
loadPrices,
|
|
27
29
|
loadRunRecords,
|
|
28
30
|
} from './report.mjs';
|
|
29
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Harnesses that grade, not answer: `gold` applies the oracle patch and `noop` does
|
|
34
|
+
* nothing (`src/bench/adapters.json`, `kind: "builtin"`). Their runs measure the
|
|
35
|
+
* grader, not the rule text, so they must never close a variant experiment by
|
|
36
|
+
* writing a lock entry (Q45). A test asserts this set matches the adapter table.
|
|
37
|
+
*/
|
|
38
|
+
export const BUILTIN_HARNESSES = new Set(['gold', 'noop']);
|
|
39
|
+
|
|
30
40
|
/**
|
|
31
41
|
* Discover every `simt-variants` slot across applicable rules for `root`.
|
|
32
42
|
*/
|
|
@@ -131,6 +141,19 @@ export async function applyVariantInWorkspace(workDir, variant) {
|
|
|
131
141
|
delete nextManifest.mode;
|
|
132
142
|
await writeFile(manifestPath, `${JSON.stringify(nextManifest, null, 2)}\n`, 'utf8');
|
|
133
143
|
|
|
144
|
+
// plan() writes the lock as a target (Q29). Left behind, it records this arm's
|
|
145
|
+
// winner as locked, and the next selectRuleVariants on the same tree reads the
|
|
146
|
+
// experiment as closed and never runs the runner. Restore it, like the manifest.
|
|
147
|
+
const lockPath = join(workDir, LOCK_FILE);
|
|
148
|
+
let originalLockBytes = null;
|
|
149
|
+
let hadLock = false;
|
|
150
|
+
try {
|
|
151
|
+
originalLockBytes = await readFile(lockPath, 'utf8');
|
|
152
|
+
hadLock = true;
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (error.code !== 'ENOENT') throw error;
|
|
155
|
+
}
|
|
156
|
+
|
|
134
157
|
const planned = await plan(workDir, {
|
|
135
158
|
variants: { [slotId]: { winner: variantKey } },
|
|
136
159
|
});
|
|
@@ -146,30 +169,36 @@ export async function applyVariantInWorkspace(workDir, variant) {
|
|
|
146
169
|
} else if (originalManifestBytes !== null) {
|
|
147
170
|
await writeFile(manifestPath, originalManifestBytes, 'utf8');
|
|
148
171
|
}
|
|
172
|
+
if (!hadLock) {
|
|
173
|
+
await unlink(lockPath).catch(() => {});
|
|
174
|
+
} else {
|
|
175
|
+
await writeFile(lockPath, originalLockBytes, 'utf8');
|
|
176
|
+
}
|
|
149
177
|
}
|
|
150
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Mean per-task R and C for one arm's evaluated runs.
|
|
181
|
+
*
|
|
182
|
+
* `collapseArmByTask` is the shared per-task aggregation: the evaluated rows carry
|
|
183
|
+
* the cost in `C`, and the old body summed `costUsd` — a field the rows never had —
|
|
184
|
+
* so C was NaN and the cost rank degenerated to a key tie-break (Q45).
|
|
185
|
+
*/
|
|
151
186
|
function summarizeEvaluated(evaluated) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
155
|
-
const byTask = new Map();
|
|
156
|
-
for (const row of evaluated) {
|
|
157
|
-
const list = byTask.get(row.task) ?? [];
|
|
158
|
-
list.push(row);
|
|
159
|
-
byTask.set(row.task, list);
|
|
160
|
-
}
|
|
187
|
+
const tasks = collapseArmByTask(evaluated);
|
|
188
|
+
if (tasks.length === 0) return { n: 0, R: 0, C: null };
|
|
161
189
|
let sumR = 0;
|
|
162
190
|
let sumC = 0;
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
191
|
+
let costComplete = true;
|
|
192
|
+
for (const t of tasks) {
|
|
193
|
+
sumR += t.R;
|
|
194
|
+
if (t.C === null) costComplete = false;
|
|
195
|
+
else sumC += t.C;
|
|
166
196
|
}
|
|
167
|
-
|
|
168
|
-
return { n, R: sumR / n, C: sumC / n };
|
|
197
|
+
return { n: tasks.length, R: sumR / tasks.length, C: costComplete ? sumC / tasks.length : null };
|
|
169
198
|
}
|
|
170
199
|
|
|
171
200
|
/**
|
|
172
|
-
* Run or evaluate variant selection on the canary suite and record winners in `simt
|
|
201
|
+
* Run or evaluate variant selection on the canary suite and record winners in `simt.lock.json`.
|
|
173
202
|
*/
|
|
174
203
|
export async function selectRuleVariants(
|
|
175
204
|
root,
|
|
@@ -267,19 +296,31 @@ export async function selectRuleVariants(
|
|
|
267
296
|
for (const { slot } of openStates) {
|
|
268
297
|
const baseArm = variantArmName(slot.slotId, slot.baseline);
|
|
269
298
|
const baseRecs = canaryEvaluated.filter((r) => r.arm === baseArm);
|
|
299
|
+
const slotComps = [];
|
|
270
300
|
for (const key of Object.keys(slot.variants)) {
|
|
271
301
|
if (key === slot.baseline) continue;
|
|
272
302
|
const candArm = variantArmName(slot.slotId, key);
|
|
273
303
|
const candRecs = canaryEvaluated.filter((r) => r.arm === candArm);
|
|
274
304
|
if (baseRecs.length > 0 && candRecs.length > 0) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
escalatedArms.add(candArm);
|
|
280
|
-
}
|
|
305
|
+
slotComps.push({
|
|
306
|
+
candArm,
|
|
307
|
+
comp: compareArms(baseArm, baseRecs, candArm, candRecs, { samples, seed }),
|
|
308
|
+
});
|
|
281
309
|
}
|
|
282
310
|
}
|
|
311
|
+
// Family = every variant of this slot versus its baseline: testing several
|
|
312
|
+
// variants at α 0.05 each inflates the false-positive rate, so the p-values
|
|
313
|
+
// are Holm-adjusted before the escalation decision (review 0003 Q46).
|
|
314
|
+
const adjusted = holmAdjust(
|
|
315
|
+
slotComps.map(({ comp }) => Math.min(comp.rTest.pValue, comp.wilcoxon.pValue)),
|
|
316
|
+
);
|
|
317
|
+
slotComps.forEach(({ candArm, comp }, i) => {
|
|
318
|
+
if (shouldEscalateCanary(comp, { alpha: CANARY_ALPHA, pValue: adjusted[i] })) {
|
|
319
|
+
includeBaseline.add(baseArm);
|
|
320
|
+
armsToEscalate.push(candArm);
|
|
321
|
+
escalatedArms.add(candArm);
|
|
322
|
+
}
|
|
323
|
+
});
|
|
283
324
|
}
|
|
284
325
|
const fullEscalationArms = [...includeBaseline, ...armsToEscalate];
|
|
285
326
|
if (fullEscalationArms.length > 0) {
|
|
@@ -321,6 +362,7 @@ export async function selectRuleVariants(
|
|
|
321
362
|
reason,
|
|
322
363
|
measured: true,
|
|
323
364
|
winner: winnerKey,
|
|
365
|
+
locked: true,
|
|
324
366
|
model: lockedEntry.model ?? resolvedModel,
|
|
325
367
|
harness: lockedEntry.harness ?? harness,
|
|
326
368
|
dR: Number(lockedEntry.dR ?? 0),
|
|
@@ -367,6 +409,7 @@ export async function selectRuleVariants(
|
|
|
367
409
|
reason,
|
|
368
410
|
measured: false,
|
|
369
411
|
winner: slot.baseline,
|
|
412
|
+
locked: false,
|
|
370
413
|
model: resolvedModel,
|
|
371
414
|
harness,
|
|
372
415
|
dR: 0,
|
|
@@ -377,13 +420,16 @@ export async function selectRuleVariants(
|
|
|
377
420
|
continue;
|
|
378
421
|
}
|
|
379
422
|
|
|
380
|
-
// Rank variants: higher R (beyond tolerance), then lower C
|
|
423
|
+
// Rank variants: higher R (beyond tolerance), then lower C (an unknown cost is
|
|
424
|
+
// worst — it cannot be ranked cheaper), then fewer bytes, then key.
|
|
381
425
|
const ranked = [...candidates].sort((a, b) => {
|
|
382
426
|
if (Math.abs(b.summary.R - a.summary.R) > DEL_R_TOLERANCE) {
|
|
383
427
|
return b.summary.R - a.summary.R;
|
|
384
428
|
}
|
|
385
|
-
|
|
386
|
-
|
|
429
|
+
const ca = a.summary.C === null ? Number.POSITIVE_INFINITY : a.summary.C;
|
|
430
|
+
const cb = b.summary.C === null ? Number.POSITIVE_INFINITY : b.summary.C;
|
|
431
|
+
if (Math.abs(ca - cb) > 1e-9) {
|
|
432
|
+
return ca - cb;
|
|
387
433
|
}
|
|
388
434
|
if (a.bytes !== b.bytes) return a.bytes - b.bytes;
|
|
389
435
|
return a.key.localeCompare(b.key);
|
|
@@ -397,10 +443,13 @@ export async function selectRuleVariants(
|
|
|
397
443
|
|
|
398
444
|
if (best.key !== slot.baseline && best.comparison) {
|
|
399
445
|
dR = Number(best.comparison.deltas.R.toFixed(4));
|
|
400
|
-
dC =
|
|
446
|
+
dC =
|
|
447
|
+
best.comparison.deltas.C === null
|
|
448
|
+
? 0
|
|
449
|
+
: Number(best.comparison.deltas.C.toFixed(4));
|
|
401
450
|
n = best.comparison.pairedTasks;
|
|
402
451
|
p = Number(
|
|
403
|
-
Math.min(best.comparison.
|
|
452
|
+
Math.min(best.comparison.rTest.pValue, best.comparison.wilcoxon.pValue).toFixed(4),
|
|
404
453
|
);
|
|
405
454
|
} else {
|
|
406
455
|
const runnerUp = ranked.find((c) => c.key !== slot.baseline && c.comparison);
|
|
@@ -409,17 +458,33 @@ export async function selectRuleVariants(
|
|
|
409
458
|
}
|
|
410
459
|
}
|
|
411
460
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
p
|
|
421
|
-
|
|
422
|
-
|
|
461
|
+
// A non-baseline winner locks only when it is significant AND non-inferior
|
|
462
|
+
// (Q45): p below the canary alpha, and dR within the deletion tolerance.
|
|
463
|
+
// Anything else locks the baseline: the experiment ran, and its conclusion is
|
|
464
|
+
// "keep what we had" - recorded, so the next run on this model and harness
|
|
465
|
+
// reads it as closed instead of re-measuring.
|
|
466
|
+
const significant =
|
|
467
|
+
best.key !== slot.baseline &&
|
|
468
|
+
best.comparison !== null &&
|
|
469
|
+
p < CANARY_ALPHA &&
|
|
470
|
+
dR >= -DEL_R_TOLERANCE;
|
|
471
|
+
const winnerKey = significant ? best.key : slot.baseline;
|
|
472
|
+
|
|
473
|
+
// Built-in harnesses grade, they do not answer: their runs must not close the
|
|
474
|
+
// experiment, so no lock entry is written for them and the next run re-measures
|
|
475
|
+
// (Q45). The result still reports the measured best, flagged `locked: false`.
|
|
476
|
+
if (!BUILTIN_HARNESSES.has(harness)) {
|
|
477
|
+
nextVariantsLock[slot.slotId] = {
|
|
478
|
+
winner: winnerKey,
|
|
479
|
+
variant: winnerKey,
|
|
480
|
+
model: resolvedModel,
|
|
481
|
+
harness,
|
|
482
|
+
dR,
|
|
483
|
+
dC,
|
|
484
|
+
n,
|
|
485
|
+
p,
|
|
486
|
+
};
|
|
487
|
+
}
|
|
423
488
|
|
|
424
489
|
results.push({
|
|
425
490
|
slotId: slot.slotId,
|
|
@@ -430,7 +495,14 @@ export async function selectRuleVariants(
|
|
|
430
495
|
reopened,
|
|
431
496
|
reason,
|
|
432
497
|
measured: true,
|
|
433
|
-
|
|
498
|
+
winner: BUILTIN_HARNESSES.has(harness) ? best.key : winnerKey,
|
|
499
|
+
locked: significant && !BUILTIN_HARNESSES.has(harness),
|
|
500
|
+
model: resolvedModel,
|
|
501
|
+
harness,
|
|
502
|
+
dR,
|
|
503
|
+
dC,
|
|
504
|
+
n,
|
|
505
|
+
p,
|
|
434
506
|
});
|
|
435
507
|
}
|
|
436
508
|
|
package/src/brief/assemble.mjs
CHANGED
|
@@ -34,7 +34,10 @@
|
|
|
34
34
|
// command and the tool leave it on. It is inert either way in a repository that declared no
|
|
35
35
|
// servers.
|
|
36
36
|
|
|
37
|
+
import { join } from 'node:path';
|
|
38
|
+
|
|
37
39
|
import { CLI, STATE_DIR } from '../name.mjs';
|
|
40
|
+
import { parse as parseRegion } from '../projection/markers.mjs';
|
|
38
41
|
import { DEFAULT_BRIEF_BYTES, buildPacket, collect } from './packet.mjs';
|
|
39
42
|
import { classifyTask, cardVerbs } from './kind.mjs';
|
|
40
43
|
import { briefAdvice } from '../advice/advice.mjs';
|
|
@@ -81,22 +84,38 @@ const MAP_FACTOR = 8;
|
|
|
81
84
|
* space is never part of a task's scope.
|
|
82
85
|
*/
|
|
83
86
|
|
|
84
|
-
/**
|
|
87
|
+
/**
|
|
88
|
+
* Every file git tracks that carries a generated region, one `git grep` away.
|
|
89
|
+
*
|
|
90
|
+
* A file that merely mentions the marker is not carrying a region: a rule file, a review
|
|
91
|
+
* and this test all quote `simt:begin`, and warning that they are generated sends a reader
|
|
92
|
+
* to edit a source file that has no generated bytes at all (review 0003 Q52). The grep is
|
|
93
|
+
* only the candidate list; the region parser decides.
|
|
94
|
+
*/
|
|
85
95
|
async function generatedRegions(root) {
|
|
86
96
|
const { execFile } = await import('node:child_process');
|
|
87
97
|
const { promisify } = await import('node:util');
|
|
98
|
+
const { readFile } = await import('node:fs/promises');
|
|
99
|
+
let candidates;
|
|
88
100
|
try {
|
|
89
101
|
const { stdout } = await promisify(execFile)(
|
|
90
102
|
'git',
|
|
91
103
|
['grep', '-lF', '--', `${'simt'}:begin`],
|
|
92
104
|
{ cwd: root, maxBuffer: 8 << 20 },
|
|
93
105
|
);
|
|
94
|
-
|
|
106
|
+
candidates = stdout.split('\n').filter((line) => line.length > 0).sort();
|
|
95
107
|
} catch {
|
|
96
108
|
// No match is exit 1, and a directory that is not a repository fails too. Both mean
|
|
97
109
|
// the same thing to a reader: no generated region was found to warn about.
|
|
98
110
|
return [];
|
|
99
111
|
}
|
|
112
|
+
const carrying = [];
|
|
113
|
+
for (const path of candidates) {
|
|
114
|
+
const text = await readFile(join(root, path), 'utf8').catch(() => null);
|
|
115
|
+
if (text === null) continue;
|
|
116
|
+
if (parseRegion(text, 'html').status === 'ok') carrying.push(path);
|
|
117
|
+
}
|
|
118
|
+
return carrying;
|
|
100
119
|
}
|
|
101
120
|
|
|
102
121
|
/**
|
|
@@ -113,6 +132,9 @@ async function generatedRegions(root) {
|
|
|
113
132
|
* @param {boolean} [options.build] whether an empty index may be built here
|
|
114
133
|
* @param {boolean} [options.federate] whether a declared external server may be asked one
|
|
115
134
|
* question; a delivered packet turns it off to keep its 300 ms bar
|
|
135
|
+
* @param {string[]} [options.commands] the dispatchable command paths (`COMMANDS`). A
|
|
136
|
+
* decision's `Affects:` token is a command only when the tool has that command, so a
|
|
137
|
+
* packet that does not know them reports every one of them as unresolved
|
|
116
138
|
* @returns a packet, or `{ error }` carrying the exit code a failure already reported, or
|
|
117
139
|
* `{ cold: true }` when the index is empty and this caller may not build one
|
|
118
140
|
*/
|
|
@@ -124,6 +146,7 @@ export async function assembleBrief({
|
|
|
124
146
|
flags = {},
|
|
125
147
|
build = true,
|
|
126
148
|
federate = true,
|
|
149
|
+
commands = [],
|
|
127
150
|
} = {}) {
|
|
128
151
|
const named = paths !== null;
|
|
129
152
|
const dirty = (await dirtyPaths(root)).filter((path) => !path.startsWith(`${STATE_DIR}/`));
|
|
@@ -132,6 +155,7 @@ export async function assembleBrief({
|
|
|
132
155
|
const graph = await loadGraph(root, { build });
|
|
133
156
|
if (graph.error) return { error: graph.error };
|
|
134
157
|
if (graph.cold) return { cold: true };
|
|
158
|
+
const graphStale = graph.stale === true;
|
|
135
159
|
const { files, symbols, edges, fts = [], embeddings = [] } = graph;
|
|
136
160
|
const filePaths = new Set(files.map((file) => file.path));
|
|
137
161
|
|
|
@@ -150,6 +174,11 @@ export async function assembleBrief({
|
|
|
150
174
|
why: ['an unasked question has no answer to be short of'],
|
|
151
175
|
});
|
|
152
176
|
}
|
|
177
|
+
if (graphStale) {
|
|
178
|
+
packet.add('CAUTIONS', 'the index predates the current tree, so files added or removed since the last build are missing or ghosted', {
|
|
179
|
+
why: ['this caller may not rebuild the index; run a command that can before quoting the file list'],
|
|
180
|
+
});
|
|
181
|
+
}
|
|
153
182
|
|
|
154
183
|
const repo = await fingerprint(root);
|
|
155
184
|
const card = await commandCard({ root, manifest: repo.manifest ?? null, stack: repo.stack, facts: repo.facts });
|
|
@@ -181,6 +210,15 @@ export async function assembleBrief({
|
|
|
181
210
|
})
|
|
182
211
|
: null;
|
|
183
212
|
const ranked = rankFiles({ files, symbols, edges, fts, bm25, embeddings, queryVector, task });
|
|
213
|
+
// An empty match is a different answer from a weak one: the ranking is then the graph
|
|
214
|
+
// hub, which is what a reader who asked about `zebracorn` must not read as a hit
|
|
215
|
+
// (review 0003 Q52).
|
|
216
|
+
const matched = ranked.filter((row) => (row.score ?? 0) > 0).length;
|
|
217
|
+
if (task.trim().length > 0 && matched === 0) {
|
|
218
|
+
packet.add('CAUTIONS', 'matched 0: no file or symbol mentions this task, so FILES is the graph hub rather than a match', {
|
|
219
|
+
why: ['an answer to a graph-shaped question is not an answer to this task'],
|
|
220
|
+
});
|
|
221
|
+
}
|
|
184
222
|
const rendered = renderMap(ranked, symbols, { level: 'names', budget: budget > 0 ? budget * MAP_FACTOR : 0 });
|
|
185
223
|
for (const line of rendered.text.split('\n')) {
|
|
186
224
|
packet.add('FILES', line, { why: ['ranked by the task words against the import graph'] });
|
|
@@ -227,7 +265,10 @@ export async function assembleBrief({
|
|
|
227
265
|
const records = await decisionRecords(root);
|
|
228
266
|
const ctx = {
|
|
229
267
|
files: filePaths,
|
|
230
|
-
|
|
268
|
+
// The card's lines are shell commands (`npm run test`); a decision's Affects token is
|
|
269
|
+
// a command of this tool. Passing only the card's list made every `simt …` token
|
|
270
|
+
// unresolved — 47 false cautions on this repository (review 0003 Q52).
|
|
271
|
+
commands: new Set([...card.lines.map((line) => line.command), ...commands]),
|
|
231
272
|
roots: sourceRoots([...filePaths]),
|
|
232
273
|
};
|
|
233
274
|
const decisions = new Map();
|
package/src/ci/workflow.mjs
CHANGED
|
@@ -52,6 +52,7 @@ import { renderLocator } from '../locate.mjs';
|
|
|
52
52
|
import { CLI } from '../name.mjs';
|
|
53
53
|
import { packageVersion } from '../package-version.mjs';
|
|
54
54
|
import { LOCAL_BIN, LOCAL_BIN_DIR, assetNameExpr, localInstall, platformLines, releaseBase } from '../distribute/release.mjs';
|
|
55
|
+
import { installCommands as nodeInstallCommands } from '../ecosystem/node-runner.mjs';
|
|
55
56
|
|
|
56
57
|
/** Directory whose presence means this repo runs GitHub Actions. */
|
|
57
58
|
export const WORKFLOWS_DIR = '.github/workflows';
|
|
@@ -166,7 +167,7 @@ function checkoutFull() {
|
|
|
166
167
|
* to. The own gate also varies the platforms and the Node versions, because this
|
|
167
168
|
* package is the one whose floor and whose ADR-012 claim have to hold on more than
|
|
168
169
|
* the minimum. */
|
|
169
|
-
function skeleton(name, steps, { matrix = null } = {}) {
|
|
170
|
+
function skeleton(name, steps, { matrix = null, timeoutMinutes = 5, gateMain = false } = {}) {
|
|
170
171
|
const axes = matrix
|
|
171
172
|
? Object.entries(matrix)
|
|
172
173
|
.map(([axis, values]) => ` ${axis}: [${values.map((v) => `'${v}'`).join(', ')}]`)
|
|
@@ -181,9 +182,22 @@ function skeleton(name, steps, { matrix = null } = {}) {
|
|
|
181
182
|
// it is written escaped, the same way the `concurrency` line below spells its own two.
|
|
182
183
|
const runner = matrix?.os ? '\${{ matrix.os }}' : 'ubuntu-latest';
|
|
183
184
|
const job = matrix?.node ? '\${{ matrix.os }} node \${{ matrix.node }}' : name;
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
185
|
+
// Two trigger blocks, because the two copies answer different questions. An
|
|
186
|
+
// adopting repository merges through pull requests, which is where its gate runs.
|
|
187
|
+
// This repository merges locally, so a merge never opens a pull request: without a
|
|
188
|
+
// push trigger the commit that lands on `main` is the one no gate ever ran on
|
|
189
|
+
// (review 0003 Q57). Each block ships only in the copy whose claim it states.
|
|
190
|
+
const trigger = gateMain
|
|
191
|
+
? `# Pull requests ask whether a change may merge; a push to \`main\` asks whether
|
|
192
|
+
# what already merged is sound. This repository merges locally, so a merge never
|
|
193
|
+
# opens a pull request — without the second trigger the commit that lands on
|
|
194
|
+
# \`main\` is the one no gate ran on.
|
|
195
|
+
on:
|
|
196
|
+
pull_request:
|
|
197
|
+
push:
|
|
198
|
+
branches: [main]
|
|
199
|
+
`
|
|
200
|
+
: `# Pull requests only, and that is a decision rather than a default.
|
|
187
201
|
#
|
|
188
202
|
# A gate that runs on every push re-runs itself for every commit that pushes a
|
|
189
203
|
# branch forward, including the ones nobody asked it about, and its verdict has to
|
|
@@ -193,7 +207,10 @@ function skeleton(name, steps, { matrix = null } = {}) {
|
|
|
193
207
|
# on \`main\` is whatever CI the repository already has there.
|
|
194
208
|
on:
|
|
195
209
|
pull_request:
|
|
210
|
+
`;
|
|
211
|
+
return `name: ${name}
|
|
196
212
|
|
|
213
|
+
${trigger}
|
|
197
214
|
# Read-only is enough to check a repo, and a gate job does not need to write to it.
|
|
198
215
|
permissions:
|
|
199
216
|
contents: read
|
|
@@ -206,7 +223,7 @@ jobs:
|
|
|
206
223
|
check:
|
|
207
224
|
name: ${job}
|
|
208
225
|
runs-on: ${runner}
|
|
209
|
-
${strategy} timeout-minutes:
|
|
226
|
+
${strategy} timeout-minutes: ${timeoutMinutes}
|
|
210
227
|
steps:
|
|
211
228
|
${steps}`;
|
|
212
229
|
}
|
|
@@ -253,7 +270,7 @@ function indent(text, spaces) {
|
|
|
253
270
|
* @param {{own?: boolean}} [options]
|
|
254
271
|
* @returns {string} YAML, LF-terminated
|
|
255
272
|
*/
|
|
256
|
-
export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
273
|
+
export function renderWorkflow({ own = false, npm = true, manager = null, version } = {}) {
|
|
257
274
|
if (own) {
|
|
258
275
|
const floor = nodeFloor();
|
|
259
276
|
return `${header(true)}${skeleton(
|
|
@@ -267,7 +284,7 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
267
284
|
' - name: tests',
|
|
268
285
|
' run: npm test',
|
|
269
286
|
' - name: skill registry lint',
|
|
270
|
-
' run:
|
|
287
|
+
' run: npm run lint',
|
|
271
288
|
' - name: the gate copy in this repo is current',
|
|
272
289
|
// The drift this exception exists to prevent, checked where it would be caught.
|
|
273
290
|
// Without it a template change lands with the copy still describing the old
|
|
@@ -281,11 +298,24 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
281
298
|
// sentence was false in the one repository that could have made it true, and no
|
|
282
299
|
// check read it - the package declares `os` for npm to enforce, and nothing here
|
|
283
300
|
// ever ran a test on a second platform.
|
|
284
|
-
|
|
301
|
+
// The slowest leg (macOS, full suite plus the registry lint) ran 4 min 28 s
|
|
302
|
+
// against a 5-minute timeout, so the timeout that was meant to bound a hang
|
|
303
|
+
// was about to fail a passing run instead. 15 leaves the hang bounded and the
|
|
304
|
+
// run alone; the sharding that would make it short again is Q92, not this row.
|
|
305
|
+
//
|
|
306
|
+
// `gateMain` because this repository merges locally: `pull_request` alone left
|
|
307
|
+
// the one commit that reached `main` ungated (review 0003 Q57).
|
|
308
|
+
{ matrix: { os: ['ubuntu-latest', 'macos-latest'], node: [floor, '24', '26'] }, timeoutMinutes: 15, gateMain: true },
|
|
285
309
|
)}\n`;
|
|
286
310
|
}
|
|
287
311
|
|
|
288
312
|
const v = version ?? packageVersion();
|
|
313
|
+
// Q31: the install step names the repo's own manager. `manager` wins over the
|
|
314
|
+
// legacy `npm` boolean so old callers keep their bytes; a Node repo installs
|
|
315
|
+
// with its lockfile's frozen command, anything else takes the release binary.
|
|
316
|
+
// The resolver row also decides setup-node (bun ships its own runtime).
|
|
317
|
+
const nodeManager = manager ?? (npm ? 'npm' : null);
|
|
318
|
+
const nodeInstall = nodeManager ? nodeInstallCommands(nodeManager) : null;
|
|
289
319
|
// The install step is the whole of S35's consumer CI: a repository with npm pins
|
|
290
320
|
// simt-dev as a devDependency and `npm ci` restores it; one without (Gradle,
|
|
291
321
|
// Connect IQ) has no node_modules to pin in, so CI downloads the release binary
|
|
@@ -301,11 +331,11 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
301
331
|
// `${{ runner.* }}` expression: an expression substituted into `$( … )` is a
|
|
302
332
|
// command substitution trying to execute "Linux", which set -eu refuses - the
|
|
303
333
|
// install would fail on every run in the one repository kind this channel is for.
|
|
304
|
-
const install =
|
|
334
|
+
const install = nodeInstall
|
|
305
335
|
? [
|
|
306
336
|
' - name: install dependencies',
|
|
307
|
-
|
|
308
|
-
|
|
337
|
+
` # \`${nodeInstall.install}\` needs a lockfile; a repo without one is still gated, not skipped.`,
|
|
338
|
+
` run: ${nodeInstall.install} || ${nodeInstall.fallback}`,
|
|
309
339
|
]
|
|
310
340
|
: [
|
|
311
341
|
' - name: install the pinned gate binary',
|
|
@@ -329,8 +359,9 @@ mkdir -p ${LOCAL_BIN_DIR}
|
|
|
329
359
|
mv "$asset" ${LOCAL_BIN}
|
|
330
360
|
chmod +x ${LOCAL_BIN}`, 10),
|
|
331
361
|
];
|
|
332
|
-
const
|
|
333
|
-
|
|
362
|
+
const addDev = { npm: `npm i -D ${PACKAGE_NAME}`, pnpm: `pnpm add -D ${PACKAGE_NAME}`, yarn: `yarn add -D ${PACKAGE_NAME}`, bun: `bun add -d ${PACKAGE_NAME}` }[nodeManager] ?? null;
|
|
363
|
+
const remedy = nodeInstall && addDev
|
|
364
|
+
? `REMEDY run: ${addDev} (the gate runs from the pinned dependency)`
|
|
334
365
|
: `REMEDY run: ${localInstall(v)} (the gate runs from the release binary in .simt/bin)`;
|
|
335
366
|
const locator = renderLocator('check --root "$ROOT"', { remedy });
|
|
336
367
|
return `${header(false)}${skeleton(
|
|
@@ -338,7 +369,8 @@ chmod +x ${LOCAL_BIN}`, 10),
|
|
|
338
369
|
[
|
|
339
370
|
uses('checkout'),
|
|
340
371
|
// The binary branch has no setup-node: see the install step above.
|
|
341
|
-
|
|
372
|
+
// Q31: bun ships its own runtime, so only managers that need Node get it.
|
|
373
|
+
...(nodeInstall && nodeInstall.setupNode ? [setupNode(nodeFloor())] : []),
|
|
342
374
|
...install,
|
|
343
375
|
` - name: ${CLI} check`,
|
|
344
376
|
' run: |',
|
package/src/cli.mjs
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// The CLI proper, imported by `bin/simt.mjs` after the runtime floor is checked.
|
|
2
|
+
//
|
|
3
|
+
// It lives here rather than in `bin/` because a static import in the entry point is
|
|
4
|
+
// evaluated before the entry point's own code: the version check could not run first,
|
|
5
|
+
// and an old Node failed inside the module graph instead of saying which version it
|
|
6
|
+
// needed (review 0003 Q54).
|
|
7
|
+
|
|
8
|
+
import { parse, rawJsonFlag } from './argv.mjs';
|
|
9
|
+
import { EXIT, fail, jsonFailMode, jsonIndentDefault } from './output/index.mjs';
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
const argv = process.argv.slice(2);
|
|
13
|
+
const parsed = parse(argv);
|
|
14
|
+
if (!parsed.ok) {
|
|
15
|
+
// The flags do not exist yet, so the JSON mode is read off the raw words:
|
|
16
|
+
// a parse failure under `--json` is a JSON document on stdout, not text on
|
|
17
|
+
// stderr with an empty stdout (Q11, ADR-070 ruling 4).
|
|
18
|
+
jsonFailMode(rawJsonFlag(argv));
|
|
19
|
+
process.exit(fail(parsed.error.message, parsed.error.code));
|
|
20
|
+
}
|
|
21
|
+
const { flags, positional } = parsed.value;
|
|
22
|
+
|
|
23
|
+
// Set before anything can fail: `no command given` under `--json` is a JSON
|
|
24
|
+
// document too, and a failure that prints as text with an empty stdout breaks
|
|
25
|
+
// the same ruling the parse failure above honours (Q11, ADR-070 ruling 4).
|
|
26
|
+
jsonIndentDefault(flags.pretty === true);
|
|
27
|
+
jsonFailMode(flags.json === true);
|
|
28
|
+
|
|
29
|
+
// No arguments at all is a usage error, not a silent help dump: an agent that
|
|
30
|
+
// ran the wrong thing should be told so rather than handed 40 lines of prose.
|
|
31
|
+
if (positional.length === 0 && !flags.help && !flags.version) {
|
|
32
|
+
process.exit(fail('no command given', EXIT.USAGE));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const { dispatch } = await import('./commands/index.mjs');
|
|
36
|
+
const code = (await dispatch(argv)) ?? EXIT.OK;
|
|
37
|
+
// Assigned, not forced: a large document on stdout (a `--json --diff` over a
|
|
38
|
+
// skill-sized body exceeds the 64 KiB pipe buffer) is still draining when the
|
|
39
|
+
// command resolves, and `process.exit()` cuts it — exit 0 with half a JSON
|
|
40
|
+
// document on stdout. Letting the loop empty first makes the exit honest
|
|
41
|
+
// (plan 0003 R50; the S10 contract asserts the document parses).
|
|
42
|
+
process.exitCode = code;
|
|
43
|
+
} catch (error) {
|
|
44
|
+
process.exit(fail(error?.message ?? String(error), EXIT.FAIL));
|
|
45
|
+
}
|