simt-dev 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +2 -2
- package/agents/simt-agent.md +84 -0
- package/docs/decisions/ADR-009-skill-layout.md +1 -1
- package/docs/decisions/ADR-086-the-stable-prefix-comes-first-and-the-lock-records-both-digests.md +1 -1
- package/docs/decisions/ADR-106-the-harness-hook-runs-through-a-repo-shim.md +46 -0
- package/docs/decisions/ADR-107-an-install-can-name-one-harness-and-the-skipped-ones-are-named.md +51 -0
- package/docs/decisions/ADR-108-an-adoption-builds-the-index-and-a-command-refreshes-it-on-read.md +59 -0
- package/docs/decisions/ADR-109-simt-dev-is-its-own-consumer-and-the-guards-that-kept-it-from-being-one-are-retired.md +47 -0
- package/docs/decisions/ADR-110-the-lock-file-follows-the-manifest-name-and-the-old-name-is-read-until-the-next-write.md +43 -0
- package/docs/decisions/ADR-111-the-tool-ships-one-agent-rendered-per-harness-and-installs-it-with-init-and-a-command.md +75 -0
- package/docs/decisions/ADR-112-the-registry-is-what-simt-dev-ships-and-a-repo-owns-its-own-skills.md +81 -0
- package/docs/decisions/ADR-113-a-pre-push-that-carries-only-documentation-skips-the-test-gate-and-says-so.md +87 -0
- package/docs/decisions/ADR-114-the-bench-pairs-at-the-task-level-a-missing-cost-is-not-zero-and-an-unasked-cross-group-compare-is-refused.md +74 -0
- package/docs/decisions/ADR-115-the-index-stamps-its-schema-version-migrates-older-databases-in-place-and-serves-newer-ones-from-a-sidecar.md +69 -0
- package/docs/decisions/ADR-116-rust-support-reaches-the-example-path-the-indent-and-the-bench-runner.md +100 -0
- package/docs/decisions/README.md +9 -0
- package/package.json +4 -2
- package/rules/core.md +2 -2
- package/rules/mdc/test-gate.mdc +2 -0
- package/rules/stacks/android.md +3 -3
- package/rules/stacks/connectiq.md +2 -2
- package/rules/stacks/dart.md +1 -1
- package/rules/stacks/deno.md +1 -1
- package/rules/stacks/dotnet.md +1 -1
- package/rules/stacks/elixir.md +1 -1
- package/rules/stacks/go.md +1 -1
- package/rules/stacks/jvm.md +3 -2
- package/rules/stacks/node.md +2 -2
- package/rules/stacks/python.md +7 -6
- package/rules/stacks/ruby.md +1 -1
- package/rules/stacks/rust.md +1 -1
- package/rules/stacks/swift.md +1 -1
- package/rules/stacks/typescript.md +2 -2
- package/rules/stacks/uv.md +3 -2
- package/skills/simt-commands/SKILL.md +70 -0
- package/src/advice/advice.mjs +7 -3
- package/src/agenthooks/decide.mjs +141 -9
- package/src/agenthooks/opencode.mjs +72 -0
- package/src/agenthooks/prompt.mjs +6 -2
- package/src/agenthooks/render.mjs +186 -19
- package/src/agenthooks/shim.mjs +55 -0
- package/src/agents/render.mjs +277 -0
- package/src/argv.mjs +8 -0
- package/src/bench/ablate.mjs +31 -14
- package/src/bench/mine.mjs +48 -1
- package/src/bench/probes.mjs +5 -6
- package/src/bench/report.mjs +243 -80
- package/src/bench/run.mjs +12 -11
- package/src/bench/variants.mjs +112 -40
- package/src/brief/assemble.mjs +6 -0
- package/src/ci/workflow.mjs +16 -7
- package/src/commands/agent.mjs +206 -0
- package/src/commands/agenthook.mjs +109 -15
- package/src/commands/bench.mjs +2 -1
- package/src/commands/compare.mjs +8 -7
- package/src/commands/decide.mjs +6 -5
- package/src/commands/evidence.mjs +13 -3
- package/src/commands/gate.mjs +85 -19
- package/src/commands/graph.mjs +18 -5
- package/src/commands/help.mjs +35 -14
- package/src/commands/hooks.mjs +68 -25
- package/src/commands/impact.mjs +4 -2
- package/src/commands/index.mjs +17 -4
- package/src/commands/init.mjs +169 -37
- package/src/commands/lease.mjs +203 -35
- package/src/commands/lessons.mjs +65 -9
- package/src/commands/map.mjs +33 -18
- package/src/commands/plugin.mjs +1 -1
- package/src/commands/repoindex.mjs +1 -0
- package/src/commands/rules.mjs +92 -18
- package/src/commands/skills.mjs +60 -25
- package/src/commands/skillscompile.mjs +8 -7
- package/src/commands/skillsinstall.mjs +12 -0
- package/src/commands/slice.mjs +377 -18
- package/src/commands/sync.mjs +37 -22
- package/src/commands/tests.mjs +5 -2
- package/src/commands/undo.mjs +24 -16
- package/src/commands/upgrade.mjs +33 -1
- package/src/commands/why.mjs +10 -4
- package/src/compare/hooksstate.mjs +23 -1
- package/src/compare/learn.mjs +82 -57
- package/src/compare/parallel.mjs +50 -0
- package/src/compare/row.mjs +26 -1
- package/src/completions.mjs +5 -0
- package/src/decisions/dir.mjs +84 -0
- package/src/decisions/index.mjs +109 -11
- package/src/distill/distill.mjs +3 -1
- package/src/ecosystem/node-runner.mjs +144 -0
- package/src/explain.mjs +18 -6
- package/src/graph/load.mjs +38 -19
- package/src/hooks/capability.mjs +14 -1
- package/src/hooks/dispatcher.mjs +335 -13
- package/src/hooks/resolve.mjs +9 -3
- package/src/hooks/shim.mjs +12 -7
- package/src/identity.mjs +67 -14
- package/src/index/extract.mjs +192 -11
- package/src/index/persist.mjs +307 -47
- package/src/index/store.mjs +263 -60
- package/src/index/tests.mjs +6 -35
- package/src/journal.mjs +296 -77
- package/src/kernel/card.mjs +37 -12
- package/src/kernel/checks.mjs +44 -9
- package/src/kernel/variants.mjs +27 -16
- package/src/lease.mjs +97 -15
- package/src/ledger/index.mjs +91 -25
- package/src/mcp/resources.mjs +5 -3
- package/src/mcp/server.mjs +81 -22
- package/src/mcp/tools.mjs +45 -9
- package/src/memory/decide.mjs +98 -43
- package/src/memory/lessons.mjs +168 -33
- package/src/name.mjs +9 -0
- package/src/plan/index.mjs +31 -8
- package/src/plan/template.mjs +27 -5
- package/src/plugins.mjs +6 -2
- package/src/prbody.mjs +7 -5
- package/src/projection/baselines.mjs +1 -1
- package/src/projection/disclosure.mjs +2 -2
- package/src/projection/loaders.mjs +41 -1
- package/src/projection/lock.mjs +21 -14
- package/src/projection/manifest.mjs +69 -19
- package/src/projection/markers.mjs +43 -3
- package/src/projection/mdc.mjs +34 -7
- package/src/projection/project.mjs +162 -18
- package/src/projection/rules.mjs +54 -3
- package/src/projection/surfaces.mjs +9 -1
- package/src/repo-root.mjs +69 -0
- package/src/resolve/body.mjs +4 -0
- package/src/resolve/fingerprint.mjs +5 -0
- package/src/resolve/local.mjs +58 -0
- package/src/review/critic.mjs +4 -2
- package/src/secrets.mjs +165 -22
- package/src/state.mjs +76 -0
- package/src/status.mjs +9 -4
- package/src/toolwords.mjs +5 -0
- package/src/tsconfig.mjs +39 -99
- package/templates/plan/tracker.json +1 -1
package/src/bench/run.mjs
CHANGED
|
@@ -10,6 +10,7 @@ import { extractOpenedSkills, matchSkillsForPaths } from '../graph/compile.mjs';
|
|
|
10
10
|
import { matchGlob } from '../index/govern.mjs';
|
|
11
11
|
import { formatLessonParagraph, listPromotedLessons } from '../memory/lessons.mjs';
|
|
12
12
|
import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
13
|
+
import { LOCAL_DIR } from '../state.mjs';
|
|
13
14
|
import { readManifest } from '../projection/manifest.mjs';
|
|
14
15
|
import { plan } from '../projection/project.mjs';
|
|
15
16
|
import { parse as parseRegion } from '../projection/markers.mjs';
|
|
@@ -119,7 +120,7 @@ function firstNumeric(obj, paths = []) {
|
|
|
119
120
|
function emptyUsage() {
|
|
120
121
|
return {
|
|
121
122
|
usage: { input: 0, cacheWrite: 0, cacheRead: 0, output: 0, turns: 0, toolCalls: 0 },
|
|
122
|
-
costUsd:
|
|
123
|
+
costUsd: null,
|
|
123
124
|
};
|
|
124
125
|
}
|
|
125
126
|
|
|
@@ -163,7 +164,7 @@ function parseJsonObject(text) {
|
|
|
163
164
|
* @param {string|object} adapterOrId
|
|
164
165
|
* @param {string} stdoutText
|
|
165
166
|
* @param {string} [analyticsText]
|
|
166
|
-
* @returns {{usage: {input: number, cacheWrite: number, cacheRead: number, output: number, turns: number, toolCalls: number}, costUsd: number}}
|
|
167
|
+
* @returns {{usage: {input: number, cacheWrite: number, cacheRead: number, output: number, turns: number, toolCalls: number}, costUsd: number|null}}
|
|
167
168
|
*/
|
|
168
169
|
export function parseAdapterUsage(adapterOrId, stdoutText = '', analyticsText = '') {
|
|
169
170
|
const result = emptyUsage();
|
|
@@ -192,7 +193,7 @@ export function parseAdapterUsage(adapterOrId, stdoutText = '', analyticsText =
|
|
|
192
193
|
result.usage.output = firstNumeric(obj, fields.output) ?? 0;
|
|
193
194
|
result.usage.turns = firstNumeric(obj, fields.turns) ?? 0;
|
|
194
195
|
result.usage.toolCalls = firstNumeric(obj, fields.toolCalls) ?? 0;
|
|
195
|
-
result.costUsd = firstNumeric(obj, fields.costUsd) ??
|
|
196
|
+
result.costUsd = firstNumeric(obj, fields.costUsd) ?? null;
|
|
196
197
|
|
|
197
198
|
// Gemini CLI also nests per-model token totals under `stats.models.<name>.tokens`.
|
|
198
199
|
const models = obj?.stats?.models;
|
|
@@ -265,7 +266,7 @@ export function parseAdapterUsage(adapterOrId, stdoutText = '', analyticsText =
|
|
|
265
266
|
}
|
|
266
267
|
}
|
|
267
268
|
|
|
268
|
-
result.costUsd = Number(result.costUsd.toFixed(6));
|
|
269
|
+
if (result.costUsd !== null) result.costUsd = Number(result.costUsd.toFixed(6));
|
|
269
270
|
return result;
|
|
270
271
|
}
|
|
271
272
|
|
|
@@ -513,7 +514,7 @@ async function writeTaskBrief(workDir, task) {
|
|
|
513
514
|
: []),
|
|
514
515
|
'',
|
|
515
516
|
];
|
|
516
|
-
const briefPath = join(workDir,
|
|
517
|
+
const briefPath = join(workDir, LOCAL_DIR, 'brief.md');
|
|
517
518
|
await mkdir(dirname(briefPath), { recursive: true });
|
|
518
519
|
await writeFile(briefPath, lines.join('\n'), 'utf8');
|
|
519
520
|
}
|
|
@@ -741,7 +742,7 @@ export async function runBench(
|
|
|
741
742
|
throw new Error(
|
|
742
743
|
filterTaskIds.length > 0
|
|
743
744
|
? `no mined tasks matched: ${filterTaskIds.join(', ')}`
|
|
744
|
-
: `no mined tasks found in
|
|
745
|
+
: `no mined tasks found in .simt/bench/tasks`,
|
|
745
746
|
);
|
|
746
747
|
}
|
|
747
748
|
|
|
@@ -781,14 +782,14 @@ export async function runBench(
|
|
|
781
782
|
resolved: null,
|
|
782
783
|
gatesPassed: null,
|
|
783
784
|
usage: emptyUsage().usage,
|
|
784
|
-
costUsd:
|
|
785
|
+
costUsd: null,
|
|
785
786
|
diff: { files: 0, outsideBlastRadius: 0 },
|
|
786
787
|
probe: null,
|
|
787
788
|
})),
|
|
788
789
|
};
|
|
789
790
|
}
|
|
790
791
|
|
|
791
|
-
const runsBaseDir = join(root,
|
|
792
|
+
const runsBaseDir = join(root, LOCAL_DIR, 'bench', 'runs');
|
|
792
793
|
for (const armSpec of armSpecs) {
|
|
793
794
|
for (let rep = 1; rep <= reps; rep += 1) {
|
|
794
795
|
const runDir = join(runsBaseDir, runIdFor(harness, armSpec.name, rep));
|
|
@@ -877,7 +878,7 @@ export async function runBench(
|
|
|
877
878
|
}
|
|
878
879
|
|
|
879
880
|
async function resolveRegradeTargets(root, target) {
|
|
880
|
-
const runsBaseDir = join(root,
|
|
881
|
+
const runsBaseDir = join(root, LOCAL_DIR, 'bench', 'runs');
|
|
881
882
|
const direct = resolve(root, target);
|
|
882
883
|
const candidate = existsSync(direct) ? direct : join(runsBaseDir, target);
|
|
883
884
|
|
|
@@ -933,8 +934,8 @@ export async function regradeRun(
|
|
|
933
934
|
throw new Error(`no stored task runs found under ${target}`);
|
|
934
935
|
}
|
|
935
936
|
|
|
936
|
-
const tasksDir = join(root,
|
|
937
|
-
const taskDefsDir = join(root,
|
|
937
|
+
const tasksDir = join(root, LOCAL_DIR, 'bench', 'runs');
|
|
938
|
+
const taskDefsDir = join(root, '.simt', 'bench', 'tasks');
|
|
938
939
|
const parentDir = await mkdtemp(join(tmpdir(), 'bench-regrade-'));
|
|
939
940
|
|
|
940
941
|
try {
|
package/src/bench/variants.mjs
CHANGED
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
// Discovers `simt-variants` slots across applicable rules, runs candidate
|
|
4
4
|
// variants on the 10-task canary suite (escalating when significant), records
|
|
5
5
|
// the winning variant with `{winner, variant, model, harness, dR, dC, n, p}`
|
|
6
|
-
// in `simt
|
|
6
|
+
// in `simt.lock.json`, and re-opens the experiment whenever the target
|
|
7
7
|
// model or harness changes.
|
|
8
8
|
|
|
9
9
|
import { mkdir, readdir, readFile, unlink, writeFile } from 'node:fs/promises';
|
|
10
10
|
import { dirname, join, resolve } from 'node:path';
|
|
11
11
|
|
|
12
12
|
import { extractVariantsFromRule, readLockedVariants } from '../kernel/variants.mjs';
|
|
13
|
-
import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
13
|
+
import { LOCK_FILE, MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
14
14
|
import { writeLockVariants } from '../projection/lock.mjs';
|
|
15
15
|
import { readManifest } from '../projection/manifest.mjs';
|
|
16
16
|
import { loadMdcRules } from '../projection/mdc.mjs';
|
|
@@ -21,12 +21,22 @@ import { CANARY_ALPHA, DEFAULT_CANARY_SIZE, DEL_R_TOLERANCE, shouldEscalateCanar
|
|
|
21
21
|
import {
|
|
22
22
|
DEFAULT_BOOTSTRAP_SAMPLES,
|
|
23
23
|
DEFAULT_BOOTSTRAP_SEED,
|
|
24
|
+
collapseArmByTask,
|
|
24
25
|
compareArms,
|
|
25
26
|
evaluateRunRecord,
|
|
27
|
+
holmAdjust,
|
|
26
28
|
loadPrices,
|
|
27
29
|
loadRunRecords,
|
|
28
30
|
} from './report.mjs';
|
|
29
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Harnesses that grade, not answer: `gold` applies the oracle patch and `noop` does
|
|
34
|
+
* nothing (`src/bench/adapters.json`, `kind: "builtin"`). Their runs measure the
|
|
35
|
+
* grader, not the rule text, so they must never close a variant experiment by
|
|
36
|
+
* writing a lock entry (Q45). A test asserts this set matches the adapter table.
|
|
37
|
+
*/
|
|
38
|
+
export const BUILTIN_HARNESSES = new Set(['gold', 'noop']);
|
|
39
|
+
|
|
30
40
|
/**
|
|
31
41
|
* Discover every `simt-variants` slot across applicable rules for `root`.
|
|
32
42
|
*/
|
|
@@ -131,6 +141,19 @@ export async function applyVariantInWorkspace(workDir, variant) {
|
|
|
131
141
|
delete nextManifest.mode;
|
|
132
142
|
await writeFile(manifestPath, `${JSON.stringify(nextManifest, null, 2)}\n`, 'utf8');
|
|
133
143
|
|
|
144
|
+
// plan() writes the lock as a target (Q29). Left behind, it records this arm's
|
|
145
|
+
// winner as locked, and the next selectRuleVariants on the same tree reads the
|
|
146
|
+
// experiment as closed and never runs the runner. Restore it, like the manifest.
|
|
147
|
+
const lockPath = join(workDir, LOCK_FILE);
|
|
148
|
+
let originalLockBytes = null;
|
|
149
|
+
let hadLock = false;
|
|
150
|
+
try {
|
|
151
|
+
originalLockBytes = await readFile(lockPath, 'utf8');
|
|
152
|
+
hadLock = true;
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (error.code !== 'ENOENT') throw error;
|
|
155
|
+
}
|
|
156
|
+
|
|
134
157
|
const planned = await plan(workDir, {
|
|
135
158
|
variants: { [slotId]: { winner: variantKey } },
|
|
136
159
|
});
|
|
@@ -146,30 +169,36 @@ export async function applyVariantInWorkspace(workDir, variant) {
|
|
|
146
169
|
} else if (originalManifestBytes !== null) {
|
|
147
170
|
await writeFile(manifestPath, originalManifestBytes, 'utf8');
|
|
148
171
|
}
|
|
172
|
+
if (!hadLock) {
|
|
173
|
+
await unlink(lockPath).catch(() => {});
|
|
174
|
+
} else {
|
|
175
|
+
await writeFile(lockPath, originalLockBytes, 'utf8');
|
|
176
|
+
}
|
|
149
177
|
}
|
|
150
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Mean per-task R and C for one arm's evaluated runs.
|
|
181
|
+
*
|
|
182
|
+
* `collapseArmByTask` is the shared per-task aggregation: the evaluated rows carry
|
|
183
|
+
* the cost in `C`, and the old body summed `costUsd` — a field the rows never had —
|
|
184
|
+
* so C was NaN and the cost rank degenerated to a key tie-break (Q45).
|
|
185
|
+
*/
|
|
151
186
|
function summarizeEvaluated(evaluated) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
155
|
-
const byTask = new Map();
|
|
156
|
-
for (const row of evaluated) {
|
|
157
|
-
const list = byTask.get(row.task) ?? [];
|
|
158
|
-
list.push(row);
|
|
159
|
-
byTask.set(row.task, list);
|
|
160
|
-
}
|
|
187
|
+
const tasks = collapseArmByTask(evaluated);
|
|
188
|
+
if (tasks.length === 0) return { n: 0, R: 0, C: null };
|
|
161
189
|
let sumR = 0;
|
|
162
190
|
let sumC = 0;
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
191
|
+
let costComplete = true;
|
|
192
|
+
for (const t of tasks) {
|
|
193
|
+
sumR += t.R;
|
|
194
|
+
if (t.C === null) costComplete = false;
|
|
195
|
+
else sumC += t.C;
|
|
166
196
|
}
|
|
167
|
-
|
|
168
|
-
return { n, R: sumR / n, C: sumC / n };
|
|
197
|
+
return { n: tasks.length, R: sumR / tasks.length, C: costComplete ? sumC / tasks.length : null };
|
|
169
198
|
}
|
|
170
199
|
|
|
171
200
|
/**
|
|
172
|
-
* Run or evaluate variant selection on the canary suite and record winners in `simt
|
|
201
|
+
* Run or evaluate variant selection on the canary suite and record winners in `simt.lock.json`.
|
|
173
202
|
*/
|
|
174
203
|
export async function selectRuleVariants(
|
|
175
204
|
root,
|
|
@@ -267,19 +296,31 @@ export async function selectRuleVariants(
|
|
|
267
296
|
for (const { slot } of openStates) {
|
|
268
297
|
const baseArm = variantArmName(slot.slotId, slot.baseline);
|
|
269
298
|
const baseRecs = canaryEvaluated.filter((r) => r.arm === baseArm);
|
|
299
|
+
const slotComps = [];
|
|
270
300
|
for (const key of Object.keys(slot.variants)) {
|
|
271
301
|
if (key === slot.baseline) continue;
|
|
272
302
|
const candArm = variantArmName(slot.slotId, key);
|
|
273
303
|
const candRecs = canaryEvaluated.filter((r) => r.arm === candArm);
|
|
274
304
|
if (baseRecs.length > 0 && candRecs.length > 0) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
escalatedArms.add(candArm);
|
|
280
|
-
}
|
|
305
|
+
slotComps.push({
|
|
306
|
+
candArm,
|
|
307
|
+
comp: compareArms(baseArm, baseRecs, candArm, candRecs, { samples, seed }),
|
|
308
|
+
});
|
|
281
309
|
}
|
|
282
310
|
}
|
|
311
|
+
// Family = every variant of this slot versus its baseline: testing several
|
|
312
|
+
// variants at α 0.05 each inflates the false-positive rate, so the p-values
|
|
313
|
+
// are Holm-adjusted before the escalation decision (review 0003 Q46).
|
|
314
|
+
const adjusted = holmAdjust(
|
|
315
|
+
slotComps.map(({ comp }) => Math.min(comp.rTest.pValue, comp.wilcoxon.pValue)),
|
|
316
|
+
);
|
|
317
|
+
slotComps.forEach(({ candArm, comp }, i) => {
|
|
318
|
+
if (shouldEscalateCanary(comp, { alpha: CANARY_ALPHA, pValue: adjusted[i] })) {
|
|
319
|
+
includeBaseline.add(baseArm);
|
|
320
|
+
armsToEscalate.push(candArm);
|
|
321
|
+
escalatedArms.add(candArm);
|
|
322
|
+
}
|
|
323
|
+
});
|
|
283
324
|
}
|
|
284
325
|
const fullEscalationArms = [...includeBaseline, ...armsToEscalate];
|
|
285
326
|
if (fullEscalationArms.length > 0) {
|
|
@@ -321,6 +362,7 @@ export async function selectRuleVariants(
|
|
|
321
362
|
reason,
|
|
322
363
|
measured: true,
|
|
323
364
|
winner: winnerKey,
|
|
365
|
+
locked: true,
|
|
324
366
|
model: lockedEntry.model ?? resolvedModel,
|
|
325
367
|
harness: lockedEntry.harness ?? harness,
|
|
326
368
|
dR: Number(lockedEntry.dR ?? 0),
|
|
@@ -367,6 +409,7 @@ export async function selectRuleVariants(
|
|
|
367
409
|
reason,
|
|
368
410
|
measured: false,
|
|
369
411
|
winner: slot.baseline,
|
|
412
|
+
locked: false,
|
|
370
413
|
model: resolvedModel,
|
|
371
414
|
harness,
|
|
372
415
|
dR: 0,
|
|
@@ -377,13 +420,16 @@ export async function selectRuleVariants(
|
|
|
377
420
|
continue;
|
|
378
421
|
}
|
|
379
422
|
|
|
380
|
-
// Rank variants: higher R (beyond tolerance), then lower C
|
|
423
|
+
// Rank variants: higher R (beyond tolerance), then lower C (an unknown cost is
|
|
424
|
+
// worst — it cannot be ranked cheaper), then fewer bytes, then key.
|
|
381
425
|
const ranked = [...candidates].sort((a, b) => {
|
|
382
426
|
if (Math.abs(b.summary.R - a.summary.R) > DEL_R_TOLERANCE) {
|
|
383
427
|
return b.summary.R - a.summary.R;
|
|
384
428
|
}
|
|
385
|
-
|
|
386
|
-
|
|
429
|
+
const ca = a.summary.C === null ? Number.POSITIVE_INFINITY : a.summary.C;
|
|
430
|
+
const cb = b.summary.C === null ? Number.POSITIVE_INFINITY : b.summary.C;
|
|
431
|
+
if (Math.abs(ca - cb) > 1e-9) {
|
|
432
|
+
return ca - cb;
|
|
387
433
|
}
|
|
388
434
|
if (a.bytes !== b.bytes) return a.bytes - b.bytes;
|
|
389
435
|
return a.key.localeCompare(b.key);
|
|
@@ -397,10 +443,13 @@ export async function selectRuleVariants(
|
|
|
397
443
|
|
|
398
444
|
if (best.key !== slot.baseline && best.comparison) {
|
|
399
445
|
dR = Number(best.comparison.deltas.R.toFixed(4));
|
|
400
|
-
dC =
|
|
446
|
+
dC =
|
|
447
|
+
best.comparison.deltas.C === null
|
|
448
|
+
? 0
|
|
449
|
+
: Number(best.comparison.deltas.C.toFixed(4));
|
|
401
450
|
n = best.comparison.pairedTasks;
|
|
402
451
|
p = Number(
|
|
403
|
-
Math.min(best.comparison.
|
|
452
|
+
Math.min(best.comparison.rTest.pValue, best.comparison.wilcoxon.pValue).toFixed(4),
|
|
404
453
|
);
|
|
405
454
|
} else {
|
|
406
455
|
const runnerUp = ranked.find((c) => c.key !== slot.baseline && c.comparison);
|
|
@@ -409,17 +458,33 @@ export async function selectRuleVariants(
|
|
|
409
458
|
}
|
|
410
459
|
}
|
|
411
460
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
p
|
|
421
|
-
|
|
422
|
-
|
|
461
|
+
// A non-baseline winner locks only when it is significant AND non-inferior
|
|
462
|
+
// (Q45): p below the canary alpha, and dR within the deletion tolerance.
|
|
463
|
+
// Anything else locks the baseline: the experiment ran, and its conclusion is
|
|
464
|
+
// "keep what we had" - recorded, so the next run on this model and harness
|
|
465
|
+
// reads it as closed instead of re-measuring.
|
|
466
|
+
const significant =
|
|
467
|
+
best.key !== slot.baseline &&
|
|
468
|
+
best.comparison !== null &&
|
|
469
|
+
p < CANARY_ALPHA &&
|
|
470
|
+
dR >= -DEL_R_TOLERANCE;
|
|
471
|
+
const winnerKey = significant ? best.key : slot.baseline;
|
|
472
|
+
|
|
473
|
+
// Built-in harnesses grade, they do not answer: their runs must not close the
|
|
474
|
+
// experiment, so no lock entry is written for them and the next run re-measures
|
|
475
|
+
// (Q45). The result still reports the measured best, flagged `locked: false`.
|
|
476
|
+
if (!BUILTIN_HARNESSES.has(harness)) {
|
|
477
|
+
nextVariantsLock[slot.slotId] = {
|
|
478
|
+
winner: winnerKey,
|
|
479
|
+
variant: winnerKey,
|
|
480
|
+
model: resolvedModel,
|
|
481
|
+
harness,
|
|
482
|
+
dR,
|
|
483
|
+
dC,
|
|
484
|
+
n,
|
|
485
|
+
p,
|
|
486
|
+
};
|
|
487
|
+
}
|
|
423
488
|
|
|
424
489
|
results.push({
|
|
425
490
|
slotId: slot.slotId,
|
|
@@ -430,7 +495,14 @@ export async function selectRuleVariants(
|
|
|
430
495
|
reopened,
|
|
431
496
|
reason,
|
|
432
497
|
measured: true,
|
|
433
|
-
|
|
498
|
+
winner: BUILTIN_HARNESSES.has(harness) ? best.key : winnerKey,
|
|
499
|
+
locked: significant && !BUILTIN_HARNESSES.has(harness),
|
|
500
|
+
model: resolvedModel,
|
|
501
|
+
harness,
|
|
502
|
+
dR,
|
|
503
|
+
dC,
|
|
504
|
+
n,
|
|
505
|
+
p,
|
|
434
506
|
});
|
|
435
507
|
}
|
|
436
508
|
|
package/src/brief/assemble.mjs
CHANGED
|
@@ -132,6 +132,7 @@ export async function assembleBrief({
|
|
|
132
132
|
const graph = await loadGraph(root, { build });
|
|
133
133
|
if (graph.error) return { error: graph.error };
|
|
134
134
|
if (graph.cold) return { cold: true };
|
|
135
|
+
const graphStale = graph.stale === true;
|
|
135
136
|
const { files, symbols, edges, fts = [], embeddings = [] } = graph;
|
|
136
137
|
const filePaths = new Set(files.map((file) => file.path));
|
|
137
138
|
|
|
@@ -150,6 +151,11 @@ export async function assembleBrief({
|
|
|
150
151
|
why: ['an unasked question has no answer to be short of'],
|
|
151
152
|
});
|
|
152
153
|
}
|
|
154
|
+
if (graphStale) {
|
|
155
|
+
packet.add('CAUTIONS', 'the index predates the current tree, so files added or removed since the last build are missing or ghosted', {
|
|
156
|
+
why: ['this caller may not rebuild the index; run a command that can before quoting the file list'],
|
|
157
|
+
});
|
|
158
|
+
}
|
|
153
159
|
|
|
154
160
|
const repo = await fingerprint(root);
|
|
155
161
|
const card = await commandCard({ root, manifest: repo.manifest ?? null, stack: repo.stack, facts: repo.facts });
|
package/src/ci/workflow.mjs
CHANGED
|
@@ -52,6 +52,7 @@ import { renderLocator } from '../locate.mjs';
|
|
|
52
52
|
import { CLI } from '../name.mjs';
|
|
53
53
|
import { packageVersion } from '../package-version.mjs';
|
|
54
54
|
import { LOCAL_BIN, LOCAL_BIN_DIR, assetNameExpr, localInstall, platformLines, releaseBase } from '../distribute/release.mjs';
|
|
55
|
+
import { installCommands as nodeInstallCommands } from '../ecosystem/node-runner.mjs';
|
|
55
56
|
|
|
56
57
|
/** Directory whose presence means this repo runs GitHub Actions. */
|
|
57
58
|
export const WORKFLOWS_DIR = '.github/workflows';
|
|
@@ -253,7 +254,7 @@ function indent(text, spaces) {
|
|
|
253
254
|
* @param {{own?: boolean}} [options]
|
|
254
255
|
* @returns {string} YAML, LF-terminated
|
|
255
256
|
*/
|
|
256
|
-
export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
257
|
+
export function renderWorkflow({ own = false, npm = true, manager = null, version } = {}) {
|
|
257
258
|
if (own) {
|
|
258
259
|
const floor = nodeFloor();
|
|
259
260
|
return `${header(true)}${skeleton(
|
|
@@ -286,6 +287,12 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
286
287
|
}
|
|
287
288
|
|
|
288
289
|
const v = version ?? packageVersion();
|
|
290
|
+
// Q31: the install step names the repo's own manager. `manager` wins over the
|
|
291
|
+
// legacy `npm` boolean so old callers keep their bytes; a Node repo installs
|
|
292
|
+
// with its lockfile's frozen command, anything else takes the release binary.
|
|
293
|
+
// The resolver row also decides setup-node (bun ships its own runtime).
|
|
294
|
+
const nodeManager = manager ?? (npm ? 'npm' : null);
|
|
295
|
+
const nodeInstall = nodeManager ? nodeInstallCommands(nodeManager) : null;
|
|
289
296
|
// The install step is the whole of S35's consumer CI: a repository with npm pins
|
|
290
297
|
// simt-dev as a devDependency and `npm ci` restores it; one without (Gradle,
|
|
291
298
|
// Connect IQ) has no node_modules to pin in, so CI downloads the release binary
|
|
@@ -301,11 +308,11 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
301
308
|
// `${{ runner.* }}` expression: an expression substituted into `$( … )` is a
|
|
302
309
|
// command substitution trying to execute "Linux", which set -eu refuses - the
|
|
303
310
|
// install would fail on every run in the one repository kind this channel is for.
|
|
304
|
-
const install =
|
|
311
|
+
const install = nodeInstall
|
|
305
312
|
? [
|
|
306
313
|
' - name: install dependencies',
|
|
307
|
-
|
|
308
|
-
|
|
314
|
+
` # \`${nodeInstall.install}\` needs a lockfile; a repo without one is still gated, not skipped.`,
|
|
315
|
+
` run: ${nodeInstall.install} || ${nodeInstall.fallback}`,
|
|
309
316
|
]
|
|
310
317
|
: [
|
|
311
318
|
' - name: install the pinned gate binary',
|
|
@@ -329,8 +336,9 @@ mkdir -p ${LOCAL_BIN_DIR}
|
|
|
329
336
|
mv "$asset" ${LOCAL_BIN}
|
|
330
337
|
chmod +x ${LOCAL_BIN}`, 10),
|
|
331
338
|
];
|
|
332
|
-
const
|
|
333
|
-
|
|
339
|
+
const addDev = { npm: `npm i -D ${PACKAGE_NAME}`, pnpm: `pnpm add -D ${PACKAGE_NAME}`, yarn: `yarn add -D ${PACKAGE_NAME}`, bun: `bun add -d ${PACKAGE_NAME}` }[nodeManager] ?? null;
|
|
340
|
+
const remedy = nodeInstall && addDev
|
|
341
|
+
? `REMEDY run: ${addDev} (the gate runs from the pinned dependency)`
|
|
334
342
|
: `REMEDY run: ${localInstall(v)} (the gate runs from the release binary in .simt/bin)`;
|
|
335
343
|
const locator = renderLocator('check --root "$ROOT"', { remedy });
|
|
336
344
|
return `${header(false)}${skeleton(
|
|
@@ -338,7 +346,8 @@ chmod +x ${LOCAL_BIN}`, 10),
|
|
|
338
346
|
[
|
|
339
347
|
uses('checkout'),
|
|
340
348
|
// The binary branch has no setup-node: see the install step above.
|
|
341
|
-
|
|
349
|
+
// Q31: bun ships its own runtime, so only managers that need Node get it.
|
|
350
|
+
...(nodeInstall && nodeInstall.setupNode ? [setupNode(nodeFloor())] : []),
|
|
342
351
|
...install,
|
|
343
352
|
` - name: ${CLI} check`,
|
|
344
353
|
' run: |',
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
// simt agents install | simt agents status
|
|
2
|
+
//
|
|
3
|
+
// Two commands, one subject: the one agent this tool ships, written into each
|
|
4
|
+
// harness's own agents directory. `install` writes the file, and `status`
|
|
5
|
+
// reports what each harness carries and what still has to be installed.
|
|
6
|
+
//
|
|
7
|
+
// An agent is a file the tool owns whole — markdown with a frontmatter the
|
|
8
|
+
// harness documents, or a TOML file for Codex — not a config entry to merge.
|
|
9
|
+
// So the install is a byte comparison: created, unchanged, updated. There is no
|
|
10
|
+
// parse that can fail and no partial state, which is what makes this command
|
|
11
|
+
// shorter than `agent-hooks install`.
|
|
12
|
+
|
|
13
|
+
import { readFile } from 'node:fs/promises';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
|
|
16
|
+
import { commitJournal } from '../journal.mjs';
|
|
17
|
+
import { diffField, emitDiffs, readBefore } from '../diff.mjs';
|
|
18
|
+
import { EXIT, emit, emitJson, fail, isJson, note, writeMode } from '../output/index.mjs';
|
|
19
|
+
import { CLI } from '../name.mjs';
|
|
20
|
+
import {
|
|
21
|
+
adapter,
|
|
22
|
+
adapters,
|
|
23
|
+
agentPath,
|
|
24
|
+
agentRows,
|
|
25
|
+
loadAgent,
|
|
26
|
+
planAgents,
|
|
27
|
+
rendered,
|
|
28
|
+
writeAgentPlan,
|
|
29
|
+
} from '../agents/render.mjs';
|
|
30
|
+
|
|
31
|
+
export async function agentsCommand(sub, args, flags) {
|
|
32
|
+
const root = flags.root ?? args[0] ?? process.cwd();
|
|
33
|
+
if (sub === 'install') return installAgents(root, flags);
|
|
34
|
+
if (sub === 'status') return agentsStatus(root, flags);
|
|
35
|
+
return fail(`unknown agents subcommand ${sub}`, EXIT.USAGE, `run: ${CLI} agents status`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// The label for a reader the run did not write, by the state that names why:
|
|
39
|
+
// `none` documents no agents file, `other` is a reader this tool does not write,
|
|
40
|
+
// `skipped` was an adapter the caller did not ask for (ADR-107).
|
|
41
|
+
const OTHER_LABEL = { none: 'NO FILE ', other: 'NOT OURS', skipped: 'SKIPPED ' };
|
|
42
|
+
|
|
43
|
+
async function installAgents(root, flags) {
|
|
44
|
+
const mode = writeMode(flags);
|
|
45
|
+
// `--harness` narrows the install to one reader (ADR-107). The id is validated
|
|
46
|
+
// against the same table the loop reads, so a typo fails before anything is
|
|
47
|
+
// planned; the remedy names the ids the way `agent-hook` does.
|
|
48
|
+
const selected = flags.harness;
|
|
49
|
+
if (selected !== undefined && adapter(selected) === null) {
|
|
50
|
+
return fail(
|
|
51
|
+
`unknown harness \`${selected}\``,
|
|
52
|
+
EXIT.USAGE,
|
|
53
|
+
`--harness <id>, one of: ${adapters().map((a) => a.id).join(', ')}`,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const plan = await planAgents(root, { harness: selected });
|
|
58
|
+
|
|
59
|
+
// The readers this tool does not write a file for, reported rather than skipped in
|
|
60
|
+
// silence: a command that writes six files and says nothing about the two harnesses
|
|
61
|
+
// that document no agents file reads as "those are covered too". A narrowed install
|
|
62
|
+
// adds a third state: an adapter that is this tool's but was not requested (ADR-107).
|
|
63
|
+
const others = [];
|
|
64
|
+
for (const r of agentRows()) {
|
|
65
|
+
if (r.path === null) others.push({ id: r.id, path: null, state: 'none', note: r.note });
|
|
66
|
+
else if (adapter(r.id) === null) others.push({ id: r.id, path: r.path, state: 'other', note: r.note });
|
|
67
|
+
else if (selected !== undefined && r.id !== selected) {
|
|
68
|
+
others.push({ id: r.id, path: r.path, state: 'skipped', note: 'not requested; install without --harness to write every harness' });
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// The `--diff` pairs, read before anything is written so the before side is
|
|
73
|
+
// true in preview and in apply alike (R50).
|
|
74
|
+
const diffs =
|
|
75
|
+
flags.diff === true
|
|
76
|
+
? await Promise.all(
|
|
77
|
+
plan
|
|
78
|
+
.filter((p) => p.action !== 'unchanged')
|
|
79
|
+
.map(async (p) => ({ path: p.path, before: await readBefore(join(root, p.path)), after: p.text })),
|
|
80
|
+
)
|
|
81
|
+
: [];
|
|
82
|
+
|
|
83
|
+
if (!mode.apply) {
|
|
84
|
+
if (isJson(flags)) {
|
|
85
|
+
emitJson({
|
|
86
|
+
ok: true,
|
|
87
|
+
command: 'agents install',
|
|
88
|
+
root,
|
|
89
|
+
dryRun: true,
|
|
90
|
+
plan: plan.map(({ text, ...rest }) => rest),
|
|
91
|
+
notWritten: others,
|
|
92
|
+
filesWritten: false,
|
|
93
|
+
...diffField(diffs, flags),
|
|
94
|
+
});
|
|
95
|
+
return EXIT.OK;
|
|
96
|
+
}
|
|
97
|
+
emit(
|
|
98
|
+
[
|
|
99
|
+
`repo ${root}`,
|
|
100
|
+
...plan.map((p) => `${p.action.toUpperCase().padEnd(9)} ${p.id.padEnd(12)} ${p.path}`),
|
|
101
|
+
...others.map((o) => `${OTHER_LABEL[o.state]} ${o.id.padEnd(12)} ${o.path ?? '(no agents file is documented)'}`),
|
|
102
|
+
'',
|
|
103
|
+
`PLAN nothing written; apply with: ${CLI} agents install --yes`,
|
|
104
|
+
].join('\n'),
|
|
105
|
+
{ maxBytes: flags.maxBytes ?? 0 },
|
|
106
|
+
);
|
|
107
|
+
// After the report: the report says what, the diff shows it (R50).
|
|
108
|
+
emitDiffs(diffs, flags);
|
|
109
|
+
return EXIT.OK;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Journaled so `undo` restores this run (R49).
|
|
113
|
+
const journal = [];
|
|
114
|
+
const written = await writeAgentPlan(root, journal, plan);
|
|
115
|
+
await commitJournal(root, 'agents install', journal);
|
|
116
|
+
|
|
117
|
+
if (isJson(flags)) {
|
|
118
|
+
emitJson({
|
|
119
|
+
ok: true,
|
|
120
|
+
command: 'agents install',
|
|
121
|
+
root,
|
|
122
|
+
plan: plan.map(({ text, ...rest }) => rest),
|
|
123
|
+
notWritten: others,
|
|
124
|
+
written,
|
|
125
|
+
filesWritten: true,
|
|
126
|
+
...diffField(diffs, flags),
|
|
127
|
+
});
|
|
128
|
+
return EXIT.OK;
|
|
129
|
+
}
|
|
130
|
+
emit(
|
|
131
|
+
[
|
|
132
|
+
`repo ${root}`,
|
|
133
|
+
...plan.map((p) => `${p.action.toUpperCase().padEnd(9)} ${p.id.padEnd(12)} ${p.path}`),
|
|
134
|
+
...others.map((o) => `${OTHER_LABEL[o.state]} ${o.id.padEnd(12)} ${o.path ?? '(no agents file is documented)'}`),
|
|
135
|
+
'',
|
|
136
|
+
`WROTE ${written.length} agent file(s)`,
|
|
137
|
+
].join('\n'),
|
|
138
|
+
{ maxBytes: flags.maxBytes ?? 0 },
|
|
139
|
+
);
|
|
140
|
+
// After the report: the report says what, the diff shows it (R50).
|
|
141
|
+
emitDiffs(diffs, flags);
|
|
142
|
+
for (const o of others) note(`note ${o.id}: not written - ${o.note}`);
|
|
143
|
+
if (written.length === 0) note('note every documented agents file already carried this agent');
|
|
144
|
+
note(`note never committed or pushed (ADR-001); review, then commit yourself`);
|
|
145
|
+
return EXIT.OK;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function agentsStatus(root, flags) {
|
|
149
|
+
const agent = await loadAgent();
|
|
150
|
+
const rows = [];
|
|
151
|
+
const problems = [];
|
|
152
|
+
|
|
153
|
+
// The file this tool owns whole is compared against what `install` would write
|
|
154
|
+
// right now, so status reads the same bytes install reads: a file that matches
|
|
155
|
+
// the preview is present, anything else is drift an install will rewrite.
|
|
156
|
+
for (const row of adapters()) {
|
|
157
|
+
const path = agentPath(row.id);
|
|
158
|
+
const existing = await readIfExists(join(root, path));
|
|
159
|
+
if (existing === null) {
|
|
160
|
+
rows.push({ id: row.id, path, state: 'absent', note: null });
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
const present = rendered(row, agent, existing).action === 'unchanged';
|
|
164
|
+
rows.push({
|
|
165
|
+
id: row.id,
|
|
166
|
+
path,
|
|
167
|
+
state: present ? 'present' : 'drift',
|
|
168
|
+
note: present ? null : 'differs from what install would write; run install to restore it',
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// The readers whose agents are not a file this tool can write — a vendor page
|
|
173
|
+
// that documents none — are reported from the same table that documents them,
|
|
174
|
+
// rather than from a second list of the harnesses this command happens to know.
|
|
175
|
+
for (const r of agentRows()) {
|
|
176
|
+
if (r.path === null) rows.push({ id: r.id, path: null, state: 'none', note: r.note });
|
|
177
|
+
else if (adapter(r.id) === null) rows.push({ id: r.id, path: r.path, state: 'other', note: r.note });
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
if (isJson(flags)) {
|
|
181
|
+
emitJson({ ok: problems.length === 0, command: 'agents status', root, harnesses: rows, problems });
|
|
182
|
+
return problems.length === 0 ? EXIT.OK : EXIT.FAIL;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const lines = [`repo ${root}`];
|
|
186
|
+
for (const r of rows) {
|
|
187
|
+
const state = { present: 'PRESENT ', absent: 'ABSENT ', drift: 'DRIFT ', none: 'NO FILE ', other: 'NOT OURS' }[r.state];
|
|
188
|
+
lines.push(`${state} ${r.id.padEnd(12)} ${r.path ?? '(no agents file is documented)'}`);
|
|
189
|
+
if (r.state === 'absent') lines.push(` ${' '.repeat(12)} install: ${CLI} agents install --yes`);
|
|
190
|
+
if (r.note) lines.push(` ${' '.repeat(12)} ${r.note}`);
|
|
191
|
+
}
|
|
192
|
+
if (problems.length > 0) {
|
|
193
|
+
lines.push('', 'problems:');
|
|
194
|
+
for (const p of problems) lines.push(` ${p}`);
|
|
195
|
+
}
|
|
196
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
197
|
+
return problems.length === 0 ? EXIT.OK : EXIT.FAIL;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
async function readIfExists(path) {
|
|
201
|
+
try {
|
|
202
|
+
return await readFile(path, 'utf8');
|
|
203
|
+
} catch {
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
}
|