simt-dev 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,229 @@
1
+ // Model and effort advice, read from the repository's own bench history (plan 0003 R40).
2
+ //
3
+ // The mapping the design row asks for — task features to the cheapest tier that resolved
4
+ // similar tasks — is data, not opinion: the records `bench run` stored on disk already say
5
+ // which model classes resolved which mined tasks and what those runs cost. So the tier
6
+ // ladder is the estate's own evidence sorted by price, and no vendor name is written into
7
+ // the code: a repository whose history has one class of run gets advice about that class,
8
+ // and one whose history has three gets a ladder with three steps.
9
+ //
10
+ // The properties this exists to keep:
11
+ //
12
+ // * Advice, never a choice. The output is a sentence in the brief header along with the
13
+ // bench numbers behind it; the agent (or the human) names the model, not simt-dev
14
+ // (P2). Nothing here writes a model id into a config, a lock or a command.
15
+ // * Deterministic. No model, no network, no clock (P1): the same runs directory yields
16
+ // the same advice, so a reviewable claim about cost is possible.
17
+ // * Quiet in thin evidence. A neighborhood with no resolved run yields a caution, not a
18
+ // made-up tier; a history with no runs at all yields no advice line.
19
+ //
20
+ // The measure of "cheapest" follows the bench's own report: cost in dollars when the run
21
+ // records or a prices table price the tokens, total tokens otherwise. Ordering by a number
22
+ // that does not exist for any run would be nondeterminism, not economics.
23
+ import { classifyTask } from '../brief/kind.mjs';
24
+ import { computeRunCost, isRunResolved, loadPrices, loadRunRecords } from '../bench/report.mjs';
25
+ import { loadTasks } from '../bench/tasks.mjs';
26
+ import { CONFIG_DIR } from '../name.mjs';
27
+
28
+ /**
29
+ * Every advice report carries its row's accept bar (ADR-076: a metric is a measurement or
30
+ * it is not claimed).
31
+ */
32
+ export const METRIC_NOTE =
33
+ 'advice is advisory until a held-out bench shows >= 25% cost cut at delta R >= -2 (plan 0003 R40)';
34
+
35
+ const TIERS = ['small', 'default', 'frontier'];
36
+
37
+ /**
38
+ * The ladder can specialize beyond kind only while there is still evidence to stand on:
39
+ * below this many same-kind tasks the kind is the whole neighborhood, and a narrower
40
+ * window makes evidence worse, not "more similar".
41
+ */
42
+ const NARROW_MIN_POOL = 6;
43
+
44
+ /** The narrowed window needs at least these resolved runs or it is not evidence either. */
45
+ const NARROW_MIN_RESOLVED = 3;
46
+
47
+ /** A file window is a step from the estimate in either direction — a factor of two in size. */
48
+ const WINDOW_HALF = 2;
49
+
50
+ function median(values) {
51
+ const sorted = [...values].sort((a, b) => a - b);
52
+ if (sorted.length === 0) return 0;
53
+ const mid = Math.floor(sorted.length / 2);
54
+ return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
55
+ }
56
+
57
+ /**
58
+ * Tier for a rank in the cheapest-first ladder: thirds, not absolute cost — an estate's
59
+ * "frontier" relative to its own history means more than a number the code guessed.
60
+ * The cheapest of two observed classes is 'small' and the dearer one 'default'; a class
61
+ * is 'frontier' only once a ladder has three steps, because a two-step ladder cannot
62
+ * name a top third honestly.
63
+ */
64
+ export function tierOfRank(rank, count) {
65
+ if (count < 3) return rank === 0 ? TIERS[0] : TIERS[1];
66
+ if (rank * 3 < count) return TIERS[0];
67
+ if (rank * 3 >= count * 2) return TIERS[2];
68
+ return TIERS[1];
69
+ }
70
+
71
+ /**
72
+ * Total tokens a usage report carries — the measure that exists even with no prices file.
73
+ */
74
+ function tokensOf(usage = {}) {
75
+ const num = (v) => Number(v ?? 0);
76
+ return num(usage.input) + num(usage.cacheWrite ?? usage.cache_write) + num(usage.cacheRead ?? usage.cache_read) + num(usage.output);
77
+ }
78
+
79
+ /**
80
+ * Load bench history as advice-ready run rows.
81
+ *
82
+ * A descriptor missing for a run leaves the row with `kind: null`; such a run can join no
83
+ * neighborhood, which is the honest answer — a task whose prompt was never mined is a
84
+ * task whose kind would be a guess.
85
+ */
86
+ export async function loadHistory(root) {
87
+ const [records, prices, tasks] = await Promise.all([
88
+ loadRunRecords(root),
89
+ // An unreadable prices file would throw out of a brief, whose callers only asked for
90
+ // a packet; the bench has its own report for surfacing a broken prices file loudly.
91
+ loadPrices(root).catch(() => null),
92
+ loadTasks(root),
93
+ ]);
94
+ const byId = new Map(tasks.map((task) => [task.id, task]));
95
+ const runs = records.map((record) => {
96
+ const descriptor = byId.get(record.task) ?? null;
97
+ return {
98
+ task: record.task,
99
+ model: record.model,
100
+ resolved: isRunResolved(record),
101
+ cost: computeRunCost(record, prices),
102
+ tokens: tokensOf(record.usage),
103
+ toolCalls: Number(record?.usage?.toolCalls ?? record?.usage?.tool_calls ?? 0),
104
+ files: Number(descriptor?.gold?.files ?? record?.diff?.files ?? 0),
105
+ kind: descriptor ? classifyTask(descriptor.prompt ?? '', { paths: [] }).kind : null,
106
+ };
107
+ });
108
+ return { runs };
109
+ }
110
+
111
+ /**
112
+ * Advise from bench history for one task's features.
113
+ *
114
+ * @param {{kind: string, files: number}} features the task kind, and the estimate of how
115
+ * many files it will touch (the brief's scope size; a history run carries the mined
116
+ * task's gold stat in the same role)
117
+ * @param {{runs: object[]}} history rows from loadHistory
118
+ * @returns null when the history holds no runs; otherwise `{ picked, caution }` where
119
+ * `picked` is `{ line, basis, metric }` when a tier was named (and `caution` null), and
120
+ * `caution` explains the missing neighborhood when no tier can honestly be named
121
+ */
122
+ export function advise(features, history) {
123
+ const runs = history?.runs ?? [];
124
+ if (runs.length === 0) return null;
125
+ const kind = features.kind;
126
+ const sameKind = runs.filter((run) => run.kind === kind);
127
+ const taskCount = new Set(sameKind.map((run) => run.task)).size;
128
+ if (sameKind.length === 0) {
129
+ return {
130
+ picked: null,
131
+ caution: `bench history holds no ${kind} task, so no cheap tier was advised`,
132
+ };
133
+ }
134
+
135
+ // The measure is chosen once, across all resolved history: a ladder built on dollars
136
+ // while the pick ran on tokens would compare a number with a different number's name.
137
+ const measure = runs.some((run) => run.resolved && run.cost > 0) ? 'cost' : 'tokens';
138
+
139
+ // Kind decides the neighborhood. The file window only specializes a neighborhood that
140
+ // has already got evidence to spare (NARROW_MIN_*): a factor-of-two window around the
141
+ // estimate refines "bugfix" to "small bugfix" when the kind has runs to lend, and says
142
+ // nothing when it has not.
143
+ let pool = sameKind;
144
+ if (features.files > 0 && sameKind.length >= NARROW_MIN_POOL) {
145
+ const near = sameKind.filter(
146
+ (run) =>
147
+ run.files > 0 &&
148
+ run.files >= Math.ceil(features.files / WINDOW_HALF) &&
149
+ run.files <= features.files * WINDOW_HALF,
150
+ );
151
+ if (near.filter((run) => run.resolved).length >= NARROW_MIN_RESOLVED) pool = near;
152
+ }
153
+ const poolTaskCount = new Set(pool.map((run) => run.task)).size;
154
+
155
+ // The ladder orders every model class the estate ever ran, cheapest first, so the
156
+ // advice can say where its pick sits in the estate's whole history. Uncost evidence is
157
+ // still cost position: a class that never resolved one of these tasks occupies a rung
158
+ // in the price ladder, and the resolved floor below — not the ladder — decides whether
159
+ // the pick is allowed to stand on it.
160
+ const ladderValues = new Map();
161
+ for (const run of runs) {
162
+ const values = ladderValues.get(run.model) ?? [];
163
+ values.push(measure === 'cost' ? run.cost : run.tokens);
164
+ ladderValues.set(run.model, values);
165
+ }
166
+ const ladderNames = [...ladderValues.keys()].sort(
167
+ (a, b) => median(ladderValues.get(a)) - median(ladderValues.get(b)) || a.localeCompare(b),
168
+ );
169
+
170
+ const resolvedByModel = new Map();
171
+ for (const run of pool) {
172
+ if (!run.resolved) continue;
173
+ const values = resolvedByModel.get(run.model) ?? [];
174
+ values.push(measure === 'cost' ? run.cost : run.tokens);
175
+ resolvedByModel.set(run.model, values);
176
+ }
177
+
178
+ // "Resolved similar tasks" is a floor, not a style: a class that never resolved one of
179
+ // these tasks is cheaper because it failed for less money.
180
+ const qualifying = ladderNames.filter((model) => resolvedByModel.has(model));
181
+ if (qualifying.length === 0) {
182
+ return {
183
+ picked: null,
184
+ caution: `no model resolved any of the ${taskCount} ${kind} task${taskCount === 1 ? '' : 's'} in bench history, so no cheap tier was advised`,
185
+ };
186
+ }
187
+
188
+ // Ties break on the model name, so the same history always yields the same pick (P4).
189
+ const cheapest = [...qualifying].sort(
190
+ (a, b) => median(resolvedByModel.get(a)) - median(resolvedByModel.get(b)) || a.localeCompare(b),
191
+ )[0];
192
+ const ladderRank = ladderNames.indexOf(cheapest);
193
+ const tier = tierOfRank(ladderRank, ladderNames.length);
194
+
195
+ // Effort advice is relative to the estate too: heavier turns of tool use than the
196
+ // estate's own resolved median is what "plan first" means here.
197
+ const resolvedRuns = runs.filter((run) => run.resolved);
198
+ const estateToolCalls = median(resolvedRuns.map((run) => run.toolCalls));
199
+ const resolvedRunsOfCheapest = pool.filter((run) => run.resolved && run.model === cheapest);
200
+ const pickToolCalls = median(resolvedRunsOfCheapest.map((run) => run.toolCalls));
201
+ const effort = estateToolCalls > 0 && pickToolCalls > estateToolCalls ? 'plan first' : 'no plan mode';
202
+
203
+ const pickMedian = median(resolvedByModel.get(cheapest));
204
+ const pickUnit = measure === 'cost' ? `cost $${pickMedian.toFixed(4)}` : `tokens ${Math.round(pickMedian)}`;
205
+
206
+ return {
207
+ picked: {
208
+ line: `${tier} model, ${effort}`,
209
+ basis: `from ${poolTaskCount} ${kind} task${poolTaskCount === 1 ? '' : 's'} in bench history; cheapest that resolved: ${cheapest} (median ${pickUnit}); a human picks the model, this only ranks classes`,
210
+ metric: METRIC_NOTE,
211
+ },
212
+ caution: null,
213
+ };
214
+ }
215
+
216
+ /**
217
+ * The brief's door into advice (plan 0003 R40): history on disk in, advice out, or null
218
+ * when the history has no runs at all — a brief in a repository without bench history
219
+ * carries every other section and no made-up opinion about models.
220
+ *
221
+ * `kind` comes in already classified when the caller is the brief itself, so the packet
222
+ * never carries two opinions about one task.
223
+ */
224
+ export async function briefAdvice(root, { task = '', scope = [], kind = null } = {}) {
225
+ const history = await loadHistory(root);
226
+ if (history.runs.length === 0) return null;
227
+ const known = kind ?? classifyTask(task, { paths: scope }).kind;
228
+ return advise({ kind: known, files: scope.length }, history);
229
+ }
package/src/argv.mjs CHANGED
@@ -80,10 +80,22 @@ export const BOOL_FLAGS = new Set([
80
80
  // index / map / brief: opt into local vector embeddings configured under local.embed
81
81
  // (plan 0003 R28, R37). Off by default unless recall@10 gains at least 10 points.
82
82
  'embed',
83
+ // distill: regenerate a proposal that is already cached. Off by default because
84
+ // the cache is the point - the same source at the same budget is the same answer,
85
+ // and re-reading it is cheaper than re-deriving it (plan 0003 R38).
86
+ 'refresh',
87
+ // distill: propose the summary from the configured local chat model instead of
88
+ // the deterministic outline. Off by default: no model in the default path (P1),
89
+ // and a model that is not asked for should not be opened against (ADR-090).
90
+ 'local',
83
91
  ]);
84
92
 
85
93
  export const VALUE_FLAGS = new Set([
86
94
  'budget',
95
+ // bench critic: the labelled set of historical diffs to score (plan 0003 R39).
96
+ // A path, not a switch, because the labels themselves come from a human under
97
+ // ADR-008 — the tool never annotates for them.
98
+ 'labels',
87
99
  'max-bytes',
88
100
  'paths',
89
101
  'skill',
package/src/bench/run.mjs CHANGED
@@ -17,6 +17,7 @@ import { fingerprint } from '../resolve/fingerprint.mjs';
17
17
  import { rankSkills, tokenize } from '../resolve/resolve.mjs';
18
18
  import { gradeWorkspace } from './grade.mjs';
19
19
  import { resolveVaultDir } from './mine.mjs';
20
+ import { loadTasks } from './tasks.mjs';
20
21
  import { ablateSectionInWorkspace, parseAblationArmName } from './ablate.mjs';
21
22
  import { applyVariantInWorkspace, parseVariantArmName } from './variants.mjs';
22
23
 
@@ -678,21 +679,8 @@ async function executeAdapter(adapter, workDir, task, { vaultRoot, model, maxTur
678
679
  return { ...parseAdapterUsage(adapter, stdoutText, analyticsText), transcript: `${stdoutText}\n${analyticsText}` };
679
680
  }
680
681
 
681
- async function loadTasks(root, filterIds = []) {
682
- const tasksDir = join(root, STATE_DIR, 'bench', 'tasks');
683
- const entries = (await readdir(tasksDir).catch(() => []))
684
- .filter((f) => f.endsWith('.json'))
685
- .sort();
686
-
687
- const filterSet = new Set(filterIds);
688
- const tasks = [];
689
- for (const entry of entries) {
690
- const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
691
- if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
692
- tasks.push(data);
693
- }
694
- return tasks.sort((a, b) => a.id.localeCompare(b.id));
695
- }
682
+ // loadTasks lives in tasks.mjs (the advice engine became a second reader, so the loader
683
+ // was extracted rather than copied — ADR-002).
696
684
 
697
685
  function runIdFor(harness, armName, rep) {
698
686
  return `${harness}-${armName}-r${rep}`;
@@ -0,0 +1,38 @@
1
+ // Mined task descriptors, read back from the tool state's `bench/tasks/` directory (plan 0003 R01).
2
+ //
3
+ // Extracted from bench/run.mjs on its second real reader (ADR-002): the R40 advice engine
4
+ // needs the two facts a run record does not carry — the prompt, which classifies the task,
5
+ // and the gold stat, which says how many files the change touched. A second private loader
6
+ // beside the first would drift the first time one learned a filter the other lacked.
7
+ //
8
+ // Deterministic by construction: the directory is read sorted, the filter is exact, and no
9
+ // part of this touches a model, a network or a clock.
10
+ import { readdir, readFile } from 'node:fs/promises';
11
+ import { join, resolve } from 'node:path';
12
+ import { CONFIG_DIR } from '../name.mjs';
13
+
14
+ const STATE_DIR = `.${CONFIG_DIR}`;
15
+
16
+ /**
17
+ * Load mined benchmark task descriptors.
18
+ *
19
+ * @param {string} root the repository
20
+ * @param {string[]} [filterIds] exact task ids to keep; empty means load them all
21
+ * @returns descriptor objects sorted by id (id, base, sourceCommit, prompt, testCmd,
22
+ * f2p, p2p, gold {files, linesAdded, linesRemoved}, vault)
23
+ */
24
+ export async function loadTasks(root, filterIds = []) {
25
+ const tasksDir = join(resolve(root), STATE_DIR, 'bench', 'tasks');
26
+ const entries = (await readdir(tasksDir).catch(() => []))
27
+ .filter((f) => f.endsWith('.json'))
28
+ .sort();
29
+
30
+ const filterSet = new Set(filterIds);
31
+ const tasks = [];
32
+ for (const entry of entries) {
33
+ const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
34
+ if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
35
+ tasks.push(data);
36
+ }
37
+ return tasks.sort((a, b) => a.id.localeCompare(b.id));
38
+ }
@@ -37,6 +37,7 @@
37
37
  import { CLI } from '../name.mjs';
38
38
  import { DEFAULT_BRIEF_BYTES, buildPacket, collect } from './packet.mjs';
39
39
  import { classifyTask, cardVerbs } from './kind.mjs';
40
+ import { briefAdvice } from '../advice/advice.mjs';
40
41
  import { commandCard } from '../kernel/card.mjs';
41
42
  import { decisionHits, scopeHits, sourceRoots } from '../index/govern.mjs';
42
43
  import { embedTaskQuery, rankFiles, renderMap } from '../index/map.mjs';
@@ -192,6 +193,27 @@ export async function assembleBrief({
192
193
  });
193
194
  }
194
195
 
196
+ // Model and effort advice comes from the repository's bench history, not from anything a
197
+ // model produced, so it stays deterministic. The tier exists only if the history resolved
198
+ // a similar task, and the claim travels with its own numbers and its unmeasured accept
199
+ // bar; simt-dev never selects the model an agent runs (plan 0003 R40).
200
+ const advice = await briefAdvice(root, { task, scope, kind });
201
+ if (advice?.picked) {
202
+ packet.add('ADVICE', advice.picked.line, {
203
+ why: ['the tier is the estate’s own bench ladder, and a human still names the model'],
204
+ });
205
+ packet.add('ADVICE', advice.picked.basis, {
206
+ why: ['an advice claim names the neighborhood that backs it, so the reader can re-derive it'],
207
+ });
208
+ packet.add('ADVICE', advice.picked.metric, {
209
+ why: ['the row has an accept bar, and nothing in a packet may claim the bar was met'],
210
+ });
211
+ } else if (advice?.caution) {
212
+ packet.add('CAUTIONS', advice.caution, {
213
+ why: ['advice without a neighborhood is a guess dressed as history'],
214
+ });
215
+ }
216
+
195
217
  const likely = ranked.slice(0, LIKELY_FILES).map((row) => row.path);
196
218
  const known = testEdges({ files, imports: edges.filter((edge) => edge.kind === 'import') });
197
219
  const untested = [];
@@ -30,7 +30,7 @@ export const PROMPT_CAP_BYTES = 2048;
30
30
  * The order is fixed and identical in every packet. A reader who has to search for the
31
31
  * command is paying the cost the packet exists to remove.
32
32
  */
33
- const SECTIONS = ['TASK', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
33
+ const SECTIONS = ['TASK', 'ADVICE', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
34
34
 
35
35
  /**
36
36
  * What each section is worth when something has to go.
@@ -49,6 +49,9 @@ const SECTIONS = ['TASK', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION
49
49
  */
50
50
  const SECTION_VALUE = {
51
51
  CAUTIONS: 1,
52
+ // Bench evidence, but advice rather than fact: losing it costs an opinion, not a
53
+ // command or a path to check (plan 0003 R40).
54
+ ADVICE: 2,
52
55
  LESSONS: 2,
53
56
  SKILLS: 3,
54
57
  DECISIONS: 4,
@@ -16,6 +16,7 @@ import { formatRecall, measureRecall } from '../index/map.mjs';
16
16
  import { ablateBench } from '../bench/ablate.mjs';
17
17
  import { runBenchProbes } from '../bench/probes.mjs';
18
18
  import { selectRuleVariants } from '../bench/variants.mjs';
19
+ import { criticBench } from '../review/critic.mjs';
19
20
  import { listFlag } from './skills.mjs';
20
21
 
21
22
  export async function benchMineCommand(positional, flags) {
@@ -394,3 +395,34 @@ export async function benchVariantsCommand(positional, flags) {
394
395
 
395
396
 
396
397
 
398
+
399
+ // `simt bench critic` (plan 0003 R39).
400
+ //
401
+ // The row's acceptance measured, or reported missing rather than inferred: the
402
+ // critic runs over a labelled set of historical diffs and the bench reports
403
+ // precision and recall for the BLOCK binary. The metric never gates the critic —
404
+ // the row says precision >= 0.8 on >= 30 labelled diffs promotes it to a gate,
405
+ // and until a bench here shows that, every report calls the critic advisory.
406
+ export async function benchCriticCommand(positional, flags, commands = []) {
407
+ const root = resolve(flags.root ?? positional[0] ?? process.cwd());
408
+ const labelsPath = flags.labels ?? '.simt-dev/critic-labels.jsonl';
409
+ const result = await criticBench(root, { labelsPath, flags, commands });
410
+ if (typeof result === 'number') return result;
411
+
412
+ if (isJson(flags)) {
413
+ emitJson(result, { pretty: flags.pretty === true, maxBytes: flags.maxBytes ?? 0 });
414
+ return EXIT.OK;
415
+ }
416
+ const lines = [
417
+ `bench critic ${root}`,
418
+ `labels ${result.labels} (${labelsPath}) measured ${result.measured} errors ${result.errors.length}`,
419
+ `counts tp=${result.tp} fp=${result.fp} fn=${result.fn} tn=${result.tn}`,
420
+ `precision ${result.precision === null ? 'unmeasured' : result.precision.toFixed(3)}`,
421
+ `recall ${result.recall === null ? 'unmeasured' : result.recall.toFixed(3)}`,
422
+ `gated ${result.gated ? 'yes' : 'no'}`,
423
+ ];
424
+ if (result.labelsNote) lines.push(`labels ${result.labelsNote}`);
425
+ for (const error of result.errors) lines.push(`error ${error}`);
426
+ emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
427
+ return EXIT.OK;
428
+ }
@@ -0,0 +1,36 @@
1
+ // `simt distill <skill|doc>` (plan 0003 R38).
2
+ //
3
+ // A budgeted summary with citations to source line ranges, written to
4
+ // `.simt-dev/distilled/` as a proposal a human merges. The deterministic engine
5
+ // is the default; a local model proposes only when the caller asks for it and the
6
+ // manifest names one (ADR-090, ADR-075). Model output never lands anywhere
7
+ // projected or committed text does — that is the direction this command cannot
8
+ // take (principle P2).
9
+
10
+ import { CLI } from '../name.mjs';
11
+ import { EXIT, emit, emitJson, isJson } from '../output/index.mjs';
12
+ import { distillTarget } from '../distill/distill.mjs';
13
+
14
+ export async function distillCommand(args, flags) {
15
+ const result = await distillTarget(args, flags);
16
+ if (typeof result === 'number') return result;
17
+
18
+ if (isJson(flags)) {
19
+ emitJson(result, { maxBytes: flags.maxBytes ?? 0 });
20
+ return EXIT.OK;
21
+ }
22
+
23
+ emit(
24
+ [
25
+ `source ${result.target.kind} ${result.target.source} (${result.target.bytes} bytes, ${result.target.lines} lines)`,
26
+ `digest ${result.digest}`,
27
+ `engine ${result.engine}`,
28
+ `budget ${result.budget} bytes`,
29
+ `citations ${result.citations}`,
30
+ ...(result.verified === true ? [] : result.faults.map((f) => `unverified ${f}`)),
31
+ `proposal ${result.proposal.path} (${result.proposal.state}, ${result.proposal.bytes} bytes)`,
32
+ ].join('\n'),
33
+ { maxBytes: flags.maxBytes ?? 0 },
34
+ );
35
+ return EXIT.OK;
36
+ }
@@ -103,6 +103,21 @@ ${CLI} decide [<title>] scaffold an ADR with the next free number, a Stat
103
103
  ${CLI} local doctor probe configured local model endpoints (chat and embed)
104
104
  (reads local in ${MANIFEST_FILE}; opens no socket when
105
105
  unset; refuses non-loopback URLs without --allow-remote)
106
+ ${CLI} distill <skill|doc> a budgeted proposal summary of a skill body or doc,
107
+ every claim citing the source line range it rests on
108
+ (writes .simt-dev/distilled/<digest>.md; a proposal a
109
+ human merges - never projected or committed;
110
+ deterministic outline by default; --local proposes
111
+ from the configured chat model instead; the cache is
112
+ keyed by source digest and budget, --refresh regenerates)
113
+ ${CLI} review [--local] an ADR-008 verdict skeleton for the staged diff,
114
+ the brief listing everything that governs it (R25)
115
+ (deterministic without --local: the brief and its
116
+ governing ids only; --local proposes from the
117
+ configured chat model and writes
118
+ .simt-dev/reviews/<digest>.md — a proposal a human
119
+ reads, verified against the verdict vocabulary and
120
+ the brief's ids; advisory always, never a gate)
106
121
  ${CLI} shared plan what a workspace's repeated rules cost to carry
107
122
  once: the shared set, its bytes, and every path that would be written
108
123
  (read-only; touches nothing, not even your home directory)
@@ -141,6 +156,11 @@ ${CLI} bench probes [--yes] 12 synthetic rule-compliance probes with determin
141
156
  ${CLI} bench variants [--yes] evaluate simt-variants rule slots on the canary suite and lock
142
157
  winning variants in simt-dev.lock.json (dry-run unless --yes;
143
158
  re-opens when model or harness changes)
159
+ ${CLI} bench critic precision and recall of the local critic over a labelled
160
+ set of historical diffs (.simt-dev/critic-labels.jsonl;
161
+ overrides --labels; labels are human-written verdicts
162
+ under ADR-008; the critic is advisory until precision
163
+ reaches 0.8 on 30 labelled diffs)
144
164
  ${CLI} mcp the read-only surface as MCP tools over stdio
145
165
  (--list prints it without reading stdin; init, sync,
146
166
  upgrade and the installers stay CLI-only by design)`;
@@ -204,9 +224,17 @@ const FLAGS = `--json structured output (compact; one document,
204
224
  --state <name> plan done: target state (default: Complete)
205
225
  --level <name> map: detail level (outline, names, signatures)
206
226
  --supersedes <spec> decide: superseded ADR or ruling (ADR-NNN or ADR-NNN:<ruling>[:<kind>]; repeatable)
207
- --allow-remote local doctor: allow non-loopback model URLs (refused by default)
227
+ --allow-remote local doctor / distill / review: allow non-loopback model
228
+ URLs (refused by default)
208
229
  --embed index / map / brief: compute or query optional local vector embeddings
209
230
  (off by default unless recall@10 gains at least 10 points)
231
+ --labels <path> bench critic: the labelled set of historical diffs to score
232
+ (default .simt-dev/critic-labels.jsonl; one human-written
233
+ {sha, verdict} pair per line)
234
+ --refresh distill / review: regenerate a proposal that is already cached
235
+ --local distill / review: propose the answer from the configured
236
+ local chat model (local.chat); off by default, which
237
+ uses the deterministic path instead
210
238
  --help this text
211
239
  --version the version, and the SHA it was built from`;
212
240
 
@@ -46,6 +46,7 @@ import {
46
46
  benchAblateCommand,
47
47
  benchProbesCommand,
48
48
  benchVariantsCommand,
49
+ benchCriticCommand,
49
50
  } from './bench.mjs';
50
51
  import { planCommand } from './plan.mjs';
51
52
  import { sharedPlanCommand, sharedInstallCommand, sharedWarning } from './shared.mjs';
@@ -58,7 +59,9 @@ import { gateCommand } from './gate.mjs';
58
59
  import { impactCommand } from './impact.mjs';
59
60
  import { lessonsListCommand, lessonsReviewCommand } from './lessons.mjs';
60
61
  import { decideCommand } from './decide.mjs';
62
+ import { reviewCommand } from './review.mjs';
61
63
  import { localDoctorCommand } from './local.mjs';
64
+ import { distillCommand } from './distill.mjs';
62
65
  import { parse } from '../argv.mjs';
63
66
  import { CLI } from '../name.mjs';
64
67
  import { fingerprint } from '../resolve/fingerprint.mjs';
@@ -135,6 +138,10 @@ export const IMPLEMENTED = {
135
138
  'lessons list': lessonsListCommand,
136
139
  'lessons review': lessonsReviewCommand,
137
140
  decide: decideCommand,
141
+ distill: distillCommand,
142
+ // R25 machinery needs the runnable command names for Affects command tokens,
143
+ // same wiring `why` and `impact` get through their lambdas.
144
+ review: (args, flags) => reviewCommand(args, flags, COMMANDS),
138
145
  'local doctor': localDoctorCommand,
139
146
  'evidence check': (args, flags) => evidenceCommand(['check', ...args], flags),
140
147
  init: initCommand,
@@ -155,6 +162,11 @@ export const IMPLEMENTED = {
155
162
  'bench ablate': benchAblateCommand,
156
163
  'bench probes': benchProbesCommand,
157
164
  'bench variants': benchVariantsCommand,
165
+ // plan 0003 R39. The critic's acceptance bench measures BLOCK-precision and
166
+ // recall over a labelled set; until it reports its floor, the critic entries
167
+ // stay advisory and no gate reads them. COMMANDS rides along so the brief
168
+ // resolves Affects command tokens the way `simt why` does.
169
+ 'bench critic': (args, flags) => benchCriticCommand(args, flags, COMMANDS),
158
170
  upgrade: upgradeCommand,
159
171
  mcp: mcpCommand,
160
172
  };
@@ -0,0 +1,65 @@
1
+ // `simt review [--local]` (plan 0003 R39).
2
+ //
3
+ // The critic's report layer: the deterministic path prints the brief (staged diff
4
+ // plus what governs it); `--local` adds the model verdict, which always arrives as
5
+ // a proposal a human reads — nothing here is a gate, and the exit code never
6
+ // carries the verdict, because a critic that fails the run would be a gate by
7
+ // another name (ADR-090, principle P2: the reply's text is model output and never
8
+ // lands in projected or committed text).
9
+
10
+ import { CLI } from '../name.mjs';
11
+ import { EXIT, emit, emitJson, isJson } from '../output/index.mjs';
12
+ import { reviewTarget } from '../review/critic.mjs';
13
+
14
+ function governsLine(items) {
15
+ if (items.length === 0) return 'none';
16
+ const ids = items.slice(0, 8).map((item) => item.id);
17
+ const more = items.length > ids.length ? ` …+${items.length - ids.length}` : '';
18
+ return ids.join(', ') + more;
19
+ }
20
+
21
+ export async function reviewCommand(args, flags, commands = []) {
22
+ const result = await reviewTarget(args, flags, { commands });
23
+ if (typeof result === 'number') return result;
24
+
25
+ if (isJson(flags)) {
26
+ emitJson(result, { maxBytes: flags.maxBytes ?? 0 });
27
+ return EXIT.OK;
28
+ }
29
+
30
+ const lines = [
31
+ `staged ${result.staged.files.length} files, ${result.staged.bytes} bytes`,
32
+ `governs ${governsLine(result.governs)}`,
33
+ ];
34
+ // The branch keys on the path, not on the verdict: a local run whose reply
35
+ // lacked a verdict line still carries faults and a proposal, and hiding them
36
+ // behind the "needs --local" reading would erase the model's failure.
37
+ if (result.brief !== undefined) {
38
+ // The nothing-staged run is a plain answer, not an invitation to rerun with
39
+ // --local: there is no brief yet, so there is nothing for the critic to read.
40
+ if (result.staged.files.length === 0) {
41
+ lines.push('verdict none (nothing is staged)');
42
+ } else {
43
+ lines.push(
44
+ `verdict needs ${CLI} review --local (advisory; a critic is never a gate)`,
45
+ );
46
+ }
47
+ } else {
48
+ if (result.verdict !== null) {
49
+ lines.push(
50
+ `verdict ${result.verdict} (${result.verified ? 'verified' : 'unverified'})`,
51
+ `findings ${result.findings}`,
52
+ );
53
+ } else {
54
+ lines.push('verdict absent (unverified)');
55
+ }
56
+ if (!result.verified) {
57
+ for (const fault of result.faults) lines.push(`unverified ${fault}`);
58
+ }
59
+ lines.push(`proposal ${result.proposal.path} (${result.proposal.state}, ${result.proposal.bytes} bytes)`);
60
+ }
61
+ if (result.metric) lines.push(`metric ${result.metric.note}`);
62
+
63
+ emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
64
+ return EXIT.OK;
65
+ }