simt-dev 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +267 -68
- package/package.json +5 -1
- package/src/advice/advice.mjs +229 -0
- package/src/argv.mjs +12 -0
- package/src/bench/run.mjs +3 -15
- package/src/bench/tasks.mjs +38 -0
- package/src/brief/assemble.mjs +22 -0
- package/src/brief/packet.mjs +4 -1
- package/src/commands/bench.mjs +32 -0
- package/src/commands/distill.mjs +36 -0
- package/src/commands/help.mjs +29 -1
- package/src/commands/index.mjs +12 -0
- package/src/commands/review.mjs +65 -0
- package/src/commands/skills.mjs +2 -33
- package/src/commands/why.mjs +5 -2
- package/src/distill/distill.mjs +246 -0
- package/src/distill/engine.mjs +187 -0
- package/src/local/config.mjs +35 -0
- package/src/resolve/body.mjs +46 -0
- package/src/review/critic.mjs +509 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// Model and effort advice, read from the repository's own bench history (plan 0003 R40).
|
|
2
|
+
//
|
|
3
|
+
// The mapping the design row asks for — task features to the cheapest tier that resolved
|
|
4
|
+
// similar tasks — is data, not opinion: the records `bench run` stored on disk already say
|
|
5
|
+
// which model classes resolved which mined tasks and what those runs cost. So the tier
|
|
6
|
+
// ladder is the estate's own evidence sorted by price, and no vendor name is written into
|
|
7
|
+
// the code: a repository whose history has one class of run gets advice about that class,
|
|
8
|
+
// and one whose history has three gets a ladder with three steps.
|
|
9
|
+
//
|
|
10
|
+
// The properties this exists to keep:
|
|
11
|
+
//
|
|
12
|
+
// * Advice, never a choice. The output is a sentence in the brief header along with the
|
|
13
|
+
// bench numbers behind it; the agent (or the human) names the model, not simt-dev
|
|
14
|
+
// (P2). Nothing here writes a model id into a config, a lock or a command.
|
|
15
|
+
// * Deterministic. No model, no network, no clock (P1): the same runs directory yields
|
|
16
|
+
// the same advice, so a reviewable claim about cost is possible.
|
|
17
|
+
// * Quiet in thin evidence. A neighborhood with no resolved run yields a caution, not a
|
|
18
|
+
// made-up tier; a history with no runs at all yields no advice line.
|
|
19
|
+
//
|
|
20
|
+
// The measure of "cheapest" follows the bench's own report: cost in dollars when the run
|
|
21
|
+
// records or a prices table price the tokens, total tokens otherwise. Ordering by a number
|
|
22
|
+
// that does not exist for any run would be nondeterminism, not economics.
|
|
23
|
+
import { classifyTask } from '../brief/kind.mjs';
|
|
24
|
+
import { computeRunCost, isRunResolved, loadPrices, loadRunRecords } from '../bench/report.mjs';
|
|
25
|
+
import { loadTasks } from '../bench/tasks.mjs';
|
|
26
|
+
import { CONFIG_DIR } from '../name.mjs';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Every advice report carries its row's accept bar (ADR-076: a metric is a measurement or
|
|
30
|
+
* it is not claimed).
|
|
31
|
+
*/
|
|
32
|
+
export const METRIC_NOTE =
|
|
33
|
+
'advice is advisory until a held-out bench shows >= 25% cost cut at delta R >= -2 (plan 0003 R40)';
|
|
34
|
+
|
|
35
|
+
const TIERS = ['small', 'default', 'frontier'];
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The ladder can specialize beyond kind only while there is still evidence to stand on:
|
|
39
|
+
* below this many same-kind tasks the kind is the whole neighborhood, and a narrower
|
|
40
|
+
* window makes evidence worse, not "more similar".
|
|
41
|
+
*/
|
|
42
|
+
const NARROW_MIN_POOL = 6;
|
|
43
|
+
|
|
44
|
+
/** The narrowed window needs at least these resolved runs or it is not evidence either. */
|
|
45
|
+
const NARROW_MIN_RESOLVED = 3;
|
|
46
|
+
|
|
47
|
+
/** A file window is a step from the estimate in either direction — a factor of two in size. */
|
|
48
|
+
const WINDOW_HALF = 2;
|
|
49
|
+
|
|
50
|
+
function median(values) {
|
|
51
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
52
|
+
if (sorted.length === 0) return 0;
|
|
53
|
+
const mid = Math.floor(sorted.length / 2);
|
|
54
|
+
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Tier for a rank in the cheapest-first ladder: thirds, not absolute cost — an estate's
|
|
59
|
+
* "frontier" relative to its own history means more than a number the code guessed.
|
|
60
|
+
* The cheapest of two observed classes is 'small' and the dearer one 'default'; a class
|
|
61
|
+
* is 'frontier' only once a ladder has three steps, because a two-step ladder cannot
|
|
62
|
+
* name a top third honestly.
|
|
63
|
+
*/
|
|
64
|
+
export function tierOfRank(rank, count) {
|
|
65
|
+
if (count < 3) return rank === 0 ? TIERS[0] : TIERS[1];
|
|
66
|
+
if (rank * 3 < count) return TIERS[0];
|
|
67
|
+
if (rank * 3 >= count * 2) return TIERS[2];
|
|
68
|
+
return TIERS[1];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Total tokens a usage report carries — the measure that exists even with no prices file.
|
|
73
|
+
*/
|
|
74
|
+
function tokensOf(usage = {}) {
|
|
75
|
+
const num = (v) => Number(v ?? 0);
|
|
76
|
+
return num(usage.input) + num(usage.cacheWrite ?? usage.cache_write) + num(usage.cacheRead ?? usage.cache_read) + num(usage.output);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Load bench history as advice-ready run rows.
|
|
81
|
+
*
|
|
82
|
+
* A descriptor missing for a run leaves the row with `kind: null`; such a run can join no
|
|
83
|
+
* neighborhood, which is the honest answer — a task whose prompt was never mined is a
|
|
84
|
+
* task whose kind would be a guess.
|
|
85
|
+
*/
|
|
86
|
+
export async function loadHistory(root) {
|
|
87
|
+
const [records, prices, tasks] = await Promise.all([
|
|
88
|
+
loadRunRecords(root),
|
|
89
|
+
// An unreadable prices file would throw out of a brief, whose callers only asked for
|
|
90
|
+
// a packet; the bench has its own report for surfacing a broken prices file loudly.
|
|
91
|
+
loadPrices(root).catch(() => null),
|
|
92
|
+
loadTasks(root),
|
|
93
|
+
]);
|
|
94
|
+
const byId = new Map(tasks.map((task) => [task.id, task]));
|
|
95
|
+
const runs = records.map((record) => {
|
|
96
|
+
const descriptor = byId.get(record.task) ?? null;
|
|
97
|
+
return {
|
|
98
|
+
task: record.task,
|
|
99
|
+
model: record.model,
|
|
100
|
+
resolved: isRunResolved(record),
|
|
101
|
+
cost: computeRunCost(record, prices),
|
|
102
|
+
tokens: tokensOf(record.usage),
|
|
103
|
+
toolCalls: Number(record?.usage?.toolCalls ?? record?.usage?.tool_calls ?? 0),
|
|
104
|
+
files: Number(descriptor?.gold?.files ?? record?.diff?.files ?? 0),
|
|
105
|
+
kind: descriptor ? classifyTask(descriptor.prompt ?? '', { paths: [] }).kind : null,
|
|
106
|
+
};
|
|
107
|
+
});
|
|
108
|
+
return { runs };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Advise from bench history for one task's features.
|
|
113
|
+
*
|
|
114
|
+
* @param {{kind: string, files: number}} features the task kind, and the estimate of how
|
|
115
|
+
* many files it will touch (the brief's scope size; a history run carries the mined
|
|
116
|
+
* task's gold stat in the same role)
|
|
117
|
+
* @param {{runs: object[]}} history rows from loadHistory
|
|
118
|
+
* @returns null when the history holds no runs; otherwise `{ picked, caution }` where
|
|
119
|
+
* `picked` is `{ line, basis, metric }` when a tier was named (and `caution` null), and
|
|
120
|
+
* `caution` explains the missing neighborhood when no tier can honestly be named
|
|
121
|
+
*/
|
|
122
|
+
export function advise(features, history) {
|
|
123
|
+
const runs = history?.runs ?? [];
|
|
124
|
+
if (runs.length === 0) return null;
|
|
125
|
+
const kind = features.kind;
|
|
126
|
+
const sameKind = runs.filter((run) => run.kind === kind);
|
|
127
|
+
const taskCount = new Set(sameKind.map((run) => run.task)).size;
|
|
128
|
+
if (sameKind.length === 0) {
|
|
129
|
+
return {
|
|
130
|
+
picked: null,
|
|
131
|
+
caution: `bench history holds no ${kind} task, so no cheap tier was advised`,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// The measure is chosen once, across all resolved history: a ladder built on dollars
|
|
136
|
+
// while the pick ran on tokens would compare a number with a different number's name.
|
|
137
|
+
const measure = runs.some((run) => run.resolved && run.cost > 0) ? 'cost' : 'tokens';
|
|
138
|
+
|
|
139
|
+
// Kind decides the neighborhood. The file window only specializes a neighborhood that
|
|
140
|
+
// has already got evidence to spare (NARROW_MIN_*): a factor-of-two window around the
|
|
141
|
+
// estimate refines "bugfix" to "small bugfix" when the kind has runs to lend, and says
|
|
142
|
+
// nothing when it has not.
|
|
143
|
+
let pool = sameKind;
|
|
144
|
+
if (features.files > 0 && sameKind.length >= NARROW_MIN_POOL) {
|
|
145
|
+
const near = sameKind.filter(
|
|
146
|
+
(run) =>
|
|
147
|
+
run.files > 0 &&
|
|
148
|
+
run.files >= Math.ceil(features.files / WINDOW_HALF) &&
|
|
149
|
+
run.files <= features.files * WINDOW_HALF,
|
|
150
|
+
);
|
|
151
|
+
if (near.filter((run) => run.resolved).length >= NARROW_MIN_RESOLVED) pool = near;
|
|
152
|
+
}
|
|
153
|
+
const poolTaskCount = new Set(pool.map((run) => run.task)).size;
|
|
154
|
+
|
|
155
|
+
// The ladder orders every model class the estate ever ran, cheapest first, so the
|
|
156
|
+
// advice can say where its pick sits in the estate's whole history. Uncost evidence is
|
|
157
|
+
// still cost position: a class that never resolved one of these tasks occupies a rung
|
|
158
|
+
// in the price ladder, and the resolved floor below — not the ladder — decides whether
|
|
159
|
+
// the pick is allowed to stand on it.
|
|
160
|
+
const ladderValues = new Map();
|
|
161
|
+
for (const run of runs) {
|
|
162
|
+
const values = ladderValues.get(run.model) ?? [];
|
|
163
|
+
values.push(measure === 'cost' ? run.cost : run.tokens);
|
|
164
|
+
ladderValues.set(run.model, values);
|
|
165
|
+
}
|
|
166
|
+
const ladderNames = [...ladderValues.keys()].sort(
|
|
167
|
+
(a, b) => median(ladderValues.get(a)) - median(ladderValues.get(b)) || a.localeCompare(b),
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
const resolvedByModel = new Map();
|
|
171
|
+
for (const run of pool) {
|
|
172
|
+
if (!run.resolved) continue;
|
|
173
|
+
const values = resolvedByModel.get(run.model) ?? [];
|
|
174
|
+
values.push(measure === 'cost' ? run.cost : run.tokens);
|
|
175
|
+
resolvedByModel.set(run.model, values);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// "Resolved similar tasks" is a floor, not a style: a class that never resolved one of
|
|
179
|
+
// these tasks is cheaper because it failed for less money.
|
|
180
|
+
const qualifying = ladderNames.filter((model) => resolvedByModel.has(model));
|
|
181
|
+
if (qualifying.length === 0) {
|
|
182
|
+
return {
|
|
183
|
+
picked: null,
|
|
184
|
+
caution: `no model resolved any of the ${taskCount} ${kind} task${taskCount === 1 ? '' : 's'} in bench history, so no cheap tier was advised`,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Ties break on the model name, so the same history always yields the same pick (P4).
|
|
189
|
+
const cheapest = [...qualifying].sort(
|
|
190
|
+
(a, b) => median(resolvedByModel.get(a)) - median(resolvedByModel.get(b)) || a.localeCompare(b),
|
|
191
|
+
)[0];
|
|
192
|
+
const ladderRank = ladderNames.indexOf(cheapest);
|
|
193
|
+
const tier = tierOfRank(ladderRank, ladderNames.length);
|
|
194
|
+
|
|
195
|
+
// Effort advice is relative to the estate too: heavier turns of tool use than the
|
|
196
|
+
// estate's own resolved median is what "plan first" means here.
|
|
197
|
+
const resolvedRuns = runs.filter((run) => run.resolved);
|
|
198
|
+
const estateToolCalls = median(resolvedRuns.map((run) => run.toolCalls));
|
|
199
|
+
const resolvedRunsOfCheapest = pool.filter((run) => run.resolved && run.model === cheapest);
|
|
200
|
+
const pickToolCalls = median(resolvedRunsOfCheapest.map((run) => run.toolCalls));
|
|
201
|
+
const effort = estateToolCalls > 0 && pickToolCalls > estateToolCalls ? 'plan first' : 'no plan mode';
|
|
202
|
+
|
|
203
|
+
const pickMedian = median(resolvedByModel.get(cheapest));
|
|
204
|
+
const pickUnit = measure === 'cost' ? `cost $${pickMedian.toFixed(4)}` : `tokens ${Math.round(pickMedian)}`;
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
picked: {
|
|
208
|
+
line: `${tier} model, ${effort}`,
|
|
209
|
+
basis: `from ${poolTaskCount} ${kind} task${poolTaskCount === 1 ? '' : 's'} in bench history; cheapest that resolved: ${cheapest} (median ${pickUnit}); a human picks the model, this only ranks classes`,
|
|
210
|
+
metric: METRIC_NOTE,
|
|
211
|
+
},
|
|
212
|
+
caution: null,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* The brief's door into advice (plan 0003 R40): history on disk in, advice out, or null
|
|
218
|
+
* when the history has no runs at all — a brief in a repository without bench history
|
|
219
|
+
* carries every other section and no made-up opinion about models.
|
|
220
|
+
*
|
|
221
|
+
* `kind` comes in already classified when the caller is the brief itself, so the packet
|
|
222
|
+
* never carries two opinions about one task.
|
|
223
|
+
*/
|
|
224
|
+
export async function briefAdvice(root, { task = '', scope = [], kind = null } = {}) {
|
|
225
|
+
const history = await loadHistory(root);
|
|
226
|
+
if (history.runs.length === 0) return null;
|
|
227
|
+
const known = kind ?? classifyTask(task, { paths: scope }).kind;
|
|
228
|
+
return advise({ kind: known, files: scope.length }, history);
|
|
229
|
+
}
|
package/src/argv.mjs
CHANGED
|
@@ -80,10 +80,22 @@ export const BOOL_FLAGS = new Set([
|
|
|
80
80
|
// index / map / brief: opt into local vector embeddings configured under local.embed
|
|
81
81
|
// (plan 0003 R28, R37). Off by default unless recall@10 gains at least 10 points.
|
|
82
82
|
'embed',
|
|
83
|
+
// distill: regenerate a proposal that is already cached. Off by default because
|
|
84
|
+
// the cache is the point - the same source at the same budget is the same answer,
|
|
85
|
+
// and re-reading it is cheaper than re-deriving it (plan 0003 R38).
|
|
86
|
+
'refresh',
|
|
87
|
+
// distill: propose the summary from the configured local chat model instead of
|
|
88
|
+
// the deterministic outline. Off by default: no model in the default path (P1),
|
|
89
|
+
// and a model that is not asked for should not be opened against (ADR-090).
|
|
90
|
+
'local',
|
|
83
91
|
]);
|
|
84
92
|
|
|
85
93
|
export const VALUE_FLAGS = new Set([
|
|
86
94
|
'budget',
|
|
95
|
+
// bench critic: the labelled set of historical diffs to score (plan 0003 R39).
|
|
96
|
+
// A path, not a switch, because the labels themselves come from a human under
|
|
97
|
+
// ADR-008 — the tool never annotates for them.
|
|
98
|
+
'labels',
|
|
87
99
|
'max-bytes',
|
|
88
100
|
'paths',
|
|
89
101
|
'skill',
|
package/src/bench/run.mjs
CHANGED
|
@@ -17,6 +17,7 @@ import { fingerprint } from '../resolve/fingerprint.mjs';
|
|
|
17
17
|
import { rankSkills, tokenize } from '../resolve/resolve.mjs';
|
|
18
18
|
import { gradeWorkspace } from './grade.mjs';
|
|
19
19
|
import { resolveVaultDir } from './mine.mjs';
|
|
20
|
+
import { loadTasks } from './tasks.mjs';
|
|
20
21
|
import { ablateSectionInWorkspace, parseAblationArmName } from './ablate.mjs';
|
|
21
22
|
import { applyVariantInWorkspace, parseVariantArmName } from './variants.mjs';
|
|
22
23
|
|
|
@@ -678,21 +679,8 @@ async function executeAdapter(adapter, workDir, task, { vaultRoot, model, maxTur
|
|
|
678
679
|
return { ...parseAdapterUsage(adapter, stdoutText, analyticsText), transcript: `${stdoutText}\n${analyticsText}` };
|
|
679
680
|
}
|
|
680
681
|
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
const entries = (await readdir(tasksDir).catch(() => []))
|
|
684
|
-
.filter((f) => f.endsWith('.json'))
|
|
685
|
-
.sort();
|
|
686
|
-
|
|
687
|
-
const filterSet = new Set(filterIds);
|
|
688
|
-
const tasks = [];
|
|
689
|
-
for (const entry of entries) {
|
|
690
|
-
const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
|
|
691
|
-
if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
|
|
692
|
-
tasks.push(data);
|
|
693
|
-
}
|
|
694
|
-
return tasks.sort((a, b) => a.id.localeCompare(b.id));
|
|
695
|
-
}
|
|
682
|
+
// loadTasks lives in tasks.mjs (the advice engine became a second reader, so the loader
|
|
683
|
+
// was extracted rather than copied — ADR-002).
|
|
696
684
|
|
|
697
685
|
function runIdFor(harness, armName, rep) {
|
|
698
686
|
return `${harness}-${armName}-r${rep}`;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Mined task descriptors, read back from the tool state's `bench/tasks/` directory (plan 0003 R01).
|
|
2
|
+
//
|
|
3
|
+
// Extracted from bench/run.mjs on its second real reader (ADR-002): the R40 advice engine
|
|
4
|
+
// needs the two facts a run record does not carry — the prompt, which classifies the task,
|
|
5
|
+
// and the gold stat, which says how many files the change touched. A second private loader
|
|
6
|
+
// beside the first would drift the first time one learned a filter the other lacked.
|
|
7
|
+
//
|
|
8
|
+
// Deterministic by construction: the directory is read sorted, the filter is exact, and no
|
|
9
|
+
// part of this touches a model, a network or a clock.
|
|
10
|
+
import { readdir, readFile } from 'node:fs/promises';
|
|
11
|
+
import { join, resolve } from 'node:path';
|
|
12
|
+
import { CONFIG_DIR } from '../name.mjs';
|
|
13
|
+
|
|
14
|
+
const STATE_DIR = `.${CONFIG_DIR}`;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Load mined benchmark task descriptors.
|
|
18
|
+
*
|
|
19
|
+
* @param {string} root the repository
|
|
20
|
+
* @param {string[]} [filterIds] exact task ids to keep; empty means load them all
|
|
21
|
+
* @returns descriptor objects sorted by id (id, base, sourceCommit, prompt, testCmd,
|
|
22
|
+
* f2p, p2p, gold {files, linesAdded, linesRemoved}, vault)
|
|
23
|
+
*/
|
|
24
|
+
export async function loadTasks(root, filterIds = []) {
|
|
25
|
+
const tasksDir = join(resolve(root), STATE_DIR, 'bench', 'tasks');
|
|
26
|
+
const entries = (await readdir(tasksDir).catch(() => []))
|
|
27
|
+
.filter((f) => f.endsWith('.json'))
|
|
28
|
+
.sort();
|
|
29
|
+
|
|
30
|
+
const filterSet = new Set(filterIds);
|
|
31
|
+
const tasks = [];
|
|
32
|
+
for (const entry of entries) {
|
|
33
|
+
const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
|
|
34
|
+
if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
|
|
35
|
+
tasks.push(data);
|
|
36
|
+
}
|
|
37
|
+
return tasks.sort((a, b) => a.id.localeCompare(b.id));
|
|
38
|
+
}
|
package/src/brief/assemble.mjs
CHANGED
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
import { CLI } from '../name.mjs';
|
|
38
38
|
import { DEFAULT_BRIEF_BYTES, buildPacket, collect } from './packet.mjs';
|
|
39
39
|
import { classifyTask, cardVerbs } from './kind.mjs';
|
|
40
|
+
import { briefAdvice } from '../advice/advice.mjs';
|
|
40
41
|
import { commandCard } from '../kernel/card.mjs';
|
|
41
42
|
import { decisionHits, scopeHits, sourceRoots } from '../index/govern.mjs';
|
|
42
43
|
import { embedTaskQuery, rankFiles, renderMap } from '../index/map.mjs';
|
|
@@ -192,6 +193,27 @@ export async function assembleBrief({
|
|
|
192
193
|
});
|
|
193
194
|
}
|
|
194
195
|
|
|
196
|
+
// Model and effort advice comes from the repository's bench history, not from anything a
|
|
197
|
+
// model produced, so it stays deterministic. The tier exists only if the history resolved
|
|
198
|
+
// a similar task, and the claim travels with its own numbers and its unmeasured accept
|
|
199
|
+
// bar; simt-dev never selects the model an agent runs (plan 0003 R40).
|
|
200
|
+
const advice = await briefAdvice(root, { task, scope, kind });
|
|
201
|
+
if (advice?.picked) {
|
|
202
|
+
packet.add('ADVICE', advice.picked.line, {
|
|
203
|
+
why: ['the tier is the estate’s own bench ladder, and a human still names the model'],
|
|
204
|
+
});
|
|
205
|
+
packet.add('ADVICE', advice.picked.basis, {
|
|
206
|
+
why: ['an advice claim names the neighborhood that backs it, so the reader can re-derive it'],
|
|
207
|
+
});
|
|
208
|
+
packet.add('ADVICE', advice.picked.metric, {
|
|
209
|
+
why: ['the row has an accept bar, and nothing in a packet may claim the bar was met'],
|
|
210
|
+
});
|
|
211
|
+
} else if (advice?.caution) {
|
|
212
|
+
packet.add('CAUTIONS', advice.caution, {
|
|
213
|
+
why: ['advice without a neighborhood is a guess dressed as history'],
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
195
217
|
const likely = ranked.slice(0, LIKELY_FILES).map((row) => row.path);
|
|
196
218
|
const known = testEdges({ files, imports: edges.filter((edge) => edge.kind === 'import') });
|
|
197
219
|
const untested = [];
|
package/src/brief/packet.mjs
CHANGED
|
@@ -30,7 +30,7 @@ export const PROMPT_CAP_BYTES = 2048;
|
|
|
30
30
|
* The order is fixed and identical in every packet. A reader who has to search for the
|
|
31
31
|
* command is paying the cost the packet exists to remove.
|
|
32
32
|
*/
|
|
33
|
-
const SECTIONS = ['TASK', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
|
|
33
|
+
const SECTIONS = ['TASK', 'ADVICE', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
36
|
* What each section is worth when something has to go.
|
|
@@ -49,6 +49,9 @@ const SECTIONS = ['TASK', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION
|
|
|
49
49
|
*/
|
|
50
50
|
const SECTION_VALUE = {
|
|
51
51
|
CAUTIONS: 1,
|
|
52
|
+
// Bench evidence, but advice rather than fact: losing it costs an opinion, not a
|
|
53
|
+
// command or a path to check (plan 0003 R40).
|
|
54
|
+
ADVICE: 2,
|
|
52
55
|
LESSONS: 2,
|
|
53
56
|
SKILLS: 3,
|
|
54
57
|
DECISIONS: 4,
|
package/src/commands/bench.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { formatRecall, measureRecall } from '../index/map.mjs';
|
|
|
16
16
|
import { ablateBench } from '../bench/ablate.mjs';
|
|
17
17
|
import { runBenchProbes } from '../bench/probes.mjs';
|
|
18
18
|
import { selectRuleVariants } from '../bench/variants.mjs';
|
|
19
|
+
import { criticBench } from '../review/critic.mjs';
|
|
19
20
|
import { listFlag } from './skills.mjs';
|
|
20
21
|
|
|
21
22
|
export async function benchMineCommand(positional, flags) {
|
|
@@ -394,3 +395,34 @@ export async function benchVariantsCommand(positional, flags) {
|
|
|
394
395
|
|
|
395
396
|
|
|
396
397
|
|
|
398
|
+
|
|
399
|
+
// `simt bench critic` (plan 0003 R39).
|
|
400
|
+
//
|
|
401
|
+
// The row's acceptance measured, or reported missing rather than inferred: the
|
|
402
|
+
// critic runs over a labelled set of historical diffs and the bench reports
|
|
403
|
+
// precision and recall for the BLOCK binary. The metric never gates the critic —
|
|
404
|
+
// the row says precision >= 0.8 on >= 30 labelled diffs promotes it to a gate,
|
|
405
|
+
// and until a bench here shows that, every report calls the critic advisory.
|
|
406
|
+
export async function benchCriticCommand(positional, flags, commands = []) {
|
|
407
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
408
|
+
const labelsPath = flags.labels ?? '.simt-dev/critic-labels.jsonl';
|
|
409
|
+
const result = await criticBench(root, { labelsPath, flags, commands });
|
|
410
|
+
if (typeof result === 'number') return result;
|
|
411
|
+
|
|
412
|
+
if (isJson(flags)) {
|
|
413
|
+
emitJson(result, { pretty: flags.pretty === true, maxBytes: flags.maxBytes ?? 0 });
|
|
414
|
+
return EXIT.OK;
|
|
415
|
+
}
|
|
416
|
+
const lines = [
|
|
417
|
+
`bench critic ${root}`,
|
|
418
|
+
`labels ${result.labels} (${labelsPath}) measured ${result.measured} errors ${result.errors.length}`,
|
|
419
|
+
`counts tp=${result.tp} fp=${result.fp} fn=${result.fn} tn=${result.tn}`,
|
|
420
|
+
`precision ${result.precision === null ? 'unmeasured' : result.precision.toFixed(3)}`,
|
|
421
|
+
`recall ${result.recall === null ? 'unmeasured' : result.recall.toFixed(3)}`,
|
|
422
|
+
`gated ${result.gated ? 'yes' : 'no'}`,
|
|
423
|
+
];
|
|
424
|
+
if (result.labelsNote) lines.push(`labels ${result.labelsNote}`);
|
|
425
|
+
for (const error of result.errors) lines.push(`error ${error}`);
|
|
426
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
427
|
+
return EXIT.OK;
|
|
428
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// `simt distill <skill|doc>` (plan 0003 R38).
|
|
2
|
+
//
|
|
3
|
+
// A budgeted summary with citations to source line ranges, written to
|
|
4
|
+
// `.simt-dev/distilled/` as a proposal a human merges. The deterministic engine
|
|
5
|
+
// is the default; a local model proposes only when the caller asks for it and the
|
|
6
|
+
// manifest names one (ADR-090, ADR-075). Model output never lands anywhere
|
|
7
|
+
// projected or committed text does — that is the direction this command cannot
|
|
8
|
+
// take (principle P2).
|
|
9
|
+
|
|
10
|
+
import { CLI } from '../name.mjs';
|
|
11
|
+
import { EXIT, emit, emitJson, isJson } from '../output/index.mjs';
|
|
12
|
+
import { distillTarget } from '../distill/distill.mjs';
|
|
13
|
+
|
|
14
|
+
export async function distillCommand(args, flags) {
|
|
15
|
+
const result = await distillTarget(args, flags);
|
|
16
|
+
if (typeof result === 'number') return result;
|
|
17
|
+
|
|
18
|
+
if (isJson(flags)) {
|
|
19
|
+
emitJson(result, { maxBytes: flags.maxBytes ?? 0 });
|
|
20
|
+
return EXIT.OK;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
emit(
|
|
24
|
+
[
|
|
25
|
+
`source ${result.target.kind} ${result.target.source} (${result.target.bytes} bytes, ${result.target.lines} lines)`,
|
|
26
|
+
`digest ${result.digest}`,
|
|
27
|
+
`engine ${result.engine}`,
|
|
28
|
+
`budget ${result.budget} bytes`,
|
|
29
|
+
`citations ${result.citations}`,
|
|
30
|
+
...(result.verified === true ? [] : result.faults.map((f) => `unverified ${f}`)),
|
|
31
|
+
`proposal ${result.proposal.path} (${result.proposal.state}, ${result.proposal.bytes} bytes)`,
|
|
32
|
+
].join('\n'),
|
|
33
|
+
{ maxBytes: flags.maxBytes ?? 0 },
|
|
34
|
+
);
|
|
35
|
+
return EXIT.OK;
|
|
36
|
+
}
|
package/src/commands/help.mjs
CHANGED
|
@@ -103,6 +103,21 @@ ${CLI} decide [<title>] scaffold an ADR with the next free number, a Stat
|
|
|
103
103
|
${CLI} local doctor probe configured local model endpoints (chat and embed)
|
|
104
104
|
(reads local in ${MANIFEST_FILE}; opens no socket when
|
|
105
105
|
unset; refuses non-loopback URLs without --allow-remote)
|
|
106
|
+
${CLI} distill <skill|doc> a budgeted proposal summary of a skill body or doc,
|
|
107
|
+
every claim citing the source line range it rests on
|
|
108
|
+
(writes .simt-dev/distilled/<digest>.md; a proposal a
|
|
109
|
+
human merges - never projected or committed;
|
|
110
|
+
deterministic outline by default; --local proposes
|
|
111
|
+
from the configured chat model instead; the cache is
|
|
112
|
+
keyed by source digest and budget, --refresh regenerates)
|
|
113
|
+
${CLI} review [--local] an ADR-008 verdict skeleton for the staged diff,
|
|
114
|
+
the brief listing everything that governs it (R25)
|
|
115
|
+
(deterministic without --local: the brief and its
|
|
116
|
+
governing ids only; --local proposes from the
|
|
117
|
+
configured chat model and writes
|
|
118
|
+
.simt-dev/reviews/<digest>.md — a proposal a human
|
|
119
|
+
reads, verified against the verdict vocabulary and
|
|
120
|
+
the brief's ids; advisory always, never a gate)
|
|
106
121
|
${CLI} shared plan what a workspace's repeated rules cost to carry
|
|
107
122
|
once: the shared set, its bytes, and every path that would be written
|
|
108
123
|
(read-only; touches nothing, not even your home directory)
|
|
@@ -141,6 +156,11 @@ ${CLI} bench probes [--yes] 12 synthetic rule-compliance probes with determin
|
|
|
141
156
|
${CLI} bench variants [--yes] evaluate simt-variants rule slots on the canary suite and lock
|
|
142
157
|
winning variants in simt-dev.lock.json (dry-run unless --yes;
|
|
143
158
|
re-opens when model or harness changes)
|
|
159
|
+
${CLI} bench critic precision and recall of the local critic over a labelled
|
|
160
|
+
set of historical diffs (.simt-dev/critic-labels.jsonl;
|
|
161
|
+
overrides --labels; labels are human-written verdicts
|
|
162
|
+
under ADR-008; the critic is advisory until precision
|
|
163
|
+
reaches 0.8 on 30 labelled diffs)
|
|
144
164
|
${CLI} mcp the read-only surface as MCP tools over stdio
|
|
145
165
|
(--list prints it without reading stdin; init, sync,
|
|
146
166
|
upgrade and the installers stay CLI-only by design)`;
|
|
@@ -204,9 +224,17 @@ const FLAGS = `--json structured output (compact; one document,
|
|
|
204
224
|
--state <name> plan done: target state (default: Complete)
|
|
205
225
|
--level <name> map: detail level (outline, names, signatures)
|
|
206
226
|
--supersedes <spec> decide: superseded ADR or ruling (ADR-NNN or ADR-NNN:<ruling>[:<kind>]; repeatable)
|
|
207
|
-
--allow-remote local doctor: allow non-loopback model
|
|
227
|
+
--allow-remote local doctor / distill / review: allow non-loopback model
|
|
228
|
+
URLs (refused by default)
|
|
208
229
|
--embed index / map / brief: compute or query optional local vector embeddings
|
|
209
230
|
(off by default unless recall@10 gains at least 10 points)
|
|
231
|
+
--labels <path> bench critic: the labelled set of historical diffs to score
|
|
232
|
+
(default .simt-dev/critic-labels.jsonl; one human-written
|
|
233
|
+
{sha, verdict} pair per line)
|
|
234
|
+
--refresh distill / review: regenerate a proposal that is already cached
|
|
235
|
+
--local distill / review: propose the answer from the configured
|
|
236
|
+
local chat model (local.chat); off by default, which
|
|
237
|
+
uses the deterministic path instead
|
|
210
238
|
--help this text
|
|
211
239
|
--version the version, and the SHA it was built from`;
|
|
212
240
|
|
package/src/commands/index.mjs
CHANGED
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
benchAblateCommand,
|
|
47
47
|
benchProbesCommand,
|
|
48
48
|
benchVariantsCommand,
|
|
49
|
+
benchCriticCommand,
|
|
49
50
|
} from './bench.mjs';
|
|
50
51
|
import { planCommand } from './plan.mjs';
|
|
51
52
|
import { sharedPlanCommand, sharedInstallCommand, sharedWarning } from './shared.mjs';
|
|
@@ -58,7 +59,9 @@ import { gateCommand } from './gate.mjs';
|
|
|
58
59
|
import { impactCommand } from './impact.mjs';
|
|
59
60
|
import { lessonsListCommand, lessonsReviewCommand } from './lessons.mjs';
|
|
60
61
|
import { decideCommand } from './decide.mjs';
|
|
62
|
+
import { reviewCommand } from './review.mjs';
|
|
61
63
|
import { localDoctorCommand } from './local.mjs';
|
|
64
|
+
import { distillCommand } from './distill.mjs';
|
|
62
65
|
import { parse } from '../argv.mjs';
|
|
63
66
|
import { CLI } from '../name.mjs';
|
|
64
67
|
import { fingerprint } from '../resolve/fingerprint.mjs';
|
|
@@ -135,6 +138,10 @@ export const IMPLEMENTED = {
|
|
|
135
138
|
'lessons list': lessonsListCommand,
|
|
136
139
|
'lessons review': lessonsReviewCommand,
|
|
137
140
|
decide: decideCommand,
|
|
141
|
+
distill: distillCommand,
|
|
142
|
+
// R25 machinery needs the runnable command names for Affects command tokens,
|
|
143
|
+
// same wiring `why` and `impact` get through their lambdas.
|
|
144
|
+
review: (args, flags) => reviewCommand(args, flags, COMMANDS),
|
|
138
145
|
'local doctor': localDoctorCommand,
|
|
139
146
|
'evidence check': (args, flags) => evidenceCommand(['check', ...args], flags),
|
|
140
147
|
init: initCommand,
|
|
@@ -155,6 +162,11 @@ export const IMPLEMENTED = {
|
|
|
155
162
|
'bench ablate': benchAblateCommand,
|
|
156
163
|
'bench probes': benchProbesCommand,
|
|
157
164
|
'bench variants': benchVariantsCommand,
|
|
165
|
+
// plan 0003 R39. The critic's acceptance bench measures BLOCK-precision and
|
|
166
|
+
// recall over a labelled set; until it reports its floor, the critic entries
|
|
167
|
+
// stay advisory and no gate reads them. COMMANDS rides along so the brief
|
|
168
|
+
// resolves Affects command tokens the way `simt why` does.
|
|
169
|
+
'bench critic': (args, flags) => benchCriticCommand(args, flags, COMMANDS),
|
|
158
170
|
upgrade: upgradeCommand,
|
|
159
171
|
mcp: mcpCommand,
|
|
160
172
|
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// `simt review [--local]` (plan 0003 R39).
|
|
2
|
+
//
|
|
3
|
+
// The critic's report layer: the deterministic path prints the brief (staged diff
|
|
4
|
+
// plus what governs it); `--local` adds the model verdict, which always arrives as
|
|
5
|
+
// a proposal a human reads — nothing here is a gate, and the exit code never
|
|
6
|
+
// carries the verdict, because a critic that fails the run would be a gate by
|
|
7
|
+
// another name (ADR-090, principle P2: the reply's text is model output and never
|
|
8
|
+
// lands in projected or committed text).
|
|
9
|
+
|
|
10
|
+
import { CLI } from '../name.mjs';
|
|
11
|
+
import { EXIT, emit, emitJson, isJson } from '../output/index.mjs';
|
|
12
|
+
import { reviewTarget } from '../review/critic.mjs';
|
|
13
|
+
|
|
14
|
+
function governsLine(items) {
|
|
15
|
+
if (items.length === 0) return 'none';
|
|
16
|
+
const ids = items.slice(0, 8).map((item) => item.id);
|
|
17
|
+
const more = items.length > ids.length ? ` …+${items.length - ids.length}` : '';
|
|
18
|
+
return ids.join(', ') + more;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function reviewCommand(args, flags, commands = []) {
|
|
22
|
+
const result = await reviewTarget(args, flags, { commands });
|
|
23
|
+
if (typeof result === 'number') return result;
|
|
24
|
+
|
|
25
|
+
if (isJson(flags)) {
|
|
26
|
+
emitJson(result, { maxBytes: flags.maxBytes ?? 0 });
|
|
27
|
+
return EXIT.OK;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const lines = [
|
|
31
|
+
`staged ${result.staged.files.length} files, ${result.staged.bytes} bytes`,
|
|
32
|
+
`governs ${governsLine(result.governs)}`,
|
|
33
|
+
];
|
|
34
|
+
// The branch keys on the path, not on the verdict: a local run whose reply
|
|
35
|
+
// lacked a verdict line still carries faults and a proposal, and hiding them
|
|
36
|
+
// behind the "needs --local" reading would erase the model's failure.
|
|
37
|
+
if (result.brief !== undefined) {
|
|
38
|
+
// The nothing-staged run is a plain answer, not an invitation to rerun with
|
|
39
|
+
// --local: there is no brief yet, so there is nothing for the critic to read.
|
|
40
|
+
if (result.staged.files.length === 0) {
|
|
41
|
+
lines.push('verdict none (nothing is staged)');
|
|
42
|
+
} else {
|
|
43
|
+
lines.push(
|
|
44
|
+
`verdict needs ${CLI} review --local (advisory; a critic is never a gate)`,
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
if (result.verdict !== null) {
|
|
49
|
+
lines.push(
|
|
50
|
+
`verdict ${result.verdict} (${result.verified ? 'verified' : 'unverified'})`,
|
|
51
|
+
`findings ${result.findings}`,
|
|
52
|
+
);
|
|
53
|
+
} else {
|
|
54
|
+
lines.push('verdict absent (unverified)');
|
|
55
|
+
}
|
|
56
|
+
if (!result.verified) {
|
|
57
|
+
for (const fault of result.faults) lines.push(`unverified ${fault}`);
|
|
58
|
+
}
|
|
59
|
+
lines.push(`proposal ${result.proposal.path} (${result.proposal.state}, ${result.proposal.bytes} bytes)`);
|
|
60
|
+
}
|
|
61
|
+
if (result.metric) lines.push(`metric ${result.metric.note}`);
|
|
62
|
+
|
|
63
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
64
|
+
return EXIT.OK;
|
|
65
|
+
}
|