simt-dev 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +157 -0
- package/README.md +271 -62
- package/package.json +6 -2
- package/rules/core.md +1 -1
- package/src/advice/advice.mjs +229 -0
- package/src/agenthooks/policy.mjs +10 -2
- package/src/agenthooks/prompt.mjs +81 -0
- package/src/agenthooks/render.mjs +138 -17
- package/src/argv.mjs +28 -1
- package/src/bench/grade.mjs +3 -0
- package/src/bench/run.mjs +88 -54
- package/src/bench/tasks.mjs +38 -0
- package/src/bench/variants.mjs +456 -0
- package/src/brief/assemble.mjs +368 -0
- package/src/brief/kind.mjs +134 -0
- package/src/brief/packet.mjs +200 -0
- package/src/commands/agenthook.mjs +84 -5
- package/src/commands/bench.mjs +72 -15
- package/src/commands/brief.mjs +78 -0
- package/src/commands/compare.mjs +10 -2
- package/src/commands/decide.mjs +116 -0
- package/src/commands/distill.mjs +36 -0
- package/src/commands/help.mjs +62 -3
- package/src/commands/index.mjs +29 -1
- package/src/commands/lessons.mjs +127 -0
- package/src/commands/local.mjs +97 -0
- package/src/commands/map.mjs +42 -7
- package/src/commands/repoindex.mjs +4 -1
- package/src/commands/review.mjs +65 -0
- package/src/commands/rules.mjs +17 -2
- package/src/commands/skills.mjs +5 -50
- package/src/commands/skillscompile.mjs +255 -0
- package/src/commands/sync.mjs +15 -0
- package/src/commands/tests.mjs +2 -49
- package/src/commands/why.mjs +23 -25
- package/src/compare/learn.mjs +680 -0
- package/src/decisions/index.mjs +116 -65
- package/src/distill/distill.mjs +246 -0
- package/src/distill/engine.mjs +187 -0
- package/src/federation/client.mjs +188 -0
- package/src/federation/config.mjs +134 -0
- package/src/federation/query.mjs +48 -0
- package/src/graph/compile.mjs +475 -0
- package/src/index/gate.mjs +1 -13
- package/src/index/govern.mjs +23 -11
- package/src/index/map.mjs +251 -17
- package/src/index/persist.mjs +74 -7
- package/src/index/store.mjs +222 -8
- package/src/kernel/variants.mjs +353 -0
- package/src/ledger/index.mjs +36 -1
- package/src/local/client.mjs +482 -0
- package/src/local/config.mjs +269 -0
- package/src/mcp/tools.mjs +138 -1
- package/src/memory/decide.mjs +462 -0
- package/src/memory/lessons.mjs +858 -0
- package/src/projection/lock.mjs +39 -7
- package/src/projection/rules.mjs +32 -8
- package/src/resolve/body.mjs +46 -0
- package/src/review/critic.mjs +509 -0
- package/src/toolwords.mjs +6 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// Model and effort advice, read from the repository's own bench history (plan 0003 R40).
|
|
2
|
+
//
|
|
3
|
+
// The mapping the design row asks for — task features to the cheapest tier that resolved
|
|
4
|
+
// similar tasks — is data, not opinion: the records `bench run` stored on disk already say
|
|
5
|
+
// which model classes resolved which mined tasks and what those runs cost. So the tier
|
|
6
|
+
// ladder is the estate's own evidence sorted by price, and no vendor name is written into
|
|
7
|
+
// the code: a repository whose history has one class of run gets advice about that class,
|
|
8
|
+
// and one whose history has three gets a ladder with three steps.
|
|
9
|
+
//
|
|
10
|
+
// The properties this exists to keep:
|
|
11
|
+
//
|
|
12
|
+
// * Advice, never a choice. The output is a sentence in the brief header along with the
|
|
13
|
+
// bench numbers behind it; the agent (or the human) names the model, not simt-dev
|
|
14
|
+
// (P2). Nothing here writes a model id into a config, a lock or a command.
|
|
15
|
+
// * Deterministic. No model, no network, no clock (P1): the same runs directory yields
|
|
16
|
+
// the same advice, so a reviewable claim about cost is possible.
|
|
17
|
+
// * Quiet in thin evidence. A neighborhood with no resolved run yields a caution, not a
|
|
18
|
+
// made-up tier; a history with no runs at all yields no advice line.
|
|
19
|
+
//
|
|
20
|
+
// The measure of "cheapest" follows the bench's own report: cost in dollars when the run
|
|
21
|
+
// records or a prices table price the tokens, total tokens otherwise. Ordering by a number
|
|
22
|
+
// that does not exist for any run would be nondeterminism, not economics.
|
|
23
|
+
import { classifyTask } from '../brief/kind.mjs';
|
|
24
|
+
import { computeRunCost, isRunResolved, loadPrices, loadRunRecords } from '../bench/report.mjs';
|
|
25
|
+
import { loadTasks } from '../bench/tasks.mjs';
|
|
26
|
+
import { CONFIG_DIR } from '../name.mjs';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Every advice report carries its row's accept bar (ADR-076: a metric is a measurement or
|
|
30
|
+
* it is not claimed).
|
|
31
|
+
*/
|
|
32
|
+
export const METRIC_NOTE =
|
|
33
|
+
'advice is advisory until a held-out bench shows >= 25% cost cut at delta R >= -2 (plan 0003 R40)';
|
|
34
|
+
|
|
35
|
+
const TIERS = ['small', 'default', 'frontier'];
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The ladder can specialize beyond kind only while there is still evidence to stand on:
|
|
39
|
+
* below this many same-kind tasks the kind is the whole neighborhood, and a narrower
|
|
40
|
+
* window makes evidence worse, not "more similar".
|
|
41
|
+
*/
|
|
42
|
+
const NARROW_MIN_POOL = 6;
|
|
43
|
+
|
|
44
|
+
/** The narrowed window needs at least these resolved runs or it is not evidence either. */
|
|
45
|
+
const NARROW_MIN_RESOLVED = 3;
|
|
46
|
+
|
|
47
|
+
/** A file window is a step from the estimate in either direction — a factor of two in size. */
|
|
48
|
+
const WINDOW_HALF = 2;
|
|
49
|
+
|
|
50
|
+
function median(values) {
|
|
51
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
52
|
+
if (sorted.length === 0) return 0;
|
|
53
|
+
const mid = Math.floor(sorted.length / 2);
|
|
54
|
+
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Tier for a rank in the cheapest-first ladder: thirds, not absolute cost — an estate's
|
|
59
|
+
* "frontier" relative to its own history means more than a number the code guessed.
|
|
60
|
+
* The cheapest of two observed classes is 'small' and the dearer one 'default'; a class
|
|
61
|
+
* is 'frontier' only once a ladder has three steps, because a two-step ladder cannot
|
|
62
|
+
* name a top third honestly.
|
|
63
|
+
*/
|
|
64
|
+
export function tierOfRank(rank, count) {
|
|
65
|
+
if (count < 3) return rank === 0 ? TIERS[0] : TIERS[1];
|
|
66
|
+
if (rank * 3 < count) return TIERS[0];
|
|
67
|
+
if (rank * 3 >= count * 2) return TIERS[2];
|
|
68
|
+
return TIERS[1];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Total tokens a usage report carries — the measure that exists even with no prices file.
|
|
73
|
+
*/
|
|
74
|
+
function tokensOf(usage = {}) {
|
|
75
|
+
const num = (v) => Number(v ?? 0);
|
|
76
|
+
return num(usage.input) + num(usage.cacheWrite ?? usage.cache_write) + num(usage.cacheRead ?? usage.cache_read) + num(usage.output);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Load bench history as advice-ready run rows.
|
|
81
|
+
*
|
|
82
|
+
* A descriptor missing for a run leaves the row with `kind: null`; such a run can join no
|
|
83
|
+
* neighborhood, which is the honest answer — a task whose prompt was never mined is a
|
|
84
|
+
* task whose kind would be a guess.
|
|
85
|
+
*/
|
|
86
|
+
export async function loadHistory(root) {
|
|
87
|
+
const [records, prices, tasks] = await Promise.all([
|
|
88
|
+
loadRunRecords(root),
|
|
89
|
+
// An unreadable prices file would throw out of a brief, whose callers only asked for
|
|
90
|
+
// a packet; the bench has its own report for surfacing a broken prices file loudly.
|
|
91
|
+
loadPrices(root).catch(() => null),
|
|
92
|
+
loadTasks(root),
|
|
93
|
+
]);
|
|
94
|
+
const byId = new Map(tasks.map((task) => [task.id, task]));
|
|
95
|
+
const runs = records.map((record) => {
|
|
96
|
+
const descriptor = byId.get(record.task) ?? null;
|
|
97
|
+
return {
|
|
98
|
+
task: record.task,
|
|
99
|
+
model: record.model,
|
|
100
|
+
resolved: isRunResolved(record),
|
|
101
|
+
cost: computeRunCost(record, prices),
|
|
102
|
+
tokens: tokensOf(record.usage),
|
|
103
|
+
toolCalls: Number(record?.usage?.toolCalls ?? record?.usage?.tool_calls ?? 0),
|
|
104
|
+
files: Number(descriptor?.gold?.files ?? record?.diff?.files ?? 0),
|
|
105
|
+
kind: descriptor ? classifyTask(descriptor.prompt ?? '', { paths: [] }).kind : null,
|
|
106
|
+
};
|
|
107
|
+
});
|
|
108
|
+
return { runs };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Advise from bench history for one task's features.
|
|
113
|
+
*
|
|
114
|
+
* @param {{kind: string, files: number}} features the task kind, and the estimate of how
|
|
115
|
+
* many files it will touch (the brief's scope size; a history run carries the mined
|
|
116
|
+
* task's gold stat in the same role)
|
|
117
|
+
* @param {{runs: object[]}} history rows from loadHistory
|
|
118
|
+
* @returns null when the history holds no runs; otherwise `{ picked, caution }` where
|
|
119
|
+
* `picked` is `{ line, basis, metric }` when a tier was named (and `caution` null), and
|
|
120
|
+
* `caution` explains the missing neighborhood when no tier can honestly be named
|
|
121
|
+
*/
|
|
122
|
+
export function advise(features, history) {
|
|
123
|
+
const runs = history?.runs ?? [];
|
|
124
|
+
if (runs.length === 0) return null;
|
|
125
|
+
const kind = features.kind;
|
|
126
|
+
const sameKind = runs.filter((run) => run.kind === kind);
|
|
127
|
+
const taskCount = new Set(sameKind.map((run) => run.task)).size;
|
|
128
|
+
if (sameKind.length === 0) {
|
|
129
|
+
return {
|
|
130
|
+
picked: null,
|
|
131
|
+
caution: `bench history holds no ${kind} task, so no cheap tier was advised`,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// The measure is chosen once, across all resolved history: a ladder built on dollars
|
|
136
|
+
// while the pick ran on tokens would compare a number with a different number's name.
|
|
137
|
+
const measure = runs.some((run) => run.resolved && run.cost > 0) ? 'cost' : 'tokens';
|
|
138
|
+
|
|
139
|
+
// Kind decides the neighborhood. The file window only specializes a neighborhood that
|
|
140
|
+
// has already got evidence to spare (NARROW_MIN_*): a factor-of-two window around the
|
|
141
|
+
// estimate refines "bugfix" to "small bugfix" when the kind has runs to lend, and says
|
|
142
|
+
// nothing when it has not.
|
|
143
|
+
let pool = sameKind;
|
|
144
|
+
if (features.files > 0 && sameKind.length >= NARROW_MIN_POOL) {
|
|
145
|
+
const near = sameKind.filter(
|
|
146
|
+
(run) =>
|
|
147
|
+
run.files > 0 &&
|
|
148
|
+
run.files >= Math.ceil(features.files / WINDOW_HALF) &&
|
|
149
|
+
run.files <= features.files * WINDOW_HALF,
|
|
150
|
+
);
|
|
151
|
+
if (near.filter((run) => run.resolved).length >= NARROW_MIN_RESOLVED) pool = near;
|
|
152
|
+
}
|
|
153
|
+
const poolTaskCount = new Set(pool.map((run) => run.task)).size;
|
|
154
|
+
|
|
155
|
+
// The ladder orders every model class the estate ever ran, cheapest first, so the
|
|
156
|
+
// advice can say where its pick sits in the estate's whole history. Uncost evidence is
|
|
157
|
+
// still cost position: a class that never resolved one of these tasks occupies a rung
|
|
158
|
+
// in the price ladder, and the resolved floor below — not the ladder — decides whether
|
|
159
|
+
// the pick is allowed to stand on it.
|
|
160
|
+
const ladderValues = new Map();
|
|
161
|
+
for (const run of runs) {
|
|
162
|
+
const values = ladderValues.get(run.model) ?? [];
|
|
163
|
+
values.push(measure === 'cost' ? run.cost : run.tokens);
|
|
164
|
+
ladderValues.set(run.model, values);
|
|
165
|
+
}
|
|
166
|
+
const ladderNames = [...ladderValues.keys()].sort(
|
|
167
|
+
(a, b) => median(ladderValues.get(a)) - median(ladderValues.get(b)) || a.localeCompare(b),
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
const resolvedByModel = new Map();
|
|
171
|
+
for (const run of pool) {
|
|
172
|
+
if (!run.resolved) continue;
|
|
173
|
+
const values = resolvedByModel.get(run.model) ?? [];
|
|
174
|
+
values.push(measure === 'cost' ? run.cost : run.tokens);
|
|
175
|
+
resolvedByModel.set(run.model, values);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// "Resolved similar tasks" is a floor, not a style: a class that never resolved one of
|
|
179
|
+
// these tasks is cheaper because it failed for less money.
|
|
180
|
+
const qualifying = ladderNames.filter((model) => resolvedByModel.has(model));
|
|
181
|
+
if (qualifying.length === 0) {
|
|
182
|
+
return {
|
|
183
|
+
picked: null,
|
|
184
|
+
caution: `no model resolved any of the ${taskCount} ${kind} task${taskCount === 1 ? '' : 's'} in bench history, so no cheap tier was advised`,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Ties break on the model name, so the same history always yields the same pick (P4).
|
|
189
|
+
const cheapest = [...qualifying].sort(
|
|
190
|
+
(a, b) => median(resolvedByModel.get(a)) - median(resolvedByModel.get(b)) || a.localeCompare(b),
|
|
191
|
+
)[0];
|
|
192
|
+
const ladderRank = ladderNames.indexOf(cheapest);
|
|
193
|
+
const tier = tierOfRank(ladderRank, ladderNames.length);
|
|
194
|
+
|
|
195
|
+
// Effort advice is relative to the estate too: heavier turns of tool use than the
|
|
196
|
+
// estate's own resolved median is what "plan first" means here.
|
|
197
|
+
const resolvedRuns = runs.filter((run) => run.resolved);
|
|
198
|
+
const estateToolCalls = median(resolvedRuns.map((run) => run.toolCalls));
|
|
199
|
+
const resolvedRunsOfCheapest = pool.filter((run) => run.resolved && run.model === cheapest);
|
|
200
|
+
const pickToolCalls = median(resolvedRunsOfCheapest.map((run) => run.toolCalls));
|
|
201
|
+
const effort = estateToolCalls > 0 && pickToolCalls > estateToolCalls ? 'plan first' : 'no plan mode';
|
|
202
|
+
|
|
203
|
+
const pickMedian = median(resolvedByModel.get(cheapest));
|
|
204
|
+
const pickUnit = measure === 'cost' ? `cost $${pickMedian.toFixed(4)}` : `tokens ${Math.round(pickMedian)}`;
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
picked: {
|
|
208
|
+
line: `${tier} model, ${effort}`,
|
|
209
|
+
basis: `from ${poolTaskCount} ${kind} task${poolTaskCount === 1 ? '' : 's'} in bench history; cheapest that resolved: ${cheapest} (median ${pickUnit}); a human picks the model, this only ranks classes`,
|
|
210
|
+
metric: METRIC_NOTE,
|
|
211
|
+
},
|
|
212
|
+
caution: null,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* The brief's door into advice (plan 0003 R40): history on disk in, advice out, or null
|
|
218
|
+
* when the history has no runs at all — a brief in a repository without bench history
|
|
219
|
+
* carries every other section and no made-up opinion about models.
|
|
220
|
+
*
|
|
221
|
+
* `kind` comes in already classified when the caller is the brief itself, so the packet
|
|
222
|
+
* never carries two opinions about one task.
|
|
223
|
+
*/
|
|
224
|
+
export async function briefAdvice(root, { task = '', scope = [], kind = null } = {}) {
|
|
225
|
+
const history = await loadHistory(root);
|
|
226
|
+
if (history.runs.length === 0) return null;
|
|
227
|
+
const known = kind ?? classifyTask(task, { paths: scope }).kind;
|
|
228
|
+
return advise({ kind: known, files: scope.length }, history);
|
|
229
|
+
}
|
|
@@ -21,8 +21,16 @@
|
|
|
21
21
|
|
|
22
22
|
import { CLI } from '../name.mjs';
|
|
23
23
|
|
|
24
|
-
/**
|
|
25
|
-
|
|
24
|
+
/**
|
|
25
|
+
* The events an agent-time hook can answer, normalised across harnesses.
|
|
26
|
+
*
|
|
27
|
+
* `pre-tool-use` is a decision: a payload in, a deny or nothing out, and exit 2 means
|
|
28
|
+
* block. `prompt-submit` is a delivery: the same payload shape in, context out, and it
|
|
29
|
+
* never denies anything — a hook that could stop a person's prompt would be a hook whose
|
|
30
|
+
* own bug is a person who cannot type (R29). `post-tool-use` recalls a lesson as one
|
|
31
|
+
* paragraph when a tool call fails with a signature matching `docs/lessons/*.md` (R33).
|
|
32
|
+
*/
|
|
33
|
+
export const EVENTS = ['pre-tool-use', 'prompt-submit', 'post-tool-use'];
|
|
26
34
|
|
|
27
35
|
/** The environment variable that lifts named policies for one session. */
|
|
28
36
|
export const ALLOW_ENV = 'SIMT_ALLOW';
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// The prompt-submit surface: the packet delivered where the agent will read it.
|
|
2
|
+
//
|
|
3
|
+
// This module must stay off the tool-call path, and the code's shape is what keeps it
|
|
4
|
+
// there: `src/commands/agenthook.mjs` reaches it through a dynamic `import()` that only a
|
|
5
|
+
// prompt event takes. The pre-tool hook is a latency promise — it answers before every
|
|
6
|
+
// shell command an agent runs — so it imports the policy table, the decision and the
|
|
7
|
+
// renderer and nothing that walks the repository, and `test/agenthooks.test.mjs` asserts
|
|
8
|
+
// exactly that set. A delivery, by contrast, is the one thing here that must read the
|
|
9
|
+
// repository, so it lives behind a door the pre-tool path never opens.
|
|
10
|
+
//
|
|
11
|
+
// Three rules shape what gets delivered:
|
|
12
|
+
//
|
|
13
|
+
// * The repository opts in. The opt-in is a `brief` key in the repository's manifest,
|
|
14
|
+
// not the presence of a hook in a harness config: a repository that declared nothing
|
|
15
|
+
// is not spending its owner's context on a packet they did not ask for, and the
|
|
16
|
+
// manifest is the one file a person reviews before they commit the change.
|
|
17
|
+
// * The packet is capped, at PROMPT_CAP_BYTES. A packet is delivered into a prompt that
|
|
18
|
+
// already exists, and the ceiling is the measured bar of this slice rather than a
|
|
19
|
+
// value a repository is free to raise.
|
|
20
|
+
// * A delivery never blocks and never fails loudly. Everything that could go wrong —
|
|
21
|
+
// no index, no prompt in the payload, a registry that will not load — ends in no
|
|
22
|
+
// injection and a line on stderr. A hook that stopped a person's prompt because this
|
|
23
|
+
// tool could not answer would be worse than a hook that said nothing.
|
|
24
|
+
|
|
25
|
+
import { readManifest } from '../projection/manifest.mjs';
|
|
26
|
+
import { DEFAULT_BRIEF_BYTES, PROMPT_CAP_BYTES } from '../brief/packet.mjs';
|
|
27
|
+
import { assembleBrief } from '../brief/assemble.mjs';
|
|
28
|
+
import { deliveryJson, factsFrom, promptText } from './render.mjs';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Whether this repository asked for a delivered packet, and the ceiling it asked for.
|
|
32
|
+
*
|
|
33
|
+
* Presence is the opt-in and the value is a preference, so `"brief": {}` means "yes, at
|
|
34
|
+
* whatever you measured" and `"brief": { "bytes": 900 }` means "yes, and keep it small".
|
|
35
|
+
* A declared zero is not honoured as a ceiling: zero is this tool's word for "no ceiling"
|
|
36
|
+
* on the command line, and a delivered packet that meant "the whole repository" would be
|
|
37
|
+
* the one thing the cap exists to prevent. It is read from the manifest rather than from
|
|
38
|
+
* the harness config because the manifest is the file the repository reviews.
|
|
39
|
+
*/
|
|
40
|
+
export async function promptOptIn(root) {
|
|
41
|
+
const { manifest, exists, error } = await readManifest(root);
|
|
42
|
+
if (!exists || error) {
|
|
43
|
+
return { optedIn: false, bytes: DEFAULT_BRIEF_BYTES, why: `${exists ? error : 'no manifest'}` };
|
|
44
|
+
}
|
|
45
|
+
if (manifest?.brief === undefined || manifest.brief === null) {
|
|
46
|
+
return { optedIn: false, bytes: DEFAULT_BRIEF_BYTES, why: 'no `brief` key in the manifest' };
|
|
47
|
+
}
|
|
48
|
+
const declared = manifest.brief?.bytes;
|
|
49
|
+
const asked = Number.isFinite(declared) && declared > 0 ? declared : PROMPT_CAP_BYTES;
|
|
50
|
+
return { optedIn: true, bytes: Math.min(asked, PROMPT_CAP_BYTES) };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The context one prompt-submit call should deliver, or the reason it delivers nothing.
|
|
55
|
+
*
|
|
56
|
+
* `build: false` is the latency promise in one argument: a delivery reports a cold index
|
|
57
|
+
* rather than building one, so the first prompt in a fresh clone gets nothing and the
|
|
58
|
+
* second gets the packet. The alternative — a person watching their first prompt sit for
|
|
59
|
+
* a minute while a repository is walked — is how the hook gets uninstalled, and the
|
|
60
|
+
* packet is not worth a prompt that cannot be typed.
|
|
61
|
+
*/
|
|
62
|
+
export async function deliverPrompt(row, payload) {
|
|
63
|
+
const task = promptText(row, payload);
|
|
64
|
+
if (task === null) return { skipped: 'the payload carries no prompt this harness documents' };
|
|
65
|
+
|
|
66
|
+
// The root is the same dialect fact for both events, so it is read the same way.
|
|
67
|
+
const root = factsFrom(row, payload).root;
|
|
68
|
+
const optIn = await promptOptIn(root);
|
|
69
|
+
if (!optIn.optedIn) return { skipped: optIn.why };
|
|
70
|
+
|
|
71
|
+
// Federation is off on this path and on no other: a delivery has 300 ms to answer, and
|
|
72
|
+
// starting a declared server is a wait a prompt submit must not take. The task is the
|
|
73
|
+
// question a federation would ask, and a person is not made to wait for it here.
|
|
74
|
+
const result = await assembleBrief({ root, task, budget: optIn.bytes, build: false, federate: false });
|
|
75
|
+
if (result.error) return { skipped: 'the packet could not be assembled' };
|
|
76
|
+
if (result.cold) return { skipped: 'the index is empty, and a delivered packet never builds one' };
|
|
77
|
+
|
|
78
|
+
const json = deliveryJson(row, result.packed.text);
|
|
79
|
+
if (json === null) return { skipped: `${row.id} has no prompt event that can inject` };
|
|
80
|
+
return { json, root, bytes: result.packed.after };
|
|
81
|
+
}
|
|
@@ -11,6 +11,26 @@
|
|
|
11
11
|
// cursor .cursor/hooks.json preToolUse {"version":1,"hooks":{E:[h]}}
|
|
12
12
|
// antigravity .agents/hooks.json PreToolUse {"<name>":{E:[{matcher,hooks:[h]}]}}
|
|
13
13
|
//
|
|
14
|
+
// The prompt-submit event (R29) is a second table, and it is shorter because the vendors
|
|
15
|
+
// are: two of the four document a prompt event that can put text into the model's context,
|
|
16
|
+
// and a delivery is the only thing this tool does there. Neither takes a matcher — both
|
|
17
|
+
// document that field for tool events, and a matcher a harness ignores is a matcher the
|
|
18
|
+
// next person believes is filtering something.
|
|
19
|
+
//
|
|
20
|
+
// claude .claude/settings.json UserPromptSubmit hookSpecificOutput.additionalContext
|
|
21
|
+
// codex .codex/hooks.json UserPromptSubmit hookSpecificOutput.additionalContext
|
|
22
|
+
// cursor .cursor/hooks.json beforeSubmitPrompt CANNOT INJECT: block-only
|
|
23
|
+
// antigravity .agents/hooks.json (no prompt event documented)
|
|
24
|
+
//
|
|
25
|
+
// Cursor is the one that would be a mistake to write anyway. Its `beforeSubmitPrompt` hook
|
|
26
|
+
// can only continue or block, so the entry would carry no packet and a broken one would
|
|
27
|
+
// stop a person from typing — and its only injection point, `sessionStart.additional_context`,
|
|
28
|
+
// fires once when the session is created, which is a different lifetime from the brief a
|
|
29
|
+
// task needs. Gemini CLI documents `BeforeAgent`, which does inject, but the product was
|
|
30
|
+
// replaced by Antigravity CLI on 2026-06-18 and its config is not in the reader table this
|
|
31
|
+
// module takes its paths from; a harness this tool did not verify is a harness it does not
|
|
32
|
+
// write to.
|
|
33
|
+
//
|
|
14
34
|
// The paths are not written here. They come from the reader table (R14) because that is
|
|
15
35
|
// where each one was verified against its vendor page, and a second copy of a verified
|
|
16
36
|
// fact is a copy that drifts the first time a vendor moves the file.
|
|
@@ -36,8 +56,8 @@ const TIMEOUT_SECONDS = 10;
|
|
|
36
56
|
* harness's dialect when a payload arrived in an unexpected shape, and the harness would
|
|
37
57
|
* read a decision it does not understand as no decision at all.
|
|
38
58
|
*/
|
|
39
|
-
export function hookCommand(id) {
|
|
40
|
-
return `${CLI} agent-hook ${
|
|
59
|
+
export function hookCommand(id, event = EVENTS[0]) {
|
|
60
|
+
return `${CLI} agent-hook ${event} --harness ${id}`;
|
|
41
61
|
}
|
|
42
62
|
|
|
43
63
|
/** The path a config file lives at, from the reader table rather than a second list. */
|
|
@@ -73,6 +93,20 @@ const ADAPTERS = {
|
|
|
73
93
|
deny: (message) => ({
|
|
74
94
|
hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'deny', permissionDecisionReason: message },
|
|
75
95
|
}),
|
|
96
|
+
// The delivery door. `additionalContext` is documented for this event and is
|
|
97
|
+
// wrapped into the conversation at the point the hook fired, which is why the
|
|
98
|
+
// packet goes here rather than into a system prompt: the cached prefix survives.
|
|
99
|
+
prompt: {
|
|
100
|
+
event: 'UserPromptSubmit',
|
|
101
|
+
verified: 'code.claude.com/docs/en/hooks, read 2026-10-19',
|
|
102
|
+
context: (text) => ({ hookSpecificOutput: { hookEventName: 'UserPromptSubmit', additionalContext: text } }),
|
|
103
|
+
},
|
|
104
|
+
post: {
|
|
105
|
+
event: 'PostToolUse',
|
|
106
|
+
matcher: 'Bash',
|
|
107
|
+
verified: 'code.claude.com/docs/en/hooks, read 2026-10-19',
|
|
108
|
+
context: (text) => ({ hookSpecificOutput: { hookEventName: 'PostToolUse', additionalContext: text } }),
|
|
109
|
+
},
|
|
76
110
|
},
|
|
77
111
|
codex: {
|
|
78
112
|
id: 'codex',
|
|
@@ -96,6 +130,17 @@ const ADAPTERS = {
|
|
|
96
130
|
deny: (message) => ({
|
|
97
131
|
hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'deny', permissionDecisionReason: message },
|
|
98
132
|
}),
|
|
133
|
+
prompt: {
|
|
134
|
+
event: 'UserPromptSubmit',
|
|
135
|
+
verified: 'developers.openai.com/codex/hooks, read 2026-10-19',
|
|
136
|
+
context: (text) => ({ hookSpecificOutput: { hookEventName: 'UserPromptSubmit', additionalContext: text } }),
|
|
137
|
+
},
|
|
138
|
+
post: {
|
|
139
|
+
event: 'PostToolUse',
|
|
140
|
+
matcher: 'Bash',
|
|
141
|
+
verified: 'developers.openai.com/codex/hooks, read 2026-10-19',
|
|
142
|
+
context: (text) => ({ hookSpecificOutput: { hookEventName: 'PostToolUse', additionalContext: text } }),
|
|
143
|
+
},
|
|
99
144
|
},
|
|
100
145
|
cursor: {
|
|
101
146
|
id: 'cursor',
|
|
@@ -111,6 +156,11 @@ const ADAPTERS = {
|
|
|
111
156
|
: [],
|
|
112
157
|
}),
|
|
113
158
|
deny: (message) => ({ permission: 'deny', user_message: message, agent_message: message }),
|
|
159
|
+
// beforeSubmitPrompt is documented as continue/block and nothing else, so there is no
|
|
160
|
+
// key to deliver a packet in. Writing the entry anyway would put a command that reads
|
|
161
|
+
// a repository on the path of every prompt a person types, to say nothing.
|
|
162
|
+
prompt: null,
|
|
163
|
+
post: null,
|
|
114
164
|
},
|
|
115
165
|
antigravity: {
|
|
116
166
|
id: 'antigravity',
|
|
@@ -133,6 +183,10 @@ const ADAPTERS = {
|
|
|
133
183
|
};
|
|
134
184
|
},
|
|
135
185
|
deny: (message) => ({ decision: 'deny', reason: message }),
|
|
186
|
+
// No prompt event is documented for this harness's hooks file, and a guessed event
|
|
187
|
+
// name is a hook that never fires while its config says it does.
|
|
188
|
+
prompt: null,
|
|
189
|
+
post: null,
|
|
136
190
|
},
|
|
137
191
|
};
|
|
138
192
|
|
|
@@ -160,6 +214,37 @@ export function decisionJson(row, decision) {
|
|
|
160
214
|
return row.deny(decision.messages.join('\n'));
|
|
161
215
|
}
|
|
162
216
|
|
|
217
|
+
/**
|
|
218
|
+
* The document to print for a delivery, or null when the harness takes none.
|
|
219
|
+
*
|
|
220
|
+
* A delivery is never a decision: nothing here can deny, and the caller exits 0 whatever
|
|
221
|
+
* this returns. That is the whole difference between the two events, and it is why the
|
|
222
|
+
* prompt event has no `denyExit` — there is no exit code that means "here is some context".
|
|
223
|
+
*/
|
|
224
|
+
export function deliveryJson(row, text) {
|
|
225
|
+
return row.prompt ? row.prompt.context(text) : null;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** The document to print for a post-tool lesson recall delivery, or null when the harness takes none. */
|
|
229
|
+
export function postDeliveryJson(row, text) {
|
|
230
|
+
return row.post ? row.post.context(text) : null;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* The submitted text a prompt payload carries, or null when it carries none we recognise.
|
|
235
|
+
*
|
|
236
|
+
* Marked inferred rather than documented: both harnesses that take this event place the
|
|
237
|
+
* prompt at the top level, but only in the example on each page, not as a stated contract.
|
|
238
|
+
* An unrecognised payload therefore delivers nothing and says so on stderr. The failure
|
|
239
|
+
* this must not have is the opposite one — a packet delivered as though it answered a task
|
|
240
|
+
* it never saw, which reads as a confident answer about the wrong question.
|
|
241
|
+
*/
|
|
242
|
+
export function promptText(row, payload) {
|
|
243
|
+
if (!row.prompt) return null;
|
|
244
|
+
const text = payload?.prompt ?? payload?.user_prompt ?? payload?.message;
|
|
245
|
+
return typeof text === 'string' && text.trim().length > 0 ? text : null;
|
|
246
|
+
}
|
|
247
|
+
|
|
163
248
|
// ---------------------------------------------------------------------------
|
|
164
249
|
// The config file: reading ours out of somebody else's JSON, and putting it back.
|
|
165
250
|
// ---------------------------------------------------------------------------
|
|
@@ -180,9 +265,36 @@ function container(row, config, { create = false } = {}) {
|
|
|
180
265
|
return config.hooks ?? null;
|
|
181
266
|
}
|
|
182
267
|
|
|
268
|
+
/** The vendor's name for one of our events on this row, or null when it has none. */
|
|
269
|
+
function vendorEvent(row, event) {
|
|
270
|
+
if (event === EVENTS[0]) return row.event;
|
|
271
|
+
if (event === 'prompt-submit') return row.prompt?.event ?? null;
|
|
272
|
+
if (event === 'post-tool-use') return row.post?.event ?? null;
|
|
273
|
+
return null;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** The events a row answers, and the name each one goes under in its config file. */
|
|
277
|
+
function eventsOf(row, { prompt = false, post = false } = {}) {
|
|
278
|
+
const pairs = [[EVENTS[0], row.event]];
|
|
279
|
+
if (prompt && row.prompt) pairs.push(['prompt-submit', row.prompt.event]);
|
|
280
|
+
if (post && row.post) pairs.push(['post-tool-use', row.post.event]);
|
|
281
|
+
return pairs;
|
|
282
|
+
}
|
|
283
|
+
|
|
183
284
|
/** The group this tool owns for one event, in that harness's shape. */
|
|
184
|
-
function group(row) {
|
|
185
|
-
const command = hookCommand(row.id);
|
|
285
|
+
function group(row, event = EVENTS[0]) {
|
|
286
|
+
const command = hookCommand(row.id, event);
|
|
287
|
+
if (event === 'prompt-submit') {
|
|
288
|
+
// No matcher: both vendors document that field for tool events, and a matcher a
|
|
289
|
+
// harness ignores is a matcher the next reader believes is filtering something.
|
|
290
|
+
return { hooks: [{ type: 'command', command, timeout: TIMEOUT_SECONDS }] };
|
|
291
|
+
}
|
|
292
|
+
if (event === 'post-tool-use') {
|
|
293
|
+
const matcher = row.post?.matcher;
|
|
294
|
+
return matcher
|
|
295
|
+
? { matcher, hooks: [{ type: 'command', command, timeout: TIMEOUT_SECONDS }] }
|
|
296
|
+
: { hooks: [{ type: 'command', command, timeout: TIMEOUT_SECONDS }] };
|
|
297
|
+
}
|
|
186
298
|
if (row.id === 'cursor') {
|
|
187
299
|
// failClosed is set because a hooks file this tool wrote should not silently
|
|
188
300
|
// degrade to fail-open on its own crash: the vendor's default turns a broken hook
|
|
@@ -197,35 +309,44 @@ function ours(entry) {
|
|
|
197
309
|
return JSON.stringify(entry).includes(`${CLI} agent-hook`);
|
|
198
310
|
}
|
|
199
311
|
|
|
200
|
-
function entries(row, config) {
|
|
201
|
-
const
|
|
202
|
-
|
|
312
|
+
function entries(row, config, event = EVENTS[0]) {
|
|
313
|
+
const name = vendorEvent(row, event);
|
|
314
|
+
if (name === null) return [];
|
|
315
|
+
const list = container(row, config)?.[name];
|
|
203
316
|
return Array.isArray(list) ? list : [];
|
|
204
317
|
}
|
|
205
318
|
|
|
206
|
-
/** Our entries in this document, for `status`; the document is not modified. */
|
|
207
|
-
export function installedIn(row, config) {
|
|
208
|
-
return entries(row, config).filter(ours);
|
|
319
|
+
/** Our entries in this document for one event, for `status`; the document is not modified. */
|
|
320
|
+
export function installedIn(row, config, event = EVENTS[0]) {
|
|
321
|
+
return entries(row, config, event).filter(ours);
|
|
209
322
|
}
|
|
210
323
|
|
|
211
324
|
/**
|
|
212
|
-
* The document with this tool's
|
|
325
|
+
* The document with this tool's hooks added, everything else left alone.
|
|
213
326
|
*
|
|
214
327
|
* Written as JSON with two spaces and a trailing newline because that is what a file in
|
|
215
328
|
* a repository reads like; a config that is rewritten with different indentation on
|
|
216
329
|
* every run is a diff nobody can review. Keys this tool does not own are preserved
|
|
217
330
|
* exactly, including their order, because the file belongs to the repository.
|
|
331
|
+
*
|
|
332
|
+
* One file can carry both events — Claude and Codex each hold a tool hook and a prompt
|
|
333
|
+
* hook in the same document — so this writes one group per event the row answers rather
|
|
334
|
+
* than one group per file. `prompt` is the caller's opt-in: a repository that has not
|
|
335
|
+
* asked for a delivered packet gets the tool hook and nothing extra, and the file it never
|
|
336
|
+
* asked to change is a file this tool does not touch.
|
|
218
337
|
*/
|
|
219
|
-
function withHook(row, config) {
|
|
338
|
+
function withHook(row, config, { prompt = false, post = false } = {}) {
|
|
220
339
|
const box = container(row, config, { create: true });
|
|
221
|
-
const
|
|
222
|
-
|
|
340
|
+
for (const [event, name] of eventsOf(row, { prompt, post })) {
|
|
341
|
+
const kept = entries(row, config, event).filter((e) => !ours(e));
|
|
342
|
+
box[name] = [...kept, group(row, event)];
|
|
343
|
+
}
|
|
223
344
|
return config;
|
|
224
345
|
}
|
|
225
346
|
|
|
226
347
|
/** The text `install` would write, or the reason the existing file cannot be read. */
|
|
227
|
-
export function rendered(row, existing) {
|
|
228
|
-
if (existing === null) return { text: `${JSON.stringify(withHook(row, {}), null, 2)}\n`, action: 'created' };
|
|
348
|
+
export function rendered(row, existing, options = {}) {
|
|
349
|
+
if (existing === null) return { text: `${JSON.stringify(withHook(row, {}, options), null, 2)}\n`, action: 'created' };
|
|
229
350
|
let config;
|
|
230
351
|
try {
|
|
231
352
|
config = JSON.parse(existing);
|
|
@@ -235,6 +356,6 @@ export function rendered(row, existing) {
|
|
|
235
356
|
if (config === null || typeof config !== 'object' || Array.isArray(config)) {
|
|
236
357
|
return { error: 'not a JSON object; left alone' };
|
|
237
358
|
}
|
|
238
|
-
const text = `${JSON.stringify(withHook(row, config), null, 2)}\n`;
|
|
359
|
+
const text = `${JSON.stringify(withHook(row, config, options), null, 2)}\n`;
|
|
239
360
|
return { text, action: text === existing ? 'unchanged' : 'updated' };
|
|
240
361
|
}
|
package/src/argv.mjs
CHANGED
|
@@ -74,10 +74,28 @@ export const BOOL_FLAGS = new Set([
|
|
|
74
74
|
// Off by default because a path is the change the caller named, and the diff
|
|
75
75
|
// is a different question.
|
|
76
76
|
'changed',
|
|
77
|
+
// local doctor: allow non-loopback model URLs. Off by default because nothing leaves
|
|
78
|
+
// the machine unless explicitly overridden (ADR-075, plan 0003 R37).
|
|
79
|
+
'allow-remote',
|
|
80
|
+
// index / map / brief: opt into local vector embeddings configured under local.embed
|
|
81
|
+
// (plan 0003 R28, R37). Off by default unless recall@10 gains at least 10 points.
|
|
82
|
+
'embed',
|
|
83
|
+
// distill: regenerate a proposal that is already cached. Off by default because
|
|
84
|
+
// the cache is the point - the same source at the same budget is the same answer,
|
|
85
|
+
// and re-reading it is cheaper than re-deriving it (plan 0003 R38).
|
|
86
|
+
'refresh',
|
|
87
|
+
// distill: propose the summary from the configured local chat model instead of
|
|
88
|
+
// the deterministic outline. Off by default: no model in the default path (P1),
|
|
89
|
+
// and a model that is not asked for should not be opened against (ADR-090).
|
|
90
|
+
'local',
|
|
77
91
|
]);
|
|
78
92
|
|
|
79
93
|
export const VALUE_FLAGS = new Set([
|
|
80
94
|
'budget',
|
|
95
|
+
// bench critic: the labelled set of historical diffs to score (plan 0003 R39).
|
|
96
|
+
// A path, not a switch, because the labels themselves come from a human under
|
|
97
|
+
// ADR-008 — the tool never annotates for them.
|
|
98
|
+
'labels',
|
|
81
99
|
'max-bytes',
|
|
82
100
|
'paths',
|
|
83
101
|
'skill',
|
|
@@ -118,6 +136,8 @@ export const VALUE_FLAGS = new Set([
|
|
|
118
136
|
// map: outline (paths), names (exported symbols) or signatures (the line).
|
|
119
137
|
// A boolean would not say which of the three the caller wanted.
|
|
120
138
|
'level',
|
|
139
|
+
// decide: structured supersession spec (ADR-NNN or ADR-NNN:<ruling>[:<kind>], repeatable).
|
|
140
|
+
'supersedes',
|
|
121
141
|
]);
|
|
122
142
|
|
|
123
143
|
/**
|
|
@@ -133,7 +153,7 @@ export const VALUE_FLAGS = new Set([
|
|
|
133
153
|
* is a superset of the old behaviour, not a replacement: `--skill a,b` parsed before it
|
|
134
154
|
* parses now.
|
|
135
155
|
*/
|
|
136
|
-
export const REPEATABLE_FLAGS = new Set(['skill', 'arm', 'compare']);
|
|
156
|
+
export const REPEATABLE_FLAGS = new Set(['skill', 'arm', 'compare', 'supersedes']);
|
|
137
157
|
|
|
138
158
|
/**
|
|
139
159
|
* @param {string[]} argv raw arguments, excluding node and the script path
|
|
@@ -207,6 +227,13 @@ export function parse(argv) {
|
|
|
207
227
|
return { ok: true, value: { flags, positional } };
|
|
208
228
|
}
|
|
209
229
|
|
|
230
|
+
export function parsePathList(value) {
|
|
231
|
+
return String(value ?? '')
|
|
232
|
+
.split(',')
|
|
233
|
+
.map((s) => s.trim())
|
|
234
|
+
.filter(Boolean);
|
|
235
|
+
}
|
|
236
|
+
|
|
210
237
|
/** `--max-bytes` -> `maxBytes`. Callers use camelCase; users type kebab-case. */
|
|
211
238
|
function camel(name) {
|
|
212
239
|
return name.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
|
package/src/bench/grade.mjs
CHANGED
|
@@ -173,6 +173,7 @@ export async function gradeWorkspace(workDir, task, { vaultRoot, initialHead, ti
|
|
|
173
173
|
resolved: false,
|
|
174
174
|
gatesPassed: false,
|
|
175
175
|
diff,
|
|
176
|
+
goldFiles: [...goldFiles],
|
|
176
177
|
agentPatch: diffPatchRes.stdout,
|
|
177
178
|
};
|
|
178
179
|
}
|
|
@@ -183,6 +184,7 @@ export async function gradeWorkspace(workDir, task, { vaultRoot, initialHead, ti
|
|
|
183
184
|
resolved: false,
|
|
184
185
|
gatesPassed: false,
|
|
185
186
|
diff,
|
|
187
|
+
goldFiles: [...goldFiles],
|
|
186
188
|
agentPatch: diffPatchRes.stdout,
|
|
187
189
|
};
|
|
188
190
|
}
|
|
@@ -224,6 +226,7 @@ export async function gradeWorkspace(workDir, task, { vaultRoot, initialHead, ti
|
|
|
224
226
|
resolved,
|
|
225
227
|
gatesPassed,
|
|
226
228
|
diff,
|
|
229
|
+
goldFiles: [...goldFiles],
|
|
227
230
|
agentPatch: diffPatchRes.stdout,
|
|
228
231
|
};
|
|
229
232
|
}
|