simt-dev 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +157 -0
- package/README.md +271 -62
- package/package.json +6 -2
- package/rules/core.md +1 -1
- package/src/advice/advice.mjs +229 -0
- package/src/agenthooks/policy.mjs +10 -2
- package/src/agenthooks/prompt.mjs +81 -0
- package/src/agenthooks/render.mjs +138 -17
- package/src/argv.mjs +28 -1
- package/src/bench/grade.mjs +3 -0
- package/src/bench/run.mjs +88 -54
- package/src/bench/tasks.mjs +38 -0
- package/src/bench/variants.mjs +456 -0
- package/src/brief/assemble.mjs +368 -0
- package/src/brief/kind.mjs +134 -0
- package/src/brief/packet.mjs +200 -0
- package/src/commands/agenthook.mjs +84 -5
- package/src/commands/bench.mjs +72 -15
- package/src/commands/brief.mjs +78 -0
- package/src/commands/compare.mjs +10 -2
- package/src/commands/decide.mjs +116 -0
- package/src/commands/distill.mjs +36 -0
- package/src/commands/help.mjs +62 -3
- package/src/commands/index.mjs +29 -1
- package/src/commands/lessons.mjs +127 -0
- package/src/commands/local.mjs +97 -0
- package/src/commands/map.mjs +42 -7
- package/src/commands/repoindex.mjs +4 -1
- package/src/commands/review.mjs +65 -0
- package/src/commands/rules.mjs +17 -2
- package/src/commands/skills.mjs +5 -50
- package/src/commands/skillscompile.mjs +255 -0
- package/src/commands/sync.mjs +15 -0
- package/src/commands/tests.mjs +2 -49
- package/src/commands/why.mjs +23 -25
- package/src/compare/learn.mjs +680 -0
- package/src/decisions/index.mjs +116 -65
- package/src/distill/distill.mjs +246 -0
- package/src/distill/engine.mjs +187 -0
- package/src/federation/client.mjs +188 -0
- package/src/federation/config.mjs +134 -0
- package/src/federation/query.mjs +48 -0
- package/src/graph/compile.mjs +475 -0
- package/src/index/gate.mjs +1 -13
- package/src/index/govern.mjs +23 -11
- package/src/index/map.mjs +251 -17
- package/src/index/persist.mjs +74 -7
- package/src/index/store.mjs +222 -8
- package/src/kernel/variants.mjs +353 -0
- package/src/ledger/index.mjs +36 -1
- package/src/local/client.mjs +482 -0
- package/src/local/config.mjs +269 -0
- package/src/mcp/tools.mjs +138 -1
- package/src/memory/decide.mjs +462 -0
- package/src/memory/lessons.mjs +858 -0
- package/src/projection/lock.mjs +39 -7
- package/src/projection/rules.mjs +32 -8
- package/src/resolve/body.mjs +46 -0
- package/src/review/critic.mjs +509 -0
- package/src/toolwords.mjs +6 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
// The brief packet: one document inside a byte ceiling (plan 0003 R27).
|
|
2
|
+
//
|
|
3
|
+
// This module is pure. It reads no filesystem, no environment, no clock and no locale —
|
|
4
|
+
// a packet built twice from the same items is the same bytes, which is the property the
|
|
5
|
+
// accept arm needs and the one a test can check.
|
|
6
|
+
//
|
|
7
|
+
// The packing rule is the kernel's, not a second one invented here: items go to
|
|
8
|
+
// `packToBudget` in the order they should be LOST, and the class marked `hard` is the
|
|
9
|
+
// floor. A brief is not a packet silently cut to size; it is a packet that says, in the
|
|
10
|
+
// packet, what it did not get.
|
|
11
|
+
|
|
12
|
+
import { packToBudget } from '../kernel/budget.mjs';
|
|
13
|
+
|
|
14
|
+
/** The ceiling a brief packs to unless it is told otherwise. */
|
|
15
|
+
export const DEFAULT_BRIEF_BYTES = 2048;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The most this tool will ever inject into somebody else's prompt (plan 0003 R29).
|
|
19
|
+
*
|
|
20
|
+
* The same number as the default and a different fact: the default is what a packet packs
|
|
21
|
+
* to, and this is the bar the delivery slice was accepted against. A repository may lower
|
|
22
|
+
* its own ceiling; it may not raise it past a number that was measured, because a manifest
|
|
23
|
+
* that could raise it would make an unmeasured figure the promise the tool keeps.
|
|
24
|
+
*/
|
|
25
|
+
export const PROMPT_CAP_BYTES = 2048;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The sections, in the order a reader meets them.
|
|
29
|
+
*
|
|
30
|
+
* The order is fixed and identical in every packet. A reader who has to search for the
|
|
31
|
+
* command is paying the cost the packet exists to remove.
|
|
32
|
+
*/
|
|
33
|
+
const SECTIONS = ['TASK', 'ADVICE', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* What each section is worth when something has to go.
|
|
37
|
+
*
|
|
38
|
+
* The numbers are a decision, so they are written here where a reader can argue with
|
|
39
|
+
* them, rather than spread through the packing code as conditions. They answer one
|
|
40
|
+
* question: what would a reader rather lose? Cautions they can rediscover, then lessons,
|
|
41
|
+
* then skill pointers (one command answers all three), then decisions, then tests, then
|
|
42
|
+
* files. Commands and the task are why the packet exists.
|
|
43
|
+
*
|
|
44
|
+
* Federation sits with the decisions, and for the same reason: an external server's answer
|
|
45
|
+
* names a source a reader can go back to, so losing it costs a question rather than a fact.
|
|
46
|
+
* A section added later takes a value beside its neighbours rather than renumbering them —
|
|
47
|
+
* the numbers are a standing answer to "what would a reader rather lose", and a new section
|
|
48
|
+
* is not a reason to change what an old one is worth.
|
|
49
|
+
*/
|
|
50
|
+
const SECTION_VALUE = {
|
|
51
|
+
CAUTIONS: 1,
|
|
52
|
+
// Bench evidence, but advice rather than fact: losing it costs an opinion, not a
|
|
53
|
+
// command or a path to check (plan 0003 R40).
|
|
54
|
+
ADVICE: 2,
|
|
55
|
+
LESSONS: 2,
|
|
56
|
+
SKILLS: 3,
|
|
57
|
+
DECISIONS: 4,
|
|
58
|
+
FEDERATION: 4,
|
|
59
|
+
TESTS: 5,
|
|
60
|
+
FILES: 6,
|
|
61
|
+
COMMANDS: 7,
|
|
62
|
+
TASK: 8,
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The floor. An item marked `hard` is never cut, whatever the ceiling.
|
|
67
|
+
*
|
|
68
|
+
* The list is short on purpose: the task itself and the commands that say whether the
|
|
69
|
+
* change worked. Everything else is recoverable with one more command; these are the ones
|
|
70
|
+
* whose absence turns the packet into a quiet lie about what the repository needs.
|
|
71
|
+
*/
|
|
72
|
+
const HARD = 'hard';
|
|
73
|
+
const SOFT = 'soft';
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A packet under construction.
|
|
77
|
+
*
|
|
78
|
+
* Within a section the insertion order IS the relevance order — the packet loses from the
|
|
79
|
+
* end of a section — so a caller adds the most valuable item first instead of inventing a
|
|
80
|
+
* score. A number per item would be a second thing to keep consistent with the order, and
|
|
81
|
+
* the two would drift.
|
|
82
|
+
*/
|
|
83
|
+
export function collect() {
|
|
84
|
+
const items = [];
|
|
85
|
+
return {
|
|
86
|
+
items,
|
|
87
|
+
/**
|
|
88
|
+
* Add one line. An empty or non-string line is not added: a blank item is a byte the
|
|
89
|
+
* packet spends saying nothing.
|
|
90
|
+
*/
|
|
91
|
+
add(section, text, { hard = false, why = [] } = {}) {
|
|
92
|
+
if (!SECTIONS.includes(section)) throw new Error(`unknown brief section ${section}`);
|
|
93
|
+
if (typeof text !== 'string' || text.length === 0) return this;
|
|
94
|
+
const order = items.length;
|
|
95
|
+
items.push({ id: String(order), section, text, priority: hard ? HARD : SOFT, order, why });
|
|
96
|
+
return this;
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The order items are lost in: least valuable first.
|
|
103
|
+
*
|
|
104
|
+
* Between sections by `SECTION_VALUE`; inside a section the last one added goes first.
|
|
105
|
+
* `hard` items sort to the end, where `packToBudget` refuses to cut them. The order is
|
|
106
|
+
* made total by the insertion index, because a comparator that returns 0 for two
|
|
107
|
+
* different items leaves the result to the sort implementation — a determinism hole in a
|
|
108
|
+
* command whose whole claim is determinism.
|
|
109
|
+
*/
|
|
110
|
+
function cutOrder(items) {
|
|
111
|
+
const ranked = [...items].sort((a, b) => {
|
|
112
|
+
if (a.priority !== b.priority) return a.priority === HARD ? 1 : -1;
|
|
113
|
+
const value = SECTION_VALUE[a.section] - SECTION_VALUE[b.section];
|
|
114
|
+
if (value !== 0) return value;
|
|
115
|
+
return b.order - a.order;
|
|
116
|
+
});
|
|
117
|
+
return ranked.map(({ id, priority }) => ({ id, priority }));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Render the packet, minus the items that were cut.
|
|
122
|
+
*
|
|
123
|
+
* A section with nothing left prints nothing at all, not an empty heading. The `cut` line
|
|
124
|
+
* is part of the document rather than a footnote, because the reader is the one who has to
|
|
125
|
+
* decide whether to ask for more — and a packet that shrank without saying so is the
|
|
126
|
+
* failure this command was written to avoid.
|
|
127
|
+
*/
|
|
128
|
+
function renderPacket(items, { cut = new Set(), budget = DEFAULT_BRIEF_BYTES } = {}) {
|
|
129
|
+
const lines = [];
|
|
130
|
+
for (const section of SECTIONS) {
|
|
131
|
+
const kept = items.filter((item) => item.section === section && !cut.has(item.id));
|
|
132
|
+
if (kept.length === 0) continue;
|
|
133
|
+
lines.push(section);
|
|
134
|
+
for (const item of kept) for (const line of item.text.split('\n')) lines.push(` ${line}`);
|
|
135
|
+
}
|
|
136
|
+
const lost = items.filter((item) => cut.has(item.id));
|
|
137
|
+
if (lost.length > 0) {
|
|
138
|
+
const where = SECTIONS.filter((section) => lost.some((item) => item.section === section)).map(
|
|
139
|
+
(section) => `${section.toLowerCase()} ${lost.filter((item) => item.section === section).length}`,
|
|
140
|
+
);
|
|
141
|
+
lines.push(`cut ${lost.length} of ${items.length} items to fit ${budget}: ${where.join(', ')}`);
|
|
142
|
+
}
|
|
143
|
+
return lines.join('\n');
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The byte length of a rendering. The ceiling is measured in these, not in characters. */
|
|
147
|
+
function packetBytes(items, { cut = new Set(), budget = DEFAULT_BRIEF_BYTES } = {}) {
|
|
148
|
+
return Buffer.byteLength(renderPacket(items, { cut, budget }));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Pack the items into a packet.
|
|
153
|
+
*
|
|
154
|
+
* Returns the text, the byte count before and after, and whether the floor itself is over
|
|
155
|
+
* the line. `floorHeld` is reported to the caller rather than printed: it is the one fact
|
|
156
|
+
* the packet cannot state about itself without the statement changing the thing it
|
|
157
|
+
* describes, so it belongs to `--explain` and the JSON payload.
|
|
158
|
+
*
|
|
159
|
+
* A budget of 0 means no ceiling at all — the reader asked for the whole packet — and the
|
|
160
|
+
* packing step is skipped rather than run against a number that means "unlimited".
|
|
161
|
+
*/
|
|
162
|
+
export function buildPacket(items, { budget = DEFAULT_BRIEF_BYTES } = {}) {
|
|
163
|
+
const before = packetBytes(items, { budget });
|
|
164
|
+
if (!(budget > 0)) {
|
|
165
|
+
return { cut: [], before, after: before, floorHeld: false, text: renderPacket(items, { budget }) };
|
|
166
|
+
}
|
|
167
|
+
const result = packToBudget({
|
|
168
|
+
candidates: cutOrder(items),
|
|
169
|
+
ceiling: budget,
|
|
170
|
+
costs: (cut) => packetBytes(items, { cut: new Set(cut), budget }),
|
|
171
|
+
});
|
|
172
|
+
const cut = new Set(result.cut);
|
|
173
|
+
return { ...result, text: renderPacket(items, { cut, budget }) };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* One line per item, for `--explain`: what was kept, what was cut, and the two facts that
|
|
178
|
+
* decided it.
|
|
179
|
+
*
|
|
180
|
+
* The explanation is not part of the packet and is never measured against the ceiling —
|
|
181
|
+
* it exists so a reader can tell a packet that is complete from one that was trimmed,
|
|
182
|
+
* which is the question the packet's own byte count cannot answer for them.
|
|
183
|
+
*/
|
|
184
|
+
export function explainPacket(items, { cut = new Set() } = {}) {
|
|
185
|
+
const counts = new Map();
|
|
186
|
+
for (const item of items) counts.set(item.section, (counts.get(item.section) ?? 0) + 1);
|
|
187
|
+
const seen = new Map();
|
|
188
|
+
const lines = [];
|
|
189
|
+
for (const item of items) {
|
|
190
|
+
const position = (seen.get(item.section) ?? 0) + 1;
|
|
191
|
+
seen.set(item.section, position);
|
|
192
|
+
const verdict = cut.has(item.id) ? 'cut' : 'keep';
|
|
193
|
+
const reason = item.why.length > 0 ? item.why.join('; ') : 'carried by its section';
|
|
194
|
+
lines.push(
|
|
195
|
+
`${verdict} ${item.section} ${position}/${counts.get(item.section)} worth ${SECTION_VALUE[item.section]}: ${item.text.split('\n')[0]}`,
|
|
196
|
+
);
|
|
197
|
+
lines.push(` ${reason}`);
|
|
198
|
+
}
|
|
199
|
+
return lines;
|
|
200
|
+
}
|
|
@@ -11,6 +11,12 @@
|
|
|
11
11
|
// walks the repository. A hook that measured the tree before answering would be exactly
|
|
12
12
|
// the latency the plan's acceptance criterion exists to prevent.
|
|
13
13
|
//
|
|
14
|
+
// A second event, `prompt-submit`, delivers rather than decides: it needs the manifest,
|
|
15
|
+
// the index and the packet assembly, so it is reached through a dynamic `import()` that
|
|
16
|
+
// only that event takes. A tool call therefore still pays for nothing that walks the
|
|
17
|
+
// repository, and the one path that does pays for it once per prompt instead of once per
|
|
18
|
+
// command.
|
|
19
|
+
//
|
|
14
20
|
// A denial exits 2, which is what every harness in the table reads as "block". A mistake
|
|
15
21
|
// in the hook's own invocation must NOT exit 2: a typo in a config would then block every
|
|
16
22
|
// tool call. Failures of this command exit 1, which a harness reports as a failed hook,
|
|
@@ -60,6 +66,32 @@ export async function agentHookCommand(args, flags) {
|
|
|
60
66
|
return EXIT.OK;
|
|
61
67
|
}
|
|
62
68
|
|
|
69
|
+
// Delivery, not decision: the packet goes into the prompt and the exit code stays 0
|
|
70
|
+
// whatever happens. The module behind it is the one thing here that reads the
|
|
71
|
+
// repository, and the tool-call path above must not pay for it, so the door is a
|
|
72
|
+
// dynamic import that only this event opens.
|
|
73
|
+
if (event === 'prompt-submit') {
|
|
74
|
+
const { deliverPrompt } = await import('../agenthooks/prompt.mjs');
|
|
75
|
+
const delivered = await deliverPrompt(row, payload);
|
|
76
|
+
if (delivered.skipped) {
|
|
77
|
+
note(`note ${id}: no packet delivered (${delivered.skipped})`);
|
|
78
|
+
return EXIT.OK;
|
|
79
|
+
}
|
|
80
|
+
emit(JSON.stringify(delivered.json));
|
|
81
|
+
return EXIT.OK;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (event === 'post-tool-use') {
|
|
85
|
+
const { deliverLessonRecall } = await import('../memory/lessons.mjs');
|
|
86
|
+
const recalled = await deliverLessonRecall(row, payload, { root: flags.root });
|
|
87
|
+
if (recalled.skipped) {
|
|
88
|
+
note(`note ${id}: no lesson recalled (${recalled.skipped})`);
|
|
89
|
+
return EXIT.OK;
|
|
90
|
+
}
|
|
91
|
+
emit(JSON.stringify(recalled.json));
|
|
92
|
+
return EXIT.OK;
|
|
93
|
+
}
|
|
94
|
+
|
|
63
95
|
const { root, command, writes } = factsFrom(row, payload);
|
|
64
96
|
const facts = await factsFor(root, { command, writes });
|
|
65
97
|
const decision = decide({ event, facts, env: process.env });
|
|
@@ -77,6 +109,22 @@ export async function agentHookCommand(args, flags) {
|
|
|
77
109
|
const json = decisionJson(row, decision);
|
|
78
110
|
if (json === null) return EXIT.OK;
|
|
79
111
|
|
|
112
|
+
if (root) {
|
|
113
|
+
try {
|
|
114
|
+
const { recordHookDenial } = await import('../memory/lessons.mjs');
|
|
115
|
+
await recordHookDenial(root, {
|
|
116
|
+
harness: row.id,
|
|
117
|
+
event,
|
|
118
|
+
policies: decision.blocked,
|
|
119
|
+
messages: decision.messages,
|
|
120
|
+
command,
|
|
121
|
+
writes,
|
|
122
|
+
});
|
|
123
|
+
} catch {
|
|
124
|
+
// Recording a lesson candidate must never block or alter the hook decision.
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
80
128
|
emit(JSON.stringify(json));
|
|
81
129
|
for (const message of decision.messages) note(`blocked ${message}`);
|
|
82
130
|
return row.denyExit;
|
|
@@ -91,14 +139,37 @@ export async function agentHooksCommand(sub, args, flags) {
|
|
|
91
139
|
|
|
92
140
|
async function installHooks(root, flags) {
|
|
93
141
|
const mode = writeMode(flags);
|
|
142
|
+
// The delivery is a second entry in the same config, and a repository asks for it by
|
|
143
|
+
// declaring a `brief` key rather than by having the entry. Read through a dynamic
|
|
144
|
+
// import for the same reason the hook delivers through one.
|
|
145
|
+
const { promptOptIn } = await import('../agenthooks/prompt.mjs');
|
|
146
|
+
const { listPromotedLessons } = await import('../memory/lessons.mjs');
|
|
147
|
+
const [optIn, promotedLessons] = await Promise.all([
|
|
148
|
+
promptOptIn(root),
|
|
149
|
+
listPromotedLessons(root).catch(() => []),
|
|
150
|
+
]);
|
|
151
|
+
const hasLessons = promotedLessons.length > 0;
|
|
94
152
|
const plan = [];
|
|
95
153
|
for (const row of adapters()) {
|
|
96
154
|
const path = hookPath(row.id);
|
|
97
155
|
const existing = await readIfExists(join(root, path));
|
|
98
|
-
const result = rendered(row, existing);
|
|
99
|
-
plan.push({
|
|
156
|
+
const result = rendered(row, existing, { prompt: optIn.optedIn, post: hasLessons });
|
|
157
|
+
plan.push({
|
|
158
|
+
id: row.id,
|
|
159
|
+
path,
|
|
160
|
+
action: result.error ? 'refused' : result.action,
|
|
161
|
+
error: result.error,
|
|
162
|
+
text: result.text,
|
|
163
|
+
prompt: row.prompt === null || row.prompt === undefined ? null : optIn.optedIn ? 'declared' : 'not declared',
|
|
164
|
+
});
|
|
100
165
|
}
|
|
101
166
|
|
|
167
|
+
// One line for the repository, not one per adapter: the opt-in is a property of the
|
|
168
|
+
// repository, and someone who has not declared it is told how once.
|
|
169
|
+
const promptLine = plan.some((p) => p.prompt !== null) && !optIn.optedIn
|
|
170
|
+
? [`PROMPT not opted in (${optIn.why}); add "brief": {} to the manifest to deliver the packet on every prompt`]
|
|
171
|
+
: [];
|
|
172
|
+
|
|
102
173
|
// The readers this tool does not write an entry for, reported rather than skipped in
|
|
103
174
|
// silence: a command that writes four files and says nothing about the fifth harness
|
|
104
175
|
// in the repository reads as "that one is covered too".
|
|
@@ -128,6 +199,7 @@ async function installHooks(root, flags) {
|
|
|
128
199
|
`repo ${root}`,
|
|
129
200
|
...plan.map((p) => `${p.action.toUpperCase().padEnd(9)} ${p.id.padEnd(12)} ${p.path}${p.error ? ` (${p.error})` : ''}`),
|
|
130
201
|
...others.map((o) => `${o.state === 'none' ? 'NO FILE ' : 'NOT OURS'} ${o.id.padEnd(12)} ${o.path ?? '(no hooks file is documented)'}`),
|
|
202
|
+
...promptLine,
|
|
131
203
|
'',
|
|
132
204
|
refused.length > 0
|
|
133
205
|
? `REFUSED ${refused.length} config file(s) could not be read; nothing written`
|
|
@@ -177,6 +249,7 @@ async function installHooks(root, flags) {
|
|
|
177
249
|
`repo ${root}`,
|
|
178
250
|
...plan.map((p) => `${p.action.toUpperCase().padEnd(9)} ${p.id.padEnd(12)} ${p.path}`),
|
|
179
251
|
...others.map((o) => `${o.state === 'none' ? 'NO FILE ' : 'NOT OURS'} ${o.id.padEnd(12)} ${o.path ?? '(no hooks file is documented)'}`),
|
|
252
|
+
...promptLine,
|
|
180
253
|
'',
|
|
181
254
|
`WROTE ${written.length} hooks file(s); each one runs ${CLI} agent-hook before the tool call`,
|
|
182
255
|
].join('\n'),
|
|
@@ -196,14 +269,14 @@ async function hooksStatus(root, flags) {
|
|
|
196
269
|
const path = hookPath(row.id);
|
|
197
270
|
const existing = await readIfExists(join(root, path));
|
|
198
271
|
if (existing === null) {
|
|
199
|
-
rows.push({ id: row.id, path, state: 'absent', entries: 0, note: null });
|
|
272
|
+
rows.push({ id: row.id, path, state: 'absent', entries: 0, prompt: row.prompt ? 'absent' : null, note: null });
|
|
200
273
|
continue;
|
|
201
274
|
}
|
|
202
275
|
let config;
|
|
203
276
|
try {
|
|
204
277
|
config = JSON.parse(existing);
|
|
205
278
|
} catch (e) {
|
|
206
|
-
rows.push({ id: row.id, path, state: 'unreadable', entries: 0, note: e.message });
|
|
279
|
+
rows.push({ id: row.id, path, state: 'unreadable', entries: 0, prompt: row.prompt ? 'unknown' : null, note: e.message });
|
|
207
280
|
problems.push(`${path} is not valid JSON: ${e.message}`);
|
|
208
281
|
continue;
|
|
209
282
|
}
|
|
@@ -211,16 +284,18 @@ async function hooksStatus(root, flags) {
|
|
|
211
284
|
// refuses to write into one. Status has to agree with it, or one question gets two
|
|
212
285
|
// answers and the quiet one is the one a reader acts on.
|
|
213
286
|
if (config === null || typeof config !== 'object' || Array.isArray(config)) {
|
|
214
|
-
rows.push({ id: row.id, path, state: 'unreadable', entries: 0, note: 'not a JSON object' });
|
|
287
|
+
rows.push({ id: row.id, path, state: 'unreadable', entries: 0, prompt: row.prompt ? 'unknown' : null, note: 'not a JSON object' });
|
|
215
288
|
problems.push(`${path} is not a JSON object, so this tool will not write into it`);
|
|
216
289
|
continue;
|
|
217
290
|
}
|
|
218
291
|
const found = installedIn(row, config).length;
|
|
292
|
+
const prompts = row.prompt ? installedIn(row, config, EVENTS[1]).length : 0;
|
|
219
293
|
rows.push({
|
|
220
294
|
id: row.id,
|
|
221
295
|
path,
|
|
222
296
|
state: found > 0 ? 'present' : 'absent',
|
|
223
297
|
entries: found,
|
|
298
|
+
prompt: row.prompt ? (prompts > 0 ? 'present' : 'absent') : null,
|
|
224
299
|
note: found > 0 ? (row.trust ?? null) : null,
|
|
225
300
|
});
|
|
226
301
|
}
|
|
@@ -243,6 +318,10 @@ async function hooksStatus(root, flags) {
|
|
|
243
318
|
const state = { present: 'PRESENT ', absent: 'ABSENT ', unreadable: 'INVALID ', none: 'NO FILE ', other: 'NOT OURS' }[r.state];
|
|
244
319
|
lines.push(`${state} ${r.id.padEnd(12)} ${r.path ?? '(no hooks file is documented)'}`);
|
|
245
320
|
if (r.state === 'absent') lines.push(` ${' '.repeat(12)} install: ${CLI} agent-hooks install --yes`);
|
|
321
|
+
// The second event is a separate answer to a separate question: a harness can carry
|
|
322
|
+
// the tool hook and not the delivery, and a reader who is told "PRESENT" would
|
|
323
|
+
// otherwise conclude the packet arrives on every prompt.
|
|
324
|
+
if (r.prompt) lines.push(` ${' '.repeat(12)} prompt-submit: ${r.prompt}`);
|
|
246
325
|
if (r.note) lines.push(` ${' '.repeat(12)} ${r.note}`);
|
|
247
326
|
}
|
|
248
327
|
if (problems.length > 0) {
|
package/src/commands/bench.mjs
CHANGED
|
@@ -15,6 +15,8 @@ import { buildBenchReport } from '../bench/report.mjs';
|
|
|
15
15
|
import { formatRecall, measureRecall } from '../index/map.mjs';
|
|
16
16
|
import { ablateBench } from '../bench/ablate.mjs';
|
|
17
17
|
import { runBenchProbes } from '../bench/probes.mjs';
|
|
18
|
+
import { selectRuleVariants } from '../bench/variants.mjs';
|
|
19
|
+
import { criticBench } from '../review/critic.mjs';
|
|
18
20
|
import { listFlag } from './skills.mjs';
|
|
19
21
|
|
|
20
22
|
export async function benchMineCommand(positional, flags) {
|
|
@@ -280,16 +282,8 @@ export async function benchAblateCommand(positional, flags) {
|
|
|
280
282
|
emitJson({
|
|
281
283
|
ok: true,
|
|
282
284
|
command: 'bench ablate',
|
|
283
|
-
root: result.root,
|
|
284
285
|
dryRun,
|
|
285
|
-
|
|
286
|
-
pricesLoaded: result.pricesLoaded,
|
|
287
|
-
canarySize: result.canarySize,
|
|
288
|
-
canaryTasks: result.canaryTasks,
|
|
289
|
-
totalTasks: result.totalTasks,
|
|
290
|
-
sections: result.sections,
|
|
291
|
-
ranking: result.ranking,
|
|
292
|
-
deletionCandidates: result.deletionCandidates,
|
|
286
|
+
...result,
|
|
293
287
|
});
|
|
294
288
|
return EXIT.OK;
|
|
295
289
|
}
|
|
@@ -340,13 +334,8 @@ export async function benchProbesCommand(positional, flags) {
|
|
|
340
334
|
emitJson({
|
|
341
335
|
ok: true,
|
|
342
336
|
command: 'bench probes',
|
|
343
|
-
root: result.root,
|
|
344
337
|
dryRun,
|
|
345
|
-
|
|
346
|
-
probeCount: result.probeCount,
|
|
347
|
-
probes: result.probes,
|
|
348
|
-
arms: result.arms,
|
|
349
|
-
runs: result.runs,
|
|
338
|
+
...result,
|
|
350
339
|
});
|
|
351
340
|
return EXIT.OK;
|
|
352
341
|
}
|
|
@@ -367,5 +356,73 @@ export async function benchProbesCommand(positional, flags) {
|
|
|
367
356
|
return EXIT.OK;
|
|
368
357
|
}
|
|
369
358
|
|
|
359
|
+
export async function benchVariantsCommand(positional, flags) {
|
|
360
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
361
|
+
const { dryRun, apply } = writeMode(flags);
|
|
362
|
+
let result;
|
|
363
|
+
try {
|
|
364
|
+
result = await selectRuleVariants(root, {
|
|
365
|
+
apply,
|
|
366
|
+
harness: flags.harness ?? 'gold',
|
|
367
|
+
model: flags.model,
|
|
368
|
+
reps: flags.reps ?? 1,
|
|
369
|
+
includeAll: flags.all === true,
|
|
370
|
+
});
|
|
371
|
+
} catch (error) {
|
|
372
|
+
return fail(
|
|
373
|
+
`${root}: ${error.message ?? error}`,
|
|
374
|
+
EXIT.CONFIG,
|
|
375
|
+
`mine tasks first with \`${CLI} bench mine --yes\` and rerun \`${CLI} bench variants --yes\``,
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
if (isJson(flags)) {
|
|
379
|
+
emitJson({ ok: true, command: 'bench variants', dryRun, ...result });
|
|
380
|
+
return EXIT.OK;
|
|
381
|
+
}
|
|
382
|
+
const lines = [
|
|
383
|
+
`bench variants ${result.root}`,
|
|
384
|
+
`mode ${result.applied ? 'applied' : dryRun ? 'dry-run (writes nothing)' : 'preview / stored runs (pass --yes to execute canary and lock winners)'}`,
|
|
385
|
+
`harness ${result.harness} model=${result.model}`,
|
|
386
|
+
`slots ${result.slots.length} (canary: ${result.canaryTasks}/${result.totalTasks} tasks)`,
|
|
387
|
+
...result.slots.map(
|
|
388
|
+
(s) => `slot ${s.slotId} status=${s.status} winner=${s.winner} dR=${s.dR} dC=${s.dC} n=${s.n} p=${s.p}`,
|
|
389
|
+
),
|
|
390
|
+
];
|
|
391
|
+
if (!result.applied) lines.push(`remedy run: ${CLI} bench variants --yes`);
|
|
392
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
393
|
+
return EXIT.OK;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
|
|
370
398
|
|
|
399
|
+
// `simt bench critic` (plan 0003 R39).
|
|
400
|
+
//
|
|
401
|
+
// The row's acceptance measured, or reported missing rather than inferred: the
|
|
402
|
+
// critic runs over a labelled set of historical diffs and the bench reports
|
|
403
|
+
// precision and recall for the BLOCK binary. The metric never gates the critic —
|
|
404
|
+
// the row says precision >= 0.8 on >= 30 labelled diffs promotes it to a gate,
|
|
405
|
+
// and until a bench here shows that, every report calls the critic advisory.
|
|
406
|
+
export async function benchCriticCommand(positional, flags, commands = []) {
|
|
407
|
+
const root = resolve(flags.root ?? positional[0] ?? process.cwd());
|
|
408
|
+
const labelsPath = flags.labels ?? '.simt-dev/critic-labels.jsonl';
|
|
409
|
+
const result = await criticBench(root, { labelsPath, flags, commands });
|
|
410
|
+
if (typeof result === 'number') return result;
|
|
371
411
|
|
|
412
|
+
if (isJson(flags)) {
|
|
413
|
+
emitJson(result, { pretty: flags.pretty === true, maxBytes: flags.maxBytes ?? 0 });
|
|
414
|
+
return EXIT.OK;
|
|
415
|
+
}
|
|
416
|
+
const lines = [
|
|
417
|
+
`bench critic ${root}`,
|
|
418
|
+
`labels ${result.labels} (${labelsPath}) measured ${result.measured} errors ${result.errors.length}`,
|
|
419
|
+
`counts tp=${result.tp} fp=${result.fp} fn=${result.fn} tn=${result.tn}`,
|
|
420
|
+
`precision ${result.precision === null ? 'unmeasured' : result.precision.toFixed(3)}`,
|
|
421
|
+
`recall ${result.recall === null ? 'unmeasured' : result.recall.toFixed(3)}`,
|
|
422
|
+
`gated ${result.gated ? 'yes' : 'no'}`,
|
|
423
|
+
];
|
|
424
|
+
if (result.labelsNote) lines.push(`labels ${result.labelsNote}`);
|
|
425
|
+
for (const error of result.errors) lines.push(`error ${error}`);
|
|
426
|
+
emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
|
|
427
|
+
return EXIT.OK;
|
|
428
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// `simt brief "<task>"` — print the packet (plan 0003 R27).
|
|
2
|
+
//
|
|
3
|
+
// What this is for: an agent that has to explore a repository to find out how to run its
|
|
4
|
+
// tests, which files the task will touch and what governs them spends turns discovering
|
|
5
|
+
// what the repository already knows. The packet is that answer, assembled from the index
|
|
6
|
+
// the repository already has, in the bytes a prompt can carry.
|
|
7
|
+
//
|
|
8
|
+
// The assembly itself is in `../brief/assemble.mjs`, because three doors open onto it —
|
|
9
|
+
// this command, MCP's `brief` tool and the prompt-submit hook — and one implementation
|
|
10
|
+
// behind three doors is the only way the three cannot disagree. This file owns the
|
|
11
|
+
// command line: what a task may be, what the ceiling may be, and how the packet is
|
|
12
|
+
// rendered for a terminal or for `--json`.
|
|
13
|
+
//
|
|
14
|
+
// One thing is deliberately missing: `skills brief` keeps its own behaviour, and MCP's
|
|
15
|
+
// `skills_brief` keeps its own shape. Superseding them changes two accepted decisions
|
|
16
|
+
// (ADR-035, ADR-052) and the MCP contract, and this slice carries no ADR to do it.
|
|
17
|
+
|
|
18
|
+
import { CLI } from '../name.mjs';
|
|
19
|
+
import { EXIT, emit, emitJson, fail, isJson } from '../output/index.mjs';
|
|
20
|
+
import { DEFAULT_BRIEF_BYTES, explainPacket } from '../brief/packet.mjs';
|
|
21
|
+
import { assembleBrief } from '../brief/assemble.mjs';
|
|
22
|
+
|
|
23
|
+
export async function briefCommand(args, flags) {
|
|
24
|
+
if (args.length > 1) {
|
|
25
|
+
return fail(`brief takes one task, not ${args.length}`, EXIT.USAGE, `run: ${CLI} brief "<task>"`);
|
|
26
|
+
}
|
|
27
|
+
if (args.length === 1 && flags.task !== undefined) {
|
|
28
|
+
return fail('the task was passed twice, as an argument and as --task', EXIT.USAGE, `run: ${CLI} brief "<task>"`);
|
|
29
|
+
}
|
|
30
|
+
const budget = flags.budget === undefined ? DEFAULT_BRIEF_BYTES : Number(flags.budget);
|
|
31
|
+
if (!Number.isInteger(budget) || budget < 0) {
|
|
32
|
+
return fail(`--budget must be a byte count, not ${flags.budget}`, EXIT.USAGE, `run: ${CLI} brief --budget ${DEFAULT_BRIEF_BYTES}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const result = await assembleBrief({
|
|
36
|
+
root: flags.root ?? process.cwd(),
|
|
37
|
+
task: String(args[0] ?? flags.task ?? ''),
|
|
38
|
+
paths: flags.paths === undefined ? null : flags.paths,
|
|
39
|
+
budget,
|
|
40
|
+
flags,
|
|
41
|
+
});
|
|
42
|
+
if (result.error) return result.error;
|
|
43
|
+
// Unreachable from here — only a delivered brief asks for a cold index to be reported
|
|
44
|
+
// rather than built — but a silent empty packet would be the worst way to find out.
|
|
45
|
+
if (result.cold) {
|
|
46
|
+
return fail('the index is empty, and this command will not assemble a packet from it', EXIT.UNSUPPORTED, `run ${CLI} index`);
|
|
47
|
+
}
|
|
48
|
+
const { root, kind, basis, task, scope, budget: ceiling, items, packed, sections } = result;
|
|
49
|
+
const cut = new Set(packed.cut);
|
|
50
|
+
|
|
51
|
+
if (isJson(flags)) {
|
|
52
|
+
emitJson(
|
|
53
|
+
{
|
|
54
|
+
ok: true,
|
|
55
|
+
command: 'brief',
|
|
56
|
+
root,
|
|
57
|
+
kind,
|
|
58
|
+
basis,
|
|
59
|
+
task,
|
|
60
|
+
scope,
|
|
61
|
+
budget: ceiling,
|
|
62
|
+
bytes: packed.after,
|
|
63
|
+
before: packed.before,
|
|
64
|
+
floorHeld: packed.floorHeld,
|
|
65
|
+
cut: packed.cut,
|
|
66
|
+
sections,
|
|
67
|
+
explain: flags.explain === true ? explainPacket(items, { cut }) : undefined,
|
|
68
|
+
text: packed.text,
|
|
69
|
+
},
|
|
70
|
+
{ pretty: flags.pretty === true, maxBytes: Number(flags.maxBytes ?? 0) || 0 },
|
|
71
|
+
);
|
|
72
|
+
return EXIT.OK;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const body = flags.explain === true ? [packed.text, '', 'EXPLAIN', ...explainPacket(items, { cut })] : [packed.text];
|
|
76
|
+
emit(body.join('\n'), { maxBytes: Number(flags.maxBytes ?? 0) || 0 });
|
|
77
|
+
return EXIT.OK;
|
|
78
|
+
}
|
package/src/commands/compare.mjs
CHANGED
|
@@ -26,6 +26,7 @@ import { provenanceDigest } from '../projection/project.mjs';
|
|
|
26
26
|
import { GATES } from '../hooks/gates.mjs';
|
|
27
27
|
import { buildRow, summarise } from '../compare/row.mjs';
|
|
28
28
|
import { discover } from '../compare/discover.mjs';
|
|
29
|
+
import { learnAcrossEstate, renderLearningLines } from '../compare/learn.mjs';
|
|
29
30
|
import { targetRoot, WORKSPACE_REMEDY } from '../workspace.mjs';
|
|
30
31
|
|
|
31
32
|
/** Gate kinds, so a row can tell a command gate with no command from a script gate. */
|
|
@@ -65,6 +66,7 @@ export async function compareCommand(positional, flags) {
|
|
|
65
66
|
// bound. Determinism of the output matters more than wall clock on a dashboard.
|
|
66
67
|
const rows = [];
|
|
67
68
|
for (const repo of found.repos) rows.push(await buildRow(repo, context));
|
|
69
|
+
const learning = await learnAcrossEstate(found.repos);
|
|
68
70
|
|
|
69
71
|
const summary = summarise(rows, {
|
|
70
72
|
root,
|
|
@@ -106,6 +108,7 @@ export async function compareCommand(positional, flags) {
|
|
|
106
108
|
shas: summary.shas,
|
|
107
109
|
},
|
|
108
110
|
rows,
|
|
111
|
+
learning,
|
|
109
112
|
worktrees: found.worktrees.map((w) => ({
|
|
110
113
|
name: w.name,
|
|
111
114
|
path: w.path,
|
|
@@ -117,7 +120,7 @@ export async function compareCommand(positional, flags) {
|
|
|
117
120
|
return rows.some((r) => r.problems.length > 0) ? EXIT.FAIL : EXIT.OK;
|
|
118
121
|
}
|
|
119
122
|
|
|
120
|
-
emit(render(rows, summary, found), { maxBytes: flags.maxBytes ?? 0 });
|
|
123
|
+
emit(render(rows, summary, found, learning), { maxBytes: flags.maxBytes ?? 0 });
|
|
121
124
|
if (summary.shape === 'repo') {
|
|
122
125
|
note('note one repository compared; pass --root at a workspace directory to compare its siblings');
|
|
123
126
|
} else if (target.source === 'none') {
|
|
@@ -146,7 +149,7 @@ export async function compareCommand(positional, flags) {
|
|
|
146
149
|
* the longest value, and an agent cannot tell which column is which without
|
|
147
150
|
* counting spaces.
|
|
148
151
|
*/
|
|
149
|
-
function render(rows, summary, found) {
|
|
152
|
+
function render(rows, summary, found, learning = null) {
|
|
150
153
|
const lines = [
|
|
151
154
|
// `workspace` and `repo` are the same field with different names, and the name is the
|
|
152
155
|
// honest one: a reader who sees "workspace" and gets one repo was told a lie. `root`
|
|
@@ -204,6 +207,11 @@ function render(rows, summary, found) {
|
|
|
204
207
|
lines.push('');
|
|
205
208
|
}
|
|
206
209
|
|
|
210
|
+
const learningLines = renderLearningLines(learning);
|
|
211
|
+
if (learningLines.length > 0) {
|
|
212
|
+
lines.push(...learningLines, '');
|
|
213
|
+
}
|
|
214
|
+
|
|
207
215
|
return lines.join('\n').trimEnd();
|
|
208
216
|
}
|
|
209
217
|
|