simt-dev 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +170 -0
- package/README.md +300 -82
- package/docs/decisions/ADR-058-one-tool-three-channels-and-the-order-is-forced.md +13 -0
- package/docs/decisions/ADR-059-the-version-a-repository-runs-is-the-one-it-pinned.md +1 -1
- package/docs/decisions/ADR-097-the-channels-publish-by-hand-and-the-version-rides-in-as-an-input.md +71 -0
- package/docs/decisions/ADR-098-the-fourth-channel-is-one-file-and-it-needs-no-toolchain.md +149 -0
- package/package.json +8 -4
- package/src/advice/advice.mjs +229 -0
- package/src/agenthooks/render.mjs +87 -21
- package/src/argv.mjs +25 -0
- package/src/bench/ablate.mjs +1 -3
- package/src/bench/grade.mjs +1 -1
- package/src/bench/mine.mjs +3 -3
- package/src/bench/probes.mjs +1 -3
- package/src/bench/report.mjs +1 -3
- package/src/bench/run.mjs +4 -18
- package/src/bench/tasks.mjs +36 -0
- package/src/bench/variants.mjs +1 -3
- package/src/brief/assemble.mjs +24 -4
- package/src/brief/packet.mjs +4 -1
- package/src/ci/dependabot.mjs +1 -1
- package/src/ci/workflow.mjs +40 -12
- package/src/commands/agenthook.mjs +6 -2
- package/src/commands/bench.mjs +35 -3
- package/src/commands/ci.mjs +1 -1
- package/src/commands/compare.mjs +1 -1
- package/src/commands/distill.mjs +36 -0
- package/src/commands/gate.mjs +2 -2
- package/src/commands/graph.mjs +2 -2
- package/src/commands/harness.mjs +87 -14
- package/src/commands/help.mjs +54 -4
- package/src/commands/hooks.mjs +1 -1
- package/src/commands/index.mjs +29 -3
- package/src/commands/init.mjs +2 -2
- package/src/commands/local.mjs +11 -0
- package/src/commands/mcp.mjs +1 -1
- package/src/commands/pack.mjs +246 -0
- package/src/commands/plugin.mjs +61 -0
- package/src/commands/review.mjs +65 -0
- package/src/commands/rules.mjs +1 -1
- package/src/commands/skills.mjs +3 -34
- package/src/commands/skillscompile.mjs +1 -3
- package/src/commands/skillsinstall.mjs +1 -1
- package/src/commands/sync.mjs +8 -8
- package/src/commands/tokens.mjs +1 -1
- package/src/commands/upgrade.mjs +1 -1
- package/src/commands/why.mjs +6 -3
- package/src/compare/discover.mjs +1 -1
- package/src/compare/hooksstate.mjs +19 -7
- package/src/compare/learn.mjs +2 -2
- package/src/compare/row.mjs +3 -3
- package/src/decisions/index.mjs +1 -1
- package/src/distill/distill.mjs +246 -0
- package/src/distill/engine.mjs +187 -0
- package/src/distribute/release.mjs +142 -0
- package/src/graph/load.mjs +9 -8
- package/src/hooks/dispatcher.mjs +3 -2
- package/src/hooks/shim.mjs +11 -6
- package/src/index/govern.mjs +1 -1
- package/src/index/map.mjs +1 -1
- package/src/index/persist.mjs +2 -2
- package/src/index/store.mjs +50 -20
- package/src/ledger/index.mjs +2 -2
- package/src/local/client.mjs +22 -8
- package/src/local/config.mjs +52 -0
- package/src/locate.mjs +22 -1
- package/src/mcp/prompts.mjs +118 -0
- package/src/mcp/resources.mjs +96 -22
- package/src/mcp/server.mjs +33 -6
- package/src/mcp/tools.mjs +12 -12
- package/src/measure/session.mjs +1 -1
- package/src/memory/decide.mjs +1 -1
- package/src/memory/lessons.mjs +14 -14
- package/src/name.mjs +11 -1
- package/src/package-version.mjs +12 -10
- package/src/packs/manage.mjs +563 -0
- package/src/packs.mjs +15 -5
- package/src/plugins.mjs +164 -0
- package/src/projection/loaders.mjs +84 -10
- package/src/projection/manifest.mjs +4 -4
- package/src/projection/project.mjs +3 -3
- package/src/projection/rules.mjs +24 -4
- package/src/projection/surfaces.mjs +20 -2
- package/src/resolve/body.mjs +46 -0
- package/src/review/critic.mjs +509 -0
- package/src/toolwords.mjs +8 -5
- package/src/tsconfig.mjs +3 -3
- package/src/upgrade/plan.mjs +1 -1
- /package/bin/{simt-dev.mjs → simt.mjs} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// One renderer per harness: the payload in, the decision out, and the config written.
|
|
2
2
|
//
|
|
3
|
-
//
|
|
3
|
+
// Five harnesses read a hook and no two agree on anything. The config path, the event
|
|
4
4
|
// name, the shape of the document, the tool name for a shell command, the key holding
|
|
5
5
|
// the command, and the JSON a hook answers with are all different, and none of them is
|
|
6
6
|
// the same fact. So this module owns the differences and `policy.mjs` owns none of
|
|
@@ -10,15 +10,17 @@
|
|
|
10
10
|
// codex .codex/hooks.json PreToolUse {"hooks":{E:[{matcher,hooks:[h]}]}}
|
|
11
11
|
// cursor .cursor/hooks.json preToolUse {"version":1,"hooks":{E:[h]}}
|
|
12
12
|
// antigravity .agents/hooks.json PreToolUse {"<name>":{E:[{matcher,hooks:[h]}]}}
|
|
13
|
+
// gemini .gemini/settings.json BeforeTool {"hooks":{E:[{matcher,hooks:[h]}]}}
|
|
13
14
|
//
|
|
14
15
|
// The prompt-submit event (R29) is a second table, and it is shorter because the vendors
|
|
15
|
-
// are:
|
|
16
|
-
// and a delivery is the only thing this tool does there.
|
|
16
|
+
// are: three of the five document a prompt event that can put text into the model's context,
|
|
17
|
+
// and a delivery is the only thing this tool does there. None takes a matcher — all
|
|
17
18
|
// document that field for tool events, and a matcher a harness ignores is a matcher the
|
|
18
19
|
// next person believes is filtering something.
|
|
19
20
|
//
|
|
20
21
|
// claude .claude/settings.json UserPromptSubmit hookSpecificOutput.additionalContext
|
|
21
22
|
// codex .codex/hooks.json UserPromptSubmit hookSpecificOutput.additionalContext
|
|
23
|
+
// gemini .gemini/settings.json BeforeAgent hookSpecificOutput.additionalContext
|
|
22
24
|
// cursor .cursor/hooks.json beforeSubmitPrompt CANNOT INJECT: block-only
|
|
23
25
|
// antigravity .agents/hooks.json (no prompt event documented)
|
|
24
26
|
//
|
|
@@ -26,10 +28,11 @@
|
|
|
26
28
|
// can only continue or block, so the entry would carry no packet and a broken one would
|
|
27
29
|
// stop a person from typing — and its only injection point, `sessionStart.additional_context`,
|
|
28
30
|
// fires once when the session is created, which is a different lifetime from the brief a
|
|
29
|
-
// task needs. Gemini CLI
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
31
|
+
// task needs. Gemini CLI was replaced by Antigravity CLI on 2026-06-18, which is the row
|
|
32
|
+
// that covers the successor; the gemini row stays because the extension is still documented
|
|
33
|
+
// and installable and R43 ships a package for it, and its hooks file is in the reader table
|
|
34
|
+
// this module takes its paths from — a harness this tool did not verify is a harness it
|
|
35
|
+
// does not write to.
|
|
33
36
|
//
|
|
34
37
|
// The paths are not written here. They come from the reader table (R14) because that is
|
|
35
38
|
// where each one was verified against its vendor page, and a second copy of a verified
|
|
@@ -188,9 +191,43 @@ const ADAPTERS = {
|
|
|
188
191
|
prompt: null,
|
|
189
192
|
post: null,
|
|
190
193
|
},
|
|
194
|
+
gemini: {
|
|
195
|
+
id: 'gemini',
|
|
196
|
+
event: 'BeforeTool',
|
|
197
|
+
matcher: 'run_shell_command|write_file|replace',
|
|
198
|
+
// The vendor's timeout unit is milliseconds (default 60000), not the seconds every
|
|
199
|
+
// other row uses; a bare 10 here would be a ten-millisecond budget, which is a hook
|
|
200
|
+
// that never finishes. The row carries its own value so the shared constant stays
|
|
201
|
+
// honest for the four vendors that measure in seconds.
|
|
202
|
+
timeout: 10000,
|
|
203
|
+
// The reference documents the decision JSON and an exit-2 system block; the denial
|
|
204
|
+
// travels in the JSON, as antigravity does.
|
|
205
|
+
denyExit: 0,
|
|
206
|
+
verified: 'geminicli.com/docs/hooks/reference/, read 2026-10-26',
|
|
207
|
+
root: (p) => p.cwd,
|
|
208
|
+
extract: (p) => ({
|
|
209
|
+
command: p.tool_name === 'run_shell_command' ? (p.tool_input?.command ?? null) : null,
|
|
210
|
+
writes: ['write_file', 'replace'].includes(p.tool_name) ? [p.tool_input?.file_path].filter(Boolean) : [],
|
|
211
|
+
}),
|
|
212
|
+
deny: (message) => ({ decision: 'deny', reason: message }),
|
|
213
|
+
// BeforeAgent appends text to the prompt for this turn only — the same lifetime as a
|
|
214
|
+
// brief. The documented output has no hookEventName field, unlike Claude's.
|
|
215
|
+
prompt: {
|
|
216
|
+
event: 'BeforeAgent',
|
|
217
|
+
verified: 'geminicli.com/docs/hooks/reference/, read 2026-10-26',
|
|
218
|
+
context: (text) => ({ hookSpecificOutput: { additionalContext: text } }),
|
|
219
|
+
},
|
|
220
|
+
// AfterTool appends to the tool result; its input carries tool_response, which is
|
|
221
|
+
// what the lesson recall reads.
|
|
222
|
+
post: {
|
|
223
|
+
event: 'AfterTool',
|
|
224
|
+
verified: 'geminicli.com/docs/hooks/reference/, read 2026-10-26',
|
|
225
|
+
context: (text) => ({ hookSpecificOutput: { additionalContext: text } }),
|
|
226
|
+
},
|
|
227
|
+
},
|
|
191
228
|
};
|
|
192
229
|
|
|
193
|
-
/** The
|
|
230
|
+
/** The five adapters, in the order the reader table lists them. */
|
|
194
231
|
export function adapters() {
|
|
195
232
|
return hookRows()
|
|
196
233
|
.filter((r) => r.path !== null)
|
|
@@ -203,9 +240,19 @@ export function adapter(id) {
|
|
|
203
240
|
return row && hookPath(id) !== null ? row : null;
|
|
204
241
|
}
|
|
205
242
|
|
|
206
|
-
/**
|
|
207
|
-
|
|
208
|
-
|
|
243
|
+
/**
|
|
244
|
+
* The facts a payload carries, and the root it names. An unknown shape yields none.
|
|
245
|
+
*
|
|
246
|
+
* `root` is a caller-supplied override that outranks the payload: a packaged hook runs
|
|
247
|
+
* from the plugin directory and the harness's environment variable names the project, so
|
|
248
|
+
* `--root` (from `$CLAUDE_PROJECT_DIR` and friends) must win over a payload cwd that is
|
|
249
|
+
* the plugin's own working directory. An empty override is treated as absent rather than
|
|
250
|
+
* trusted: an env var the harness did not set expands to `""` in the shell, and a root of
|
|
251
|
+
* the empty string would resolve every fact against the filesystem root.
|
|
252
|
+
*/
|
|
253
|
+
export function factsFrom(row, payload, { root } = {}) {
|
|
254
|
+
const override = typeof root === 'string' && root.trim() !== '' ? root : null;
|
|
255
|
+
return { root: override ?? row.root(payload) ?? process.cwd(), ...row.extract(payload ?? {}) };
|
|
209
256
|
}
|
|
210
257
|
|
|
211
258
|
/** The document to print for a decision, or null when there is nothing to say. */
|
|
@@ -281,27 +328,33 @@ function eventsOf(row, { prompt = false, post = false } = {}) {
|
|
|
281
328
|
return pairs;
|
|
282
329
|
}
|
|
283
330
|
|
|
284
|
-
/**
|
|
285
|
-
|
|
286
|
-
|
|
331
|
+
/**
|
|
332
|
+
* The group this tool owns for one event, in that harness's shape.
|
|
333
|
+
*
|
|
334
|
+
* `command` overrides the entry point for one build: a package ships a self-contained
|
|
335
|
+
* runtime and its config names that copy rather than the binary on PATH, so the shape
|
|
336
|
+
* comes from this table and only the path differs.
|
|
337
|
+
*/
|
|
338
|
+
function group(row, event = EVENTS[0], command = hookCommand(row.id, event)) {
|
|
339
|
+
const timeout = row.timeout ?? TIMEOUT_SECONDS;
|
|
287
340
|
if (event === 'prompt-submit') {
|
|
288
341
|
// No matcher: both vendors document that field for tool events, and a matcher a
|
|
289
342
|
// harness ignores is a matcher the next reader believes is filtering something.
|
|
290
|
-
return { hooks: [{ type: 'command', command, timeout
|
|
343
|
+
return { hooks: [{ type: 'command', command, timeout }] };
|
|
291
344
|
}
|
|
292
345
|
if (event === 'post-tool-use') {
|
|
293
346
|
const matcher = row.post?.matcher;
|
|
294
347
|
return matcher
|
|
295
|
-
? { matcher, hooks: [{ type: 'command', command, timeout
|
|
296
|
-
: { hooks: [{ type: 'command', command, timeout
|
|
348
|
+
? { matcher, hooks: [{ type: 'command', command, timeout }] }
|
|
349
|
+
: { hooks: [{ type: 'command', command, timeout }] };
|
|
297
350
|
}
|
|
298
351
|
if (row.id === 'cursor') {
|
|
299
352
|
// failClosed is set because a hooks file this tool wrote should not silently
|
|
300
353
|
// degrade to fail-open on its own crash: the vendor's default turns a broken hook
|
|
301
354
|
// into an absent one, and the whole point of writing it is that it runs.
|
|
302
|
-
return { command, timeout
|
|
355
|
+
return { command, timeout, matcher: row.matcher, failClosed: true };
|
|
303
356
|
}
|
|
304
|
-
return { matcher: row.matcher, hooks: [{ type: 'command', command, timeout
|
|
357
|
+
return { matcher: row.matcher, hooks: [{ type: 'command', command, timeout }] };
|
|
305
358
|
}
|
|
306
359
|
|
|
307
360
|
/** Whether a group is this tool's, judged by the command it runs and not by position. */
|
|
@@ -335,11 +388,11 @@ export function installedIn(row, config, event = EVENTS[0]) {
|
|
|
335
388
|
* asked for a delivered packet gets the tool hook and nothing extra, and the file it never
|
|
336
389
|
* asked to change is a file this tool does not touch.
|
|
337
390
|
*/
|
|
338
|
-
function withHook(row, config, { prompt = false, post = false } = {}) {
|
|
391
|
+
function withHook(row, config, { prompt = false, post = false, command } = {}) {
|
|
339
392
|
const box = container(row, config, { create: true });
|
|
340
393
|
for (const [event, name] of eventsOf(row, { prompt, post })) {
|
|
341
394
|
const kept = entries(row, config, event).filter((e) => !ours(e));
|
|
342
|
-
box[name] = [...kept, group(row, event)];
|
|
395
|
+
box[name] = [...kept, group(row, event, event === EVENTS[0] ? command : undefined)];
|
|
343
396
|
}
|
|
344
397
|
return config;
|
|
345
398
|
}
|
|
@@ -359,3 +412,16 @@ export function rendered(row, existing, options = {}) {
|
|
|
359
412
|
const text = `${JSON.stringify(withHook(row, config, options), null, 2)}\n`;
|
|
360
413
|
return { text, action: text === existing ? 'unchanged' : 'updated' };
|
|
361
414
|
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* A fresh config document carrying only this tool's hooks, for a package that ships its
|
|
418
|
+
* own runtime: the shape comes from the same table `install` merges into a repository,
|
|
419
|
+
* and only the entry point differs — the package's copy of the binary, named by `command`.
|
|
420
|
+
* Returns null for a harness this tool has no adapter for or that documents no hooks file.
|
|
421
|
+
*/
|
|
422
|
+
export function hookDocument(id, { command, prompt = false, post = false } = {}) {
|
|
423
|
+
const row = adapter(id);
|
|
424
|
+
if (!row) return null;
|
|
425
|
+
const config = withHook(row, {}, { prompt, post, command });
|
|
426
|
+
return `${JSON.stringify(config, null, 2)}\n`;
|
|
427
|
+
}
|
package/src/argv.mjs
CHANGED
|
@@ -80,10 +80,27 @@ export const BOOL_FLAGS = new Set([
|
|
|
80
80
|
// index / map / brief: opt into local vector embeddings configured under local.embed
|
|
81
81
|
// (plan 0003 R28, R37). Off by default unless recall@10 gains at least 10 points.
|
|
82
82
|
'embed',
|
|
83
|
+
// distill: regenerate a proposal that is already cached. Off by default because
|
|
84
|
+
// the cache is the point - the same source at the same budget is the same answer,
|
|
85
|
+
// and re-reading it is cheaper than re-deriving it (plan 0003 R38).
|
|
86
|
+
'refresh',
|
|
87
|
+
// pack lint: reject a pack whose bench summary is missing or unmeasured.
|
|
88
|
+
// Off by default because a scaffold is born unmeasured - `pack new` writes a
|
|
89
|
+
// bench.json marked unmeasured, and a lint that failed it on arrival would make
|
|
90
|
+
// every new pack red before its author wrote the rule (plan 0003 R45).
|
|
91
|
+
'require-measured',
|
|
92
|
+
// distill: propose the summary from the configured local chat model instead of
|
|
93
|
+
// the deterministic outline. Off by default: no model in the default path (P1),
|
|
94
|
+
// and a model that is not asked for should not be opened against (ADR-090).
|
|
95
|
+
'local',
|
|
83
96
|
]);
|
|
84
97
|
|
|
85
98
|
export const VALUE_FLAGS = new Set([
|
|
86
99
|
'budget',
|
|
100
|
+
// bench critic: the labelled set of historical diffs to score (plan 0003 R39).
|
|
101
|
+
// A path, not a switch, because the labels themselves come from a human under
|
|
102
|
+
// ADR-008 — the tool never annotates for them.
|
|
103
|
+
'labels',
|
|
87
104
|
'max-bytes',
|
|
88
105
|
'paths',
|
|
89
106
|
'skill',
|
|
@@ -126,6 +143,14 @@ export const VALUE_FLAGS = new Set([
|
|
|
126
143
|
'level',
|
|
127
144
|
// decide: structured supersession spec (ADR-NNN or ADR-NNN:<ruling>[:<kind>], repeatable).
|
|
128
145
|
'supersedes',
|
|
146
|
+
// pack new: where the pack's bytes come from. Required, because a pack without
|
|
147
|
+
// a source is bytes nobody can obtain again - and a source nobody can act on
|
|
148
|
+
// is refused by lint rather than blessed (plan 0003 R45).
|
|
149
|
+
'source',
|
|
150
|
+
// plugin: where the generated package lands. A value rather than a boolean because
|
|
151
|
+
// "generate a package" with no destination would have to guess, and a guess about where
|
|
152
|
+
// a directory tree goes is the one failure that leaves files behind.
|
|
153
|
+
'out',
|
|
129
154
|
]);
|
|
130
155
|
|
|
131
156
|
/**
|
package/src/bench/ablate.mjs
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { mkdir, readFile, rm, unlink, writeFile } from 'node:fs/promises';
|
|
11
11
|
import { dirname, join, resolve } from 'node:path';
|
|
12
12
|
|
|
13
|
-
import {
|
|
13
|
+
import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
14
14
|
import { readManifest } from '../projection/manifest.mjs';
|
|
15
15
|
import { inject, parse as parseRegion } from '../projection/markers.mjs';
|
|
16
16
|
import { loadMdcRules, renderMdc, ruleName } from '../projection/mdc.mjs';
|
|
@@ -27,8 +27,6 @@ import {
|
|
|
27
27
|
loadRunRecords,
|
|
28
28
|
} from './report.mjs';
|
|
29
29
|
|
|
30
|
-
const STATE_DIR = `.${CONFIG_DIR}`;
|
|
31
|
-
|
|
32
30
|
export const DEFAULT_CANARY_SIZE = 10;
|
|
33
31
|
export const DEL_R_TOLERANCE = 0.01;
|
|
34
32
|
export const CANARY_ALPHA = 0.05;
|
package/src/bench/grade.mjs
CHANGED
|
@@ -148,7 +148,7 @@ export async function gradeWorkspace(workDir, task, { vaultRoot, initialHead, ti
|
|
|
148
148
|
.toString('utf8')
|
|
149
149
|
.split('\n')
|
|
150
150
|
.map((s) => s.trim())
|
|
151
|
-
.filter((f) => Boolean(f) && !f.startsWith('.simt
|
|
151
|
+
.filter((f) => Boolean(f) && !f.startsWith('.simt/'));
|
|
152
152
|
|
|
153
153
|
const goldFiles = new Set(extractPatchFiles(goldPatch));
|
|
154
154
|
const outsideBlastRadius =
|
package/src/bench/mine.mjs
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
//
|
|
9
9
|
// Hidden test patches live in `~/.cache/simt-dev/vault/<repo-id>/<task-id>/`
|
|
10
10
|
// outside every worktree (RepoTrials / PatchGym), while deterministic task
|
|
11
|
-
// descriptors live in `.simt
|
|
11
|
+
// descriptors live in `.simt/bench/tasks/<id>.json`.
|
|
12
12
|
|
|
13
13
|
import { execFile, spawn } from 'node:child_process';
|
|
14
14
|
import { createHash } from 'node:crypto';
|
|
@@ -661,7 +661,7 @@ async function validateCandidate(root, worktree, candidate, { timeoutMs = 5000 }
|
|
|
661
661
|
* Mine verified benchmark tasks from a repository's git history.
|
|
662
662
|
*
|
|
663
663
|
* When `apply` is false (the default without `--yes`, or under `--dry-run`),
|
|
664
|
-
* candidates are discovered and reported without writing to `.simt
|
|
664
|
+
* candidates are discovered and reported without writing to `.simt/bench/tasks/`
|
|
665
665
|
* or the vault. When `apply` is true, candidates are validated in isolated
|
|
666
666
|
* temporary worktrees until `target` valid tasks are collected, and written
|
|
667
667
|
* deterministically to disk.
|
|
@@ -685,7 +685,7 @@ export async function mineTasks(
|
|
|
685
685
|
return { ok: false, error: listed.error };
|
|
686
686
|
}
|
|
687
687
|
|
|
688
|
-
const tasksDir = join(absRoot, '.simt
|
|
688
|
+
const tasksDir = join(absRoot, '.simt', 'bench', 'tasks');
|
|
689
689
|
const vaultRoot = resolveVaultDir(absRoot, { env });
|
|
690
690
|
|
|
691
691
|
if (!apply) {
|
package/src/bench/probes.mjs
CHANGED
|
@@ -25,12 +25,10 @@ import { chmod, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs
|
|
|
25
25
|
import { tmpdir } from 'node:os';
|
|
26
26
|
import { dirname, join, resolve } from 'node:path';
|
|
27
27
|
|
|
28
|
-
import {
|
|
28
|
+
import { STATE_DIR } from '../name.mjs';
|
|
29
29
|
import { parse as parseRegion } from '../projection/markers.mjs';
|
|
30
30
|
import { materialiseArm, parseArmSpec } from './run.mjs';
|
|
31
31
|
|
|
32
|
-
const STATE_DIR = `.${CONFIG_DIR}`;
|
|
33
|
-
|
|
34
32
|
function sanitizedEnv(extra = {}) {
|
|
35
33
|
const env = { ...process.env, SIMT_DEV_WORKSPACE: 'off', ...extra };
|
|
36
34
|
for (const key of Object.keys(env)) {
|
package/src/bench/report.mjs
CHANGED
|
@@ -8,9 +8,7 @@
|
|
|
8
8
|
import { readdir, readFile } from 'node:fs/promises';
|
|
9
9
|
import { join, resolve } from 'node:path';
|
|
10
10
|
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
const STATE_DIR = `.${CONFIG_DIR}`;
|
|
11
|
+
import { STATE_DIR } from '../name.mjs';
|
|
14
12
|
|
|
15
13
|
export const DEFAULT_BOOTSTRAP_SAMPLES = 1000;
|
|
16
14
|
export const DEFAULT_BOOTSTRAP_SEED = 0x51772026;
|
package/src/bench/run.mjs
CHANGED
|
@@ -9,7 +9,7 @@ import { loadRegistry } from '../graph/load.mjs';
|
|
|
9
9
|
import { extractOpenedSkills, matchSkillsForPaths } from '../graph/compile.mjs';
|
|
10
10
|
import { matchGlob } from '../index/govern.mjs';
|
|
11
11
|
import { formatLessonParagraph, listPromotedLessons } from '../memory/lessons.mjs';
|
|
12
|
-
import {
|
|
12
|
+
import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
13
13
|
import { readManifest } from '../projection/manifest.mjs';
|
|
14
14
|
import { plan } from '../projection/project.mjs';
|
|
15
15
|
import { parse as parseRegion } from '../projection/markers.mjs';
|
|
@@ -17,11 +17,10 @@ import { fingerprint } from '../resolve/fingerprint.mjs';
|
|
|
17
17
|
import { rankSkills, tokenize } from '../resolve/resolve.mjs';
|
|
18
18
|
import { gradeWorkspace } from './grade.mjs';
|
|
19
19
|
import { resolveVaultDir } from './mine.mjs';
|
|
20
|
+
import { loadTasks } from './tasks.mjs';
|
|
20
21
|
import { ablateSectionInWorkspace, parseAblationArmName } from './ablate.mjs';
|
|
21
22
|
import { applyVariantInWorkspace, parseVariantArmName } from './variants.mjs';
|
|
22
23
|
|
|
23
|
-
const STATE_DIR = `.${CONFIG_DIR}`;
|
|
24
|
-
|
|
25
24
|
const BENCH_DIR = dirname(fileURLToPath(import.meta.url));
|
|
26
25
|
const ADAPTERS_PATH = join(BENCH_DIR, 'adapters.json');
|
|
27
26
|
|
|
@@ -678,21 +677,8 @@ async function executeAdapter(adapter, workDir, task, { vaultRoot, model, maxTur
|
|
|
678
677
|
return { ...parseAdapterUsage(adapter, stdoutText, analyticsText), transcript: `${stdoutText}\n${analyticsText}` };
|
|
679
678
|
}
|
|
680
679
|
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
const entries = (await readdir(tasksDir).catch(() => []))
|
|
684
|
-
.filter((f) => f.endsWith('.json'))
|
|
685
|
-
.sort();
|
|
686
|
-
|
|
687
|
-
const filterSet = new Set(filterIds);
|
|
688
|
-
const tasks = [];
|
|
689
|
-
for (const entry of entries) {
|
|
690
|
-
const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
|
|
691
|
-
if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
|
|
692
|
-
tasks.push(data);
|
|
693
|
-
}
|
|
694
|
-
return tasks.sort((a, b) => a.id.localeCompare(b.id));
|
|
695
|
-
}
|
|
680
|
+
// loadTasks lives in tasks.mjs (the advice engine became a second reader, so the loader
|
|
681
|
+
// was extracted rather than copied — ADR-002).
|
|
696
682
|
|
|
697
683
|
function runIdFor(harness, armName, rep) {
|
|
698
684
|
return `${harness}-${armName}-r${rep}`;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Mined task descriptors, read back from the tool state's `bench/tasks/` directory (plan 0003 R01).
|
|
2
|
+
//
|
|
3
|
+
// Extracted from bench/run.mjs on its second real reader (ADR-002): the R40 advice engine
|
|
4
|
+
// needs the two facts a run record does not carry — the prompt, which classifies the task,
|
|
5
|
+
// and the gold stat, which says how many files the change touched. A second private loader
|
|
6
|
+
// beside the first would drift the first time one learned a filter the other lacked.
|
|
7
|
+
//
|
|
8
|
+
// Deterministic by construction: the directory is read sorted, the filter is exact, and no
|
|
9
|
+
// part of this touches a model, a network or a clock.
|
|
10
|
+
import { readdir, readFile } from 'node:fs/promises';
|
|
11
|
+
import { join, resolve } from 'node:path';
|
|
12
|
+
import { STATE_DIR } from '../name.mjs';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Load mined benchmark task descriptors.
|
|
16
|
+
*
|
|
17
|
+
* @param {string} root the repository
|
|
18
|
+
* @param {string[]} [filterIds] exact task ids to keep; empty means load them all
|
|
19
|
+
* @returns descriptor objects sorted by id (id, base, sourceCommit, prompt, testCmd,
|
|
20
|
+
* f2p, p2p, gold {files, linesAdded, linesRemoved}, vault)
|
|
21
|
+
*/
|
|
22
|
+
export async function loadTasks(root, filterIds = []) {
|
|
23
|
+
const tasksDir = join(resolve(root), STATE_DIR, 'bench', 'tasks');
|
|
24
|
+
const entries = (await readdir(tasksDir).catch(() => []))
|
|
25
|
+
.filter((f) => f.endsWith('.json'))
|
|
26
|
+
.sort();
|
|
27
|
+
|
|
28
|
+
const filterSet = new Set(filterIds);
|
|
29
|
+
const tasks = [];
|
|
30
|
+
for (const entry of entries) {
|
|
31
|
+
const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
|
|
32
|
+
if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
|
|
33
|
+
tasks.push(data);
|
|
34
|
+
}
|
|
35
|
+
return tasks.sort((a, b) => a.id.localeCompare(b.id));
|
|
36
|
+
}
|
package/src/bench/variants.mjs
CHANGED
|
@@ -10,7 +10,7 @@ import { mkdir, readdir, readFile, unlink, writeFile } from 'node:fs/promises';
|
|
|
10
10
|
import { dirname, join, resolve } from 'node:path';
|
|
11
11
|
|
|
12
12
|
import { extractVariantsFromRule, readLockedVariants } from '../kernel/variants.mjs';
|
|
13
|
-
import {
|
|
13
|
+
import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
|
|
14
14
|
import { writeLockVariants } from '../projection/lock.mjs';
|
|
15
15
|
import { readManifest } from '../projection/manifest.mjs';
|
|
16
16
|
import { loadMdcRules } from '../projection/mdc.mjs';
|
|
@@ -27,8 +27,6 @@ import {
|
|
|
27
27
|
loadRunRecords,
|
|
28
28
|
} from './report.mjs';
|
|
29
29
|
|
|
30
|
-
const STATE_DIR = `.${CONFIG_DIR}`;
|
|
31
|
-
|
|
32
30
|
/**
|
|
33
31
|
* Discover every `simt-variants` slot across applicable rules for `root`.
|
|
34
32
|
*/
|
package/src/brief/assemble.mjs
CHANGED
|
@@ -34,13 +34,13 @@
|
|
|
34
34
|
// command and the tool leave it on. It is inert either way in a repository that declared no
|
|
35
35
|
// servers.
|
|
36
36
|
|
|
37
|
-
import { CLI } from '../name.mjs';
|
|
37
|
+
import { CLI, STATE_DIR } from '../name.mjs';
|
|
38
38
|
import { DEFAULT_BRIEF_BYTES, buildPacket, collect } from './packet.mjs';
|
|
39
39
|
import { classifyTask, cardVerbs } from './kind.mjs';
|
|
40
|
+
import { briefAdvice } from '../advice/advice.mjs';
|
|
40
41
|
import { commandCard } from '../kernel/card.mjs';
|
|
41
42
|
import { decisionHits, scopeHits, sourceRoots } from '../index/govern.mjs';
|
|
42
43
|
import { embedTaskQuery, rankFiles, renderMap } from '../index/map.mjs';
|
|
43
|
-
import { SQLITE_FILE } from '../index/persist.mjs';
|
|
44
44
|
import { searchFtsBm25 } from '../index/store.mjs';
|
|
45
45
|
import { testEdges, testsFor } from '../index/tests.mjs';
|
|
46
46
|
import { loadRegistry } from '../graph/load.mjs';
|
|
@@ -80,7 +80,6 @@ const MAP_FACTOR = 8;
|
|
|
80
80
|
* between two identical commands is not the packet this promises. The tool's own scratch
|
|
81
81
|
* space is never part of a task's scope.
|
|
82
82
|
*/
|
|
83
|
-
const STATE_DIR = `${SQLITE_FILE.split('/')[0]}/`;
|
|
84
83
|
|
|
85
84
|
/** Every file git tracks that carries a generated region, one `git grep` away. */
|
|
86
85
|
async function generatedRegions(root) {
|
|
@@ -127,7 +126,7 @@ export async function assembleBrief({
|
|
|
127
126
|
federate = true,
|
|
128
127
|
} = {}) {
|
|
129
128
|
const named = paths !== null;
|
|
130
|
-
const dirty = (await dirtyPaths(root)).filter((path) => !path.startsWith(STATE_DIR));
|
|
129
|
+
const dirty = (await dirtyPaths(root)).filter((path) => !path.startsWith(`${STATE_DIR}/`));
|
|
131
130
|
const scope = named ? parsePathList(paths) : dirty;
|
|
132
131
|
|
|
133
132
|
const graph = await loadGraph(root, { build });
|
|
@@ -192,6 +191,27 @@ export async function assembleBrief({
|
|
|
192
191
|
});
|
|
193
192
|
}
|
|
194
193
|
|
|
194
|
+
// Model and effort advice comes from the repository's bench history, not from anything a
|
|
195
|
+
// model produced, so it stays deterministic. The tier exists only if the history resolved
|
|
196
|
+
// a similar task, and the claim travels with its own numbers and its unmeasured accept
|
|
197
|
+
// bar; simt-dev never selects the model an agent runs (plan 0003 R40).
|
|
198
|
+
const advice = await briefAdvice(root, { task, scope, kind });
|
|
199
|
+
if (advice?.picked) {
|
|
200
|
+
packet.add('ADVICE', advice.picked.line, {
|
|
201
|
+
why: ['the tier is the estate’s own bench ladder, and a human still names the model'],
|
|
202
|
+
});
|
|
203
|
+
packet.add('ADVICE', advice.picked.basis, {
|
|
204
|
+
why: ['an advice claim names the neighborhood that backs it, so the reader can re-derive it'],
|
|
205
|
+
});
|
|
206
|
+
packet.add('ADVICE', advice.picked.metric, {
|
|
207
|
+
why: ['the row has an accept bar, and nothing in a packet may claim the bar was met'],
|
|
208
|
+
});
|
|
209
|
+
} else if (advice?.caution) {
|
|
210
|
+
packet.add('CAUTIONS', advice.caution, {
|
|
211
|
+
why: ['advice without a neighborhood is a guess dressed as history'],
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
|
|
195
215
|
const likely = ranked.slice(0, LIKELY_FILES).map((row) => row.path);
|
|
196
216
|
const known = testEdges({ files, imports: edges.filter((edge) => edge.kind === 'import') });
|
|
197
217
|
const untested = [];
|
package/src/brief/packet.mjs
CHANGED
|
@@ -30,7 +30,7 @@ export const PROMPT_CAP_BYTES = 2048;
|
|
|
30
30
|
* The order is fixed and identical in every packet. A reader who has to search for the
|
|
31
31
|
* command is paying the cost the packet exists to remove.
|
|
32
32
|
*/
|
|
33
|
-
const SECTIONS = ['TASK', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
|
|
33
|
+
const SECTIONS = ['TASK', 'ADVICE', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION', 'SKILLS', 'LESSONS', 'CAUTIONS'];
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
36
|
* What each section is worth when something has to go.
|
|
@@ -49,6 +49,9 @@ const SECTIONS = ['TASK', 'COMMANDS', 'FILES', 'TESTS', 'DECISIONS', 'FEDERATION
|
|
|
49
49
|
*/
|
|
50
50
|
const SECTION_VALUE = {
|
|
51
51
|
CAUTIONS: 1,
|
|
52
|
+
// Bench evidence, but advice rather than fact: losing it costs an opinion, not a
|
|
53
|
+
// command or a path to check (plan 0003 R40).
|
|
54
|
+
ADVICE: 2,
|
|
52
55
|
LESSONS: 2,
|
|
53
56
|
SKILLS: 3,
|
|
54
57
|
DECISIONS: 4,
|
package/src/ci/dependabot.mjs
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
// rewritten, their file from the first commit. A repo that wants a different
|
|
22
22
|
// cadence, a private registry or an `ignore` rule edits it, and the edit
|
|
23
23
|
// stays. simt-dev's own repository is the ADR-024 exception and is refreshed
|
|
24
|
-
// by `simt
|
|
24
|
+
// by `simt ci`.
|
|
25
25
|
//
|
|
26
26
|
// The content is detection-driven, which is the ADR-013 shape: the ecosystems
|
|
27
27
|
// listed are the ones this tree actually has a manifest for. Writing an `npm`
|
package/src/ci/workflow.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// The CI gate, as a file a repo owns.
|
|
2
2
|
//
|
|
3
|
-
// `simt
|
|
3
|
+
// `simt check` is the whole gate: projection drift, an unreadable rule, an
|
|
4
4
|
// unrunnable gate, a stale manifest. The pre-commit hook runs it locally, which
|
|
5
5
|
// stops one person committing a broken region and stops nobody else - a hook is a
|
|
6
6
|
// developer's machine, and every review is somebody else's. So the same command
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
// which is why the header says so in the file rather than in this comment.
|
|
29
29
|
//
|
|
30
30
|
// The own copy also RUNS something different, and that is not a variation for taste.
|
|
31
|
-
// `simt
|
|
31
|
+
// `simt check` is red in this repository by design: AGENTS.md here is written by
|
|
32
32
|
// hand and deliberately never self-adopted (see its own "Why this repository is not
|
|
33
33
|
// self-adopted"), so the check reports it as unmanaged forever. A gate that is red on
|
|
34
34
|
// the day it lands is the one thing worse than no gate - it teaches the reader to
|
|
@@ -51,6 +51,7 @@ import { PACKAGE_NAME, WORKFLOW_FILE_NAME } from '../name.mjs';
|
|
|
51
51
|
import { renderLocator } from '../locate.mjs';
|
|
52
52
|
import { CLI } from '../name.mjs';
|
|
53
53
|
import { packageVersion } from '../package-version.mjs';
|
|
54
|
+
import { LOCAL_BIN, LOCAL_BIN_DIR, assetNameExpr, localInstall, platformLines, releaseBase } from '../distribute/release.mjs';
|
|
54
55
|
|
|
55
56
|
/** Directory whose presence means this repo runs GitHub Actions. */
|
|
56
57
|
export const WORKFLOWS_DIR = '.github/workflows';
|
|
@@ -264,7 +265,7 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
264
265
|
' - name: tests',
|
|
265
266
|
' run: npm test',
|
|
266
267
|
' - name: skill registry lint',
|
|
267
|
-
' run: node bin/simt
|
|
268
|
+
' run: node bin/simt.mjs graph lint',
|
|
268
269
|
' - name: the gate copy in this repo is current',
|
|
269
270
|
// The drift this exception exists to prevent, checked where it would be caught.
|
|
270
271
|
// Without it a template change lands with the copy still describing the old
|
|
@@ -272,7 +273,7 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
272
273
|
//
|
|
273
274
|
// `--check`, and not the bare command: `ci` previews by default and exits 0,
|
|
274
275
|
// so this step used to print the drift it exists to catch and pass anyway.
|
|
275
|
-
' run: node bin/simt
|
|
276
|
+
' run: node bin/simt.mjs ci --check',
|
|
276
277
|
].join('\n'),
|
|
277
278
|
// ADR-012 says CI verifies on macOS and Linux. Until this matrix existed that
|
|
278
279
|
// sentence was false in the one repository that could have made it true, and no
|
|
@@ -285,9 +286,19 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
285
286
|
const v = version ?? packageVersion();
|
|
286
287
|
// The install step is the whole of S35's consumer CI: a repository with npm pins
|
|
287
288
|
// simt-dev as a devDependency and `npm ci` restores it; one without (Gradle,
|
|
288
|
-
// Connect IQ) has no node_modules to pin in, so CI
|
|
289
|
-
//
|
|
290
|
-
//
|
|
289
|
+
// Connect IQ) has no node_modules to pin in, so CI downloads the release binary
|
|
290
|
+
// into the repository - the same pinned bytes the machine remedy names, verified
|
|
291
|
+
// against the checksum published beside it. The two must agree, or the gate
|
|
292
|
+
// passes in one place and cannot start in the other.
|
|
293
|
+
//
|
|
294
|
+
// The binary branch needs no setup-node at all: the SEA runs without a Node on
|
|
295
|
+
// PATH (ADR-098), and installing one to run a file that does not need it would be
|
|
296
|
+
// the toolchain this channel exists to remove, present in the one place it was not.
|
|
297
|
+
//
|
|
298
|
+
// The platform is derived in the shell (platformLines) and not from a
|
|
299
|
+
// `${{ runner.* }}` expression: an expression substituted into `$( … )` is a
|
|
300
|
+
// command substitution trying to execute "Linux", which set -eu refuses - the
|
|
301
|
+
// install would fail on every run in the one repository kind this channel is for.
|
|
291
302
|
const install = npm
|
|
292
303
|
? [
|
|
293
304
|
' - name: install dependencies',
|
|
@@ -296,19 +307,36 @@ export function renderWorkflow({ own = false, npm = true, version } = {}) {
|
|
|
296
307
|
]
|
|
297
308
|
: [
|
|
298
309
|
' - name: install the pinned gate binary',
|
|
299
|
-
' # No node_modules to pin in, so
|
|
300
|
-
' # the machine remedy names,
|
|
301
|
-
|
|
310
|
+
' # No node_modules to pin in, so the release binary into the repository:',
|
|
311
|
+
' # the same bytes the machine remedy names, checked against the',
|
|
312
|
+
' # checksum published beside it before anything runs them.',
|
|
313
|
+
' run: |',
|
|
314
|
+
indent(
|
|
315
|
+
`set -eu
|
|
316
|
+
${platformLines().join('\n')}
|
|
317
|
+
asset="${assetNameExpr(v)}"
|
|
318
|
+
base="${releaseBase(v)}"
|
|
319
|
+
curl -fsSL "$base/$asset" -o "$asset"
|
|
320
|
+
expected=$(curl -fsSL "$base/$asset.sha256" | awk '{print $1}')
|
|
321
|
+
actual=$(openssl sha256 "$asset" | grep -oE '[0-9a-f]{64}')
|
|
322
|
+
if [ "$expected" != "$actual" ]; then
|
|
323
|
+
echo "checksum mismatch for $asset: the release asset does not match its published sha256" >&2
|
|
324
|
+
exit 1
|
|
325
|
+
fi
|
|
326
|
+
mkdir -p ${LOCAL_BIN_DIR}
|
|
327
|
+
mv "$asset" ${LOCAL_BIN}
|
|
328
|
+
chmod +x ${LOCAL_BIN}`, 10),
|
|
302
329
|
];
|
|
303
330
|
const remedy = npm
|
|
304
331
|
? `REMEDY run: npm i -D ${PACKAGE_NAME} (the gate runs from the pinned dependency)`
|
|
305
|
-
: `REMEDY run:
|
|
332
|
+
: `REMEDY run: ${localInstall(v)} (the gate runs from the release binary in .simt/bin)`;
|
|
306
333
|
const locator = renderLocator('check --root "$ROOT"', { remedy });
|
|
307
334
|
return `${header(false)}${skeleton(
|
|
308
335
|
`${CLI} check`,
|
|
309
336
|
[
|
|
310
337
|
uses('checkout'),
|
|
311
|
-
|
|
338
|
+
// The binary branch has no setup-node: see the install step above.
|
|
339
|
+
...(npm ? [setupNode(nodeFloor())] : []),
|
|
312
340
|
...install,
|
|
313
341
|
` - name: ${CLI} check`,
|
|
314
342
|
' run: |',
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// simt
|
|
1
|
+
// simt agent-hook <event> | agent-hooks install | agent-hooks status
|
|
2
2
|
//
|
|
3
3
|
// Three commands, one subject: the rules the tool projects, enforced while the agent is
|
|
4
4
|
// working rather than when git is committing. `agent-hook` is the entry point a harness
|
|
@@ -92,7 +92,11 @@ export async function agentHookCommand(args, flags) {
|
|
|
92
92
|
return EXIT.OK;
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
// `--root` outranks the payload: a packaged hook runs from the plugin directory and the
|
|
96
|
+
// harness's environment variable names the project, so a payload cwd that is the plugin's
|
|
97
|
+
// own working directory must not win. The repository-install path passes no flag, and the
|
|
98
|
+
// payload's cwd is what the harness reports.
|
|
99
|
+
const { root, command, writes } = factsFrom(row, payload, { root: flags.root });
|
|
96
100
|
const facts = await factsFor(root, { command, writes });
|
|
97
101
|
const decision = decide({ event, facts, env: process.env });
|
|
98
102
|
|