simt-dev 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -1
- package/README.md +169 -80
- package/docs/decisions/ADR-014-workspace-is-configuration.md +1 -1
- package/docs/decisions/ADR-025-a-baseline-is-a-region-in-a-file-the-repo-owns.md +1 -1
- package/docs/decisions/ADR-026-a-config-without-a-region-is-a-file-the-repo-owns.md +1 -1
- package/docs/decisions/ADR-027-the-one-gitattributes-line-that-is-neither-dangerous-nor-empty.md +1 -1
- package/docs/decisions/ADR-028-a-base-file-is-a-floor-so-the-disagreement-it-cannot-repair-becomes-a-check.md +1 -1
- package/docs/decisions/ADR-029-the-pins-a-repository-declares-are-its-own.md +1 -1
- package/docs/decisions/ADR-030-what-a-plan-does-not-ship-is-a-decision-with-a-record.md +1 -1
- package/docs/decisions/ADR-048-a-baseline-carries-only-keys-a-reader-honours.md +1 -1
- package/docs/decisions/ADR-082-the-tracker-override-is-a-manifest-key.md +1 -1
- package/docs/decisions/ADR-126-the-mcp-surface-stays-a-reader-so-there-is-no-write-path-to-build.md +68 -0
- package/docs/decisions/ADR-127-the-mcp-tool-list-is-written-by-hand-and-roots-is-not-asked-for.md +65 -0
- package/docs/decisions/ADR-128-a-research-plan-is-a-plan-and-the-tracker-reads-it.md +42 -0
- package/ecosystems/bun.json +6 -0
- package/ecosystems/cargo.json +25 -0
- package/ecosystems/composer.json +6 -0
- package/ecosystems/dart.json +24 -0
- package/ecosystems/deno.json +21 -0
- package/ecosystems/devcontainers.json +6 -0
- package/ecosystems/docker-compose.json +6 -0
- package/ecosystems/docker.json +6 -0
- package/ecosystems/dotnet.json +25 -0
- package/ecosystems/elixir.json +23 -0
- package/ecosystems/gitsubmodule.json +6 -0
- package/ecosystems/go.json +25 -0
- package/ecosystems/gradle.json +13 -0
- package/ecosystems/maven.json +23 -0
- package/ecosystems/node.json +25 -0
- package/ecosystems/python.json +20 -0
- package/ecosystems/ruby.json +23 -0
- package/ecosystems/swift.json +25 -0
- package/ecosystems/terraform.json +6 -0
- package/ecosystems/uv.json +6 -0
- package/harnesses/antigravity.json +162 -0
- package/harnesses/claude.json +206 -0
- package/harnesses/codex.json +136 -0
- package/harnesses/copilot.json +88 -0
- package/harnesses/cursor.json +140 -0
- package/harnesses/deepseek.json +99 -0
- package/harnesses/gemini.json +122 -0
- package/harnesses/opencode.json +173 -0
- package/package.json +3 -1
- package/rules/mdc/intake-questions.mdc +8 -0
- package/src/advice/advice.mjs +3 -2
- package/src/agenthooks/render.mjs +86 -180
- package/src/agents/render.mjs +15 -18
- package/src/api.mjs +7 -5
- package/src/argv.mjs +19 -0
- package/src/bench/ablate.mjs +3 -4
- package/src/bench/adapters.json +155 -113
- package/src/bench/adapters.mjs +466 -0
- package/src/bench/arms.mjs +333 -0
- package/src/bench/grade.mjs +12 -26
- package/src/bench/graders.mjs +42 -0
- package/src/bench/mine.mjs +17 -535
- package/src/bench/probe-ledger.mjs +149 -0
- package/src/bench/probes.catalog.mjs +918 -0
- package/src/bench/probes.mjs +15 -1037
- package/src/bench/render.mjs +111 -0
- package/src/bench/report.mjs +116 -971
- package/src/bench/run.mjs +164 -881
- package/src/bench/variants.mjs +4 -5
- package/src/bench/workspace.mjs +160 -0
- package/src/brief/assemble.mjs +11 -13
- package/src/ci/dependabot.mjs +25 -55
- package/src/commands/agent.mjs +1 -9
- package/src/commands/agenthook.mjs +66 -35
- package/src/commands/bench.mjs +2 -2
- package/src/commands/ci.mjs +11 -13
- package/src/commands/docs.mjs +88 -0
- package/src/commands/gate.mjs +10 -13
- package/src/commands/harness.mjs +3 -3
- package/src/commands/help.mjs +44 -3
- package/src/commands/hooks.mjs +20 -20
- package/src/commands/ignore.mjs +11 -7
- package/src/commands/impact.mjs +4 -8
- package/src/commands/index.mjs +49 -9
- package/src/commands/init-scan.mjs +90 -0
- package/src/commands/init.mjs +122 -104
- package/src/commands/lease.mjs +67 -18
- package/src/commands/map.mjs +10 -5
- package/src/commands/mcp.mjs +2 -17
- package/src/commands/repoindex.mjs +8 -8
- package/src/commands/rules.mjs +8 -4
- package/src/commands/shared.mjs +12 -16
- package/src/commands/skills.mjs +5 -6
- package/src/commands/skillscompile.mjs +3 -3
- package/src/commands/skillsinstall.mjs +5 -4
- package/src/commands/slice.mjs +199 -64
- package/src/commands/spec.mjs +78 -62
- package/src/commands/status.mjs +24 -12
- package/src/commands/sync.mjs +14 -14
- package/src/commands/tests.mjs +12 -12
- package/src/commands/tokens.mjs +19 -24
- package/src/commands/upgrade.mjs +8 -4
- package/src/commands/why.mjs +6 -13
- package/src/compare/hooksstate.mjs +2 -6
- package/src/compare/learn.mjs +3 -1
- package/src/decisions/index.mjs +3 -2
- package/src/docs/commands.mjs +65 -0
- package/src/ecosystem/records.mjs +219 -0
- package/src/explain.mjs +31 -0
- package/src/gates/index.mjs +385 -0
- package/src/graph/compile.mjs +8 -4
- package/src/graph/load.mjs +2 -1
- package/src/graph/spec.mjs +10 -8
- package/src/harness/records.mjs +143 -0
- package/src/hooks/capability.mjs +5 -3
- package/src/hooks/dispatcher.mjs +26 -32
- package/src/hooks/gates.mjs +12 -2
- package/src/hooks/resolve.mjs +2 -7
- package/src/hooks/where.mjs +2 -7
- package/src/identity.mjs +2 -11
- package/src/index/coverage.mjs +58 -0
- package/src/index/extract.mjs +6 -16
- package/src/index/gate.mjs +1 -1
- package/src/index/govern.mjs +26 -9
- package/src/index/impact.mjs +4 -8
- package/src/index/map.mjs +14 -15
- package/src/index/open.mjs +173 -0
- package/src/index/persist.mjs +224 -57
- package/src/index/store.mjs +82 -39
- package/src/index/tests.mjs +2 -6
- package/src/journal.mjs +16 -11
- package/src/kernel/card.mjs +37 -63
- package/src/kernel/checks.mjs +4 -10
- package/src/lease.mjs +102 -7
- package/src/ledger/index.mjs +124 -17
- package/src/mcp/paginate.mjs +33 -0
- package/src/mcp/schemas.mjs +512 -0
- package/src/mcp/server.mjs +49 -9
- package/src/mcp/tools-facts.mjs +326 -0
- package/src/mcp/tools-read.mjs +221 -0
- package/src/mcp/tools.mjs +4 -1005
- package/src/{bench → measure}/corpus.mjs +1 -1
- package/src/measure/metrics.mjs +236 -0
- package/src/measure/records.mjs +211 -0
- package/src/measure/stats.mjs +538 -0
- package/src/measure/task-schema.mjs +125 -0
- package/src/measure/tasks.mjs +69 -0
- package/src/memory/decide.mjs +6 -15
- package/src/memory/lesson-text.mjs +61 -0
- package/src/memory/lessons.mjs +29 -82
- package/src/name.mjs +7 -0
- package/src/output/index.mjs +122 -31
- package/src/packs/manage.mjs +15 -9
- package/src/plan/index.mjs +1 -1
- package/src/plugins.mjs +22 -16
- package/src/prbody.mjs +7 -11
- package/src/projection/loaders.mjs +7 -692
- package/src/projection/manifest.mjs +10 -435
- package/src/projection/mdc.mjs +7 -8
- package/src/projection/project.mjs +126 -221
- package/src/projection/region.mjs +561 -0
- package/src/projection/rules.mjs +16 -547
- package/src/projection/surfaces.mjs +43 -210
- package/src/projection/targets.mjs +5 -5
- package/src/repo-root.mjs +13 -1
- package/src/resolve/eval.mjs +2 -1
- package/src/resolve/fingerprint.mjs +0 -7
- package/src/resolve/resolve.mjs +1 -41
- package/src/review/critic.mjs +8 -12
- package/src/run.mjs +98 -0
- package/src/secrets.mjs +14 -22
- package/src/state/events.mjs +129 -0
- package/src/status-snapshot.mjs +233 -0
- package/src/status.mjs +2 -6
- package/src/sys/git.mjs +82 -0
- package/src/{bench/process.mjs → sys/proc.mjs} +28 -5
- package/src/testing/parsers.mjs +265 -0
- package/src/testing/patches.mjs +23 -0
- package/src/testing/paths.mjs +49 -0
- package/src/testing/runners.mjs +186 -0
- package/src/testing/vault.mjs +68 -0
- package/src/text/frontmatter.mjs +49 -0
- package/src/text/tokens.mjs +52 -0
- package/src/toolwords.mjs +1 -1
- package/src/util/fs.mjs +20 -0
- package/src/util/sort.mjs +12 -0
- package/src/workspace.mjs +3 -1
- package/src/write/plan.mjs +82 -0
- package/templates/plan/tracker.json +1 -0
- package/docs/decisions/ADR-033-three-plan-rows-closed-by-decision-not-by-authoring.md +0 -82
- package/docs/decisions/ADR-102-the-two-queued-review-rows-are-closed-by-ruling.md +0 -49
- package/src/bench/tasks.mjs +0 -36
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
// Parsers for the report formats a test runner writes.
|
|
2
|
+
//
|
|
3
|
+
// A runner's verdict is read from one of four shapes: TAP, the Vitest/Jest JSON result,
|
|
4
|
+
// JUnit XML, `go test -json`, and libtest's text. They are parsers, not graders, so the
|
|
5
|
+
// gate, the index and the bench read a run the same way.
|
|
6
|
+
import { readFile, readdir, rm, stat } from 'node:fs/promises';
|
|
7
|
+
import { join } from 'node:path';
|
|
8
|
+
import { tmpdir } from 'node:os';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Sort and de-duplicate a list of ids.
|
|
12
|
+
*
|
|
13
|
+
* @param {string[]} items
|
|
14
|
+
* @returns {string[]}
|
|
15
|
+
*/
|
|
16
|
+
export function dedupeSorted(items) {
|
|
17
|
+
return [...new Set(items)].sort();
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Parse TAP version 13 output from `node --test --test-reporter=tap`.
|
|
22
|
+
*
|
|
23
|
+
* Leaf test cases carry `type: 'test'` in their YAML block; suite headers carry
|
|
24
|
+
* `type: 'suite'` and are used only to build `parent > child` qualified IDs.
|
|
25
|
+
*
|
|
26
|
+
* @param {string} text
|
|
27
|
+
* @returns {{passed: string[], failed: string[]}}
|
|
28
|
+
*/
|
|
29
|
+
export function parseTapOutput(text) {
|
|
30
|
+
const passed = [];
|
|
31
|
+
const failed = [];
|
|
32
|
+
const lines = String(text ?? '').split('\n');
|
|
33
|
+
const stack = [];
|
|
34
|
+
let pending = null;
|
|
35
|
+
|
|
36
|
+
const flush = () => {
|
|
37
|
+
if (!pending) return;
|
|
38
|
+
if (pending.type !== 'suite' && !pending.hadChild && !pending.directive) {
|
|
39
|
+
const id = [...pending.ancestors, pending.name].join(' > ');
|
|
40
|
+
if (pending.ok) passed.push(id);
|
|
41
|
+
else failed.push(id);
|
|
42
|
+
}
|
|
43
|
+
pending = null;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
for (const raw of lines) {
|
|
47
|
+
const sub = /^(\s*)# Subtest:\s+(.*)$/.exec(raw);
|
|
48
|
+
if (sub) {
|
|
49
|
+
flush();
|
|
50
|
+
const indent = sub[1].length;
|
|
51
|
+
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) stack.pop();
|
|
52
|
+
if (stack.length > 0) stack[stack.length - 1].hadChild = true;
|
|
53
|
+
stack.push({ indent, name: sub[2].trim(), hadChild: false });
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const res = /^(\s*)(not ok|ok)\s+\d+(?:\s+-\s+(.*?))?(?:\s+#\s*(SKIP|TODO)\b.*)?$/i.exec(raw);
|
|
58
|
+
if (res) {
|
|
59
|
+
flush();
|
|
60
|
+
const indent = res[1].length;
|
|
61
|
+
while (stack.length > 0 && stack[stack.length - 1].indent > indent) stack.pop();
|
|
62
|
+
let hadChild = false;
|
|
63
|
+
if (stack.length > 0 && stack[stack.length - 1].indent === indent) {
|
|
64
|
+
hadChild = stack.pop().hadChild;
|
|
65
|
+
}
|
|
66
|
+
const ancestors = stack.map((entry) => entry.name);
|
|
67
|
+
const ok = res[2].toLowerCase() === 'ok';
|
|
68
|
+
const name = (res[3] ?? '').trim();
|
|
69
|
+
const directive = res[4] ? res[4].toUpperCase() : null;
|
|
70
|
+
if (name) pending = { indent, ok, name, ancestors, hadChild, directive, type: null };
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (pending) {
|
|
75
|
+
const typeMatch = /^\s*type:\s*['"]?(test|suite)['"]?\s*$/.exec(raw);
|
|
76
|
+
if (typeMatch) pending.type = typeMatch[1];
|
|
77
|
+
else if (/^\s*\.\.\.\s*$/.test(raw)) flush();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
flush();
|
|
81
|
+
|
|
82
|
+
return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Parse Vitest `--reporter=json` output. Jest emits the same
|
|
87
|
+
* `testResults`/`assertionResults` shape, so one parser serves both (review 0003 Q48).
|
|
88
|
+
*
|
|
89
|
+
* @param {string} text
|
|
90
|
+
* @returns {{passed: string[], failed: string[]}}
|
|
91
|
+
*/
|
|
92
|
+
export function parseVitestJson(text) {
|
|
93
|
+
const passed = [];
|
|
94
|
+
const failed = [];
|
|
95
|
+
const raw = String(text ?? '').trim();
|
|
96
|
+
const start = raw.indexOf('{');
|
|
97
|
+
const end = raw.lastIndexOf('}');
|
|
98
|
+
if (start === -1 || end === -1 || end < start) return { passed, failed };
|
|
99
|
+
|
|
100
|
+
let doc;
|
|
101
|
+
try {
|
|
102
|
+
doc = JSON.parse(raw.slice(start, end + 1));
|
|
103
|
+
} catch {
|
|
104
|
+
return { passed, failed };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
for (const suite of doc.testResults ?? []) {
|
|
108
|
+
for (const assertion of suite.assertionResults ?? []) {
|
|
109
|
+
const parts = [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean);
|
|
110
|
+
const id = parts.length > 0 ? parts.join(' > ') : (assertion.fullName ?? '').trim();
|
|
111
|
+
if (!id) continue;
|
|
112
|
+
if (assertion.status === 'passed') passed.push(id);
|
|
113
|
+
else if (assertion.status === 'failed') failed.push(id);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function xmlAttr(attrs, key) {
|
|
121
|
+
const match = new RegExp(`\\b${key}=["']([^"']*)["']`).exec(attrs);
|
|
122
|
+
if (!match) return '';
|
|
123
|
+
return match[1]
|
|
124
|
+
.replaceAll('"', '"')
|
|
125
|
+
.replaceAll(''', "'")
|
|
126
|
+
.replaceAll('<', '<')
|
|
127
|
+
.replaceAll('>', '>')
|
|
128
|
+
.replaceAll('&', '&')
|
|
129
|
+
.trim();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Parse JUnit XML output (pytest `--junitxml`, Gradle and Maven).
|
|
134
|
+
*
|
|
135
|
+
* @param {string} xml
|
|
136
|
+
* @returns {{passed: string[], failed: string[]}}
|
|
137
|
+
*/
|
|
138
|
+
export function parseJunitXml(xml) {
|
|
139
|
+
const passed = [];
|
|
140
|
+
const failed = [];
|
|
141
|
+
const source = String(xml ?? '');
|
|
142
|
+
const tagRe = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
|
|
143
|
+
|
|
144
|
+
for (const match of source.matchAll(tagRe)) {
|
|
145
|
+
const attrs = match[1] ?? '';
|
|
146
|
+
const inner = match[2] ?? '';
|
|
147
|
+
const name = xmlAttr(attrs, 'name');
|
|
148
|
+
const classname = xmlAttr(attrs, 'classname');
|
|
149
|
+
if (!name) continue;
|
|
150
|
+
if (/<skipped\b/i.test(inner)) continue;
|
|
151
|
+
|
|
152
|
+
const id = classname ? `${classname} > ${name}` : name;
|
|
153
|
+
if (/<(?:failure|error)\b/i.test(inner)) failed.push(id);
|
|
154
|
+
else passed.push(id);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Parse `go test -json` JSONL event stream. Only leaf events (carrying `Test`) are
|
|
162
|
+
* results; package summaries without one are ignored.
|
|
163
|
+
*
|
|
164
|
+
* @param {string} text
|
|
165
|
+
* @returns {{passed: string[], failed: string[]}}
|
|
166
|
+
*/
|
|
167
|
+
export function parseGoTestJson(text) {
|
|
168
|
+
const passed = [];
|
|
169
|
+
const failed = [];
|
|
170
|
+
for (const line of String(text ?? '').split('\n')) {
|
|
171
|
+
const trimmed = line.trim();
|
|
172
|
+
if (!trimmed.startsWith('{')) continue;
|
|
173
|
+
let event;
|
|
174
|
+
try {
|
|
175
|
+
event = JSON.parse(trimmed);
|
|
176
|
+
} catch {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (typeof event.Test !== 'string' || event.Test.length === 0) continue;
|
|
180
|
+
const id = event.Package ? `${event.Package} > ${event.Test}` : event.Test;
|
|
181
|
+
if (event.Action === 'pass') passed.push(id);
|
|
182
|
+
else if (event.Action === 'fail') failed.push(id);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Parse libtest's text output, which `cargo test` prints for every test binary.
|
|
190
|
+
*
|
|
191
|
+
* A line is a result only when it has the `test <name> ... <status>` shape. `ignored` is
|
|
192
|
+
* neither a pass nor a failure: counting it as one would make a skipped test look like
|
|
193
|
+
* evidence the change works.
|
|
194
|
+
*
|
|
195
|
+
* @param {string} text
|
|
196
|
+
* @returns {{passed: string[], failed: string[]}}
|
|
197
|
+
*/
|
|
198
|
+
export function parseCargoTestOutput(text) {
|
|
199
|
+
const passed = [];
|
|
200
|
+
const failed = [];
|
|
201
|
+
for (const line of String(text ?? '').split('\n')) {
|
|
202
|
+
const match = /^test\s+(.+?)\s+\.\.\.\s+(ok|FAILED|ignored)\s*$/.exec(line.trimEnd());
|
|
203
|
+
if (!match || match[2] === 'ignored') continue;
|
|
204
|
+
(match[2] === 'ok' ? passed : failed).push(match[1]);
|
|
205
|
+
}
|
|
206
|
+
return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Every `*.xml` under `dir`, sorted, for a deterministic read order. */
|
|
210
|
+
async function listXml(dir) {
|
|
211
|
+
const out = [];
|
|
212
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
213
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
214
|
+
const full = join(dir, entry.name);
|
|
215
|
+
if (entry.isDirectory()) out.push(...(await listXml(full)));
|
|
216
|
+
else if (entry.name.endsWith('.xml')) out.push(full);
|
|
217
|
+
}
|
|
218
|
+
return out;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Read every JUnit XML a run left behind, from report files or report *directories*
|
|
223
|
+
* (review 0003 Q48: Gradle writes one XML per class under `build/test-results/test/`,
|
|
224
|
+
* Maven under `target/surefire-reports/`; reading a single file saw none of them).
|
|
225
|
+
*
|
|
226
|
+
* @param {string[]} paths report files and/or directories
|
|
227
|
+
* @returns {Promise<string>} the concatenated XML, '' when nothing is there
|
|
228
|
+
*/
|
|
229
|
+
export async function readReports(paths) {
|
|
230
|
+
const chunks = [];
|
|
231
|
+
for (const path of paths ?? []) {
|
|
232
|
+
let isDir = false;
|
|
233
|
+
try {
|
|
234
|
+
isDir = (await stat(path)).isDirectory();
|
|
235
|
+
} catch {
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (isDir) {
|
|
239
|
+
for (const file of await listXml(path)) {
|
|
240
|
+
chunks.push(await readFile(file, 'utf8').catch(() => ''));
|
|
241
|
+
}
|
|
242
|
+
} else {
|
|
243
|
+
chunks.push(await readFile(path, 'utf8').catch(() => ''));
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return chunks.join('\n');
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Remove the report files a runner wrote outside its workspace.
|
|
251
|
+
*
|
|
252
|
+
* A report inside the workspace belongs to the workspace and is removed with it. A report
|
|
253
|
+
* this package directed to the temporary directory has no other owner, and leaving it
|
|
254
|
+
* there is how a repository ended up with `.bench-junit.xml` beside its sources (review
|
|
255
|
+
* 0003 Q79).
|
|
256
|
+
*
|
|
257
|
+
* @param {string[]} paths
|
|
258
|
+
*/
|
|
259
|
+
export async function cleanupReports(paths = []) {
|
|
260
|
+
const temp = tmpdir();
|
|
261
|
+
for (const path of paths ?? []) {
|
|
262
|
+
if (!String(path).startsWith(temp)) continue;
|
|
263
|
+
await rm(path, { recursive: true, force: true }).catch(() => {});
|
|
264
|
+
}
|
|
265
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Which files a unified diff touches.
|
|
2
|
+
//
|
|
3
|
+
// A patch is read from its destination side (`b/`), because the destination is the file
|
|
4
|
+
// that exists after the change; a deletion names `/dev/null` and contributes no path.
|
|
5
|
+
// The grader uses it to measure the blast radius, and the skills compiler uses it to ask
|
|
6
|
+
// which files a stored run touched — neither should have to import the bench's grader to
|
|
7
|
+
// read a diff header (review 0003 Q79).
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Extract repository-relative file paths touched by a unified git diff.
|
|
11
|
+
*
|
|
12
|
+
* @param {Buffer|string} patch
|
|
13
|
+
* @returns {string[]} sorted, unique paths
|
|
14
|
+
*/
|
|
15
|
+
export function extractPatchFiles(patch) {
|
|
16
|
+
const text = Buffer.isBuffer(patch) ? patch.toString('utf8') : String(patch ?? '');
|
|
17
|
+
const files = new Set();
|
|
18
|
+
for (const line of text.split('\n')) {
|
|
19
|
+
const match = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
|
|
20
|
+
if (match) files.add(match[2]);
|
|
21
|
+
}
|
|
22
|
+
return [...files].sort();
|
|
23
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// The test-path convention: is this repository-relative path a test?
|
|
2
|
+
//
|
|
3
|
+
// The directories are structural (`test/`, `tests/`, `spec/`, `__tests__/`, `src/test/`)
|
|
4
|
+
// and the suffixes are file conventions. A language's own convention arrives as data —
|
|
5
|
+
// the ecosystem records' `testConventions` (ADR-013) — so a new language lands in a
|
|
6
|
+
// record, not in this file.
|
|
7
|
+
//
|
|
8
|
+
// This module lives outside `src/bench/` because the index, the gate and the map all ask
|
|
9
|
+
// this question. Production code importing it from the bench made the bench a
|
|
10
|
+
// requirement for commands that never run a benchmark (review 0003 Q79).
|
|
11
|
+
import { testConventions } from '../ecosystem/records.mjs';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The `testConventions` records' `base` sources, compiled once.
|
|
15
|
+
*
|
|
16
|
+
* De-duplicated because a language may share a pattern with a sibling (the JVM pattern
|
|
17
|
+
* matches both `.java` and `.kt`); a duplicate RegExp is the same check run twice.
|
|
18
|
+
*/
|
|
19
|
+
const LANG_TEST_PATTERNS = [...new Set(testConventions().flatMap((record) => record.base ?? []))].map(
|
|
20
|
+
(src) => new RegExp(src),
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
const TEST_DIRECTORIES = ['test', 'tests', 'spec', '__tests__', 'src/test'];
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Classify whether a repository-relative path is a test file.
|
|
27
|
+
*
|
|
28
|
+
* @param {string} relPath forward-slash repository-relative path
|
|
29
|
+
* @returns {boolean}
|
|
30
|
+
*/
|
|
31
|
+
export function isTestPath(relPath) {
|
|
32
|
+
const norm = String(relPath).replaceAll('\\', '/').replace(/^\.\//, '');
|
|
33
|
+
if (!norm) return false;
|
|
34
|
+
const segments = norm.split('/');
|
|
35
|
+
const base = segments[segments.length - 1];
|
|
36
|
+
|
|
37
|
+
if (
|
|
38
|
+
TEST_DIRECTORIES.some((dir) => norm.startsWith(`${dir}/`)) ||
|
|
39
|
+
segments.some((segment) => ['__tests__', 'test', 'tests', 'spec'].includes(segment))
|
|
40
|
+
) {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (/\.(?:test|spec)\.[A-Za-z0-9]+$/.test(base)) return true;
|
|
45
|
+
if (/_(?:test|spec)\.[A-Za-z0-9]+$/.test(base)) return true;
|
|
46
|
+
if (LANG_TEST_PATTERNS.some((re) => re.test(base))) return true;
|
|
47
|
+
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// Which runner a repository uses, and the command that runs one set of test files.
|
|
2
|
+
//
|
|
3
|
+
// Detection reads the repository's own manifest, never a guess from a directory name: the
|
|
4
|
+
// runner that is installed is the runner a contributor runs. This module is
|
|
5
|
+
// production-facing because `simt gate` runs the repository's tests (review 0003 Q79);
|
|
6
|
+
// the bench uses the same answer, so a benchmarked run and a gated run cannot disagree
|
|
7
|
+
// about what `test` means.
|
|
8
|
+
import { existsSync } from 'node:fs';
|
|
9
|
+
import { readFile } from 'node:fs/promises';
|
|
10
|
+
import { tmpdir } from 'node:os';
|
|
11
|
+
import { basename, dirname, join } from 'node:path';
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
dedupeSorted,
|
|
15
|
+
parseCargoTestOutput,
|
|
16
|
+
parseGoTestJson,
|
|
17
|
+
parseJunitXml,
|
|
18
|
+
parseTapOutput,
|
|
19
|
+
parseVitestJson,
|
|
20
|
+
} from './parsers.mjs';
|
|
21
|
+
|
|
22
|
+
let junitCounter = 0;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* A JUnit report path this process owns.
|
|
26
|
+
*
|
|
27
|
+
* The report used to be written beside the sources as `.bench-junit.xml`, so running the
|
|
28
|
+
* gate in a Python repository left a file in the package directory (review 0003 Q79). It
|
|
29
|
+
* belongs to the run, so it lives in the temporary directory and `cleanupReports` removes
|
|
30
|
+
* it once the run has been read.
|
|
31
|
+
*/
|
|
32
|
+
function junitReportPath() {
|
|
33
|
+
junitCounter += 1;
|
|
34
|
+
return join(tmpdir(), `${CLI}-junit-${process.pid}-${junitCounter}.xml`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Detect the test runner for a repository and build the command for a set of changed test
|
|
39
|
+
* files.
|
|
40
|
+
*
|
|
41
|
+
* @param {string} root
|
|
42
|
+
* @param {string[]} testFiles
|
|
43
|
+
* @returns {Promise<{runner: string, displayCmd: string,
|
|
44
|
+
* buildExec: (worktree: string) => {cmd: string, args: string[], reportPaths?: string[]},
|
|
45
|
+
* parse: (stdout: string, reportText?: string) => {passed: string[], failed: string[]}}|null>}
|
|
46
|
+
*/
|
|
47
|
+
export async function detectTestRunner(root, testFiles) {
|
|
48
|
+
const files = [...testFiles].sort();
|
|
49
|
+
|
|
50
|
+
if (existsSync(join(root, 'package.json'))) {
|
|
51
|
+
let pkg = {};
|
|
52
|
+
try {
|
|
53
|
+
pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
|
|
54
|
+
} catch {
|
|
55
|
+
pkg = {};
|
|
56
|
+
}
|
|
57
|
+
const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
|
|
58
|
+
const testScript = String(pkg.scripts?.test ?? '');
|
|
59
|
+
if (deps.vitest || /\bvitest\b/.test(testScript)) {
|
|
60
|
+
const bin = join('node_modules', '.bin', 'vitest');
|
|
61
|
+
return {
|
|
62
|
+
runner: 'vitest',
|
|
63
|
+
displayCmd: `vitest run --reporter=json ${files.join(' ')}`,
|
|
64
|
+
buildExec: (worktree) => ({
|
|
65
|
+
cmd: existsSync(join(worktree, bin)) ? join(worktree, bin) : 'vitest',
|
|
66
|
+
args: ['run', '--reporter=json', ...files],
|
|
67
|
+
}),
|
|
68
|
+
parse: (stdout) => parseVitestJson(stdout),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
// Jest emits the same `testResults`/`assertionResults` JSON shape as Vitest (Vitest
|
|
72
|
+
// copied it), so the JSON parser serves both (review 0003 Q48: a Jest repository fell
|
|
73
|
+
// through to `node --test` and parsed TAP out of JSON).
|
|
74
|
+
if (deps.jest || /\bjest\b/.test(testScript)) {
|
|
75
|
+
const bin = join('node_modules', '.bin', 'jest');
|
|
76
|
+
return {
|
|
77
|
+
runner: 'jest',
|
|
78
|
+
displayCmd: `jest --json ${files.join(' ')}`,
|
|
79
|
+
buildExec: (worktree) => ({
|
|
80
|
+
cmd: existsSync(join(worktree, bin)) ? join(worktree, bin) : 'jest',
|
|
81
|
+
args: ['--json', ...files],
|
|
82
|
+
}),
|
|
83
|
+
parse: (stdout) => parseVitestJson(stdout),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
runner: 'node-test',
|
|
88
|
+
displayCmd: `node --test ${files.join(' ')}`,
|
|
89
|
+
buildExec: () => ({
|
|
90
|
+
cmd: process.execPath,
|
|
91
|
+
args: ['--test', '--test-reporter=tap', ...files],
|
|
92
|
+
}),
|
|
93
|
+
parse: (stdout) => parseTapOutput(stdout),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (existsSync(join(root, 'Cargo.toml'))) {
|
|
98
|
+
// Cargo has no "run this file" argument. An integration test is a target named after
|
|
99
|
+
// its file in `tests/`, so it can be selected with `--test`. A source file carries
|
|
100
|
+
// inline unit tests and can only be reached by the crate's test binary, so one such
|
|
101
|
+
// file means the whole crate runs.
|
|
102
|
+
const targets = dedupeSorted(
|
|
103
|
+
files.filter((file) => /^tests\/[^/]+\.rs$/.test(file)).map((file) => basename(file, '.rs')),
|
|
104
|
+
);
|
|
105
|
+
const integrationOnly = files.length > 0 && files.every((file) => /^tests\/[^/]+\.rs$/.test(file));
|
|
106
|
+
const args = ['test'];
|
|
107
|
+
if (integrationOnly) {
|
|
108
|
+
for (const target of targets) args.push('--test', target);
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
runner: 'cargo-test',
|
|
112
|
+
displayCmd: ['cargo', ...args].join(' '),
|
|
113
|
+
buildExec: () => ({ cmd: 'cargo', args }),
|
|
114
|
+
parse: (stdout) => parseCargoTestOutput(stdout),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (existsSync(join(root, 'go.mod'))) {
|
|
119
|
+
const packages = dedupeSorted(
|
|
120
|
+
files.map((file) => {
|
|
121
|
+
const dir = dirname(file);
|
|
122
|
+
return dir === '.' ? './...' : `./${dir}`;
|
|
123
|
+
}),
|
|
124
|
+
);
|
|
125
|
+
return {
|
|
126
|
+
runner: 'go-test',
|
|
127
|
+
displayCmd: `go test -json ${packages.join(' ')}`,
|
|
128
|
+
buildExec: () => ({ cmd: 'go', args: ['test', '-json', ...packages] }),
|
|
129
|
+
parse: (stdout) => parseGoTestJson(stdout),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (
|
|
134
|
+
existsSync(join(root, 'pyproject.toml')) ||
|
|
135
|
+
existsSync(join(root, 'requirements.txt')) ||
|
|
136
|
+
existsSync(join(root, 'setup.py'))
|
|
137
|
+
) {
|
|
138
|
+
return {
|
|
139
|
+
runner: 'pytest',
|
|
140
|
+
displayCmd: `pytest ${files.join(' ')}`,
|
|
141
|
+
buildExec: () => {
|
|
142
|
+
const reportPath = junitReportPath();
|
|
143
|
+
return {
|
|
144
|
+
cmd: 'pytest',
|
|
145
|
+
args: [`--junitxml=${reportPath}`, ...files],
|
|
146
|
+
reportPaths: [reportPath],
|
|
147
|
+
};
|
|
148
|
+
},
|
|
149
|
+
parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (existsSync(join(root, 'build.gradle.kts')) || existsSync(join(root, 'build.gradle'))) {
|
|
154
|
+
const classes = files.map((file) => basename(file).replace(/\.(?:java|kt)$/, '')).filter(Boolean);
|
|
155
|
+
const filterArgs = classes.flatMap((name) => ['--tests', `*${name}*`]);
|
|
156
|
+
return {
|
|
157
|
+
runner: 'gradle',
|
|
158
|
+
displayCmd: `./gradlew test ${filterArgs.join(' ')}`.trim(),
|
|
159
|
+
buildExec: (worktree) => ({
|
|
160
|
+
cmd: existsSync(join(worktree, 'gradlew')) ? join(worktree, 'gradlew') : 'gradle',
|
|
161
|
+
args: ['test', ...filterArgs],
|
|
162
|
+
// One XML per test class; readReports walks the directory.
|
|
163
|
+
reportPaths: [join(worktree, 'build', 'test-results', 'test')],
|
|
164
|
+
}),
|
|
165
|
+
parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (existsSync(join(root, 'pom.xml'))) {
|
|
170
|
+
const classes = files.map((file) => basename(file).replace(/\.(?:java|kt)$/, '')).filter(Boolean);
|
|
171
|
+
const arg = classes.length > 0 ? [`-Dtest=${classes.join(',')}`] : [];
|
|
172
|
+
return {
|
|
173
|
+
runner: 'maven',
|
|
174
|
+
displayCmd: `mvn test ${arg.join(' ')}`.trim(),
|
|
175
|
+
buildExec: (worktree) => ({
|
|
176
|
+
cmd: 'mvn',
|
|
177
|
+
args: ['test', ...arg],
|
|
178
|
+
// Surefire writes one XML per class here (review 0003 Q48).
|
|
179
|
+
reportPaths: [join(worktree, 'target', 'surefire-reports')],
|
|
180
|
+
}),
|
|
181
|
+
parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Where a repository's hidden test patches live.
|
|
2
|
+
//
|
|
3
|
+
// The vault is outside every checkout on purpose: a container of test patches a
|
|
4
|
+
// bench run is graded against must not be readable from the workspace it is measuring.
|
|
5
|
+
// It is keyed on the repository's root commit, which is identical in every clone and
|
|
6
|
+
// survives remote renames, so the corpus a bench was mined from is found on another
|
|
7
|
+
// machine (review 0003 Q48: an absolute-path key made the committed corpus unrunnable
|
|
8
|
+
// from any other checkout).
|
|
9
|
+
//
|
|
10
|
+
// This lives under `src/testing/` because the map measures recall against the same
|
|
11
|
+
// patches and the index asks whether a path is a test — neither of them runs a benchmark
|
|
12
|
+
// (review 0003 Q79).
|
|
13
|
+
import { execFileSync } from 'node:child_process';
|
|
14
|
+
import { createHash } from 'node:crypto';
|
|
15
|
+
import { homedir, platform as osPlatform } from 'node:os';
|
|
16
|
+
import { basename, join, resolve } from 'node:path';
|
|
17
|
+
|
|
18
|
+
import { CONFIG_DIR } from '../name.mjs';
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The stable identity a repository's vault is keyed on: the basename for a
|
|
22
|
+
* human-readable prefix, and the root commit for the digest. Outside git it degrades to
|
|
23
|
+
* the absolute-path digest: a root with no history has no better identity, and the vault
|
|
24
|
+
* path must never fail because git is absent.
|
|
25
|
+
*/
|
|
26
|
+
function repoIdentifier(root) {
|
|
27
|
+
const full = resolve(root);
|
|
28
|
+
const name = basename(full).replace(/[^A-Za-z0-9._-]/g, '-') || 'repo';
|
|
29
|
+
try {
|
|
30
|
+
const sha = execFileSync('git', ['rev-list', '--max-parents=0', 'HEAD'], {
|
|
31
|
+
cwd: root,
|
|
32
|
+
encoding: 'utf8',
|
|
33
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
34
|
+
})
|
|
35
|
+
.trim()
|
|
36
|
+
.split(/\s/)[0];
|
|
37
|
+
if (/^[0-9a-f]{40}$/.test(sha)) return `${name}-root-${sha.slice(0, 8)}`;
|
|
38
|
+
} catch {
|
|
39
|
+
// Not a git root (or no commits): fall through to the path digest.
|
|
40
|
+
}
|
|
41
|
+
const digest = createHash('sha256').update(full).digest('hex').slice(0, 8);
|
|
42
|
+
return `${name}-${digest}`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Resolve the hidden verifier vault root outside every worktree.
|
|
47
|
+
*
|
|
48
|
+
* Precedence:
|
|
49
|
+
* 1. `SIMT_DEV_VAULT_DIR` (explicit override for hermetic tests)
|
|
50
|
+
* 2. `$XDG_CACHE_HOME/simt-dev/vault/<repo-id>`
|
|
51
|
+
* 3. `~/Library/Caches/simt-dev/vault/<repo-id>` on macOS, or `~/.cache/simt-dev/vault/<repo-id>`
|
|
52
|
+
*
|
|
53
|
+
* @param {string} root
|
|
54
|
+
* @param {{env?: NodeJS.ProcessEnv, home?: string, platform?: string}} [options]
|
|
55
|
+
* @returns {string}
|
|
56
|
+
*/
|
|
57
|
+
export function resolveVaultDir(root, { env = process.env, home = homedir(), platform = osPlatform() } = {}) {
|
|
58
|
+
const repoId = repoIdentifier(root);
|
|
59
|
+
if (env.SIMT_DEV_VAULT_DIR) {
|
|
60
|
+
return join(resolve(env.SIMT_DEV_VAULT_DIR), repoId);
|
|
61
|
+
}
|
|
62
|
+
const cacheBase = env.XDG_CACHE_HOME
|
|
63
|
+
? resolve(env.XDG_CACHE_HOME)
|
|
64
|
+
: platform === 'darwin'
|
|
65
|
+
? join(home, 'Library', 'Caches')
|
|
66
|
+
: join(home, '.cache');
|
|
67
|
+
return join(cacheBase, CONFIG_DIR, 'vault', repoId);
|
|
68
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// The one frontmatter block: an opening `---` line, a body, a closing `---` line,
|
|
2
|
+
// at the top of a document.
|
|
3
|
+
//
|
|
4
|
+
// Eleven call sites carried their own copy of the regex, and the copies disagreed
|
|
5
|
+
// about CRLF: two tolerated `\r\n`, nine assumed `\n`, so a document checked out
|
|
6
|
+
// with CRLF line endings parsed in one command and came back with the block glued
|
|
7
|
+
// to the body in another. The shape here is the superset: it tolerates CRLF, and it
|
|
8
|
+
// fixes what "the block" is, so a caller that asks for the block, the body or the
|
|
9
|
+
// stripped document gets all three from one match.
|
|
10
|
+
|
|
11
|
+
/** The block as written: the two `---` lines and what is between them. The line
|
|
12
|
+
* ending after the closing `---` is NOT part of the block, because a caller that
|
|
13
|
+
* swaps the block in place must leave that newline where the file had it. */
|
|
14
|
+
const BLOCK = /^---\r?\n([\s\S]*?)\r?\n---/;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Split a leading frontmatter block from its document.
|
|
18
|
+
*
|
|
19
|
+
* @param {string} text
|
|
20
|
+
* @returns {{front: string|null, body: string, block: string|null}}
|
|
21
|
+
* `front` is the interior between the `---` lines, `block` is the block as written,
|
|
22
|
+
* and `body` is the document after the block and the one line ending that follows
|
|
23
|
+
* it. All three are null-or-unchanged when the document carries no block.
|
|
24
|
+
*/
|
|
25
|
+
export function parseFrontmatter(text) {
|
|
26
|
+
const raw = String(text ?? '');
|
|
27
|
+
const m = BLOCK.exec(raw);
|
|
28
|
+
if (!m) return { front: null, body: raw, block: null };
|
|
29
|
+
const rest = raw.slice(m[0].length);
|
|
30
|
+
const body = rest.startsWith('\r\n') ? rest.slice(2) : rest.startsWith('\n') ? rest.slice(1) : rest;
|
|
31
|
+
return { front: m[1], body, block: m[0] };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The block as written, or null when the document carries none. */
|
|
35
|
+
export function frontmatterBlock(text) {
|
|
36
|
+
const m = BLOCK.exec(String(text ?? ''));
|
|
37
|
+
return m ? m[0] : null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The document without its block, and without the line endings that followed it.
|
|
42
|
+
*
|
|
43
|
+
* The trailing-newline swallow lives here rather than in BLOCK on purpose: a strip
|
|
44
|
+
* that left the blank lines in place would send a following `^`-anchored strip (the
|
|
45
|
+
* H1 a region renderer drops) at a bare newline, and it would silently match nothing.
|
|
46
|
+
*/
|
|
47
|
+
export function stripFrontmatter(text) {
|
|
48
|
+
return String(text ?? '').replace(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n)*/, '');
|
|
49
|
+
}
|