simt-dev 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (235) hide show
  1. package/CHANGELOG.md +191 -870
  2. package/README.md +188 -213
  3. package/bin/simt.mjs +19 -55
  4. package/docs/decisions/ADR-014-workspace-is-configuration.md +2 -2
  5. package/docs/decisions/ADR-025-a-baseline-is-a-region-in-a-file-the-repo-owns.md +1 -1
  6. package/docs/decisions/ADR-026-a-config-without-a-region-is-a-file-the-repo-owns.md +1 -1
  7. package/docs/decisions/ADR-027-the-one-gitattributes-line-that-is-neither-dangerous-nor-empty.md +1 -1
  8. package/docs/decisions/ADR-028-a-base-file-is-a-floor-so-the-disagreement-it-cannot-repair-becomes-a-check.md +1 -1
  9. package/docs/decisions/ADR-029-the-pins-a-repository-declares-are-its-own.md +1 -1
  10. package/docs/decisions/ADR-030-what-a-plan-does-not-ship-is-a-decision-with-a-record.md +1 -1
  11. package/docs/decisions/ADR-048-a-baseline-carries-only-keys-a-reader-honours.md +1 -1
  12. package/docs/decisions/ADR-082-the-tracker-override-is-a-manifest-key.md +1 -1
  13. package/docs/decisions/ADR-099-the-release-verifies-served-bytes-and-the-formula-mirrors-them.md +1 -1
  14. package/docs/decisions/ADR-117-the-darwin-release-builds-on-supported-macos-15-runners-and-a-re-run-is-idempotent.md +81 -0
  15. package/docs/decisions/ADR-118-the-bench-corpus-is-keyed-on-history-is-append-only-and-is-portable-as-a-sealed-document.md +91 -0
  16. package/docs/decisions/ADR-119-the-environment-variable-points-into-a-workspace-the-repo-declared.md +90 -0
  17. package/docs/decisions/ADR-120-a-pre-push-whose-tree-is-already-proven-green-skips-the-test-gate-on-that-evidence.md +96 -0
  18. package/docs/decisions/ADR-122-the-shims-move-into-the-state-directory-and-the-legacy-directory-removes-itself.md +94 -0
  19. package/docs/decisions/ADR-123-a-surplus-positional-is-not-refused-because-arity-is-not-in-the-source.md +57 -0
  20. package/docs/decisions/ADR-124-the-manifest-has-a-schema-in-the-tool-and-its-unknown-keys-stay-the-repositorys.md +64 -0
  21. package/docs/decisions/ADR-125-a-shared-context-belongs-to-a-read-and-a-writer-reads-directly.md +65 -0
  22. package/docs/decisions/ADR-126-the-mcp-surface-stays-a-reader-so-there-is-no-write-path-to-build.md +68 -0
  23. package/docs/decisions/ADR-127-the-mcp-tool-list-is-written-by-hand-and-roots-is-not-asked-for.md +65 -0
  24. package/docs/decisions/ADR-128-a-research-plan-is-a-plan-and-the-tracker-reads-it.md +42 -0
  25. package/ecosystems/bun.json +6 -0
  26. package/ecosystems/cargo.json +25 -0
  27. package/ecosystems/composer.json +6 -0
  28. package/ecosystems/dart.json +24 -0
  29. package/ecosystems/deno.json +21 -0
  30. package/ecosystems/devcontainers.json +6 -0
  31. package/ecosystems/docker-compose.json +6 -0
  32. package/ecosystems/docker.json +6 -0
  33. package/ecosystems/dotnet.json +25 -0
  34. package/ecosystems/elixir.json +23 -0
  35. package/ecosystems/gitsubmodule.json +6 -0
  36. package/ecosystems/go.json +25 -0
  37. package/ecosystems/gradle.json +13 -0
  38. package/ecosystems/maven.json +23 -0
  39. package/ecosystems/node.json +25 -0
  40. package/ecosystems/python.json +20 -0
  41. package/ecosystems/ruby.json +23 -0
  42. package/ecosystems/swift.json +25 -0
  43. package/ecosystems/terraform.json +6 -0
  44. package/ecosystems/uv.json +6 -0
  45. package/harnesses/antigravity.json +162 -0
  46. package/harnesses/claude.json +206 -0
  47. package/harnesses/codex.json +136 -0
  48. package/harnesses/copilot.json +88 -0
  49. package/harnesses/cursor.json +140 -0
  50. package/harnesses/deepseek.json +99 -0
  51. package/harnesses/gemini.json +122 -0
  52. package/harnesses/opencode.json +173 -0
  53. package/package.json +12 -2
  54. package/rules/core.md +7 -0
  55. package/rules/mdc/intake-questions.mdc +8 -0
  56. package/rules/mdc/slice-commit.mdc +57 -4
  57. package/src/advice/advice.mjs +3 -2
  58. package/src/agenthooks/prompt.mjs +5 -2
  59. package/src/agenthooks/render.mjs +86 -180
  60. package/src/agenthooks/shim.mjs +1 -1
  61. package/src/agents/render.mjs +15 -18
  62. package/src/api.mjs +31 -0
  63. package/src/argv.mjs +48 -3
  64. package/src/bench/ablate.mjs +3 -4
  65. package/src/bench/adapters.json +163 -105
  66. package/src/bench/adapters.mjs +466 -0
  67. package/src/bench/arms.mjs +333 -0
  68. package/src/bench/grade.mjs +33 -73
  69. package/src/bench/graders.mjs +42 -0
  70. package/src/bench/mine.mjs +121 -475
  71. package/src/bench/probe-ledger.mjs +149 -0
  72. package/src/bench/probes.catalog.mjs +918 -0
  73. package/src/bench/probes.mjs +79 -1064
  74. package/src/bench/render.mjs +111 -0
  75. package/src/bench/report.mjs +120 -962
  76. package/src/bench/run.mjs +185 -802
  77. package/src/bench/variants.mjs +4 -5
  78. package/src/bench/workspace.mjs +160 -0
  79. package/src/brief/assemble.mjs +48 -15
  80. package/src/ci/dependabot.mjs +25 -55
  81. package/src/ci/workflow.mjs +30 -7
  82. package/src/cli.mjs +45 -0
  83. package/src/commands/agent.mjs +4 -13
  84. package/src/commands/agenthook.mjs +70 -40
  85. package/src/commands/bench.mjs +191 -18
  86. package/src/commands/brief.mjs +5 -4
  87. package/src/commands/ci.mjs +13 -14
  88. package/src/commands/compare.mjs +42 -42
  89. package/src/commands/completions.mjs +2 -6
  90. package/src/commands/decide.mjs +4 -4
  91. package/src/commands/distill.mjs +4 -3
  92. package/src/commands/docs.mjs +88 -0
  93. package/src/commands/doctor.mjs +11 -7
  94. package/src/commands/evidence.mjs +6 -9
  95. package/src/commands/explain.mjs +0 -1
  96. package/src/commands/gate.mjs +26 -25
  97. package/src/commands/graph.mjs +2 -3
  98. package/src/commands/harness.mjs +6 -5
  99. package/src/commands/help.mjs +168 -19
  100. package/src/commands/hooks.mjs +119 -49
  101. package/src/commands/ignore.mjs +13 -8
  102. package/src/commands/impact.mjs +8 -15
  103. package/src/commands/index.mjs +318 -193
  104. package/src/commands/init-scan.mjs +90 -0
  105. package/src/commands/init.mjs +124 -105
  106. package/src/commands/lease.mjs +73 -25
  107. package/src/commands/lessons.mjs +10 -9
  108. package/src/commands/local.mjs +3 -3
  109. package/src/commands/map.mjs +24 -14
  110. package/src/commands/mcp.mjs +2 -18
  111. package/src/commands/pack.mjs +9 -6
  112. package/src/commands/plan.mjs +6 -5
  113. package/src/commands/plugin.mjs +2 -2
  114. package/src/commands/prbody.mjs +3 -3
  115. package/src/commands/repoindex.mjs +12 -11
  116. package/src/commands/review.mjs +2 -2
  117. package/src/commands/rules.mjs +37 -16
  118. package/src/commands/shared.mjs +14 -19
  119. package/src/commands/skills.mjs +33 -32
  120. package/src/commands/skillscompile.mjs +7 -8
  121. package/src/commands/skillsinstall.mjs +7 -6
  122. package/src/commands/slice.mjs +220 -74
  123. package/src/commands/spec.mjs +167 -0
  124. package/src/commands/status.mjs +27 -18
  125. package/src/commands/sync.mjs +27 -25
  126. package/src/commands/tests.mjs +18 -21
  127. package/src/commands/tokens.mjs +21 -26
  128. package/src/commands/undo.mjs +5 -15
  129. package/src/commands/upgrade.mjs +15 -7
  130. package/src/commands/why.mjs +15 -21
  131. package/src/compare/hooksstate.mjs +2 -6
  132. package/src/compare/learn.mjs +3 -1
  133. package/src/compare/row.mjs +0 -2
  134. package/src/context.mjs +64 -0
  135. package/src/decisions/index-doc.mjs +46 -0
  136. package/src/decisions/index.mjs +3 -2
  137. package/src/diff.mjs +1 -1
  138. package/src/distill/distill.mjs +14 -1
  139. package/src/distill/engine.mjs +5 -1
  140. package/src/docs/commands.mjs +65 -0
  141. package/src/ecosystem/records.mjs +219 -0
  142. package/src/explain.mjs +31 -0
  143. package/src/gates/index.mjs +385 -0
  144. package/src/graph/compile.mjs +8 -4
  145. package/src/graph/load.mjs +6 -2
  146. package/src/graph/spec.mjs +10 -8
  147. package/src/harness/records.mjs +143 -0
  148. package/src/hooks/capability.mjs +7 -5
  149. package/src/hooks/dispatcher.mjs +209 -60
  150. package/src/hooks/gates.mjs +10 -43
  151. package/src/hooks/resolve.mjs +4 -14
  152. package/src/hooks/shim.mjs +5 -2
  153. package/src/hooks/where.mjs +2 -7
  154. package/src/identity.mjs +31 -12
  155. package/src/index/coverage.mjs +58 -0
  156. package/src/index/extract.mjs +6 -16
  157. package/src/index/gate.mjs +1 -1
  158. package/src/index/govern.mjs +26 -9
  159. package/src/index/impact.mjs +4 -8
  160. package/src/index/map.mjs +14 -15
  161. package/src/index/open.mjs +173 -0
  162. package/src/index/persist.mjs +224 -57
  163. package/src/index/store.mjs +82 -39
  164. package/src/index/tests.mjs +2 -6
  165. package/src/journal.mjs +94 -20
  166. package/src/kernel/card.mjs +37 -63
  167. package/src/kernel/checks.mjs +4 -10
  168. package/src/lease.mjs +102 -7
  169. package/src/ledger/index.mjs +150 -14
  170. package/src/local/client.mjs +4 -6
  171. package/src/locate.mjs +33 -9
  172. package/src/manifest/schema.mjs +213 -0
  173. package/src/mcp/paginate.mjs +33 -0
  174. package/src/mcp/resources.mjs +7 -10
  175. package/src/mcp/schemas.mjs +512 -0
  176. package/src/mcp/server.mjs +119 -25
  177. package/src/mcp/tools-facts.mjs +326 -0
  178. package/src/mcp/tools-read.mjs +221 -0
  179. package/src/mcp/tools.mjs +4 -967
  180. package/src/measure/corpus.mjs +150 -0
  181. package/src/measure/metrics.mjs +236 -0
  182. package/src/measure/records.mjs +211 -0
  183. package/src/measure/session.mjs +3 -2
  184. package/src/measure/stats.mjs +538 -0
  185. package/src/measure/task-schema.mjs +125 -0
  186. package/src/measure/tasks.mjs +69 -0
  187. package/src/memory/decide.mjs +6 -15
  188. package/src/memory/lesson-text.mjs +61 -0
  189. package/src/memory/lessons.mjs +29 -82
  190. package/src/name.mjs +7 -0
  191. package/src/node-version.mjs +37 -0
  192. package/src/output/index.mjs +209 -10
  193. package/src/packs/manage.mjs +32 -12
  194. package/src/packs.mjs +20 -2
  195. package/src/plan/index.mjs +1 -1
  196. package/src/plugins.mjs +22 -16
  197. package/src/prbody.mjs +7 -11
  198. package/src/projection/loaders.mjs +7 -692
  199. package/src/projection/lock.mjs +11 -1
  200. package/src/projection/manifest.mjs +29 -468
  201. package/src/projection/mdc.mjs +7 -8
  202. package/src/projection/project.mjs +126 -221
  203. package/src/projection/region.mjs +561 -0
  204. package/src/projection/rules.mjs +20 -550
  205. package/src/projection/surfaces.mjs +43 -210
  206. package/src/projection/targets.mjs +5 -5
  207. package/src/repo-root.mjs +31 -0
  208. package/src/resolve/eval.mjs +2 -1
  209. package/src/resolve/fingerprint.mjs +19 -22
  210. package/src/resolve/resolve.mjs +1 -41
  211. package/src/review/critic.mjs +10 -13
  212. package/src/run.mjs +98 -0
  213. package/src/secrets.mjs +14 -22
  214. package/src/state/events.mjs +129 -0
  215. package/src/status-snapshot.mjs +233 -0
  216. package/src/status.mjs +2 -6
  217. package/src/sys/git.mjs +82 -0
  218. package/src/sys/proc.mjs +151 -0
  219. package/src/testing/parsers.mjs +265 -0
  220. package/src/testing/patches.mjs +23 -0
  221. package/src/testing/paths.mjs +49 -0
  222. package/src/testing/runners.mjs +186 -0
  223. package/src/testing/vault.mjs +68 -0
  224. package/src/text/frontmatter.mjs +49 -0
  225. package/src/text/tokens.mjs +52 -0
  226. package/src/toolwords.mjs +3 -0
  227. package/src/util/fs.mjs +20 -0
  228. package/src/util/sort.mjs +12 -0
  229. package/src/workspace.mjs +58 -11
  230. package/src/write/plan.mjs +82 -0
  231. package/templates/plan/tracker.json +1 -0
  232. package/docs/decisions/ADR-033-three-plan-rows-closed-by-decision-not-by-authoring.md +0 -82
  233. package/docs/decisions/ADR-102-the-two-queued-review-rows-are-closed-by-ruling.md +0 -49
  234. package/docs/decisions/README.md +0 -600
  235. package/src/bench/tasks.mjs +0 -36
@@ -0,0 +1,151 @@
1
+ // One bounded child process and the environment a child inherits, for everything
2
+ // that runs one (review 0003 Q47, Q72).
3
+ //
4
+ // `child.kill('SIGKILL')` kills the process the caller started and leaves its
5
+ // grandchildren running: an adapter that ran `(sleep 15 &); sleep 60` kept the pipe open
6
+ // and the caller waited 62.6 s against a 2 s timeout, then graded the result. A detached
7
+ // child is its own process group, so the deadline kills the group, destroys the streams
8
+ // and settles. A run cannot outlive its own deadline, whatever it spawned.
9
+ //
10
+ // Output is bounded so a runaway child cannot decide the caller's memory: past the cap
11
+ // the oldest chunk is dropped and `truncated` says so, rather than the spawn failing.
12
+ //
13
+ // This module used to live in `src/bench/` because the bench was its only caller. The
14
+ // bench is not the only thing that runs a child it does not trust, and a second copy of
15
+ // the deadline logic is a second way for a run to outlive its deadline.
16
+
17
+ import { spawn } from 'node:child_process';
18
+
19
+ /** The most output one run may accumulate before the oldest bytes are dropped. */
20
+ const DEFAULT_MAX_OUTPUT_BYTES = 8 << 20;
21
+
22
+ /** How long to wait for streams to close after the process group is killed. */
23
+ const HARD_STOP_GRACE_MS = 500;
24
+
25
+ function boundedPush(chunks, state, chunk) {
26
+ state.bytes += chunk.length;
27
+ chunks.push(chunk);
28
+ while (state.bytes > state.max && chunks.length > 1) {
29
+ state.bytes -= chunks.shift().length;
30
+ state.truncated = true;
31
+ }
32
+ }
33
+
34
+ /**
35
+ * Run `cmd` with `args`, bounded by a deadline and an output cap.
36
+ *
37
+ * @param {string} cmd
38
+ * @param {string[]} args
39
+ * @param {{cwd?: string, env?: NodeJS.ProcessEnv, input?: Buffer|string|null, timeoutMs?: number, maxOutputBytes?: number}} [options]
40
+ * @returns {Promise<{code: number, timedOut: boolean, wallMs: number, stdout: Buffer, stderr: Buffer, truncated: boolean, spawnError: string|null}>}
41
+ * `code` is 124 when the deadline fired, and `wallMs` is what the run actually took.
42
+ */
43
+ export function spawnBounded(
44
+ cmd,
45
+ args,
46
+ { cwd, env, input = null, timeoutMs = 0, maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES } = {},
47
+ ) {
48
+ return new Promise((settle) => {
49
+ const startedAt = Date.now();
50
+ let child;
51
+ try {
52
+ child = spawn(cmd, args, {
53
+ cwd,
54
+ env,
55
+ stdio: ['pipe', 'pipe', 'pipe'],
56
+ detached: true,
57
+ });
58
+ } catch (error) {
59
+ settle({
60
+ code: 1,
61
+ timedOut: false,
62
+ wallMs: 0,
63
+ stdout: Buffer.alloc(0),
64
+ stderr: Buffer.from(String(error?.message ?? error)),
65
+ truncated: false,
66
+ spawnError: String(error?.message ?? error),
67
+ });
68
+ return;
69
+ }
70
+
71
+ const out = [];
72
+ const err = [];
73
+ const outState = { bytes: 0, max: maxOutputBytes, truncated: false };
74
+ const errState = { bytes: 0, max: maxOutputBytes, truncated: false };
75
+ let timer = null;
76
+ let hardStop = null;
77
+ let timedOut = false;
78
+ let done = false;
79
+
80
+ const killGroup = () => {
81
+ try {
82
+ process.kill(-child.pid, 'SIGKILL');
83
+ } catch {
84
+ try {
85
+ child.kill('SIGKILL');
86
+ } catch {
87
+ // Already gone; the close handler settles the promise.
88
+ }
89
+ }
90
+ };
91
+
92
+ const finish = (code, spawnError = null) => {
93
+ if (done) return;
94
+ done = true;
95
+ if (timer) clearTimeout(timer);
96
+ if (hardStop) clearTimeout(hardStop);
97
+ child.stdout?.destroy();
98
+ child.stderr?.destroy();
99
+ settle({
100
+ code,
101
+ timedOut,
102
+ wallMs: Date.now() - startedAt,
103
+ stdout: Buffer.concat(out),
104
+ stderr: Buffer.concat(err),
105
+ truncated: outState.truncated || errState.truncated,
106
+ spawnError,
107
+ });
108
+ };
109
+
110
+ if (timeoutMs > 0) {
111
+ timer = setTimeout(() => {
112
+ timedOut = true;
113
+ killGroup();
114
+ // A process that escaped the group can hold the pipes open after the kill, and a
115
+ // deadline that waits on a stranger's cleanup is not a deadline.
116
+ hardStop = setTimeout(() => finish(124), HARD_STOP_GRACE_MS);
117
+ hardStop.unref?.();
118
+ }, timeoutMs);
119
+ timer.unref?.();
120
+ }
121
+
122
+ child.stdout?.on('data', (chunk) => boundedPush(out, outState, chunk));
123
+ child.stderr?.on('data', (chunk) => boundedPush(err, errState, chunk));
124
+ child.on('error', (error) => {
125
+ const message = String(error?.message ?? error);
126
+ boundedPush(err, errState, Buffer.from(message));
127
+ finish(1, message);
128
+ });
129
+ child.on('close', (code) => finish(timedOut ? 124 : (code ?? 1)));
130
+
131
+ child.stdin?.end(input ?? undefined);
132
+ });
133
+ }
134
+
135
+ /**
136
+ * What a spawned child is allowed to inherit from the operator's shell.
137
+ *
138
+ * Three bench modules grew this identically, and the differences between what a child
139
+ * needs and what the shell carries are the whole point: `SIMT_DEV_WORKSPACE` is forced
140
+ * off so a child cannot resolve THIS repository's workspace, and the `NODE_TEST*`
141
+ * variables are dropped because they steer `node --test` at a test directory the child
142
+ * does not have. Nothing here names a credential or a state directory: those come from
143
+ * the caller's own `extra`.
144
+ */
145
+ export function sanitizedEnv(extra = {}) {
146
+ const env = { ...process.env, SIMT_DEV_WORKSPACE: 'off', ...extra };
147
+ for (const key of Object.keys(env)) {
148
+ if (key.startsWith('NODE_TEST')) delete env[key];
149
+ }
150
+ return env;
151
+ }
@@ -0,0 +1,265 @@
1
+ // Parsers for the report formats a test runner writes.
2
+ //
3
+ // A runner's verdict is read from one of four shapes: TAP, the Vitest/Jest JSON result,
4
+ // JUnit XML, `go test -json`, and libtest's text. They are parsers, not graders, so the
5
+ // gate, the index and the bench read a run the same way.
6
+ import { readFile, readdir, rm, stat } from 'node:fs/promises';
7
+ import { join } from 'node:path';
8
+ import { tmpdir } from 'node:os';
9
+
10
+ /**
11
+ * Sort and de-duplicate a list of ids.
12
+ *
13
+ * @param {string[]} items
14
+ * @returns {string[]}
15
+ */
16
+ export function dedupeSorted(items) {
17
+ return [...new Set(items)].sort();
18
+ }
19
+
20
+ /**
21
+ * Parse TAP version 13 output from `node --test --test-reporter=tap`.
22
+ *
23
+ * Leaf test cases carry `type: 'test'` in their YAML block; suite headers carry
24
+ * `type: 'suite'` and are used only to build `parent > child` qualified IDs.
25
+ *
26
+ * @param {string} text
27
+ * @returns {{passed: string[], failed: string[]}}
28
+ */
29
+ export function parseTapOutput(text) {
30
+ const passed = [];
31
+ const failed = [];
32
+ const lines = String(text ?? '').split('\n');
33
+ const stack = [];
34
+ let pending = null;
35
+
36
+ const flush = () => {
37
+ if (!pending) return;
38
+ if (pending.type !== 'suite' && !pending.hadChild && !pending.directive) {
39
+ const id = [...pending.ancestors, pending.name].join(' > ');
40
+ if (pending.ok) passed.push(id);
41
+ else failed.push(id);
42
+ }
43
+ pending = null;
44
+ };
45
+
46
+ for (const raw of lines) {
47
+ const sub = /^(\s*)# Subtest:\s+(.*)$/.exec(raw);
48
+ if (sub) {
49
+ flush();
50
+ const indent = sub[1].length;
51
+ while (stack.length > 0 && stack[stack.length - 1].indent >= indent) stack.pop();
52
+ if (stack.length > 0) stack[stack.length - 1].hadChild = true;
53
+ stack.push({ indent, name: sub[2].trim(), hadChild: false });
54
+ continue;
55
+ }
56
+
57
+ const res = /^(\s*)(not ok|ok)\s+\d+(?:\s+-\s+(.*?))?(?:\s+#\s*(SKIP|TODO)\b.*)?$/i.exec(raw);
58
+ if (res) {
59
+ flush();
60
+ const indent = res[1].length;
61
+ while (stack.length > 0 && stack[stack.length - 1].indent > indent) stack.pop();
62
+ let hadChild = false;
63
+ if (stack.length > 0 && stack[stack.length - 1].indent === indent) {
64
+ hadChild = stack.pop().hadChild;
65
+ }
66
+ const ancestors = stack.map((entry) => entry.name);
67
+ const ok = res[2].toLowerCase() === 'ok';
68
+ const name = (res[3] ?? '').trim();
69
+ const directive = res[4] ? res[4].toUpperCase() : null;
70
+ if (name) pending = { indent, ok, name, ancestors, hadChild, directive, type: null };
71
+ continue;
72
+ }
73
+
74
+ if (pending) {
75
+ const typeMatch = /^\s*type:\s*['"]?(test|suite)['"]?\s*$/.exec(raw);
76
+ if (typeMatch) pending.type = typeMatch[1];
77
+ else if (/^\s*\.\.\.\s*$/.test(raw)) flush();
78
+ }
79
+ }
80
+ flush();
81
+
82
+ return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
83
+ }
84
+
85
+ /**
86
+ * Parse Vitest `--reporter=json` output. Jest emits the same
87
+ * `testResults`/`assertionResults` shape, so one parser serves both (review 0003 Q48).
88
+ *
89
+ * @param {string} text
90
+ * @returns {{passed: string[], failed: string[]}}
91
+ */
92
+ export function parseVitestJson(text) {
93
+ const passed = [];
94
+ const failed = [];
95
+ const raw = String(text ?? '').trim();
96
+ const start = raw.indexOf('{');
97
+ const end = raw.lastIndexOf('}');
98
+ if (start === -1 || end === -1 || end < start) return { passed, failed };
99
+
100
+ let doc;
101
+ try {
102
+ doc = JSON.parse(raw.slice(start, end + 1));
103
+ } catch {
104
+ return { passed, failed };
105
+ }
106
+
107
+ for (const suite of doc.testResults ?? []) {
108
+ for (const assertion of suite.assertionResults ?? []) {
109
+ const parts = [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean);
110
+ const id = parts.length > 0 ? parts.join(' > ') : (assertion.fullName ?? '').trim();
111
+ if (!id) continue;
112
+ if (assertion.status === 'passed') passed.push(id);
113
+ else if (assertion.status === 'failed') failed.push(id);
114
+ }
115
+ }
116
+
117
+ return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
118
+ }
119
+
120
+ function xmlAttr(attrs, key) {
121
+ const match = new RegExp(`\\b${key}=["']([^"']*)["']`).exec(attrs);
122
+ if (!match) return '';
123
+ return match[1]
124
+ .replaceAll('&quot;', '"')
125
+ .replaceAll('&apos;', "'")
126
+ .replaceAll('&lt;', '<')
127
+ .replaceAll('&gt;', '>')
128
+ .replaceAll('&amp;', '&')
129
+ .trim();
130
+ }
131
+
132
+ /**
133
+ * Parse JUnit XML output (pytest `--junitxml`, Gradle and Maven).
134
+ *
135
+ * @param {string} xml
136
+ * @returns {{passed: string[], failed: string[]}}
137
+ */
138
+ export function parseJunitXml(xml) {
139
+ const passed = [];
140
+ const failed = [];
141
+ const source = String(xml ?? '');
142
+ const tagRe = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
143
+
144
+ for (const match of source.matchAll(tagRe)) {
145
+ const attrs = match[1] ?? '';
146
+ const inner = match[2] ?? '';
147
+ const name = xmlAttr(attrs, 'name');
148
+ const classname = xmlAttr(attrs, 'classname');
149
+ if (!name) continue;
150
+ if (/<skipped\b/i.test(inner)) continue;
151
+
152
+ const id = classname ? `${classname} > ${name}` : name;
153
+ if (/<(?:failure|error)\b/i.test(inner)) failed.push(id);
154
+ else passed.push(id);
155
+ }
156
+
157
+ return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
158
+ }
159
+
160
+ /**
161
+ * Parse `go test -json` JSONL event stream. Only leaf events (carrying `Test`) are
162
+ * results; package summaries without one are ignored.
163
+ *
164
+ * @param {string} text
165
+ * @returns {{passed: string[], failed: string[]}}
166
+ */
167
+ export function parseGoTestJson(text) {
168
+ const passed = [];
169
+ const failed = [];
170
+ for (const line of String(text ?? '').split('\n')) {
171
+ const trimmed = line.trim();
172
+ if (!trimmed.startsWith('{')) continue;
173
+ let event;
174
+ try {
175
+ event = JSON.parse(trimmed);
176
+ } catch {
177
+ continue;
178
+ }
179
+ if (typeof event.Test !== 'string' || event.Test.length === 0) continue;
180
+ const id = event.Package ? `${event.Package} > ${event.Test}` : event.Test;
181
+ if (event.Action === 'pass') passed.push(id);
182
+ else if (event.Action === 'fail') failed.push(id);
183
+ }
184
+
185
+ return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
186
+ }
187
+
188
+ /**
189
+ * Parse libtest's text output, which `cargo test` prints for every test binary.
190
+ *
191
+ * A line is a result only when it has the `test <name> ... <status>` shape. `ignored` is
192
+ * neither a pass nor a failure: counting it as one would make a skipped test look like
193
+ * evidence the change works.
194
+ *
195
+ * @param {string} text
196
+ * @returns {{passed: string[], failed: string[]}}
197
+ */
198
+ export function parseCargoTestOutput(text) {
199
+ const passed = [];
200
+ const failed = [];
201
+ for (const line of String(text ?? '').split('\n')) {
202
+ const match = /^test\s+(.+?)\s+\.\.\.\s+(ok|FAILED|ignored)\s*$/.exec(line.trimEnd());
203
+ if (!match || match[2] === 'ignored') continue;
204
+ (match[2] === 'ok' ? passed : failed).push(match[1]);
205
+ }
206
+ return { passed: dedupeSorted(passed), failed: dedupeSorted(failed) };
207
+ }
208
+
209
+ /** Every `*.xml` under `dir`, sorted, for a deterministic read order. */
210
+ async function listXml(dir) {
211
+ const out = [];
212
+ const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
213
+ for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
214
+ const full = join(dir, entry.name);
215
+ if (entry.isDirectory()) out.push(...(await listXml(full)));
216
+ else if (entry.name.endsWith('.xml')) out.push(full);
217
+ }
218
+ return out;
219
+ }
220
+
221
+ /**
222
+ * Read every JUnit XML a run left behind, from report files or report *directories*
223
+ * (review 0003 Q48: Gradle writes one XML per class under `build/test-results/test/`,
224
+ * Maven under `target/surefire-reports/`; reading a single file saw none of them).
225
+ *
226
+ * @param {string[]} paths report files and/or directories
227
+ * @returns {Promise<string>} the concatenated XML, '' when nothing is there
228
+ */
229
+ export async function readReports(paths) {
230
+ const chunks = [];
231
+ for (const path of paths ?? []) {
232
+ let isDir = false;
233
+ try {
234
+ isDir = (await stat(path)).isDirectory();
235
+ } catch {
236
+ continue;
237
+ }
238
+ if (isDir) {
239
+ for (const file of await listXml(path)) {
240
+ chunks.push(await readFile(file, 'utf8').catch(() => ''));
241
+ }
242
+ } else {
243
+ chunks.push(await readFile(path, 'utf8').catch(() => ''));
244
+ }
245
+ }
246
+ return chunks.join('\n');
247
+ }
248
+
249
+ /**
250
+ * Remove the report files a runner wrote outside its workspace.
251
+ *
252
+ * A report inside the workspace belongs to the workspace and is removed with it. A report
253
+ * this package directed to the temporary directory has no other owner, and leaving it
254
+ * there is how a repository ended up with `.bench-junit.xml` beside its sources (review
255
+ * 0003 Q79).
256
+ *
257
+ * @param {string[]} paths
258
+ */
259
+ export async function cleanupReports(paths = []) {
260
+ const temp = tmpdir();
261
+ for (const path of paths ?? []) {
262
+ if (!String(path).startsWith(temp)) continue;
263
+ await rm(path, { recursive: true, force: true }).catch(() => {});
264
+ }
265
+ }
@@ -0,0 +1,23 @@
1
+ // Which files a unified diff touches.
2
+ //
3
+ // A patch is read from its destination side (`b/`), because the destination is the file
4
+ // that exists after the change; a deletion names `/dev/null` and contributes no path.
5
+ // The grader uses it to measure the blast radius, and the skills compiler uses it to ask
6
+ // which files a stored run touched — neither should have to import the bench's grader to
7
+ // read a diff header (review 0003 Q79).
8
+
9
+ /**
10
+ * Extract repository-relative file paths touched by a unified git diff.
11
+ *
12
+ * @param {Buffer|string} patch
13
+ * @returns {string[]} sorted, unique paths
14
+ */
15
+ export function extractPatchFiles(patch) {
16
+ const text = Buffer.isBuffer(patch) ? patch.toString('utf8') : String(patch ?? '');
17
+ const files = new Set();
18
+ for (const line of text.split('\n')) {
19
+ const match = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
20
+ if (match) files.add(match[2]);
21
+ }
22
+ return [...files].sort();
23
+ }
@@ -0,0 +1,49 @@
1
+ // The test-path convention: is this repository-relative path a test?
2
+ //
3
+ // The directories are structural (`test/`, `tests/`, `spec/`, `__tests__/`, `src/test/`)
4
+ // and the suffixes are file conventions. A language's own convention arrives as data —
5
+ // the ecosystem records' `testConventions` (ADR-013) — so a new language lands in a
6
+ // record, not in this file.
7
+ //
8
+ // This module lives outside `src/bench/` because the index, the gate and the map all ask
9
+ // this question. Production code importing it from the bench made the bench a
10
+ // requirement for commands that never run a benchmark (review 0003 Q79).
11
+ import { testConventions } from '../ecosystem/records.mjs';
12
+
13
+ /**
14
+ * The `testConventions` records' `base` sources, compiled once.
15
+ *
16
+ * De-duplicated because a language may share a pattern with a sibling (the JVM pattern
17
+ * matches both `.java` and `.kt`); a duplicate RegExp is the same check run twice.
18
+ */
19
+ const LANG_TEST_PATTERNS = [...new Set(testConventions().flatMap((record) => record.base ?? []))].map(
20
+ (src) => new RegExp(src),
21
+ );
22
+
23
+ const TEST_DIRECTORIES = ['test', 'tests', 'spec', '__tests__', 'src/test'];
24
+
25
+ /**
26
+ * Classify whether a repository-relative path is a test file.
27
+ *
28
+ * @param {string} relPath forward-slash repository-relative path
29
+ * @returns {boolean}
30
+ */
31
+ export function isTestPath(relPath) {
32
+ const norm = String(relPath).replaceAll('\\', '/').replace(/^\.\//, '');
33
+ if (!norm) return false;
34
+ const segments = norm.split('/');
35
+ const base = segments[segments.length - 1];
36
+
37
+ if (
38
+ TEST_DIRECTORIES.some((dir) => norm.startsWith(`${dir}/`)) ||
39
+ segments.some((segment) => ['__tests__', 'test', 'tests', 'spec'].includes(segment))
40
+ ) {
41
+ return true;
42
+ }
43
+
44
+ if (/\.(?:test|spec)\.[A-Za-z0-9]+$/.test(base)) return true;
45
+ if (/_(?:test|spec)\.[A-Za-z0-9]+$/.test(base)) return true;
46
+ if (LANG_TEST_PATTERNS.some((re) => re.test(base))) return true;
47
+
48
+ return false;
49
+ }
@@ -0,0 +1,186 @@
1
+ // Which runner a repository uses, and the command that runs one set of test files.
2
+ //
3
+ // Detection reads the repository's own manifest, never a guess from a directory name: the
4
+ // runner that is installed is the runner a contributor runs. This module is
5
+ // production-facing because `simt gate` runs the repository's tests (review 0003 Q79);
6
+ // the bench uses the same answer, so a benchmarked run and a gated run cannot disagree
7
+ // about what `test` means.
8
+ import { existsSync } from 'node:fs';
9
+ import { readFile } from 'node:fs/promises';
10
+ import { tmpdir } from 'node:os';
11
+ import { basename, dirname, join } from 'node:path';
12
+
13
+ import {
14
+ dedupeSorted,
15
+ parseCargoTestOutput,
16
+ parseGoTestJson,
17
+ parseJunitXml,
18
+ parseTapOutput,
19
+ parseVitestJson,
20
+ } from './parsers.mjs';
21
+
22
+ let junitCounter = 0;
23
+
24
+ /**
25
+ * A JUnit report path this process owns.
26
+ *
27
+ * The report used to be written beside the sources as `.bench-junit.xml`, so running the
28
+ * gate in a Python repository left a file in the package directory (review 0003 Q79). It
29
+ * belongs to the run, so it lives in the temporary directory and `cleanupReports` removes
30
+ * it once the run has been read.
31
+ */
32
+ function junitReportPath() {
33
+ junitCounter += 1;
34
+ return join(tmpdir(), `${CLI}-junit-${process.pid}-${junitCounter}.xml`);
35
+ }
36
+
37
+ /**
38
+ * Detect the test runner for a repository and build the command for a set of changed test
39
+ * files.
40
+ *
41
+ * @param {string} root
42
+ * @param {string[]} testFiles
43
+ * @returns {Promise<{runner: string, displayCmd: string,
44
+ * buildExec: (worktree: string) => {cmd: string, args: string[], reportPaths?: string[]},
45
+ * parse: (stdout: string, reportText?: string) => {passed: string[], failed: string[]}}|null>}
46
+ */
47
+ export async function detectTestRunner(root, testFiles) {
48
+ const files = [...testFiles].sort();
49
+
50
+ if (existsSync(join(root, 'package.json'))) {
51
+ let pkg = {};
52
+ try {
53
+ pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
54
+ } catch {
55
+ pkg = {};
56
+ }
57
+ const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
58
+ const testScript = String(pkg.scripts?.test ?? '');
59
+ if (deps.vitest || /\bvitest\b/.test(testScript)) {
60
+ const bin = join('node_modules', '.bin', 'vitest');
61
+ return {
62
+ runner: 'vitest',
63
+ displayCmd: `vitest run --reporter=json ${files.join(' ')}`,
64
+ buildExec: (worktree) => ({
65
+ cmd: existsSync(join(worktree, bin)) ? join(worktree, bin) : 'vitest',
66
+ args: ['run', '--reporter=json', ...files],
67
+ }),
68
+ parse: (stdout) => parseVitestJson(stdout),
69
+ };
70
+ }
71
+ // Jest emits the same `testResults`/`assertionResults` JSON shape as Vitest (Vitest
72
+ // copied it), so the JSON parser serves both (review 0003 Q48: a Jest repository fell
73
+ // through to `node --test` and parsed TAP out of JSON).
74
+ if (deps.jest || /\bjest\b/.test(testScript)) {
75
+ const bin = join('node_modules', '.bin', 'jest');
76
+ return {
77
+ runner: 'jest',
78
+ displayCmd: `jest --json ${files.join(' ')}`,
79
+ buildExec: (worktree) => ({
80
+ cmd: existsSync(join(worktree, bin)) ? join(worktree, bin) : 'jest',
81
+ args: ['--json', ...files],
82
+ }),
83
+ parse: (stdout) => parseVitestJson(stdout),
84
+ };
85
+ }
86
+ return {
87
+ runner: 'node-test',
88
+ displayCmd: `node --test ${files.join(' ')}`,
89
+ buildExec: () => ({
90
+ cmd: process.execPath,
91
+ args: ['--test', '--test-reporter=tap', ...files],
92
+ }),
93
+ parse: (stdout) => parseTapOutput(stdout),
94
+ };
95
+ }
96
+
97
+ if (existsSync(join(root, 'Cargo.toml'))) {
98
+ // Cargo has no "run this file" argument. An integration test is a target named after
99
+ // its file in `tests/`, so it can be selected with `--test`. A source file carries
100
+ // inline unit tests and can only be reached by the crate's test binary, so one such
101
+ // file means the whole crate runs.
102
+ const targets = dedupeSorted(
103
+ files.filter((file) => /^tests\/[^/]+\.rs$/.test(file)).map((file) => basename(file, '.rs')),
104
+ );
105
+ const integrationOnly = files.length > 0 && files.every((file) => /^tests\/[^/]+\.rs$/.test(file));
106
+ const args = ['test'];
107
+ if (integrationOnly) {
108
+ for (const target of targets) args.push('--test', target);
109
+ }
110
+ return {
111
+ runner: 'cargo-test',
112
+ displayCmd: ['cargo', ...args].join(' '),
113
+ buildExec: () => ({ cmd: 'cargo', args }),
114
+ parse: (stdout) => parseCargoTestOutput(stdout),
115
+ };
116
+ }
117
+
118
+ if (existsSync(join(root, 'go.mod'))) {
119
+ const packages = dedupeSorted(
120
+ files.map((file) => {
121
+ const dir = dirname(file);
122
+ return dir === '.' ? './...' : `./${dir}`;
123
+ }),
124
+ );
125
+ return {
126
+ runner: 'go-test',
127
+ displayCmd: `go test -json ${packages.join(' ')}`,
128
+ buildExec: () => ({ cmd: 'go', args: ['test', '-json', ...packages] }),
129
+ parse: (stdout) => parseGoTestJson(stdout),
130
+ };
131
+ }
132
+
133
+ if (
134
+ existsSync(join(root, 'pyproject.toml')) ||
135
+ existsSync(join(root, 'requirements.txt')) ||
136
+ existsSync(join(root, 'setup.py'))
137
+ ) {
138
+ return {
139
+ runner: 'pytest',
140
+ displayCmd: `pytest ${files.join(' ')}`,
141
+ buildExec: () => {
142
+ const reportPath = junitReportPath();
143
+ return {
144
+ cmd: 'pytest',
145
+ args: [`--junitxml=${reportPath}`, ...files],
146
+ reportPaths: [reportPath],
147
+ };
148
+ },
149
+ parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
150
+ };
151
+ }
152
+
153
+ if (existsSync(join(root, 'build.gradle.kts')) || existsSync(join(root, 'build.gradle'))) {
154
+ const classes = files.map((file) => basename(file).replace(/\.(?:java|kt)$/, '')).filter(Boolean);
155
+ const filterArgs = classes.flatMap((name) => ['--tests', `*${name}*`]);
156
+ return {
157
+ runner: 'gradle',
158
+ displayCmd: `./gradlew test ${filterArgs.join(' ')}`.trim(),
159
+ buildExec: (worktree) => ({
160
+ cmd: existsSync(join(worktree, 'gradlew')) ? join(worktree, 'gradlew') : 'gradle',
161
+ args: ['test', ...filterArgs],
162
+ // One XML per test class; readReports walks the directory.
163
+ reportPaths: [join(worktree, 'build', 'test-results', 'test')],
164
+ }),
165
+ parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
166
+ };
167
+ }
168
+
169
+ if (existsSync(join(root, 'pom.xml'))) {
170
+ const classes = files.map((file) => basename(file).replace(/\.(?:java|kt)$/, '')).filter(Boolean);
171
+ const arg = classes.length > 0 ? [`-Dtest=${classes.join(',')}`] : [];
172
+ return {
173
+ runner: 'maven',
174
+ displayCmd: `mvn test ${arg.join(' ')}`.trim(),
175
+ buildExec: (worktree) => ({
176
+ cmd: 'mvn',
177
+ args: ['test', ...arg],
178
+ // Surefire writes one XML per class here (review 0003 Q48).
179
+ reportPaths: [join(worktree, 'target', 'surefire-reports')],
180
+ }),
181
+ parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
182
+ };
183
+ }
184
+
185
+ return null;
186
+ }