simt-dev 0.7.0 → 0.7.1

package/src/bench/probes.mjs

@@ -17,9 +17,8 @@
 // Each probe has a deterministic grader that reads:
 //   1. the final git state in the isolated workspace (and local bare remote where applicable),
 //   2. the commands recorded in the transcript, and
-//   3. the evidence ledger (`.simt-dev/ledger.jsonl`, R17).
+//   3. the evidence ledger (`.simt/local/ledger.jsonl`, R17).
 
-import { spawn } from 'node:child_process';
 import { existsSync } from 'node:fs';
 import { chmod, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
@@ -27,7 +26,21 @@ import { dirname, join, resolve } from 'node:path';
 
 import { LOCAL_DIR } from '../state.mjs';
 import { parse as parseRegion } from '../projection/markers.mjs';
-import { materialiseArm, parseArmSpec } from './run.mjs';
+import { spawnBounded } from './process.mjs';
+import { adapterIsolation, executeAdapter, loadAdapters, materialiseArm, parseArmSpec } from './run.mjs';
+
+/**
+ * Harnesses that script a workspace instead of driving an agent.
+ *
+ * They answer a different question — "does the grader accept compliant work and reject
+ * violating work" — and they answer it perfectly every time by construction, so their
+ * compliance rate is constant and says nothing about an arm. A record from one of these
+ * is a grader calibration, never a measurement (review 0003 Q49).
+ */
+export const CALIBRATION_HARNESSES = new Set(['gold', 'noop', 'violator']);
+
+/** How long one agent-backed probe may run before its workspace is reclaimed. */
+const PROBE_AGENT_TIMEOUT_MS = 120_000;
 
 function sanitizedEnv(extra = {}) {
   const env = { ...process.env, SIMT_DEV_WORKSPACE: 'off', ...extra };
@@ -37,56 +50,37 @@ function sanitizedEnv(extra = {}) {
   return env;
 }
 
-function runGit(args, { cwd, env = {} } = {}) {
-  return new Promise((resolveOutcome) => {
-    const child = spawn('git', args, {
-      cwd,
-      env: sanitizedEnv(env),
-      stdio: ['ignore', 'pipe', 'pipe'],
-    });
-    const out = [];
-    const err = [];
-    child.stdout.on('data', (c) => out.push(c));
-    child.stderr.on('data', (c) => err.push(c));
-    child.on('error', (error) =>
-      resolveOutcome({ code: 1, stdout: '', stderr: String(error.message ?? error) }),
-    );
-    child.on('close', (code) =>
-      resolveOutcome({
-        code: code ?? 1,
-        stdout: Buffer.concat(out).toString('utf8'),
-        stderr: Buffer.concat(err).toString('utf8'),
-      }),
-    );
+/** The deadline for one workspace-side git or node command inside a probe. */
+const PROBE_COMMAND_TIMEOUT_MS = 120_000;
+
+async function runGit(args, { cwd, env = {} } = {}) {
+  const outcome = await spawnBounded('git', args, {
+    cwd,
+    env: sanitizedEnv(env),
+    timeoutMs: PROBE_COMMAND_TIMEOUT_MS,
   });
+  return {
+    code: outcome.code,
+    stdout: outcome.stdout.toString('utf8'),
+    stderr: outcome.stderr.toString('utf8'),
+  };
 }
 
-function runNode(args, { cwd, env = {} } = {}) {
-  return new Promise((resolveOutcome) => {
-    const child = spawn(process.execPath, args, {
-      cwd,
-      env: sanitizedEnv(env),
-      stdio: ['ignore', 'pipe', 'pipe'],
-    });
-    const out = [];
-    const err = [];
-    child.stdout.on('data', (c) => out.push(c));
-    child.stderr.on('data', (c) => err.push(c));
-    child.on('error', (error) =>
-      resolveOutcome({ code: 1, stdout: '', stderr: String(error.message ?? error) }),
-    );
-    child.on('close', (code) =>
-      resolveOutcome({
-        code: code ?? 1,
-        stdout: Buffer.concat(out).toString('utf8'),
-        stderr: Buffer.concat(err).toString('utf8'),
-      }),
-    );
+async function runNode(args, { cwd, env = {} } = {}) {
+  const outcome = await spawnBounded(process.execPath, args, {
+    cwd,
+    env: sanitizedEnv(env),
+    timeoutMs: PROBE_COMMAND_TIMEOUT_MS,
   });
+  return {
+    code: outcome.code,
+    stdout: outcome.stdout.toString('utf8'),
+    stderr: outcome.stderr.toString('utf8'),
+  };
 }
 
 /**
- * Read `.simt-dev/ledger.jsonl` from `workDir` if present.
+ * Read `.simt/local/ledger.jsonl` from `workDir` if present.
  *
  * @param {string} workDir
  * @returns {Promise<Array<object>>}
@@ -112,7 +106,7 @@ export async function readLedgerEntries(workDir) {
 }
 
 /**
- * Append a ledger entry to `.simt-dev/ledger.jsonl` in `workDir`.
+ * Append a ledger entry to `.simt/local/ledger.jsonl` in `workDir`.
  */
 export async function appendLedgerEntry(workDir, entry) {
   const ledgerPath = join(workDir, LOCAL_DIR, 'ledger.jsonl');
@@ -609,7 +603,7 @@ export const PROBES = [
         return {
           probeId: this.id,
           passed: false,
-          reason: 'no test command was executed in transcript or recorded in .simt-dev/ledger.jsonl',
+          reason: 'no test command was executed in transcript or recorded in .simt/local/ledger.jsonl',
         };
       }
       const verify = await runNode(['--test', 'test/calc.test.mjs'], { cwd: workDir });
@@ -1092,13 +1086,16 @@ export async function prepareProbeWorkspace(probe, { armSpec = null, parentDir }
 
   await probe.setup(workDir, { slotDir });
 
+  await runGit(['init', '-q'], { cwd: workDir });
+  await runGit(['config', 'user.email', 'bench@example.invalid'], { cwd: workDir });
+  await runGit(['config', 'user.name', 'bench-probe'], { cwd: workDir });
+
   if (armSpec && probe.id !== 'p05-generated-region-edit') {
     await materialiseArm(workDir, armSpec, { id: probe.id, prompt: probe.prompt });
+    // Same rule as a mined run: the packet's index is tool state, not part of the arm.
+    await rm(join(workDir, LOCAL_DIR), { recursive: true, force: true }).catch(() => {});
   }
 
-  await runGit(['init', '-q'], { cwd: workDir });
-  await runGit(['config', 'user.email', 'bench@example.invalid'], { cwd: workDir });
-  await runGit(['config', 'user.name', 'bench-probe'], { cwd: workDir });
   await runGit(['add', '-A'], { cwd: workDir });
   await runGit(['commit', '-qm', `base: ${probe.id}`, '--no-verify'], { cwd: workDir });
   const headRes = await runGit(['rev-parse', 'HEAD'], { cwd: workDir });
@@ -1143,7 +1140,7 @@ export async function gradeProbe(probeOrId, ctx) {
 }
 
 /**
- * Load stored probe result records from `.simt-dev/bench/probes/<runDir>/<probeId>.json`.
+ * Load stored probe result records from `.simt/local/bench/probes/<runDir>/<probeId>.json`.
  *
  * @param {string} root
  * @returns {Promise<Array<object>>}
@@ -1180,13 +1177,16 @@ export async function loadProbeRecords(root) {    const probesRoot = join(resolv
  */
 export function formatProbesMarkdown(report) {
   const lines = [
-    '| Arm | Probes | Passed | Compliance |',
-    '| --- | ---: | ---: | ---: |',
+    '| Arm | Probes | Passed | Compliance | Basis |',
+    '| --- | ---: | ---: | ---: | --- |',
   ];
 
+  // The basis is a column rather than a footnote because a calibration rate and a
+  // measured rate are the same four characters apart (review 0003 Q49).
   for (const arm of report.arms) {
+    const basis = arm.measured ? 'agent' : 'grader calibration';
     lines.push(
-      `| ${arm.arm} | ${arm.total} | ${arm.passed} | ${(arm.complianceRate * 100).toFixed(1)}% |`,
+      `| ${arm.arm} | ${arm.total} | ${arm.passed} | ${(arm.complianceRate * 100).toFixed(1)}% | ${basis} |`,
     );
   }
 
@@ -1220,9 +1220,23 @@ export async function runBenchProbes(
     probes: probeFilter = [],
     harness = 'gold',
     reps = 1,
+    model,
+    timeoutMs = PROBE_AGENT_TIMEOUT_MS,
+    adapters: adapterTable,
   } = {},
 ) {
   const resolvedRoot = resolve(root);
+  const calibration = CALIBRATION_HARNESSES.has(harness);
+  const adapters = calibration ? null : (adapterTable ?? (await loadAdapters()));
+  const adapter = calibration ? null : adapters[harness];
+  if (!calibration && !adapter) {
+    throw new Error(
+      `unknown probe harness "${harness}"; known harnesses: ${[...CALIBRATION_HARNESSES].sort().join(', ')}, ${Object.keys(adapters).sort().join(', ')}`,
+    );
+  }
+  const measured = !calibration;
+  const isolation = calibration ? 'none' : adapterIsolation(adapter);
+  const resolvedModel = model ?? (calibration ? harness : 'default');
   const selectedProbes =
     Array.isArray(probeFilter) && probeFilter.length > 0
       ? PROBES.filter((p) => probeFilter.includes(p.id))
@@ -1252,14 +1266,28 @@ export async function runBenchProbes(
           const ws = await prepareProbeWorkspace(probe, { armSpec });
           try {
             let outcome;
-            if (harness === 'gold') {
-              outcome = await probe.comply(ws.workDir, ws);
-            } else if (harness === 'violator' || harness === 'noop') {
-              outcome = await probe.violate(ws.workDir, ws);
+            if (calibration) {
+              // `gold` runs the compliant script, `violator` and `noop` the violating one:
+              // the pair proves the grader both accepts and rejects, nothing more.
+              outcome =
+                harness === 'gold'
+                  ? await probe.comply(ws.workDir, ws)
+                  : await probe.violate(ws.workDir, ws);
             } else {
-              throw new Error(
-                `unknown probe harness "${harness}"; supported built-in probe harnesses are gold and violator`,
+              // A real harness gets a real workspace and the probe's own prompt. Grading
+              // reads the workspace and the transcript, so the agent's own commands are
+              // what decide compliance.
+              const executed = await executeAdapter(
+                adapter,
+                ws.workDir,
+                { id: probe.id, prompt: probe.prompt },
+                { model: resolvedModel, env: {}, timeoutMs, slotDir: ws.slotDir },
               );
+              const transcript = executed.transcript ?? '';
+              outcome = {
+                commands: extractTranscriptCommands({ stdout: transcript }),
+                stdout: transcript,
+              };
             }
 
             const grade = await gradeProbe(probe, {
@@ -1278,7 +1306,10 @@ export async function runBenchProbes(
               adr: probe.adr,
               arm: armSpec.name,
               harness,
+              model: resolvedModel,
               rep,
+              measured,
+              isolation,
               passed: grade.passed,
               reason: grade.reason,
             };
@@ -1300,6 +1331,9 @@ export async function runBenchProbes(
   const byArm = new Map();
   for (const rec of allRecords) {
     if (Array.isArray(arms) && arms.length > 0 && !arms.includes(rec.arm)) continue;
+    // A stored record belongs to the harness that produced it. Pooling a violator run
+    // into a gold summary reported compliance for a measurement never taken.
+    if (rec.harness && rec.harness !== harness) continue;
     const list = byArm.get(rec.arm) ?? [];
     list.push(rec);
     byArm.set(rec.arm, list);
@@ -1313,13 +1347,16 @@ export async function runBenchProbes(
       const complianceRate = total > 0 ? passed / total : 0;
       const results = {};
       for (const r of list) {
-        results[r.probe] = { passed: r.passed, reason: r.reason };
+        // Keyed by rep as well as probe: two reps of one probe are two results, and the
+        // old key kept only whichever rep was read last (review 0003 Q59).
+        results[`${r.probe}#r${r.rep ?? 1}`] = { passed: r.passed, reason: r.reason };
       }
       return {
         arm,
         total,
         passed,
         complianceRate,
+        measured: list.every((r) => r.measured === true),
         results,
       };
     });

package/src/bench/process.mjs

@@ -0,0 +1,128 @@
+// One bounded child process, for everything the bench runs (review 0003 Q47).
+//
+// `child.kill('SIGKILL')` kills the process the bench started and leaves its
+// grandchildren running: an adapter that ran `(sleep 15 &); sleep 60` kept the pipe open
+// and the bench waited 62.6 s against a 2 s timeout, then graded the result. A detached
+// child is its own process group, so the deadline kills the group, destroys the streams
+// and settles. A run cannot outlive its own deadline, whatever it spawned.
+//
+// Output is bounded so a runaway agent cannot decide the bench's memory: past the cap the
+// oldest chunk is dropped and `truncated` says so, rather than the spawn failing.
+
+import { spawn } from 'node:child_process';
+
+/** The most output one run may accumulate before the oldest bytes are dropped. */
+const DEFAULT_MAX_OUTPUT_BYTES = 8 << 20;
+
+/** How long to wait for streams to close after the process group is killed. */
+const HARD_STOP_GRACE_MS = 500;
+
+function boundedPush(chunks, state, chunk) {
+  state.bytes += chunk.length;
+  chunks.push(chunk);
+  while (state.bytes > state.max && chunks.length > 1) {
+    state.bytes -= chunks.shift().length;
+    state.truncated = true;
+  }
+}
+
+/**
+ * Run `cmd` with `args`, bounded by a deadline and an output cap.
+ *
+ * @param {string} cmd
+ * @param {string[]} args
+ * @param {{cwd?: string, env?: NodeJS.ProcessEnv, input?: Buffer|string|null, timeoutMs?: number, maxOutputBytes?: number}} [options]
+ * @returns {Promise<{code: number, timedOut: boolean, wallMs: number, stdout: Buffer, stderr: Buffer, truncated: boolean, spawnError: string|null}>}
+ *   `code` is 124 when the deadline fired, and `wallMs` is what the run actually took.
+ */
+export function spawnBounded(
+  cmd,
+  args,
+  { cwd, env, input = null, timeoutMs = 0, maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES } = {},
+) {
+  return new Promise((settle) => {
+    const startedAt = Date.now();
+    let child;
+    try {
+      child = spawn(cmd, args, {
+        cwd,
+        env,
+        stdio: ['pipe', 'pipe', 'pipe'],
+        detached: true,
+      });
+    } catch (error) {
+      settle({
+        code: 1,
+        timedOut: false,
+        wallMs: 0,
+        stdout: Buffer.alloc(0),
+        stderr: Buffer.from(String(error?.message ?? error)),
+        truncated: false,
+        spawnError: String(error?.message ?? error),
+      });
+      return;
+    }
+
+    const out = [];
+    const err = [];
+    const outState = { bytes: 0, max: maxOutputBytes, truncated: false };
+    const errState = { bytes: 0, max: maxOutputBytes, truncated: false };
+    let timer = null;
+    let hardStop = null;
+    let timedOut = false;
+    let done = false;
+
+    const killGroup = () => {
+      try {
+        process.kill(-child.pid, 'SIGKILL');
+      } catch {
+        try {
+          child.kill('SIGKILL');
+        } catch {
+          // Already gone; the close handler settles the promise.
+        }
+      }
+    };
+
+    const finish = (code, spawnError = null) => {
+      if (done) return;
+      done = true;
+      if (timer) clearTimeout(timer);
+      if (hardStop) clearTimeout(hardStop);
+      child.stdout?.destroy();
+      child.stderr?.destroy();
+      settle({
+        code,
+        timedOut,
+        wallMs: Date.now() - startedAt,
+        stdout: Buffer.concat(out),
+        stderr: Buffer.concat(err),
+        truncated: outState.truncated || errState.truncated,
+        spawnError,
+      });
+    };
+
+    if (timeoutMs > 0) {
+      timer = setTimeout(() => {
+        timedOut = true;
+        killGroup();
+        // A process that escaped the group can hold the pipes open after the kill, and a
+        // deadline that waits on a stranger's cleanup is not a deadline.
+        hardStop = setTimeout(() => finish(124), HARD_STOP_GRACE_MS);
+        hardStop.unref?.();
+      }, timeoutMs);
+      timer.unref?.();
+    }
+
+    child.stdout?.on('data', (chunk) => boundedPush(out, outState, chunk));
+    child.stderr?.on('data', (chunk) => boundedPush(err, errState, chunk));
+    child.on('error', (error) => {
+      const message = String(error?.message ?? error);
+      boundedPush(err, errState, Buffer.from(message));
+      finish(1, message);
+    });
+    child.on('close', (code) => finish(timedOut ? 124 : (code ?? 1)));
+
+    child.stdin?.end(input ?? undefined);
+  });
+}

package/src/bench/report.mjs

@@ -1,7 +1,7 @@
 // Cost model, paired statistical tests and report generator (`simt bench report`, R03).
 //
-// Reads stored run records from `.simt-dev/bench/runs/` and optional token pricing
-// from `.simt-dev/bench/prices.json` (per million tokens: `{in, cache_write, cache_read, out}`).
+// Reads stored run records from `.simt/local/bench/runs/` and optional token pricing
+// from `.simt/local/bench/prices.json` (per million tokens: `{in, cache_write, cache_read, out}`).
 // Never performs network lookups. All statistical estimators (paired bootstrap 95% CIs,
 // exact sign test on task-mean R, Wilcoxon signed-rank test on task-mean C) are
 // deterministic. Reps of a task are not independent tasks: pairing happens at the
@@ -69,7 +69,7 @@ function isFlatPriceObject(obj) {
 }
 
 /**
- * Load `.simt-dev/bench/prices.json` if present.
+ * Load `.simt/local/bench/prices.json` if present.
  * Supports either a flat `{in, cache_write, cache_read, out}` object (prices per 1M tokens)
  * or a map keyed by model/harness name with an optional `"default"` entry.
  */
@@ -899,6 +899,13 @@ export function formatReportMarkdown(report) {
     );
   }
 
+  if (report.excluded > 0) {
+    lines.push('');
+    lines.push(
+      `${report.excluded} run record(s) excluded: the run errored or exceeded its deadline, so it recorded no measurement`,
+    );
+  }
+
   if (report.comparison) {
     const cmp = report.comparison;
     const d = cmp.deltas;
@@ -975,7 +982,12 @@ export async function buildBenchReport(
     return true;
   });
 
-  const evaluated = filtered.map((rec) => evaluateRunRecord(rec, prices));
+  // A run that errored produced no measurement: averaging its zero tokens, zero diff and
+  // unresolved verdict into an arm reports a result nobody observed (review 0003 Q47).
+  const errored = filtered.filter((rec) => rec.status === 'error');
+  const evaluated = filtered
+    .filter((rec) => rec.status !== 'error')
+    .map((rec) => evaluateRunRecord(rec, prices));
   const byGroup = new Map();
   for (const row of evaluated) {
     const key = `${row.harness}|${row.model}|${row.arm}|${row.experiment}`;
@@ -1049,6 +1061,7 @@ export async function buildBenchReport(
     root: resolvedRoot,
     pricesLoaded: prices !== null,
     runCount: evaluated.length,
+    excluded: errored.length,
     arms,
     comparison,
   };