simt-dev 0.2.1 → 0.3.0

package/src/bench/adapters.json

@@ -0,0 +1,149 @@
+{
+  "schemaVersion": 1,
+  "adapters": {
+    "claude": {
+      "id": "claude",
+      "label": "Claude Code",
+      "kind": "cli",
+      "bin": "claude",
+      "template": "claude -p \"{prompt}\" --output-format json --max-turns {turns}",
+      "args": ["-p", "{prompt}", "--output-format", "json", "--max-turns", "{turns}"],
+      "modelArgs": ["--model", "{model}"],
+      "format": "json",
+      "usageSource": "usage.*, total_cost_usd, num_turns",
+      "fields": {
+        "input": ["usage.input_tokens", "usage.input"],
+        "cacheWrite": ["usage.cache_creation_input_tokens", "usage.cacheWrite"],
+        "cacheRead": ["usage.cache_read_input_tokens", "usage.cacheRead"],
+        "output": ["usage.output_tokens", "usage.output"],
+        "turns": ["num_turns", "usage.turns"],
+        "toolCalls": ["num_tool_calls", "tool_calls", "usage.toolCalls"],
+        "costUsd": ["total_cost_usd", "cost_usd", "costUsd"]
+      }
+    },
+    "codex": {
+      "id": "codex",
+      "label": "Codex",
+      "kind": "cli",
+      "bin": "codex",
+      "template": "codex exec --json \"{prompt}\"",
+      "args": ["exec", "--json", "{prompt}"],
+      "modelArgs": ["--model", "{model}"],
+      "format": "jsonl",
+      "usageSource": "token events in JSONL",
+      "fields": {
+        "input": ["usage.input_tokens", "input_tokens", "usage.input"],
+        "cacheWrite": ["usage.cache_creation_input_tokens", "usage.cache_write_input_tokens", "cache_write_input_tokens"],
+        "cacheRead": ["usage.cached_input_tokens", "usage.cache_read_input_tokens", "cached_input_tokens"],
+        "output": ["usage.output_tokens", "output_tokens", "usage.output"],
+        "turns": ["turns", "num_turns"],
+        "toolCalls": ["tool_calls", "num_tool_calls"],
+        "costUsd": ["total_cost_usd", "cost_usd", "cost"]
+      }
+    },
+    "gemini": {
+      "id": "gemini",
+      "label": "Gemini CLI",
+      "kind": "cli",
+      "bin": "gemini",
+      "template": "gemini -p \"{prompt}\" --output-format json",
+      "args": ["-p", "{prompt}", "--output-format", "json"],
+      "modelArgs": ["--model", "{model}"],
+      "format": "json",
+      "usageSource": "stats block",
+      "fields": {
+        "input": ["stats.inputTokens", "stats.tokens.prompt", "stats.prompt_tokens", "usage.input_tokens"],
+        "cacheWrite": ["stats.cacheWriteTokens", "stats.tokens.cacheWrite", "usage.cache_creation_input_tokens"],
+        "cacheRead": ["stats.cachedTokens", "stats.tokens.cached", "usage.cache_read_input_tokens"],
+        "output": ["stats.outputTokens", "stats.tokens.candidates", "stats.completion_tokens", "usage.output_tokens"],
+        "turns": ["stats.turns", "stats.totalTurns", "num_turns"],
+        "toolCalls": ["stats.toolCalls", "stats.tools.totalCalls", "num_tool_calls"],
+        "costUsd": ["stats.totalCostUsd", "stats.costUsd", "total_cost_usd"]
+      }
+    },
+    "opencode": {
+      "id": "opencode",
+      "label": "opencode",
+      "kind": "cli",
+      "bin": "opencode",
+      "template": "opencode run \"{prompt}\"",
+      "args": ["run", "--format", "json", "{prompt}"],
+      "modelArgs": ["--model", "{model}"],
+      "format": "jsonl",
+      "usageSource": "session export",
+      "fields": {
+        "input": ["tokens.input", "usage.input_tokens", "part.tokens.input"],
+        "cacheWrite": ["tokens.cache.write", "usage.cache_creation_input_tokens", "part.tokens.cache.write"],
+        "cacheRead": ["tokens.cache.read", "usage.cache_read_input_tokens", "part.tokens.cache.read"],
+        "output": ["tokens.output", "usage.output_tokens", "part.tokens.output"],
+        "turns": ["turns", "num_turns"],
+        "toolCalls": ["toolCalls", "num_tool_calls"],
+        "costUsd": ["cost", "part.cost", "total_cost_usd"]
+      }
+    },
+    "cursor": {
+      "id": "cursor",
+      "label": "Cursor CLI",
+      "kind": "cli",
+      "bin": "cursor-agent",
+      "template": "cursor-agent -p \"{prompt}\" --output-format json",
+      "args": ["-p", "{prompt}", "--output-format", "json"],
+      "modelArgs": ["--model", "{model}"],
+      "format": "json",
+      "usageSource": "result JSON",
+      "fields": {
+        "input": ["usage.input_tokens", "usage.input", "stats.inputTokens"],
+        "cacheWrite": ["usage.cache_creation_input_tokens", "usage.cacheWrite", "stats.cacheWriteTokens"],
+        "cacheRead": ["usage.cache_read_input_tokens", "usage.cacheRead", "stats.cachedTokens"],
+        "output": ["usage.output_tokens", "usage.output", "stats.outputTokens"],
+        "turns": ["num_turns", "turns", "usage.turns"],
+        "toolCalls": ["num_tool_calls", "tool_calls", "usage.toolCalls", "toolCalls"],
+        "costUsd": ["total_cost_usd", "costUsd", "cost_usd"]
+      }
+    },
+    "aider": {
+      "id": "aider",
+      "label": "Aider",
+      "kind": "cli",
+      "bin": "aider",
+      "template": "aider --message \"{prompt}\" --yes-always",
+      "args": ["--message", "{prompt}", "--yes-always", "--analytics-log", "{analyticsLog}"],
+      "modelArgs": ["--model", "{model}"],
+      "format": "analytics-log",
+      "usageSource": "--analytics-log file",
+      "fields": {
+        "input": ["prompt_tokens", "properties.prompt_tokens", "usage.input_tokens"],
+        "cacheWrite": ["cache_creation_input_tokens", "properties.cache_write_tokens"],
+        "cacheRead": ["cache_read_input_tokens", "properties.cache_read_tokens"],
+        "output": ["completion_tokens", "properties.completion_tokens", "usage.output_tokens"],
+        "turns": ["turns", "properties.turns"],
+        "toolCalls": ["tool_calls", "properties.tool_calls"],
+        "costUsd": ["cost", "total_cost", "properties.cost", "properties.total_cost"]
+      }
+    },
+    "gold": {
+      "id": "gold",
+      "label": "Gold Oracle",
+      "kind": "builtin",
+      "bin": null,
+      "template": null,
+      "args": [],
+      "modelArgs": [],
+      "format": "builtin",
+      "usageSource": "vault gold.patch",
+      "fields": {}
+    },
+    "noop": {
+      "id": "noop",
+      "label": "No-op Baseline",
+      "kind": "builtin",
+      "bin": null,
+      "template": null,
+      "args": [],
+      "modelArgs": [],
+      "format": "builtin",
+      "usageSource": "none",
+      "fields": {}
+    }
+  }
+}

package/src/bench/grade.mjs

@@ -0,0 +1,229 @@
+import { spawn } from 'node:child_process';
+import { readFile, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { runGates } from '../hooks/dispatcher.mjs';
+import { packageVersion } from '../package-version.mjs';
+import { readManifest } from '../projection/manifest.mjs';
+import { detectTestRunner } from './mine.mjs';
+
+const DEFAULT_GRADE_TIMEOUT_MS = 45_000;
+
+function sanitizedEnv(extra = {}) {
+  const env = { ...process.env, SIMT_DEV_WORKSPACE: 'off', ...extra };
+  for (const key of Object.keys(env)) {
+    if (key.startsWith('NODE_TEST')) delete env[key];
+  }
+  return env;
+}
+
+function runCmd(cmd, args, { cwd, input = null, timeoutMs = DEFAULT_GRADE_TIMEOUT_MS } = {}) {
+  return new Promise((resolveOutcome) => {
+    const child = spawn(cmd, args, {
+      cwd,
+      env: sanitizedEnv(),
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const out = [];
+    const err = [];
+    let timedOut = false;
+    const timer =
+      timeoutMs > 0
+        ? setTimeout(() => {
+            timedOut = true;
+            child.kill('SIGKILL');
+          }, timeoutMs)
+        : null;
+
+    child.stdout.on('data', (chunk) => out.push(chunk));
+    child.stderr.on('data', (chunk) => err.push(chunk));
+    child.on('error', (error) => {
+      if (timer) clearTimeout(timer);
+      resolveOutcome({
+        code: 1,
+        timedOut,
+        stdout: Buffer.concat(out),
+        stderr: Buffer.from(String(error.message ?? error)),
+      });
+    });
+    child.on('close', (code) => {
+      if (timer) clearTimeout(timer);
+      resolveOutcome({
+        code: timedOut ? 124 : (code ?? 1),
+        timedOut,
+        stdout: Buffer.concat(out),
+        stderr: Buffer.concat(err),
+      });
+    });
+
+    if (input !== null) child.stdin.end(input);
+    else child.stdin.end();
+  });
+}
+
+/**
+ * Extract repository-relative file paths touched by a unified git diff.
+ *
+ * @param {Buffer|string} patch
+ * @returns {string[]}
+ */
+export function extractPatchFiles(patch) {
+  const text = Buffer.isBuffer(patch) ? patch.toString('utf8') : String(patch ?? '');
+  const files = new Set();
+  for (const line of text.split('\n')) {
+    const match = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
+    if (match) files.add(match[2]);
+  }
+  return [...files].sort();
+}
+
+async function resetFilesToHead(workDir, initialHead, files) {
+  for (const file of files) {
+    const existsAtHead = await runCmd('git', ['cat-file', '-e', `${initialHead}:${file}`], { cwd: workDir });
+    if (existsAtHead.code === 0) {
+      await runCmd('git', ['checkout', '-f', initialHead, '--', file], { cwd: workDir });
+    } else {
+      await runCmd('git', ['rm', '-f', '--ignore-unmatch', '--', file], { cwd: workDir });
+      await rm(join(workDir, file), { force: true }).catch(() => {});
+    }
+  }
+}
+
+async function evaluateDeclaredGates(workDir, testPassed) {
+  const { exists, data: manifest, error } = await readManifest(workDir);
+  if (!exists) return testPassed;
+  if (error || !manifest) return false;
+
+  const hasCustomTestCmd = Boolean(manifest?.gates?.commands?.test);
+  const filterGates = (list) =>
+    Array.isArray(list) ? list.filter((g) => g !== 'test' || hasCustomTestCmd) : [];
+
+  const alignedManifest = {
+    ...manifest,
+    projection: {
+      ...(manifest.projection ?? {}),
+      version: packageVersion(),
+    },
+    gates: {
+      ...(manifest.gates ?? {}),
+      preCommit: filterGates(manifest?.gates?.preCommit),
+      prePush: filterGates(manifest?.gates?.prePush),
+    },
+  };
+
+  await runCmd('git', ['add', '-A'], { cwd: workDir });
+  const preCommit = await runGates('pre-commit', { root: workDir, manifest: alignedManifest });
+  const prePush = await runGates('pre-push', { root: workDir, manifest: alignedManifest });
+  return testPassed && !preCommit.failed && !prePush.failed;
+}
+
+/**
+ * Grade an isolated agent workspace after the agent finishes.
+ *
+ * Measures the agent's diff before applying the hidden verifier test patch, then resets
+ * any test files touched by `test.patch` to `initialHead` so an agent cannot pass by
+ * weakening or deleting the test suite.
+ *
+ * @param {string} workDir
+ * @param {object} task
+ * @param {{vaultRoot: string, initialHead: string, timeoutMs?: number}} options
+ * @returns {Promise<{resolved: boolean, gatesPassed: boolean, diff: {files: number, outsideBlastRadius: number}, agentPatch: Buffer}>}
+ */
+export async function gradeWorkspace(workDir, task, { vaultRoot, initialHead, timeoutMs = DEFAULT_GRADE_TIMEOUT_MS }) {
+  const taskVaultDir = join(vaultRoot, task.id);
+  const [testPatch, goldPatch] = await Promise.all([
+    readFile(join(taskVaultDir, 'test.patch')),
+    readFile(join(taskVaultDir, 'gold.patch')).catch(() => Buffer.alloc(0)),
+  ]);
+
+  // Remove transient brief file if the `brief` arm or agent left one untracked, so it
+  // is not counted as an implementation edit outside the task's blast radius.
+  await runCmd('git', ['add', '-N', '.'], { cwd: workDir });
+  const [diffNamesRes, diffPatchRes] = await Promise.all([
+    runCmd('git', ['diff', '--name-only', initialHead], { cwd: workDir }),
+    runCmd('git', ['diff', '--binary', initialHead], { cwd: workDir }),
+  ]);
+
+  const changedFiles = diffNamesRes.stdout
+    .toString('utf8')
+    .split('\n')
+    .map((s) => s.trim())
+    .filter((f) => Boolean(f) && !f.startsWith('.simt-dev/'));
+
+  const goldFiles = new Set(extractPatchFiles(goldPatch));
+  const outsideBlastRadius =
+    goldFiles.size > 0
+      ? changedFiles.filter((file) => !goldFiles.has(file)).length
+      : 0;
+  const diff = {
+    files: changedFiles.length,
+    outsideBlastRadius,
+  };
+
+  const testPatchFiles = extractPatchFiles(testPatch);
+  await resetFilesToHead(workDir, initialHead, testPatchFiles);
+
+  const applyTest = await runCmd('git', ['apply', '-'], {
+    cwd: workDir,
+    input: testPatch,
+    timeoutMs,
+  });
+  if (applyTest.code !== 0) {
+    return {
+      resolved: false,
+      gatesPassed: false,
+      diff,
+      agentPatch: diffPatchRes.stdout,
+    };
+  }
+
+  const runner = await detectTestRunner(workDir, testPatchFiles);
+  if (!runner) {
+    return {
+      resolved: false,
+      gatesPassed: false,
+      diff,
+      agentPatch: diffPatchRes.stdout,
+    };
+  }
+
+  const execSpec = runner.buildExec(workDir);
+  const runRes = await runCmd(execSpec.cmd, execSpec.args, { cwd: workDir, timeoutMs });
+  const reportText = execSpec.reportPath
+    ? await readFile(execSpec.reportPath, 'utf8').catch(() => '')
+    : '';
+  const parsed = runner.parse(runRes.stdout.toString('utf8'), reportText);
+  const testRun = {
+    code: runRes.code,
+    timedOut: runRes.timedOut,
+    passed: parsed.passed,
+    failed: parsed.failed,
+  };
+  const passedSet = new Set(testRun.passed);
+  const f2pResolved =
+    Array.isArray(task.f2p) &&
+    task.f2p.length > 0 &&
+    task.f2p.every((id) => passedSet.has(id));
+  const p2pIntact =
+    Array.isArray(task.p2p) &&
+    task.p2p.every((id) => passedSet.has(id));
+
+  const resolved =
+    !testRun.timedOut &&
+    testRun.code === 0 &&
+    testRun.failed.length === 0 &&
+    f2pResolved &&
+    p2pIntact;
+
+  const gatesPassed = await evaluateDeclaredGates(
+    workDir,
+    !testRun.timedOut && testRun.code === 0 && testRun.failed.length === 0,
+  );
+
+  return {
+    resolved,
+    gatesPassed,
+    diff,
+    agentPatch: diffPatchRes.stdout,
+  };
+}