simt-dev 0.7.0 → 0.8.0

package/src/measure/corpus.mjs

@@ -0,0 +1,150 @@
+// Sealed export / import of the mined benchmark corpus (review 0003 Q48).
+//
+// The vault is keyed on the repository's root commit, outside every checkout, so
+// a clone of the repository cannot read the hidden test patches the miner wrote:
+// the committed corpus was unrunnable from any machine that was not the one that
+// mined it. A sealed document carries the task descriptors and both patches for
+// every task in a single file, so `bench export` on one machine and `bench import`
+// on another reproduce the corpus exactly.
+//
+// The document is deterministic: no timestamp, tasks sorted by id, canonical
+// (key-sorted) JSON, and a sha256 seal over everything but the seal itself. Two
+// exports of the same corpus are byte-identical, and a reader can verify the
+// seal before trusting a single byte of it.
+
+import { createHash } from 'node:crypto';
+import { chmod, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+
+import { STATE_DIR } from '../name.mjs';
+import { packageVersion } from '../package-version.mjs';
+import { resolveVaultDir } from '../testing/vault.mjs';
+
+/** The document kind stamp, so a reader can refuse a file that is not a corpus. */
+export const CORPUS_KIND = 'simt-bench-corpus';
+
+/** The document schema version. Bump when the shape changes, not when content does. */
+export const CORPUS_SCHEMA_VERSION = 1;
+
+/**
+ * Canonical serialisation: object keys sorted, no whitespace. Arrays keep their
+ * order (the caller sorts them). This is what the seal hashes, so both the
+ * exporter and the importer must reduce the payload to exactly this form.
+ *
+ * @param {*} value a JSON value
+ * @returns {string}
+ */
+function canonical(value) {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  const keys = Object.keys(value).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`).join(',')}}`;
+}
+
+/**
+ * @param {{tasks: object[]}} payload the document minus its seal
+ * @returns {string} hex sha256 of the canonical payload
+ */
+function sealOf(payload) {
+  return createHash('sha256').update(canonical(payload)).digest('hex');
+}
+
+function payloadOf(doc) {
+  const { seal, ...payload } = doc;
+  void seal;
+  return payload;
+}
+
+/**
+ * Read every mined task and its hidden patches from `root` and assemble the
+ * sealed corpus document (no file written; the command decides where it goes).
+ *
+ * @param {string} root
+ * @param {{env?: NodeJS.ProcessEnv}} [options]
+ * @returns {Promise<object>} the sealed document
+ */
+export async function buildCorpusDocument(root, { env = process.env } = {}) {
+  const absRoot = resolve(root);
+  const tasksDir = join(absRoot, STATE_DIR, 'bench', 'tasks');
+  const vaultRoot = resolveVaultDir(absRoot, { env });
+
+  const names = (await readdir(tasksDir).catch(() => [])).filter((f) => f.endsWith('.json')).sort();
+  const tasks = [];
+  for (const name of names) {
+    const task = JSON.parse(await readFile(join(tasksDir, name), 'utf8'));
+    const testPatch = await readFile(join(vaultRoot, task.id, 'test.patch'))
+      .then((b) => b.toString('base64'))
+      .catch(() => '');
+    const goldPatch = await readFile(join(vaultRoot, task.id, 'gold.patch'))
+      .then((b) => b.toString('base64'))
+      .catch(() => '');
+    tasks.push({ id: task.id, task, testPatch, goldPatch });
+  }
+  tasks.sort((a, b) => a.id.localeCompare(b.id));
+
+  const corpus = await readFile(join(dirname(tasksDir), 'corpus.json'), 'utf8')
+    .then((t) => JSON.parse(t))
+    .catch(() => null);
+
+  const payload = {
+    schemaVersion: CORPUS_SCHEMA_VERSION,
+    kind: CORPUS_KIND,
+    minedFrom: corpus?.minedFrom ?? null,
+    simtVersion: corpus?.simtVersion ?? packageVersion(),
+    taskCount: tasks.length,
+    tasks,
+  };
+  return { ...payload, seal: sealOf(payload) };
+}
+
+/**
+ * Verify a corpus document's seal. Returns a reason string when the document is
+ * not a trusted corpus; `null` when it is.
+ *
+ * @param {object} doc
+ * @returns {string|null}
+ */
+export function verifyCorpusDocument(doc) {
+  if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) return 'not a corpus document';
+  if (doc.kind !== CORPUS_KIND) return `unexpected kind \`${doc.kind}\` (want \`${CORPUS_KIND}\`)`;
+  if (doc.schemaVersion !== CORPUS_SCHEMA_VERSION) {
+    return `unsupported schemaVersion \`${doc.schemaVersion}\` (want ${CORPUS_SCHEMA_VERSION})`;
+  }
+  const payload = payloadOf(doc);
+  if (sealOf(payload) !== doc.seal) return 'seal mismatch: the document was modified after it was sealed';
+  if (!Array.isArray(payload.tasks)) return 'missing task list';
+  return null;
+}
+
+/**
+ * Write a verified corpus document into `root`: task descriptors to
+ * `.simt/bench/tasks/` and the hidden patches to the vault. Idempotent — a task
+ * already present is overwritten with the document's copy, which is the point of
+ * importing a known corpus.
+ *
+ * @param {string} root
+ * @param {object} doc a document that passed `verifyCorpusDocument`
+ * @param {{env?: NodeJS.ProcessEnv, apply?: boolean}} [options]
+ * @returns {Promise<{applied: boolean, tasks: number, tasksDir: string, vaultRoot: string}>}
+ */
+export async function writeCorpusDocument(root, doc, { env = process.env, apply = true } = {}) {
+  const absRoot = resolve(root);
+  const tasksDir = join(absRoot, STATE_DIR, 'bench', 'tasks');
+  const vaultRoot = resolveVaultDir(absRoot, { env });
+
+  if (apply) await mkdir(tasksDir, { recursive: true });
+  for (const entry of doc.tasks) {
+    if (apply) {
+      await writeFile(join(tasksDir, `${entry.id}.json`), `${JSON.stringify(entry.task, null, 2)}\n`, 'utf8');
+      const taskVault = join(vaultRoot, entry.id);
+      // An imported corpus restores an answer key, so it gets the same 0700 the miner
+      // gives one (review 0003 Q49): the import must not be the cheaper way in.
+      await mkdir(taskVault, { recursive: true, mode: 0o700 });
+      await chmod(taskVault, 0o700).catch(() => {});
+      await writeFile(join(taskVault, 'test.patch'), Buffer.from(entry.testPatch, 'base64'));
+      await writeFile(join(taskVault, 'gold.patch'), Buffer.from(entry.goldPatch, 'base64'));
+    }
+  }
+
+  return { applied: apply, tasks: doc.tasks.length, tasksDir, vaultRoot };
+}

package/src/measure/metrics.mjs

@@ -0,0 +1,236 @@
+// Derived per-run metrics: what a record costs, and how good it is.
+//
+// Split from the statistics beside it (review 0003 Q78): a metric is a function of one
+// record, a statistical test is a function of a population of them, and a reader looking
+// for the cost model should not have to open the bootstrap.
+//
+// The cost model distinguishes three answers because a missing cost must never read as a
+// zero cost: `priced` from the token table, `reported` by the harness, `missing` when
+// neither exists (review 0003 Q46).
+import { readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+import { LOCAL_DIR } from '../state.mjs';
+
+function normalizePriceEntry(raw, label) {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+    throw new Error(`${label} must be a JSON object with {in, cache_write, cache_read, out}`);
+  }
+  const pIn = Number(raw.in ?? raw.input ?? 0);
+  const pCw = Number(raw.cache_write ?? raw.cacheWrite ?? 0);
+  const pCr = Number(raw.cache_read ?? raw.cacheRead ?? 0);
+  const pOut = Number(raw.out ?? raw.output ?? 0);
+  for (const [key, value] of [
+    ['in', pIn],
+    ['cache_write', pCw],
+    ['cache_read', pCr],
+    ['out', pOut],
+  ]) {
+    if (!Number.isFinite(value) || value < 0) {
+      throw new Error(`${label}.${key} must be a non-negative number`);
+    }
+  }
+  return { in: pIn, cache_write: pCw, cache_read: pCr, out: pOut };
+}
+
+function isFlatPriceObject(obj) {
+  return (
+    'in' in obj ||
+    'input' in obj ||
+    'cache_write' in obj ||
+    'cacheWrite' in obj ||
+    'cache_read' in obj ||
+    'cacheRead' in obj ||
+    'out' in obj ||
+    'output' in obj
+  );
+}
+
+/**
+ * Load `.simt/local/bench/prices.json` if present.
+ *
+ * Supports either a flat `{in, cache_write, cache_read, out}` object (prices per 1M
+ * tokens) or a map keyed by model/harness name with an optional `"default"` entry.
+ *
+ * @param {string} root
+ * @returns {Promise<null|{kind: 'flat'|'map', entry?: object, entries?: object}>}
+ */
+export async function loadPrices(root) {
+  // The pre-move path is a fallback for one release (review 0003 Q19).
+  const pricesPath = join(resolve(root), LOCAL_DIR, 'bench', 'prices.json');
+  const legacyPath = join(resolve(root), '.simt', 'bench', 'prices.json');
+  let rawText;
+  try {
+    rawText = await readFile(pricesPath, 'utf8');
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      try {
+        rawText = await readFile(legacyPath, 'utf8');
+      } catch (legacyError) {
+        if (legacyError && legacyError.code === 'ENOENT') return null;
+        throw legacyError;
+      }
+    } else throw error;
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(rawText);
+  } catch (error) {
+    throw new Error(`invalid JSON in ${pricesPath}: ${error.message}`);
+  }
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+    throw new Error(`${pricesPath} must be a JSON object`);
+  }
+
+  if (isFlatPriceObject(parsed)) {
+    return { kind: 'flat', entry: normalizePriceEntry(parsed, pricesPath) };
+  }
+
+  const entries = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    entries[key] = normalizePriceEntry(value, `${pricesPath}[${JSON.stringify(key)}]`);
+  }
+  return { kind: 'map', entries };
+}
+
+function resolvePriceEntry(prices, run) {
+  if (!prices) return null;
+  if (prices.kind === 'flat') return normalizePriceEntry(prices.entry, 'prices');
+  if (prices.kind === 'map') {
+    const entry =
+      (run?.model && prices.entries[run.model]) ||
+      (run?.harness && prices.entries[run.harness]) ||
+      prices.entries.default ||
+      null;
+    return entry ? normalizePriceEntry(entry, 'prices') : null;
+  }
+  if (isFlatPriceObject(prices)) {
+    return normalizePriceEntry(prices, 'prices');
+  }
+  const entry =
+    (run?.model && prices[run.model]) || (run?.harness && prices[run.harness]) || prices.default || null;
+  return entry ? normalizePriceEntry(entry, 'prices') : null;
+}
+
+/**
+ * Compute billed cost in USD for a single run record.
+ *
+ * Returns `{usd, source}` where `usd` is null when the cost is unknown: an arm whose
+ * prices are absent must not report a free run.
+ *
+ * @param {object} run
+ * @param {object|null} [prices]
+ * @returns {{usd: number|null, source: 'priced'|'reported'|'missing'}}
+ */
+export function computeRunCost(run, prices = null) {
+  const usage = run?.usage ?? {};
+  const input = Number(usage.input ?? 0);
+  const cacheWrite = Number(usage.cacheWrite ?? usage.cache_write ?? 0);
+  const cacheRead = Number(usage.cacheRead ?? usage.cache_read ?? 0);
+  const output = Number(usage.output ?? 0);
+  const totalTokens = input + cacheWrite + cacheRead + output;
+
+  const entry = resolvePriceEntry(prices, run);
+  if (entry) {
+    if (totalTokens === 0 && Number(run?.costUsd ?? 0) > 0) {
+      return { usd: Number(run.costUsd), source: 'reported' };
+    }
+    return {
+      usd:
+        (input * entry.in + cacheWrite * entry.cache_write + cacheRead * entry.cache_read + output * entry.out) /
+        1_000_000,
+      source: 'priced',
+    };
+  }
+  const reported = run?.costUsd;
+  if (typeof reported === 'number' && Number.isFinite(reported)) {
+    return { usd: reported, source: 'reported' };
+  }
+  return { usd: null, source: 'missing' };
+}
+
+/** Whether a run attempt counts as resolved: hidden tests pass AND declared gates pass. */
+export function isRunResolved(run) {
+  return Boolean(run?.resolved && (run?.gatesPassed ?? true));
+}
+
+/**
+ * Quality score Q in [0, 1]: 0.5 hidden tests + 0.2 gates + 0.2 probe + 0.1 diff hygiene.
+ *
+ * @param {object} run
+ * @returns {number}
+ */
+export function computeRunQuality(run) {
+  const testsScore = run?.resolved ? 1 : 0;
+  const gatesScore = (run?.gatesPassed ?? true) ? 1 : 0;
+
+  let probeScore = 1;
+  if (run?.probe !== null && run?.probe !== undefined) {
+    if (typeof run.probe === 'boolean') probeScore = run.probe ? 1 : 0;
+    else if (typeof run.probe === 'number' && Number.isFinite(run.probe)) {
+      probeScore = Math.max(0, Math.min(1, run.probe));
+    } else if (typeof run.probe === 'object') probeScore = run.probe.passed ? 1 : 0;
+  }
+
+  const files = Number(run?.diff?.files ?? 0);
+  const outside = Number(run?.diff?.outsideBlastRadius ?? 0);
+  let hygieneScore = 1;
+  if (outside > 0) {
+    hygieneScore = files > 0 ? Math.max(0, Math.min(1, (files - outside) / files)) : 0;
+  }
+
+  return (5 * testsScore + 2 * gatesScore + 2 * probeScore + hygieneScore) / 10;
+}
+
+/**
+ * Evaluate one record into the row an arm aggregation and a paired test consume.
+ *
+ * @param {object} run
+ * @param {object|null} [prices]
+ * @returns {object}
+ */
+export function evaluateRunRecord(run, prices = null) {
+  const usage = run?.usage ?? {};
+  const input = Number(usage.input ?? 0);
+  const cacheWrite = Number(usage.cacheWrite ?? usage.cache_write ?? 0);
+  const cacheRead = Number(usage.cacheRead ?? usage.cache_read ?? 0);
+  const output = Number(usage.output ?? 0);
+  const turns = Number(usage.turns ?? 0);
+  const toolCalls = Number(usage.toolCalls ?? usage.tool_calls ?? 0);
+  const R = isRunResolved(run) ? 1 : 0;
+  const Q = computeRunQuality(run);
+  const cost = computeRunCost(run, prices);
+  const cacheDenom = input + cacheRead;
+  const cacheHitRatio = cacheDenom > 0 ? cacheRead / cacheDenom : 0;
+  // A v2 experiment is the conditions, a v1 experiment was a label. Both reduce to one
+  // string, so grouping by experiment keeps working across a stored corpus.
+  const experiment = run?.experiment;
+  const experimentKey =
+    typeof experiment === 'string'
+      ? experiment
+      : experiment && typeof experiment === 'object'
+        ? [experiment.simtVersion ?? '-', experiment.rulesDigest ?? '-', experiment.corpusDigest ?? '-'].join('/')
+        : '';
+
+  return {
+    task: String(run?.task ?? ''),
+    stratum: String(run?.stratum ?? 'unknown'),
+    arm: String(run?.arm ?? 'default'),
+    harness: String(run?.harness ?? ''),
+    model: String(run?.model ?? ''),
+    experiment: experimentKey,
+    rep: Number(run?.rep ?? 1),
+    R,
+    Q,
+    C: cost.usd,
+    costSource: cost.source,
+    turns,
+    toolCalls,
+    input,
+    cacheWrite,
+    cacheRead,
+    output,
+    cacheHitRatio,
+  };
+}

package/src/measure/records.mjs

@@ -0,0 +1,211 @@
+// Stored benchmark run records, schema v2.
+//
+// A v1 record said what an arm scored and nothing about the conditions it scored under.
+// Two records from different rule text, different corpora or different harness versions
+// were indistinguishable, so a report could compare them and call the difference an
+// effect (review 0003 Q78). The record now carries the experiment that produced it, and
+// the run id names the model whenever a model other than the harness default was used:
+// two models driven through one harness used to overwrite each other's run directory.
+//
+// Loading is tolerant in one direction only: a record that does not parse is skipped and
+// reported as a warning, because one corrupt file is not an empty corpus. A reader that
+// answers "no runs" for a directory of unreadable files hides the corruption entirely.
+import { readdir, readFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { arch, cpus, platform } from 'node:os';
+import { join, resolve } from 'node:path';
+
+import { LOCAL_DIR } from '../state.mjs';
+import { packageVersion } from '../package-version.mjs';
+
+export const RECORD_SCHEMA_VERSION = 2;
+
+/** Directory a run's records are stored under, with the pre-move path as a fallback. */
+export function runsDir(root) {
+  return join(resolve(root), LOCAL_DIR, 'bench', 'runs');
+}
+
+function legacyRunsDir(root) {
+  return join(resolve(root), '.simt', 'bench', 'runs');
+}
+
+/**
+ * The run directory name for one arm and repetition.
+ *
+ * The model is part of the name when it is a real choice. A `default` model (or a
+ * harness whose model is itself) adds nothing to the name, and a name that changes for
+ * every run would make the stored path useless to a reader.
+ *
+ * @param {{harness: string, model?: string|null, arm: string, rep: number}} spec
+ * @returns {string}
+ */
+export function runIdFor({ harness, model, arm, rep }) {
+  const named =
+    typeof model === 'string' && model.length > 0 && model !== 'default' && model !== harness;
+  return `${harness}-${named ? `${model}-` : ''}${arm}-r${rep}`;
+}
+
+/**
+ * The conditions a run was produced under, recorded so a report can refuse to pool
+ * records that were not comparable.
+ *
+ * @param {{simtVersion?: string, rulesDigest?: string|null, adapterVersion?: string|null,
+ *   corpusDigest?: string|null, seed?: number|null, isolation?: string}} input
+ * @returns {object}
+ */
+export function experimentOf({
+  simtVersion = packageVersion(),
+  rulesDigest = null,
+  adapterVersion = null,
+  corpusDigest = null,
+  seed = null,
+  isolation = 'none',
+} = {}) {
+  return {
+    simtVersion,
+    rulesDigest,
+    adapterVersion,
+    corpusDigest,
+    seed,
+    isolation,
+    hardware: `${platform()}/${arch()}/${cpus().length}cpu`,
+  };
+}
+
+/**
+ * A digest of the corpus a run was measured against.
+ *
+ * It is the sorted task ids and their base commits, hashed: a re-mine that changes which
+ * tasks exist, or where they start, changes the digest, and a report can say the two
+ * records were not measured on the same corpus.
+ *
+ * @param {object[]} tasks normalised task descriptors
+ * @returns {string}
+ */
+export function corpusDigestOf(tasks = []) {
+  const lines = [...tasks]
+    .map((task) => `${task.id}\0${task.base}`)
+    .sort()
+    .join('\n');
+  return createHash('sha256').update(lines).digest('hex').slice(0, 16);
+}
+
+/**
+ * Build one record.
+ *
+ * `status` is the record's own claim: `ok` measured something, `error` did not. An error
+ * record keeps the run's identity so the report can exclude it by name rather than by
+ * looking for a missing field.
+ *
+ * @param {object} input
+ * @returns {object}
+ */
+export function runRecord({
+  task,
+  arm,
+  harness,
+  model,
+  rep,
+  measured = true,
+  isolation = 'none',
+  experiment = null,
+  status = 'ok',
+  error = null,
+  agent = null,
+  resolved = false,
+  gatesPassed = false,
+  f2pPassed = [],
+  usage = null,
+  costUsd = null,
+  diff = { files: 0, outsideBlastRadius: 0 },
+  skills = null,
+  probe = null,
+}) {
+  const record = {
+    schemaVersion: RECORD_SCHEMA_VERSION,
+    task: String(task?.id ?? task ?? ''),
+    stratum: String(task?.stratum ?? 'unknown'),
+    arm,
+    harness,
+    model,
+    rep,
+    measured: Boolean(measured),
+    isolation,
+    experiment,
+    status,
+    agent,
+    resolved: Boolean(resolved),
+    gatesPassed: Boolean(gatesPassed),
+    f2pPassed: [...f2pPassed],
+    usage: usage ?? { input: 0, cacheWrite: 0, cacheRead: 0, output: 0, turns: 0, toolCalls: 0 },
+    costUsd: typeof costUsd === 'number' && Number.isFinite(costUsd) ? costUsd : null,
+    diff,
+    skills,
+    probe,
+  };
+  if (error !== null) record.error = String(error);
+  return record;
+}
+
+/**
+ * Read every stored record, and report the files it could not read.
+ *
+ * @param {string} root
+ * @returns {Promise<{records: object[], warnings: string[]}>}
+ */
+export async function readRunRecords(root) {
+  let base = runsDir(root);
+  let entries = await readdir(base, { withFileTypes: true }).catch(() => null);
+  if (entries === null) {
+    base = legacyRunsDir(root);
+    entries = await readdir(base, { withFileTypes: true }).catch(() => []);
+  }
+  const sorted = [...entries].sort((a, b) => a.name.localeCompare(b.name));
+
+  const records = [];
+  const warnings = [];
+  const readOne = async (file) => {
+    let text;
+    try {
+      text = await readFile(file, 'utf8');
+    } catch (error) {
+      warnings.push(`could not read ${file}: ${error.message ?? error}`);
+      return;
+    }
+    try {
+      const record = JSON.parse(text);
+      if (!record || typeof record !== 'object' || Array.isArray(record)) {
+        warnings.push(`skipped ${file}: not a record object`);
+        return;
+      }
+      records.push(record);
+    } catch (error) {
+      warnings.push(`skipped ${file}: ${error.message ?? error}`);
+    }
+  };
+
+  for (const entry of sorted) {
+    if (entry.name.startsWith('.')) continue;
+    const full = join(base, entry.name);
+    if (entry.isDirectory()) {
+      const files = (await readdir(full).catch(() => []))
+        .filter((file) => file.endsWith('.json') && !file.startsWith('.'))
+        .sort();
+      for (const file of files) await readOne(join(full, file));
+    } else if (entry.isFile() && entry.name.endsWith('.json')) {
+      await readOne(full);
+    }
+  }
+  return { records, warnings };
+}
+
+/**
+ * Every stored record. A caller that has nowhere to put a warning gets the records and
+ * no warnings; the report is the caller that shows them.
+ *
+ * @param {string} root
+ * @returns {Promise<object[]>}
+ */
+export async function loadRunRecords(root) {
+  return (await readRunRecords(root)).records;
+}

package/src/measure/session.mjs

@@ -8,6 +8,7 @@
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 
+import { CLI } from '../name.mjs';
 import { tokens } from './context.mjs';
 
 const RULE_PATH_RE =
@@ -53,14 +54,14 @@ export async function measureSession(transcriptPath) {
     return {
       ok: false,
       error: '--session requires a local transcript path',
-      remedy: 'pass a local JSONL or JSON transcript path: simt tokens --session <path>',
+      remedy: `pass a local JSONL or JSON transcript path: ${CLI} tokens --session <path>`,
     };
   }
   if (/^[a-z][a-z0-9+.-]*:\/\//i.test(transcriptPath.trim())) {
     return {
       ok: false,
       error: `${transcriptPath}: remote URLs are refused; session transcripts are read locally only (ADR-075)`,
-      remedy: 'pass a local file path on this machine: simt tokens --session ./session.jsonl',
+      remedy: `pass a local file path on this machine: ${CLI} tokens --session ./session.jsonl`,
     };
   }