simt-dev 0.7.1 → 0.8.0

package/src/{bench → measure}/corpus.mjs

@@ -18,7 +18,7 @@ import { dirname, join, resolve } from 'node:path';
 
 import { STATE_DIR } from '../name.mjs';
 import { packageVersion } from '../package-version.mjs';
-import { resolveVaultDir } from './mine.mjs';
+import { resolveVaultDir } from '../testing/vault.mjs';
 
 /** The document kind stamp, so a reader can refuse a file that is not a corpus. */
 export const CORPUS_KIND = 'simt-bench-corpus';

package/src/measure/metrics.mjs

@@ -0,0 +1,236 @@
+// Derived per-run metrics: what a record costs, and how good it is.
+//
+// Split from the statistics beside it (review 0003 Q78): a metric is a function of one
+// record, a statistical test is a function of a population of them, and a reader looking
+// for the cost model should not have to open the bootstrap.
+//
+// The cost model distinguishes three answers because a missing cost must never read as a
+// zero cost: `priced` from the token table, `reported` by the harness, `missing` when
+// neither exists (review 0003 Q46).
+import { readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+import { LOCAL_DIR } from '../state.mjs';
+
+function normalizePriceEntry(raw, label) {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+    throw new Error(`${label} must be a JSON object with {in, cache_write, cache_read, out}`);
+  }
+  const pIn = Number(raw.in ?? raw.input ?? 0);
+  const pCw = Number(raw.cache_write ?? raw.cacheWrite ?? 0);
+  const pCr = Number(raw.cache_read ?? raw.cacheRead ?? 0);
+  const pOut = Number(raw.out ?? raw.output ?? 0);
+  for (const [key, value] of [
+    ['in', pIn],
+    ['cache_write', pCw],
+    ['cache_read', pCr],
+    ['out', pOut],
+  ]) {
+    if (!Number.isFinite(value) || value < 0) {
+      throw new Error(`${label}.${key} must be a non-negative number`);
+    }
+  }
+  return { in: pIn, cache_write: pCw, cache_read: pCr, out: pOut };
+}
+
+function isFlatPriceObject(obj) {
+  return (
+    'in' in obj ||
+    'input' in obj ||
+    'cache_write' in obj ||
+    'cacheWrite' in obj ||
+    'cache_read' in obj ||
+    'cacheRead' in obj ||
+    'out' in obj ||
+    'output' in obj
+  );
+}
+
+/**
+ * Load `.simt/local/bench/prices.json` if present.
+ *
+ * Supports either a flat `{in, cache_write, cache_read, out}` object (prices per 1M
+ * tokens) or a map keyed by model/harness name with an optional `"default"` entry.
+ *
+ * @param {string} root
+ * @returns {Promise<null|{kind: 'flat'|'map', entry?: object, entries?: object}>}
+ */
+export async function loadPrices(root) {
+  // The pre-move path is a fallback for one release (review 0003 Q19).
+  const pricesPath = join(resolve(root), LOCAL_DIR, 'bench', 'prices.json');
+  const legacyPath = join(resolve(root), '.simt', 'bench', 'prices.json');
+  let rawText;
+  try {
+    rawText = await readFile(pricesPath, 'utf8');
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      try {
+        rawText = await readFile(legacyPath, 'utf8');
+      } catch (legacyError) {
+        if (legacyError && legacyError.code === 'ENOENT') return null;
+        throw legacyError;
+      }
+    } else throw error;
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(rawText);
+  } catch (error) {
+    throw new Error(`invalid JSON in ${pricesPath}: ${error.message}`);
+  }
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+    throw new Error(`${pricesPath} must be a JSON object`);
+  }
+
+  if (isFlatPriceObject(parsed)) {
+    return { kind: 'flat', entry: normalizePriceEntry(parsed, pricesPath) };
+  }
+
+  const entries = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    entries[key] = normalizePriceEntry(value, `${pricesPath}[${JSON.stringify(key)}]`);
+  }
+  return { kind: 'map', entries };
+}
+
+function resolvePriceEntry(prices, run) {
+  if (!prices) return null;
+  if (prices.kind === 'flat') return normalizePriceEntry(prices.entry, 'prices');
+  if (prices.kind === 'map') {
+    const entry =
+      (run?.model && prices.entries[run.model]) ||
+      (run?.harness && prices.entries[run.harness]) ||
+      prices.entries.default ||
+      null;
+    return entry ? normalizePriceEntry(entry, 'prices') : null;
+  }
+  if (isFlatPriceObject(prices)) {
+    return normalizePriceEntry(prices, 'prices');
+  }
+  const entry =
+    (run?.model && prices[run.model]) || (run?.harness && prices[run.harness]) || prices.default || null;
+  return entry ? normalizePriceEntry(entry, 'prices') : null;
+}
+
+/**
+ * Compute billed cost in USD for a single run record.
+ *
+ * Returns `{usd, source}` where `usd` is null when the cost is unknown: an arm whose
+ * prices are absent must not report a free run.
+ *
+ * @param {object} run
+ * @param {object|null} [prices]
+ * @returns {{usd: number|null, source: 'priced'|'reported'|'missing'}}
+ */
+export function computeRunCost(run, prices = null) {
+  const usage = run?.usage ?? {};
+  const input = Number(usage.input ?? 0);
+  const cacheWrite = Number(usage.cacheWrite ?? usage.cache_write ?? 0);
+  const cacheRead = Number(usage.cacheRead ?? usage.cache_read ?? 0);
+  const output = Number(usage.output ?? 0);
+  const totalTokens = input + cacheWrite + cacheRead + output;
+
+  const entry = resolvePriceEntry(prices, run);
+  if (entry) {
+    if (totalTokens === 0 && Number(run?.costUsd ?? 0) > 0) {
+      return { usd: Number(run.costUsd), source: 'reported' };
+    }
+    return {
+      usd:
+        (input * entry.in + cacheWrite * entry.cache_write + cacheRead * entry.cache_read + output * entry.out) /
+        1_000_000,
+      source: 'priced',
+    };
+  }
+  const reported = run?.costUsd;
+  if (typeof reported === 'number' && Number.isFinite(reported)) {
+    return { usd: reported, source: 'reported' };
+  }
+  return { usd: null, source: 'missing' };
+}
+
+/** Whether a run attempt counts as resolved: hidden tests pass AND declared gates pass. */
+export function isRunResolved(run) {
+  return Boolean(run?.resolved && (run?.gatesPassed ?? true));
+}
+
+/**
+ * Quality score Q in [0, 1]: 0.5 hidden tests + 0.2 gates + 0.2 probe + 0.1 diff hygiene.
+ *
+ * @param {object} run
+ * @returns {number}
+ */
+export function computeRunQuality(run) {
+  const testsScore = run?.resolved ? 1 : 0;
+  const gatesScore = (run?.gatesPassed ?? true) ? 1 : 0;
+
+  let probeScore = 1;
+  if (run?.probe !== null && run?.probe !== undefined) {
+    if (typeof run.probe === 'boolean') probeScore = run.probe ? 1 : 0;
+    else if (typeof run.probe === 'number' && Number.isFinite(run.probe)) {
+      probeScore = Math.max(0, Math.min(1, run.probe));
+    } else if (typeof run.probe === 'object') probeScore = run.probe.passed ? 1 : 0;
+  }
+
+  const files = Number(run?.diff?.files ?? 0);
+  const outside = Number(run?.diff?.outsideBlastRadius ?? 0);
+  let hygieneScore = 1;
+  if (outside > 0) {
+    hygieneScore = files > 0 ? Math.max(0, Math.min(1, (files - outside) / files)) : 0;
+  }
+
+  return (5 * testsScore + 2 * gatesScore + 2 * probeScore + hygieneScore) / 10;
+}
+
+/**
+ * Evaluate one record into the row an arm aggregation and a paired test consume.
+ *
+ * @param {object} run
+ * @param {object|null} [prices]
+ * @returns {object}
+ */
+export function evaluateRunRecord(run, prices = null) {
+  const usage = run?.usage ?? {};
+  const input = Number(usage.input ?? 0);
+  const cacheWrite = Number(usage.cacheWrite ?? usage.cache_write ?? 0);
+  const cacheRead = Number(usage.cacheRead ?? usage.cache_read ?? 0);
+  const output = Number(usage.output ?? 0);
+  const turns = Number(usage.turns ?? 0);
+  const toolCalls = Number(usage.toolCalls ?? usage.tool_calls ?? 0);
+  const R = isRunResolved(run) ? 1 : 0;
+  const Q = computeRunQuality(run);
+  const cost = computeRunCost(run, prices);
+  const cacheDenom = input + cacheRead;
+  const cacheHitRatio = cacheDenom > 0 ? cacheRead / cacheDenom : 0;
+  // A v2 experiment is the conditions, a v1 experiment was a label. Both reduce to one
+  // string, so grouping by experiment keeps working across a stored corpus.
+  const experiment = run?.experiment;
+  const experimentKey =
+    typeof experiment === 'string'
+      ? experiment
+      : experiment && typeof experiment === 'object'
+        ? [experiment.simtVersion ?? '-', experiment.rulesDigest ?? '-', experiment.corpusDigest ?? '-'].join('/')
+        : '';
+
+  return {
+    task: String(run?.task ?? ''),
+    stratum: String(run?.stratum ?? 'unknown'),
+    arm: String(run?.arm ?? 'default'),
+    harness: String(run?.harness ?? ''),
+    model: String(run?.model ?? ''),
+    experiment: experimentKey,
+    rep: Number(run?.rep ?? 1),
+    R,
+    Q,
+    C: cost.usd,
+    costSource: cost.source,
+    turns,
+    toolCalls,
+    input,
+    cacheWrite,
+    cacheRead,
+    output,
+    cacheHitRatio,
+  };
+}

package/src/measure/records.mjs

@@ -0,0 +1,211 @@
+// Stored benchmark run records, schema v2.
+//
+// A v1 record said what an arm scored and nothing about the conditions it scored under.
+// Two records from different rule text, different corpora or different harness versions
+// were indistinguishable, so a report could compare them and call the difference an
+// effect (review 0003 Q78). The record now carries the experiment that produced it, and
+// the run id names the model whenever a model other than the harness default was used:
+// two models driven through one harness used to overwrite each other's run directory.
+//
+// Loading is tolerant in one direction only: a record that does not parse is skipped and
+// reported as a warning, because one corrupt file is not an empty corpus. A reader that
+// answers "no runs" for a directory of unreadable files hides the corruption entirely.
+import { readdir, readFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { arch, cpus, platform } from 'node:os';
+import { join, resolve } from 'node:path';
+
+import { LOCAL_DIR } from '../state.mjs';
+import { packageVersion } from '../package-version.mjs';
+
+export const RECORD_SCHEMA_VERSION = 2;
+
+/** Directory a run's records are stored under, with the pre-move path as a fallback. */
+export function runsDir(root) {
+  return join(resolve(root), LOCAL_DIR, 'bench', 'runs');
+}
+
+function legacyRunsDir(root) {
+  return join(resolve(root), '.simt', 'bench', 'runs');
+}
+
+/**
+ * The run directory name for one arm and repetition.
+ *
+ * The model is part of the name when it is a real choice. A `default` model (or a
+ * harness whose model is itself) adds nothing to the name, and a name that changes for
+ * every run would make the stored path useless to a reader.
+ *
+ * @param {{harness: string, model?: string|null, arm: string, rep: number}} spec
+ * @returns {string}
+ */
+export function runIdFor({ harness, model, arm, rep }) {
+  const named =
+    typeof model === 'string' && model.length > 0 && model !== 'default' && model !== harness;
+  return `${harness}-${named ? `${model}-` : ''}${arm}-r${rep}`;
+}
+
+/**
+ * The conditions a run was produced under, recorded so a report can refuse to pool
+ * records that were not comparable.
+ *
+ * @param {{simtVersion?: string, rulesDigest?: string|null, adapterVersion?: string|null,
+ *   corpusDigest?: string|null, seed?: number|null, isolation?: string}} input
+ * @returns {object}
+ */
+export function experimentOf({
+  simtVersion = packageVersion(),
+  rulesDigest = null,
+  adapterVersion = null,
+  corpusDigest = null,
+  seed = null,
+  isolation = 'none',
+} = {}) {
+  return {
+    simtVersion,
+    rulesDigest,
+    adapterVersion,
+    corpusDigest,
+    seed,
+    isolation,
+    hardware: `${platform()}/${arch()}/${cpus().length}cpu`,
+  };
+}
+
+/**
+ * A digest of the corpus a run was measured against.
+ *
+ * It is the sorted task ids and their base commits, hashed: a re-mine that changes which
+ * tasks exist, or where they start, changes the digest, and a report can say the two
+ * records were not measured on the same corpus.
+ *
+ * @param {object[]} tasks normalised task descriptors
+ * @returns {string}
+ */
+export function corpusDigestOf(tasks = []) {
+  const lines = [...tasks]
+    .map((task) => `${task.id}\0${task.base}`)
+    .sort()
+    .join('\n');
+  return createHash('sha256').update(lines).digest('hex').slice(0, 16);
+}
+
+/**
+ * Build one record.
+ *
+ * `status` is the record's own claim: `ok` measured something, `error` did not. An error
+ * record keeps the run's identity so the report can exclude it by name rather than by
+ * looking for a missing field.
+ *
+ * @param {object} input
+ * @returns {object}
+ */
+export function runRecord({
+  task,
+  arm,
+  harness,
+  model,
+  rep,
+  measured = true,
+  isolation = 'none',
+  experiment = null,
+  status = 'ok',
+  error = null,
+  agent = null,
+  resolved = false,
+  gatesPassed = false,
+  f2pPassed = [],
+  usage = null,
+  costUsd = null,
+  diff = { files: 0, outsideBlastRadius: 0 },
+  skills = null,
+  probe = null,
+}) {
+  const record = {
+    schemaVersion: RECORD_SCHEMA_VERSION,
+    task: String(task?.id ?? task ?? ''),
+    stratum: String(task?.stratum ?? 'unknown'),
+    arm,
+    harness,
+    model,
+    rep,
+    measured: Boolean(measured),
+    isolation,
+    experiment,
+    status,
+    agent,
+    resolved: Boolean(resolved),
+    gatesPassed: Boolean(gatesPassed),
+    f2pPassed: [...f2pPassed],
+    usage: usage ?? { input: 0, cacheWrite: 0, cacheRead: 0, output: 0, turns: 0, toolCalls: 0 },
+    costUsd: typeof costUsd === 'number' && Number.isFinite(costUsd) ? costUsd : null,
+    diff,
+    skills,
+    probe,
+  };
+  if (error !== null) record.error = String(error);
+  return record;
+}
+
+/**
+ * Read every stored record, and report the files it could not read.
+ *
+ * @param {string} root
+ * @returns {Promise<{records: object[], warnings: string[]}>}
+ */
+export async function readRunRecords(root) {
+  let base = runsDir(root);
+  let entries = await readdir(base, { withFileTypes: true }).catch(() => null);
+  if (entries === null) {
+    base = legacyRunsDir(root);
+    entries = await readdir(base, { withFileTypes: true }).catch(() => []);
+  }
+  const sorted = [...entries].sort((a, b) => a.name.localeCompare(b.name));
+
+  const records = [];
+  const warnings = [];
+  const readOne = async (file) => {
+    let text;
+    try {
+      text = await readFile(file, 'utf8');
+    } catch (error) {
+      warnings.push(`could not read ${file}: ${error.message ?? error}`);
+      return;
+    }
+    try {
+      const record = JSON.parse(text);
+      if (!record || typeof record !== 'object' || Array.isArray(record)) {
+        warnings.push(`skipped ${file}: not a record object`);
+        return;
+      }
+      records.push(record);
+    } catch (error) {
+      warnings.push(`skipped ${file}: ${error.message ?? error}`);
+    }
+  };
+
+  for (const entry of sorted) {
+    if (entry.name.startsWith('.')) continue;
+    const full = join(base, entry.name);
+    if (entry.isDirectory()) {
+      const files = (await readdir(full).catch(() => []))
+        .filter((file) => file.endsWith('.json') && !file.startsWith('.'))
+        .sort();
+      for (const file of files) await readOne(join(full, file));
+    } else if (entry.isFile() && entry.name.endsWith('.json')) {
+      await readOne(full);
+    }
+  }
+  return { records, warnings };
+}
+
+/**
+ * Every stored record. A caller that has nowhere to put a warning gets the records and
+ * no warnings; the report is the caller that shows them.
+ *
+ * @param {string} root
+ * @returns {Promise<object[]>}
+ */
+export async function loadRunRecords(root) {
+  return (await readRunRecords(root)).records;
+}