synabun 2.0.10 → 2.1.0

package/neural-interface/public/shared/skills/ss-eval-adapters.js

@@ -0,0 +1,269 @@
+// ═══════════════════════════════════════════
+// SynaBun — evaluation formats in and out (P4, SK-086)
+// ═══════════════════════════════════════════
+//
+// The suites and results other tools read and write, each read and written as its own dialect (master plan §3.6,
+// 03a §2.2–2.5), never guessed into one:
+//
+//   in   the skill-creator's task file (`{ skill_name, evals: [{ id, prompt, expected_output, files, expectations }] }`),
+//        the Agent Skills evaluation guide's (the same with `assertions`), and the skill-creator's trigger set (a JSON
+//        array of `{ query, should_trigger }`). A case with both `assertions` and `expectations` that differ is
+//        refused (ambiguous), never merged. Duplicate requests (case, spacing and punctuation aside) become one case;
+//        the same request with two labels is refused. Each request is its own group, and the groups are split
+//        train / validation / test per stratum (task cases, positive triggers, near-misses) by a seeded shuffle, so
+//        the same file always gets the same split. The result is a synabun-evals/1 suite (ss-evals.js).
+//   out  the skill-creator's task file and trigger set (the held-out cases in a file of their own: an outside
+//        optimiser must not select on them), its benchmark.json (runs[].configuration with_skill / without_skill /
+//        old_skill, a nested result, run_summary with signed string deltas, the actual repetitions and models, and
+//        null for what was not measured), a grading.json per trial (expectations with text, passed, evidence), the
+//        reviewer's feedback (`reviews[{ run_id, feedback, timestamp }]`), and the Agent Skills guide's benchmark
+//        (assertion_results, numeric deltas).
+// No DOM.
+
+import { EVALS_SCHEMA, EVAL_SPLITS, normalizedCaseText, validateEvals } from './ss-evals.js';
+import { stats } from './ss-eval-grade.js';
+
+export const IMPORT_DIALECTS = Object.freeze(['synabun-evals/1', 'skill-creator-tasks', 'agent-skills-tasks', 'skill-creator-triggers']);
+export const SPLIT_SHARES = Object.freeze({ validation: 0.2, test: 0.2 });
+const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+
+/** Which dialect a parsed JSON value is, or null. */
+export function detectDialect(value) {
+  if (Array.isArray(value)) return value.length && value.every((x) => isObject(x) && typeof x.query === 'string' && typeof x.should_trigger === 'boolean') ? 'skill-creator-triggers' : null;
+  if (!isObject(value)) return null;
+  if (value.schema === EVALS_SCHEMA) return 'synabun-evals/1';
+  if (!Array.isArray(value.evals)) return null;
+  const hasExpectations = value.evals.some((c) => isObject(c) && Array.isArray(c.expectations));
+  const hasAssertions = value.evals.some((c) => isObject(c) && Array.isArray(c.assertions));
+  if (hasExpectations && !hasAssertions) return 'skill-creator-tasks';
+  if (hasAssertions && !hasExpectations) return 'agent-skills-tasks';
+  if (hasAssertions && hasExpectations) return 'skill-creator-tasks';   // case by case below: equal lists pass, others are refused
+  return 'agent-skills-tasks';
+}
+
+/** A lowercase slug of a text (an id from a prompt or a query), at most `max` characters. */
+export function slugOf(text, max = 48) {
+  const s = normalizedCaseText(text).normalize('NFKD').replace(/[̀-ͯ]/g, '').replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, max).replace(/-+$/, '');
+  return /^[a-z0-9]/.test(s) ? s : `case-${s || 'x'}`.slice(0, max);
+}
+
+/** A seeded pseudo-random generator (mulberry32): the same seed, the same split. */
+function seeded(seed) {
+  let a = (Number(seed) >>> 0) || 42;
+  return () => { a = (a + 0x6D2B79F5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };
+}
+function shuffle(list, rnd) { const out = [...list]; for (let i = out.length - 1; i > 0; i--) { const j = Math.floor(rnd() * (i + 1)); [out[i], out[j]] = [out[j], out[i]]; } return out; }
+
+/**
+ * The split of each item of one stratum: one item trains; two give train and validation; three or more keep at least
+ * one of each, with about a fifth held out and a fifth for validation.
+ */
+export function splitStratum(ids, { seed = 42 } = {}) {
+  const order = shuffle(ids, seeded(seed));
+  const n = order.length;
+  const out = new Map();
+  if (!n) return out;
+  let test = n >= 3 ? Math.max(1, Math.round(n * SPLIT_SHARES.test)) : 0;
+  let validation = n >= 2 ? Math.max(1, Math.round(n * SPLIT_SHARES.validation)) : 0;
+  while (n - test - validation < 1) { if (test > validation && test > 0) test -= 1; else if (validation > 0) validation -= 1; else break; }
+  order.forEach((id, i) => out.set(id, i < test ? 'test' : i < test + validation ? 'validation' : 'train'));
+  return out;
+}
+
+const textOf = (a) => (typeof a === 'string' ? a : isObject(a) && typeof a.text === 'string' ? a.text : null);
+function sameList(a, b) { return a.length === b.length && a.every((x, i) => normalizedCaseText(textOf(x)) === normalizedCaseText(textOf(b[i]))); }
+
+/**
+ * Import other tools' suites into one synabun-evals/1 suite.
+ * @param {{ tasks?: string|object, triggers?: string|object[], skillName?: string, seed?: number }} input any of the two
+ *   (text or parsed); a synabun-evals/1 suite passes through, checked
+ * @returns {{ ok: boolean, evals: object|null, dialects: string[], problems: {code, message}[], notes: string[] }}
+ */
+export function importEvals({ tasks = null, triggers = null, skillName = '', seed = 42 } = {}) {
+  const problems = [];
+  const notes = [];
+  const dialects = [];
+  const parse = (x, label) => {
+    if (x === null || x === undefined || x === '') return null;
+    if (typeof x !== 'string') return x;
+    try { return JSON.parse(x.replace(/^/, '')); } catch (e) { problems.push({ code: 'json', message: `${label} is not valid JSON (${e?.message || e}).` }); return undefined; }
+  };
+  const t = parse(tasks, 'The task file');
+  const g = parse(triggers, 'The trigger set');
+  if (t === undefined || g === undefined) return { ok: false, evals: null, dialects, problems, notes };
+  if (t === null && g === null) return { ok: false, evals: null, dialects, problems: [{ code: 'empty', message: 'Nothing to import: give a task file, a trigger set, or both.' }], notes };
+  // A SynaBun suite as it is (with a trigger set beside it, the trigger set is refused: one suite, one source).
+  if (t !== null && detectDialect(t) === 'synabun-evals/1') {
+    if (g !== null) return { ok: false, evals: null, dialects: ['synabun-evals/1'], problems: [{ code: 'two-suites', message: 'A SynaBun suite already has its trigger cases: import it alone.' }], notes };
+    const checked = validateEvals(t, { skillName });
+    return { ok: checked.ok, evals: checked.ok ? t : null, dialects: ['synabun-evals/1'], problems: checked.errors.map((e) => ({ code: e.code, message: `${e.path}: ${e.message}` })), notes: checked.warnings.map((w) => w.message) };
+  }
+  const taskCases = [];
+  if (t !== null) {
+    const d = detectDialect(t);
+    if (!d || d === 'skill-creator-triggers') problems.push({ code: 'dialect', message: d === 'skill-creator-triggers' ? 'The task file is a trigger set: give it as the trigger set.' : 'The task file is neither the skill-creator\'s nor the Agent Skills guide\'s format ({ "evals": [ … ] }).' });
+    else {
+      dialects.push(d);
+      t.evals.forEach((c, i) => {
+        if (!isObject(c)) { problems.push({ code: 'shape', message: `evals[${i}] is not an object.` }); return; }
+        const exp = Array.isArray(c.expectations) ? c.expectations : null;
+        const asr = Array.isArray(c.assertions) ? c.assertions : null;
+        if (exp && asr && !sameList(exp, asr)) { problems.push({ code: 'ambiguous-dialect', message: `evals[${i}] has both "assertions" and "expectations" and they differ: say which list is meant (keep one).` }); return; }
+        if (typeof c.prompt !== 'string' || !c.prompt.trim()) { problems.push({ code: 'empty', message: `evals[${i}] has no prompt.` }); return; }
+        const checks = (exp || asr || []).map(textOf);
+        if (checks.some((x) => x === null || !x.trim())) { problems.push({ code: 'assertion', message: `evals[${i}] has an empty or unreadable assertion.` }); return; }
+        const rawId = c.id === undefined || c.id === null ? '' : String(c.id);
+        taskCases.push({ rawId, prompt: c.prompt, expected: typeof c.expected_output === 'string' ? c.expected_output : '', files: Array.isArray(c.files) ? c.files.filter((f) => typeof f === 'string') : [], assertions: checks.map((x) => x.trim()) });
+      });
+    }
+  }
+  const triggerCases = [];
+  if (g !== null) {
+    const d = detectDialect(g);
+    if (d !== 'skill-creator-triggers') problems.push({ code: 'dialect', message: 'The trigger set is not the skill-creator\'s format (a JSON array of { "query", "should_trigger" }).' });
+    else {
+      dialects.push(d);
+      g.forEach((c) => { if (c.query.trim()) triggerCases.push({ query: c.query, shouldTrigger: c.should_trigger }); });
+    }
+  }
+  if (problems.length) return { ok: false, evals: null, dialects, problems, notes };
+  // One case per request: a repeat with the same label is dropped (said so); with another label, refused.
+  const seen = new Map();
+  const keep = (text, label, what) => {
+    const key = normalizedCaseText(text);
+    if (!seen.has(key)) { seen.set(key, label); return true; }
+    if (seen.get(key) !== label) problems.push({ code: 'conflicting-labels', message: `The same request appears twice with different labels: "${text.slice(0, 80)}".` });
+    else notes.push(`A repeated ${what} was kept once: "${text.slice(0, 80)}".`);
+    return false;
+  };
+  const tasksKept = taskCases.filter((c) => keep(c.prompt, 'task', 'task'));
+  const triggersKept = triggerCases.filter((c) => keep(c.query, c.shouldTrigger ? 'positive' : 'near-miss', 'trigger'));
+  if (problems.length) return { ok: false, evals: null, dialects, problems, notes };
+  // Stable ids: the source's own (as a slug) when it has one, else a slug of the text; never two the same.
+  const used = new Set();
+  const unique = (base) => { let id = base || 'case'; let i = 2; while (used.has(id)) id = `${base.slice(0, 58)}-${i++}`; used.add(id); return id; };
+  const tasksOut = tasksKept.map((c) => ({ ...c, id: unique(c.rawId ? (/^\d+$/.test(c.rawId) ? `eval-${c.rawId}` : slugOf(c.rawId)) : slugOf(c.prompt)) }));
+  const triggersOut = triggersKept.map((c) => ({ ...c, id: unique(slugOf(c.query)) }));
+  const splitOf = new Map([
+    ...splitStratum(tasksOut.map((c) => c.id), { seed }),
+    ...splitStratum(triggersOut.filter((c) => c.shouldTrigger).map((c) => c.id), { seed: seed + 1 }),
+    ...splitStratum(triggersOut.filter((c) => !c.shouldTrigger).map((c) => c.id), { seed: seed + 2 }),
+  ]);
+  const evals = {
+    schema: EVALS_SCHEMA,
+    skill_name: skillName || (isObject(t) && typeof t.skill_name === 'string' ? t.skill_name : ''),
+    evals: tasksOut.map((c) => ({ id: c.id, group: c.id, split: splitOf.get(c.id), prompt: c.prompt, expected_output: c.expected, files: c.files, assertions: c.assertions })),
+    triggers: triggersOut.map((c) => ({ id: c.id, group: c.id, split: splitOf.get(c.id), query: c.query, should_trigger: c.shouldTrigger })),
+  };
+  const checked = validateEvals(evals, { skillName: evals.skill_name });
+  if (!checked.ok) return { ok: false, evals: null, dialects, problems: checked.errors.map((e) => ({ code: e.code, message: `${e.path}: ${e.message}` })), notes };
+  notes.push(...checked.warnings.map((w) => w.message));
+  return { ok: true, evals, dialects, problems: [], notes };
+}
+
+/**
+ * A SynaBun suite in the skill-creator's files: the task file (expectations), the trigger set of the cases an
+ * optimiser may select on (train and validation), and the held-out trigger cases apart.
+ * → { tasks: { skill_name, evals }, triggerSet: [{ query, should_trigger }], heldOut: [{ query, should_trigger }] }
+ */
+export function exportSkillCreator(suite) {
+  const evals = Array.isArray(suite?.evals) ? suite.evals : [];
+  const triggers = Array.isArray(suite?.triggers) ? suite.triggers : [];
+  return {
+    tasks: {
+      skill_name: suite?.skill_name || '',
+      evals: evals.map((c, i) => ({ id: i + 1, prompt: c.prompt, expected_output: c.expected_output || '', files: c.files || [], expectations: (c.assertions || []).map(textOf).filter(Boolean) })),
+    },
+    triggerSet: triggers.filter((c) => c.split !== 'test').map((c) => ({ query: c.query, should_trigger: c.should_trigger })),
+    heldOut: triggers.filter((c) => c.split === 'test').map((c) => ({ query: c.query, should_trigger: c.should_trigger })),
+  };
+}
+
+const CONFIG_NAME = { candidate: 'with_skill' };
+/** The skill-creator's configuration name of a trial: with_skill, and without_skill or old_skill (a revision baseline). */
+export function configurationName(trial, run) {
+  if (trial.configuration === 'candidate') return CONFIG_NAME.candidate;
+  return run?.request?.baseline === 'revision' ? 'old_skill' : 'without_skill';
+}
+/** A stable id of a trial in the skill-creator's layout: never two the same (the trial id is part of it). */
+export const exportRunId = (trial, run) => `eval-${trial.caseId}-${configurationName(trial, run)}-run-${trial.repetition}-${trial.trialId}`;
+
+/** A trial's grading.json in the skill-creator's shape (unverifiable exported as not passed, said in the evidence). */
+export function exportGrading(trial) {
+  const rows = (trial?.grade?.assertions || []).filter((a) => a.status !== 'error');
+  const expectations = rows.map((a) => ({ text: a.text, passed: a.status === 'passed', evidence: a.status === 'unverifiable' ? `unverifiable: ${a.evidence || ''}`.trim() : (a.evidence || '') }));
+  const passed = expectations.filter((e) => e.passed).length;
+  return { expectations, summary: { passed, failed: expectations.length - passed, total: expectations.length, pass_rate: expectations.length ? passed / expectations.length : null } };
+}
+
+const signed = (x, digits) => (x === null || x === undefined ? null : `${x >= 0 ? '+' : ''}${x.toFixed(digits)}`);
+const statBlock = (s) => (s ? { mean: s.mean, stddev: s.sd, min: s.min, max: s.max } : null);
+
+/**
+ * A run in the skill-creator's benchmark.json shape: one entry per graded task trial, the actual repetitions and
+ * models, run_summary per configuration and the delta (with_skill minus the baseline configuration) as signed text.
+ * Tokens are not measured as tokens here, so they are null (never characters); a run without a baseline has no delta.
+ */
+export function exportBenchmark(run) {
+  const trials = (run?.trials || []).filter((t) => t.kind === 'task' && t.status === 'completed');
+  const runs = trials.map((t) => {
+    const g = exportGrading(t);
+    return {
+      eval_id: t.caseId, configuration: configurationName(t, run), run_number: t.repetition, run_id: exportRunId(t, run),
+      result: { pass_rate: g.summary.pass_rate, passed: g.summary.passed, failed: g.summary.failed, total: g.summary.total, time_seconds: Number.isFinite(t.durationMs) ? t.durationMs / 1000 : null, tokens: null, tool_calls: Array.isArray(t.toolCalls) ? t.toolCalls.length : null, errors: t.grade?.counts?.error ?? null, cost_usd: Number.isFinite(t.costUsd) ? t.costUsd : null },
+      expectations: g.expectations, notes: [],
+    };
+  });
+  const configs = [...new Set(runs.map((r) => r.configuration))];
+  const summary = {};
+  for (const c of configs) {
+    const mine = runs.filter((r) => r.configuration === c);
+    summary[c] = { pass_rate: statBlock(stats(mine.map((r) => r.result.pass_rate))), time_seconds: statBlock(stats(mine.map((r) => r.result.time_seconds))), tokens: null };
+  }
+  const base = configs.find((c) => c !== 'with_skill') || null;
+  if (summary.with_skill && base && summary[base]) {
+    const d = (k) => (summary.with_skill[k]?.mean !== undefined && summary[base][k]?.mean !== undefined ? summary.with_skill[k].mean - summary[base][k].mean : null);
+    summary.delta = { pass_rate: signed(d('pass_rate'), 2), time_seconds: signed(d('time_seconds'), 1), tokens: null, against: base };
+  }
+  const reps = [...new Set(trials.map((t) => t.repetition))].length || null;
+  return {
+    metadata: {
+      skill_name: run?.skill?.name || '', skill_path: null,
+      executor_model: [...new Set(trials.map((t) => { const r = t.actualRuntime || t.runtime; return `${r.provider}${r.model ? `/${r.model}` : ''}`; }))].join(', ') || null,
+      analyzer_model: [...new Set(trials.map((t) => t.grade?.grader).filter((r) => r?.provider).map((r) => `${r.provider}${r.model ? `/${r.model}` : ''}`))].join(', ') || null,
+      timestamp: run?.finishedAt || run?.createdAt || null,
+      evals_run: [...new Set(trials.map((t) => t.caseId))],
+      runs_per_configuration: reps,
+      synabun: { evalRunId: run?.evalRunId || null, revisionId: run?.skill?.revisionId || null, suiteDigest: run?.suite?.digest || null, experiment: run?.experiment?.digest || null, orientation: 'with_skill-minus-baseline' },
+    },
+    runs, run_summary: summary, notes: [],
+  };
+}
+
+/** The reviewer's notes in the skill-creator's feedback.json shape: every trial, an empty note where there is none. */
+export function exportFeedback(run) {
+  const fb = isObject(run?.feedback) ? run.feedback : {};
+  const reviews = (run?.trials || []).filter((t) => t.kind === 'task').map((t) => ({ run_id: exportRunId(t, run), feedback: fb[t.trialId]?.text || '', timestamp: fb[t.trialId]?.at || null }));
+  const reviewed = (run?.trials || []).filter((t) => t.kind === 'task').every((t) => fb[t.trialId]?.reviewed === true);
+  return { reviews, status: reviewed && reviews.length ? 'complete' : 'in_progress' };
+}
+
+/** A run in the Agent Skills evaluation guide's shape: assertion_results per run, numeric deltas. */
+export function exportAgentSkills(run) {
+  const trials = (run?.trials || []).filter((t) => t.kind === 'task' && t.status === 'completed');
+  const runs = trials.map((t) => ({
+    eval_id: t.caseId, configuration: configurationName(t, run), run: t.repetition,
+    assertion_results: (t.grade?.assertions || []).filter((a) => a.status !== 'error').map((a) => ({ assertion: a.text, passed: a.status === 'passed', evidence: a.evidence || '' })),
+    pass_rate: t.grade?.passRate ?? null,
+  }));
+  const rate = (c) => stats(runs.filter((r) => r.configuration === c).map((r) => r.pass_rate));
+  const w = rate('with_skill');
+  const baseName = runs.some((r) => r.configuration === 'old_skill') ? 'old_skill' : 'without_skill';
+  const b = rate(baseName);
+  return {
+    skill_name: run?.skill?.name || '', runs,
+    benchmark: { with_skill: { pass_rate: w?.mean ?? null, n: w?.n ?? 0 }, [baseName]: { pass_rate: b?.mean ?? null, n: b?.n ?? 0 }, delta: { pass_rate: w && b ? w.mean - b.mean : null } },
+  };
+}
+
+export { EVAL_SPLITS };

package/neural-interface/public/shared/skills/ss-eval-grade.js

@@ -0,0 +1,359 @@
+// ═══════════════════════════════════════════
+// SynaBun — a skill's evaluation: grades, verdicts and what a run shows (P4)
+// ═══════════════════════════════════════════
+//
+// How a trial's answer is graded and what a run's report says, the same on the server (lib/skills/evals.js grades
+// and keeps the record) and the page (the Test tab draws it, compares two runs). The upstream traps are handled here
+// (03a §2.3, master plan §3.6):
+//   - an assertion's status is passed, failed, unverifiable (the grader could not tell, or passed it without evidence)
+//     or error (nothing graded it: the grader failed); a pass needs evidence, and the arithmetic is recomputed here,
+//     never taken from a grader;
+//   - a missing measurement stays null (never 0), a token count is never made from characters, and a standard
+//     deviation is the sample one over the actual n (none for n < 2);
+//   - the comparison is explicit: candidate minus baseline, paired by case and repetition, with the number of pairs;
+//     a run without a baseline shows no uplift;
+//   - an execution status (a trial that failed or timed out) is not an outcome: it is counted apart, never as a
+//     failed assertion or a trigger that did not fire.
+// No DOM.
+
+export const ASSERTION_STATUSES = Object.freeze(['passed', 'failed', 'unverifiable', 'error']);
+export const TRIGGER_METHODS = Object.freeze(['native_skill_tool', 'successful_skill_file_read']);
+export const GRADING_VERSION = 'synabun-grade/1';
+export const TRIGGER_THRESHOLD = 0.5;
+const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+const clip = (s, n) => { const t = String(s ?? ''); return t.length > n ? `${t.slice(0, n - 1)}…` : t; };
+
+/** The line (1-based) of a character offset. */
+function lineAt(text, index) { let n = 1; for (let i = 0; i < index && i < text.length; i++) if (text[i] === '\n') n += 1; return n; }
+
+/**
+ * One deterministic check of an assertion, over the trial's answer and the files it wrote.
+ * @param {object} check ss-evals.js checkProblem()'s shape
+ * @param {{ answer: string, files: Map<string, string|null> }} ctx `files`: path → its text, null for a file that is
+ *   not text; a path missing from the map was not written
+ * @param {{ regexTest?: (source, flags, text) => boolean|null }} [o] how a pattern is run (the server: with a time
+ *   limit; null = it could not be run)
+ * @returns {{ status: 'passed'|'failed'|'error', evidence: string }}
+ */
+export function runCheck(check, ctx, { regexTest = null } = {}) {
+  const files = ctx?.files instanceof Map ? ctx.files : new Map();
+  const answer = typeof ctx?.answer === 'string' ? ctx.answer : '';
+  const where = check.in && check.in !== 'answer' ? check.in : null;
+  const target = () => {
+    if (!where) return { text: answer, label: 'the answer' };
+    if (!files.has(where)) return { missing: true, label: where };
+    const text = files.get(where);
+    return text === null ? { binary: true, label: where } : { text, label: where };
+  };
+  const fold = (s) => (check.ignoreCase ? s.toLowerCase() : s);
+  switch (check.type) {
+    case 'contains':
+    case 'not_contains': {
+      const t = target();
+      if (t.missing) return { status: check.type === 'contains' ? 'failed' : 'passed', evidence: `${t.label} was not written.` };
+      if (t.binary) return { status: 'error', evidence: `${t.label} is not text.` };
+      const at = fold(t.text).indexOf(fold(String(check.value)));
+      if (check.type === 'contains') return at >= 0 ? { status: 'passed', evidence: `"${clip(check.value, 80)}" is in ${t.label} (line ${lineAt(t.text, at)}).` } : { status: 'failed', evidence: `"${clip(check.value, 80)}" is not in ${t.label}.` };
+      return at >= 0 ? { status: 'failed', evidence: `"${clip(check.value, 80)}" is in ${t.label} (line ${lineAt(t.text, at)}).` } : { status: 'passed', evidence: `"${clip(check.value, 80)}" is not in ${t.label}.` };
+    }
+    case 'regex': {
+      const t = target();
+      if (t.missing) return { status: 'failed', evidence: `${t.label} was not written.` };
+      if (t.binary) return { status: 'error', evidence: `${t.label} is not text.` };
+      let hit = null;
+      try {
+        hit = typeof regexTest === 'function' ? regexTest(String(check.value), check.flags || '', t.text) : new RegExp(String(check.value), check.flags || '').test(t.text);
+      } catch { hit = null; }
+      if (hit === null) return { status: 'error', evidence: 'The pattern could not be run on this answer in time.' };
+      return hit ? { status: 'passed', evidence: `The pattern matches ${t.label}.` } : { status: 'failed', evidence: `The pattern does not match ${t.label}.` };
+    }
+    case 'file_exists': {
+      const path = check.path ?? check.in;
+      return files.has(path) ? { status: 'passed', evidence: `${path} was written.` } : { status: 'failed', evidence: `${path} was not written.` };
+    }
+    case 'json_valid': {
+      const path = check.path ?? check.in;
+      if (!files.has(path)) return { status: 'failed', evidence: `${path} was not written.` };
+      const text = files.get(path);
+      if (text === null) return { status: 'failed', evidence: `${path} is not text.` };
+      try { JSON.parse(String(text).replace(/^/, '')); return { status: 'passed', evidence: `${path} is valid JSON.` }; } catch (e) { return { status: 'failed', evidence: `${path} is not valid JSON (${clip(e?.message || e, 120)}).` }; }
+    }
+    case 'max_lines': {
+      const t = target();
+      if (t.missing) return { status: 'failed', evidence: `${t.label} was not written.` };
+      if (t.binary) return { status: 'error', evidence: `${t.label} is not text.` };
+      const lines = t.text.replace(/\n+$/, '').split('\n').length;
+      return lines <= check.value ? { status: 'passed', evidence: `${t.label} has ${lines} line${lines === 1 ? '' : 's'} (at most ${check.value}).` } : { status: 'failed', evidence: `${t.label} has ${lines} lines, more than ${check.value}.` };
+    }
+    default: return { status: 'error', evidence: 'Unknown check.' };
+  }
+}
+
+/** The JSON a grader run answers with (its output schema): one verdict per assertion it was given, with evidence. */
+export const GRADER_SCHEMA = Object.freeze({
+  type: 'object',
+  properties: {
+    assertions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: { id: { type: 'string' }, verdict: { type: 'string', enum: ['passed', 'failed', 'unverifiable'] }, evidence: { type: 'string' } },
+        required: ['id', 'verdict', 'evidence'],
+      },
+    },
+    notes: { type: 'string' },
+  },
+  required: ['assertions'],
+});
+
+/** The JSON a blind judge answers with: which of A and B is the better answer to the request, or a tie. */
+export const JUDGE_SCHEMA = Object.freeze({
+  type: 'object',
+  properties: { winner: { type: 'string', enum: ['A', 'B', 'tie'] }, reasoning: { type: 'string' } },
+  required: ['winner', 'reasoning'],
+});
+
+/**
+ * A grader's answer, checked against the assertions it was given: every one answered once (a second answer for the
+ * same id is ignored), an unknown id ignored, a pass without evidence downgraded to unverifiable, a missing answer an
+ * error. → { verdicts: Map(id → { status, evidence }), problems: string[] }
+ */
+export function readGraderAnswer(answer, assertions) {
+  const problems = [];
+  const verdicts = new Map();
+  const wanted = new Set(assertions.map((a) => a.id));
+  const list = isObject(answer) && Array.isArray(answer.assertions) ? answer.assertions : null;
+  if (!list) problems.push('The grader gave no list of verdicts.');
+  for (const item of list || []) {
+    if (!isObject(item) || typeof item.id !== 'string') { problems.push('A verdict without an id was ignored.'); continue; }
+    if (!wanted.has(item.id)) { problems.push(`A verdict for an unknown assertion (${clip(item.id, 40)}) was ignored.`); continue; }
+    if (verdicts.has(item.id)) { problems.push(`A second verdict for ${item.id} was ignored.`); continue; }
+    const evidence = typeof item.evidence === 'string' ? clip(item.evidence.trim(), 1200) : '';
+    let status = ['passed', 'failed', 'unverifiable'].includes(item.verdict) ? item.verdict : 'unverifiable';
+    if (status === 'passed' && !evidence) { status = 'unverifiable'; problems.push(`${item.id}: passed without evidence, so counted as unverifiable.`); }
+    verdicts.set(item.id, { status, evidence });
+  }
+  for (const id of wanted) if (!verdicts.has(id)) verdicts.set(id, { status: 'error', evidence: 'The grader did not answer this assertion.' });
+  return { verdicts, problems };
+}
+
+/**
+ * One trial's grade from its checks and its grader's verdicts. `rubric` is false when every assertion is a check.
+ * The pass rate is passed over what was graded (an error is no grade: it is counted apart); null when nothing was.
+ * → { assertions: [{ id, text, kind, required, by, status, evidence }], counts, passRate, requiredPassed, graded }
+ */
+export function gradeTrial(assertions, { checks = new Map(), verdicts = null, graderError = null } = {}) {
+  const out = [];
+  const counts = { passed: 0, failed: 0, unverifiable: 0, error: 0, total: assertions.length };
+  for (const a of assertions) {
+    let row;
+    if (a.check) row = checks.get(a.id) || { status: 'error', evidence: 'The check did not run.' };
+    else if (verdicts instanceof Map && verdicts.has(a.id)) row = verdicts.get(a.id);
+    else row = { status: 'error', evidence: graderError ? clip(graderError, 300) : 'No grader graded this assertion.' };
+    const status = ASSERTION_STATUSES.includes(row.status) ? row.status : 'error';
+    counts[status] += 1;
+    out.push({ id: a.id, text: a.text, kind: a.kind || 'outcome', required: a.required !== false, by: a.check ? 'check' : 'model', status, evidence: clip(row.evidence, 1200) });
+  }
+  const graded = counts.total - counts.error;
+  const requiredRows = out.filter((r) => r.required && r.status !== 'error');
+  return {
+    assertions: out, counts, graded,
+    passRate: graded > 0 ? counts.passed / graded : null,
+    requiredPassed: requiredRows.length ? requiredRows.every((r) => r.status === 'passed') : null,
+  };
+}
+
+/**
+ * A trigger case's verdict over its repetitions: how often the runtime picked the skill among the trials that
+ * completed with a reading (a failed or timed-out trial, or one whose selection could not be read, is not a "no").
+ * A positive passes at a rate of `threshold` or more, a near-miss under it.
+ * → { completed, failed, triggers, rate, expected, passed, method }
+ */
+export function triggerVerdict(trials, { expected, threshold = TRIGGER_THRESHOLD } = {}) {
+  let completed = 0; let failed = 0; let triggers = 0;
+  const methods = new Set();
+  for (const t of trials) {
+    const observed = t?.trigger?.observed;
+    if (t?.status !== 'completed' || (observed !== true && observed !== false)) { failed += 1; continue; }
+    completed += 1;
+    if (observed) { triggers += 1; if (t.trigger.method) methods.add(t.trigger.method); }
+  }
+  const rate = completed ? triggers / completed : null;
+  const passed = rate === null ? null : (expected ? rate >= threshold : rate < threshold);
+  // The weaker evidence wins the label: a file read is an inference, a native skill call is seen.
+  const method = methods.has('successful_skill_file_read') ? 'successful_skill_file_read' : methods.has('native_skill_tool') ? 'native_skill_tool' : null;
+  return { completed, failed, triggers, rate, expected: !!expected, passed, method };
+}
+
+/** mean, sample standard deviation (n - 1; null under 2), n, min, max of numbers (nulls left out); null for none. */
+export function stats(values) {
+  const v = values.filter((x) => typeof x === 'number' && Number.isFinite(x));
+  if (!v.length) return null;
+  const n = v.length;
+  const mean = v.reduce((s, x) => s + x, 0) / n;
+  const sd = n >= 2 ? Math.sqrt(v.reduce((s, x) => s + (x - mean) ** 2, 0) / (n - 1)) : null;
+  return { mean, sd, n, min: Math.min(...v), max: Math.max(...v) };
+}
+
+export const executedRuntime = (trial) => trial.actualRuntime || trial.runtime || {};
+
+const runtimeKeyOf = (r) => `${r.provider}${r.model ? `/${r.model}` : ''}`;
+const pct = (x) => (x === null || x === undefined ? null : Math.round(x * 1000) / 10);
+
+/**
+ * What a run shows, per runtime: task cases with and without the skill (paired by case and repetition: candidate
+ * minus baseline), trigger queries (positives and near-misses, precision and recall), preference fidelity, cost and
+ * time; then notes about the suite (an assertion every answer passes or fails, one that does not tell the
+ * configurations apart, a case the skill made worse). `cases` is readSuite()'s list.
+ */
+export function summarizeRun(run, cases = []) {
+  const byId = new Map(cases.map((c) => [c.id, c]));
+  const trials = Array.isArray(run?.trials) ? run.trials : [];
+  const keys = [...new Set(trials.map((t) => runtimeKeyOf(executedRuntime(t))))];
+  const runtimes = keys.map((key) => {
+    const mine = trials.filter((t) => runtimeKeyOf(executedRuntime(t)) === key);
+    const runtime = executedRuntime(mine[0] || {});
+    const taskIds = [...new Set(mine.filter((t) => t.kind === 'task').map((t) => t.caseId))];
+    const taskCases = taskIds.map((caseId) => {
+      const all = mine.filter((t) => t.kind === 'task' && t.caseId === caseId);
+      const rate = (config) => all.filter((t) => t.configuration === config && t.status === 'completed').map((t) => t.grade?.passRate ?? null);
+      const cand = stats(rate('candidate'));
+      const base = stats(rate('baseline'));
+      // Pairs: the same repetition of both configurations, both graded.
+      const pairs = [];
+      for (const t of all.filter((x) => x.configuration === 'candidate')) {
+        const b = all.find((x) => x.configuration === 'baseline' && x.repetition === t.repetition);
+        const a1 = t.status === 'completed' ? t.grade?.passRate ?? null : null;
+        const b1 = b && b.status === 'completed' ? b.grade?.passRate ?? null : null;
+        if (a1 !== null && b1 !== null) pairs.push(a1 - b1);
+      }
+      const delta = pairs.length ? pairs.reduce((s, x) => s + x, 0) / pairs.length : null;
+      const judged = (run.comparisons || []).filter((c) => c.caseId === caseId && c.runtimeKey === key);
+      const wins = { candidate: 0, baseline: 0, tie: 0, inconclusive: 0 };
+      for (const j of judged) wins[j.winner in wins ? j.winner : 'inconclusive'] += 1;
+      const failedRuns = all.filter((t) => ['failed', 'timed_out'].includes(t.status)).length;
+      return {
+        caseId, split: all[0]?.split || byId.get(caseId)?.split || null,
+        candidate: cand, baseline: base, delta, pairs: pairs.length, failedRuns,
+        outcome: delta === null ? 'n/a' : delta > 0.0001 ? 'better' : delta < -0.0001 ? 'worse' : 'same',
+        comparison: judged.length ? wins : null,
+      };
+    });
+    const allPairs = taskCases.flatMap((c) => (c.delta === null ? [] : [{ delta: c.delta, n: c.pairs }]));
+    const candRates = mine.filter((t) => t.kind === 'task' && t.configuration === 'candidate' && t.status === 'completed').map((t) => t.grade?.passRate ?? null);
+    const baseRates = mine.filter((t) => t.kind === 'task' && t.configuration === 'baseline' && t.status === 'completed').map((t) => t.grade?.passRate ?? null);
+    const triggerIds = [...new Set(mine.filter((t) => t.kind === 'trigger').map((t) => t.caseId))];
+    const triggerCases = triggerIds.map((caseId) => {
+      const c = byId.get(caseId);
+      const all = mine.filter((t) => t.kind === 'trigger' && t.caseId === caseId);
+      const expected = c ? c.shouldTrigger : !!all[0]?.expected;
+      return { caseId, split: all[0]?.split || c?.split || null, ...triggerVerdict(all, { expected }) };
+    });
+    const pos = triggerCases.filter((c) => c.expected);
+    const neg = triggerCases.filter((c) => !c.expected);
+    const read = (list) => list.filter((c) => c.passed !== null);
+    // Precision and recall over the queries that have a reading: a positive picked is a true positive, a near-miss
+    // picked a false positive, a positive not picked a false negative.
+    const tp = read(pos).filter((c) => c.passed).length;
+    const fp = read(neg).filter((c) => !c.passed).length;
+    const fn = read(pos).filter((c) => !c.passed).length;
+    const pref = mine.filter((t) => t.kind === 'task' && t.configuration === 'candidate' && t.status === 'completed')
+      .flatMap((t) => (t.grade?.assertions || []).filter((a) => a.kind === 'preference' && a.status !== 'error'));
+    const cost = mine.reduce((s, t) => s + (Number(t.costUsd) || 0) + (Number(t.grade?.costUsd) || 0), 0);
+    const durations = mine.filter((t) => Number.isFinite(t.durationMs)).map((t) => t.durationMs);
+    return {
+      key, provider: runtime.provider, model: runtime.model || null,
+      requestedRuntimes: [...new Map(mine.map((t) => [runtimeKeyOf(t.runtime), t.runtime])).values()],
+      task: taskIds.length ? {
+        candidate: stats(candRates), baseline: stats(baseRates), cases: taskCases,
+        delta: allPairs.length ? allPairs.reduce((s, x) => s + x.delta, 0) / allPairs.length : null,
+        pairs: allPairs.reduce((s, x) => s + x.n, 0), orientation: 'candidate-minus-baseline',
+      } : null,
+      trigger: triggerIds.length ? {
+        cases: triggerCases, queries: triggerCases.length, passed: triggerCases.filter((c) => c.passed).length, unread: triggerCases.filter((c) => c.passed === null).length,
+        positives: { passed: pos.filter((c) => c.passed).length, total: pos.length }, nearMisses: { passed: neg.filter((c) => c.passed).length, total: neg.length },
+        precision: tp + fp ? tp / (tp + fp) : null, recall: tp + fn ? tp / (tp + fn) : null,
+      } : null,
+      preference: pref.length ? { passed: pref.filter((a) => a.status === 'passed').length, total: pref.length } : null,
+      costUsd: Math.round(cost * 10000) / 10000,
+      durationMs: stats(durations),
+      failedRuns: mine.filter((t) => ['failed', 'timed_out'].includes(t.status)).length,
+      skipped: mine.filter((t) => t.status === 'skipped').length,
+    };
+  });
+  return { runtimes, notes: benchmarkNotes(run), gradingVersion: GRADING_VERSION };
+}
+
+/**
+ * What the run says about the suite itself (the skill-creator's analyzer notes, as data): an assertion every graded
+ * answer passes (it measures nothing), one every answer fails (a broken or impossible check), one that passes with
+ * and without the skill alike (it does not tell them apart), and a case where the skill did worse than no skill.
+ * → [{ code: 'always-pass'|'always-fail'|'not-discriminating'|'skill-worse', caseId, assertionId? }]
+ */
+export function benchmarkNotes(run) {
+  const notes = [];
+  const trials = (run?.trials || []).filter((t) => t.kind === 'task' && t.status === 'completed' && t.grade);
+  const byCase = new Map();
+  for (const t of trials) { if (!byCase.has(t.caseId)) byCase.set(t.caseId, []); byCase.get(t.caseId).push(t); }
+  for (const [caseId, list] of byCase) {
+    const ids = [...new Set(list.flatMap((t) => t.grade.assertions.map((a) => a.id)))];
+    for (const id of ids) {
+      const rows = list.map((t) => ({ config: t.configuration, a: t.grade.assertions.find((x) => x.id === id) })).filter((r) => r.a && r.a.status !== 'error' && r.a.status !== 'unverifiable');
+      if (rows.length < 2) continue;
+      const passes = rows.filter((r) => r.a.status === 'passed').length;
+      const configs = new Set(rows.map((r) => r.config));
+      if (passes === rows.length && configs.size > 1) notes.push({ code: 'not-discriminating', caseId, assertionId: id });
+      else if (passes === rows.length) notes.push({ code: 'always-pass', caseId, assertionId: id });
+      else if (passes === 0) notes.push({ code: 'always-fail', caseId, assertionId: id });
+    }
+    const rate = (config) => stats(list.filter((t) => t.configuration === config).map((t) => t.grade.passRate));
+    const c = rate('candidate'); const b = rate('baseline');
+    if (c && b && c.mean < b.mean) notes.push({ code: 'skill-worse', caseId });
+  }
+  return notes.slice(0, 50);
+}
+
+/**
+ * Two runs of a skill side by side, case by case (the same runtime key in both): the change of each case's pass rate
+ * or trigger reading. A case only one of them ran is listed with the other side empty. Says whether the two ran the
+ * same suite and the same revision (otherwise a change can come from the cases, not the skill).
+ */
+export function compareRuns(a, b, cases = []) {
+  const sa = summarizeRun(a, cases);
+  const sb = summarizeRun(b, cases);
+  const rows = [];
+  const keys = [...new Set([...sa.runtimes.map((r) => r.key), ...sb.runtimes.map((r) => r.key)])];
+  for (const key of keys) {
+    const ra = sa.runtimes.find((r) => r.key === key) || null;
+    const rb = sb.runtimes.find((r) => r.key === key) || null;
+    const taskIds = [...new Set([...(ra?.task?.cases || []), ...(rb?.task?.cases || [])].map((c) => c.caseId))];
+    for (const caseId of taskIds) {
+      const ca = ra?.task?.cases.find((c) => c.caseId === caseId) || null;
+      const cb = rb?.task?.cases.find((c) => c.caseId === caseId) || null;
+      const va = ca?.candidate?.mean ?? null; const vb = cb?.candidate?.mean ?? null;
+      rows.push({ key, kind: 'task', caseId, a: va, b: vb, change: va !== null && vb !== null ? vb - va : null });
+    }
+    const triggerIds = [...new Set([...(ra?.trigger?.cases || []), ...(rb?.trigger?.cases || [])].map((c) => c.caseId))];
+    for (const caseId of triggerIds) {
+      const ca = ra?.trigger?.cases.find((c) => c.caseId === caseId) || null;
+      const cb = rb?.trigger?.cases.find((c) => c.caseId === caseId) || null;
+      rows.push({ key, kind: 'trigger', caseId, a: ca?.passed ?? null, b: cb?.passed ?? null, rateA: ca?.rate ?? null, rateB: cb?.rate ?? null, change: ca?.passed === null || cb?.passed === null || !ca || !cb ? null : Number(cb.passed) - Number(ca.passed) });
+    }
+  }
+  return {
+    rows, sameSuite: !!a?.suite?.digest && a.suite.digest === b?.suite?.digest, sameRevision: !!a?.skill?.revisionId && a.skill.revisionId === b?.skill?.revisionId,
+    sameExperiment: !!a?.experiment?.digest && a.experiment.digest === b?.experiment?.digest,
+    summary: { a: sa, b: sb },
+  };
+}
+
+/** Why a run's result no longer describes the skill as it is now, or []: its revision is not the latest, its suite changed. */
+export function staleness(run, { latestRevisionId = null, latestSuiteDigest = null } = {}) {
+  const out = [];
+  if (latestRevisionId && run?.skill?.revisionId && run.skill.revisionId !== latestRevisionId) out.push('revision');
+  if (latestSuiteDigest && run?.suite?.digest && run.suite.digest !== latestSuiteDigest) out.push('suite');
+  return out;
+}
+
+/** A pass rate as a percentage with one decimal (null stays null). */
+export const percent = pct;