synabun 2.0.10 → 2.1.0

package/neural-interface/lib/skills/eval-sandbox.js

@@ -0,0 +1,280 @@
+// ── The OS sandbox of an evaluation's shell (P4, macOS) ──
+// A task case that asks for a shell (`shell: true`) runs untrusted skill text with a real shell. Every command of that
+// shell, and nothing else, runs under a Seatbelt profile of its own (evalShellProfile), through a small wrapper the
+// runtime is told to use for its commands: Claude Code's CLAUDE_CODE_SHELL_PREFIX (its Bash tool and any hook command
+// go through it), OpenCode's SHELL (its bash tool runs `$SHELL -c <command>`). The runtime itself stays outside, with
+// its own sign-in; its shell, inside, has:
+//
+//   network   none at all: SynaBun's ports, local servers and the internet alike (the deny is `network*`)
+//   writes    its job folder, its own temp, home and git folders, the terminal devices; inside the job folder, the
+//             folders and files a runtime reads its settings, skills or code from (.claude, .opencode, .git,
+//             opencode.json, … ss-evals.js EVAL_CONFIG_DIRS / EVAL_CONFIG_FILES, any letter case, a folder at any
+//             depth) stay as SynaBun wrote them. No symlink can be created in any writable place (the last file-write
+//             rule denies vnode-type SYMLINK): a shell step is the only thing that could plant or repoint a job-folder
+//             symlink, and that link is the one way the (unsandboxed) eval read tool could be steered outside the folder
+//             between its permission ask and its open — removing it is what makes the ask-time realpath check race-free
+//   git       a repository of its own beside the job folder, never in it: GIT_DIR is the trial's git folder and
+//             GIT_WORK_TREE the job folder, so `git init`, `add`, `commit`, `log`, `diff`, `branch` and `checkout` work on
+//             the folder's files while no `.git` (a folder, or a `gitdir:` file) can be made anywhere in it. The runtime
+//             outside runs git in the job folder too (Claude Code's status, OpenCode's project and snapshot reads):
+//             there is no repository there to read, and that side's git takes no bare repository it was not pointed at
+//             and reads no global or system configuration (lib/skills/eval-confinement.js evalGitSafetyEnv), so nothing
+//             a command writes is read by it. A commit's identity is the trial home's .gitconfig (a test may set its own)
+//   reads     nothing under /Users (every home folder), /Volumes, SynaBun's data home, /private/tmp or
+//             /private/var/folders (other programs' temp files), except the job folder, its own temp, home and git
+//             folders, the folders on PATH and a few language toolchains (runtimes, no settings), and Claude Code's
+//             shell snapshot (its shell setup, which every one of its commands sources first)
+//   others    a signal only to a process of its own sandbox (`kill -0` of any other process is refused); no launchd job
+//             (`job-creation` is denied: `launchctl submit` is refused, while `launchctl print` still reads); no Apple
+//             event (`appleevent-send` is denied: a real event, to System Events, fails with -4960, while
+//             `osascript -e 1` still runs, because it sends none); no lookup of the Mach services that hold the user's
+//             data or start programs outside it (EVAL_SHELL_MACH_DENY: a keychain read and pbcopy / pbpaste were seen
+//             to fail; LaunchServices, the window server, screen capture and TCC are denied by name)
+//   env       a short allowlist (locale, terminal, PATH and what the runtime's own command line needs): no token,
+//             key or password of the runtime's environment reaches a command; HOME and TMPDIR are the trial's own, and
+//             git's variables are the wrapper's (none of the runtime's)
+//
+// The `others` line is what the P4 verification saw. tests/skills-p4-eval-sandbox.test.mjs repeats the signal, keychain
+// and launchd checks (a job its `launchctl submit` did make would be removed at once, and the test would fail); it
+// sends no Apple event and asks no screen or app service again, since one that got through would act on the owner's
+// session (a TCC prompt, an app started).
+// Seatbelt is last-match-wins and matches resolved paths (lib/worker-sandbox.js: canonicalPath), so every path here is
+// canonical and every allow comes after the deny it opens. This module only builds and writes; whether sandbox-exec
+// works is lib/worker-sandbox.js sandboxExecAvailable(). A shell that cannot be set up this way is not given: the run
+// fails closed (lib/skills/evals.js, server.js).
+
+import { chmodSync, existsSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs';
+import { basename, dirname, isAbsolute, join, sep } from 'node:path';
+import { SANDBOX_EXEC, canonicalPath, sandboxExecAvailable, sandboxSupported } from '../worker-sandbox.js';
+import { EVAL_CONFIG_DIRS, EVAL_CONFIG_FILES } from '../../public/shared/skills/ss-evals.js';
+
+/** Part of an evaluation's confinement version: a change to what an evaluation's shell may do is a new one. */
+export const EVAL_SHELL_VERSION = 'skill-eval-shell/3';
+
+/** Mach services an evaluation's shell never reaches (the keychain, the clipboard, apps, the screen). */
+export const EVAL_SHELL_MACH_DENY = Object.freeze([
+  ['global-name', 'com.apple.SecurityServer'], ['global-name', 'com.apple.securityd'], ['global-name', 'com.apple.securityd.xpc'],
+  ['global-name', 'com.apple.security.agent'], ['global-name', 'com.apple.security.authhost'], ['global-name', 'com.apple.pasteboard.1'],
+  ['global-name-prefix', 'com.apple.coreservices.launchservicesd'], ['global-name-prefix', 'com.apple.lsd.'],
+  ['global-name', 'com.apple.windowserver.active'], ['global-name-prefix', 'com.apple.screencapture'], ['global-name', 'com.apple.tccd'],
+]);
+
+/** Folders under a home that hold language runtimes only: readable from an evaluation's shell when present. */
+export const EVAL_SHELL_TOOLCHAINS = Object.freeze([
+  '.nvm/versions', '.volta/tools/image', '.fnm/node-versions', '.local/share/fnm/node-versions', '.pyenv/versions', '.rbenv/versions',
+  '.rustup/toolchains', '.asdf/installs', '.local/share/mise/installs', '.bun/bin', '.deno/bin',
+]);
+
+/** What of the runtime's environment a command keeps (everything else, its tokens and keys among it, is left out). */
+export const EVAL_SHELL_ENV_KEEP = Object.freeze([
+  'PATH', 'LANG', 'LC_ALL', 'LC_CTYPE', 'LC_MESSAGES', 'LC_COLLATE', 'LC_NUMERIC', 'LC_TIME', 'LC_MONETARY', 'TERM', 'COLORTERM',
+  'NO_COLOR', 'FORCE_COLOR', 'COLUMNS', 'LINES', 'USER', 'LOGNAME', 'TZ', 'SHLVL',
+  // Claude Code's command line reads these (it sets BUN_OPTIONS from the first, and refuses to run without it).
+  '__CLAUDE_CC_BUN_OPTIONS', 'CLAUDECODE', 'CLAUDE_CODE_ENTRYPOINT',
+]);
+
+/** The identity of a commit a shell step makes, in the trial home's .gitconfig (its lowest place: a test's own wins). */
+export const EVAL_SHELL_GIT_IDENTITY = Object.freeze({ name: 'SynaBun evaluation', email: 'evaluation@synabun.invalid' });
+
+const SHELLS = Object.freeze({ zsh: { path: '/bin/zsh', args: ['-f'] }, bash: { path: '/bin/bash', args: ['--noprofile', '--norc'] }, sh: { path: '/bin/sh', args: [] } });
+
+/** Whether an evaluation's shell can be held here: macOS with a working sandbox-exec. */
+export function evalShellSupported({ platform = process.platform, probe = sandboxExecAvailable } = {}) {
+  return sandboxSupported(platform) && probe({ platform }) === true;
+}
+
+/** A Seatbelt string literal. */
+function lit(text) { return `"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`; }
+/**
+ * Text as a Seatbelt regular expression that matches it and nothing else. A #"…" literal is raw (a backslash is not an
+ * escape there), so ASCII punctuation goes in a bracket ([.] is a dot); letters, digits and other characters stay as
+ * they are. A path holding a quote, a backslash or a control character cannot be named this way: refused.
+ */
+function regexText(text) {
+  let out = '';
+  for (const c of String(text)) {
+    if (c === '"' || c === '\\' || /[\u0000-\u001f\u007f]/.test(c)) throw new Error('The evaluation folder\'s path has a character the sandbox cannot name.');
+    if (/[A-Za-z0-9_/ -]/.test(c) || c.charCodeAt(0) > 0x7f) out += c;
+    else if (c === ']') out += '[]]';
+    else if (c === '^') out += '[*^]';
+    else out += `[${c}]`;
+  }
+  return out;
+}
+/** A Seatbelt regular expression literal (#"…", raw). */
+const rx = (source) => `#"${source}"`;
+/** A name for a regex in any letter case ([cC][lL]…), every other character as itself. */
+function anyCase(name) { return [...String(name)].map((c) => (/[a-z]/i.test(c) ? `[${c.toLowerCase()}${c.toUpperCase()}]` : regexText(c))).join(''); }
+/** The deepest existing ancestor resolved, and absolute: what Seatbelt sees. */
+const canon = (p) => canonicalPath(isAbsolute(String(p)) ? String(p) : join(process.cwd(), String(p)));
+const inside = (p, root) => p === root || p.startsWith(root.endsWith(sep) ? root : `${root}${sep}`);
+
+/**
+ * The directory nodes on the way to each allowed path that fall in a denied tree: `stat` of those nodes only (a
+ * literal, never their contents), so a command can resolve its way into the job folder without listing a home.
+ */
+function metadataAncestors(allows, denies) {
+  const out = new Set();
+  for (const allow of allows) {
+    for (let dir = dirname(allow), i = 0; i < 256; i++) {
+      if (denies.some((d) => inside(dir, d))) out.add(dir);
+      const up = dirname(dir);
+      if (up === dir) break;
+      dir = up;
+    }
+  }
+  return [...out].sort();
+}
+
+/**
+ * The profile of one evaluation's shell.
+ * @param {object} o
+ * @param {string} o.root the job folder (the run's cwd)
+ * @param {string} o.temp the trial's own temp folder
+ * @param {string} o.home the trial's own home folder (HOME of every command)
+ * @param {string} o.git the trial's own git folder (GIT_DIR of every command: the repository of the job folder's files)
+ * @param {string[]} [o.denyReadTrees] trees nothing is read from (the home folders, SynaBun's data home, …), on top of
+ *   /Users, /Volumes, /private/tmp and /private/var/folders
+ * @param {string[]} [o.readOnly] paths read back after the denies (PATH folders, toolchains, Claude Code's snapshots)
+ * @param {string} [o.note] a comment line
+ * @returns {string}
+ */
+export function evalShellProfile({ root, temp, home, git, denyReadTrees = [], readOnly = [], note = '' } = {}) {
+  if (!root || !temp || !home || !git) throw new Error('An evaluation shell profile needs its job, temp, home and git folders.');
+  const job = canon(root);
+  const tmp = canon(temp);
+  const own = canon(home);
+  const repo = canon(git);
+  if (inside(repo, job) || inside(job, repo)) throw new Error('An evaluation shell\'s git folder is beside its job folder, never in it or around it.');
+  const denies = [...new Set(['/Users', '/Volumes', '/private/tmp', '/private/var/folders', ...denyReadTrees.filter(Boolean).map(canon)])];
+  const reads = [...new Set(readOnly.filter(Boolean).map(canon))].filter((p) => p !== '/' && !inside(job, p));
+  const lines = ['(version 1)', `; synabun ${EVAL_SHELL_VERSION}${note ? `: ${String(note).replace(/[\r\n]+/g, ' ').slice(0, 200)}` : ''}`, '(allow default)'];
+  lines.push('; no network at all (SynaBun, local servers, the internet)');
+  lines.push('(deny network*)');
+  lines.push('; a signal only to a process of this sandbox; no launchd job (launchctl submit refused); no Apple event sent');
+  lines.push('(deny signal)', '(allow signal (target same-sandbox))', '(deny job-creation)', '(deny appleevent-send)');
+  lines.push('; the keychain, the clipboard, LaunchServices, the window server, screen capture and TCC, by service name');
+  lines.push(`(deny mach-lookup ${EVAL_SHELL_MACH_DENY.map(([kind, name]) => `(${kind} ${lit(name)})`).join(' ')})`);
+  lines.push('; writes: the job folder, the trial\'s temp, home and git folders, the terminal devices');
+  lines.push('(deny file-write*)');
+  lines.push(`(allow file-write* (subpath ${lit(job)}) (subpath ${lit(tmp)}) (subpath ${lit(own)}) (subpath ${lit(repo)}) (literal "/dev/null") (literal "/dev/zero") (literal "/dev/random") (literal "/dev/urandom") (literal "/dev/dtracehelper") (literal "/dev/tty") (literal "/dev/stdout") (literal "/dev/stderr") (regex #"^/dev/ttys[0-9]+$") (regex #"^/dev/fd/[0-9]+$"))`);
+  lines.push('; the folders (at any depth: .git among them, so no repository is ever made here) and top files a runtime reads its settings or code from stay as SynaBun wrote them (any letter case)');
+  const base = regexText(job);
+  const dirRules = Object.keys(EVAL_CONFIG_DIRS).map((name) => `(regex ${rx(`^${base}/(.+/)?${anyCase(name)}(/|$)`)})`);
+  const fileRules = Object.keys(EVAL_CONFIG_FILES).map((name) => `(regex ${rx(`^${base}/${anyCase(name)}$`)})`);
+  lines.push(`(deny file-write* ${[...dirRules, ...fileRules].join(' ')})`);
+  lines.push('; reads: no home folder, other volume, other temp files or SynaBun data');
+  lines.push(`(deny file-read* ${denies.map((d) => `(subpath ${lit(d)})`).join(' ')})`);
+  lines.push('; read back: the job folder, the trial\'s temp, home and git folders, the tools on PATH and the runtime\'s shell setup');
+  lines.push(`(allow file-read* (subpath ${lit(job)}) (subpath ${lit(tmp)}) (subpath ${lit(own)}) (subpath ${lit(repo)})${reads.map((p) => ` (subpath ${lit(p)})`).join('')})`);
+  // Apple's command-line tool shims (git, python3 from the Command Line Tools) keep an SDK lookup cache in the user's
+  // temp folder whatever TMPDIR says: that one file name only.
+  lines.push('(allow file-read* file-write* (regex #"^/private/var/folders/[^/]+/[^/]+/T/xcrun_db"))');
+  // No symlink may be created in any writable place (last of the file-write rules, so it wins over every allow above,
+  // xcrun_db included). A symlink is the one way a shell step of a trial could repoint a path inside the job folder at
+  // a file OUTSIDE it between the read tool's permission ask (evalOpenCodeDecision, which resolves the real path and
+  // refuses one leaving the folder) and the tool's open — a TOCTOU the ask alone cannot close, because the eval read
+  // tool runs in the runtime's own process, not under this sandbox, so SynaBun cannot bind the path it finally opens.
+  // The job folder is created fresh with no symlink, SynaBun's skill copy follows none, and Write/Edit make plain files
+  // only, so the shell is the sole writer that could introduce one; deny it here and no symlink can ever exist in the
+  // folder, which is what makes the ask-time realpath check race-free. Regular writes, mkdir, rename (mv) and hardlinks
+  // to in-folder files are untouched; a hardlink to an outside file already fails (its lookup needs a read the denies
+  // above refuse). Verified on macOS 26 (scratch probes, 2026-10-25): ln -s / ln -sf are refused, mv and mkdir work.
+  lines.push('(deny file-write* (vnode-type SYMLINK))');
+  const ancestors = metadataAncestors([job, tmp, own, repo, ...reads], denies);
+  if (ancestors.length) lines.push(`(allow file-read-metadata ${ancestors.map((d) => `(literal ${lit(d)})`).join(' ')})`);
+  return `${lines.join('\n')}\n`;
+}
+
+/**
+ * The folders of a trial an evaluation's shell uses: the job folder, and beside it its temp, home, git (the repository
+ * of the job folder's files, GIT_DIR) and sandbox folders.
+ */
+export function evalShellPaths(cwd) {
+  const root = realpathSync(String(cwd || ''));
+  const trialDir = dirname(root);
+  return { root, trialDir, temp: join(trialDir, 'tmp'), home: join(trialDir, 'home'), git: join(trialDir, 'git'), dir: join(trialDir, 'sandbox') };
+}
+
+/** The real shell a command runs in (the runtime's shell's family: Claude Code's snapshot is written for it). */
+export function evalInnerShell(shell = process.env.SHELL) {
+  const name = basename(String(shell || '')).toLowerCase();
+  return name === 'bash' ? 'bash' : name === 'sh' ? 'sh' : 'zsh';
+}
+
+/**
+ * The wrapper one command goes through. Claude Code calls it as `wrapper '<command>'` (its shell prefix), OpenCode as
+ * `wrapper -c '<command>'` (its shell); `-l`/`-i` before `-c` are let go. Anything else is refused (exit 126). Its git
+ * works on the job folder's files from the trial's git folder (GIT_DIR, GIT_WORK_TREE), never looks above the trial's
+ * folder for another repository and reads no system configuration; the global one is the trial home's.
+ */
+export function evalShellWrapper({ profilePath, shell = 'zsh', home, temp, root, git }) {
+  if (!root || !git) throw new Error('An evaluation shell\'s wrapper needs its job and git folders.');
+  const sh = (s) => `'${String(s).replace(/'/g, `'\\''`)}'`;
+  const inner = SHELLS[shell] || SHELLS.zsh;
+  const keep = EVAL_SHELL_ENV_KEEP.map((name) => `\${${name}+"${name}=$${name}"}`).join(' ');
+  const gitEnv = `GIT_DIR=${sh(git)} GIT_WORK_TREE=${sh(root)} GIT_CEILING_DIRECTORIES=${sh(dirname(root))} GIT_CONFIG_NOSYSTEM=1`;
+  return [
+    '#!/bin/sh',
+    `# SynaBun (${EVAL_SHELL_VERSION}): one command of a skill evaluation, under the evaluation's sandbox. Written per trial.`,
+    'cmd=',
+    'if [ $# -eq 1 ]; then cmd=$1',
+    'else',
+    '  while [ $# -gt 0 ]; do',
+    '    case "$1" in',
+    '      -c|-lc|-cl|-ic|-ci|-ilc|-lic) shift; cmd=${1-}; break ;;',
+    '      -l|-i|--login|--interactive|--noprofile|--norc) shift ;;',
+    '      *) break ;;',
+    '    esac',
+    '  done',
+    'fi',
+    'if [ -z "$cmd" ]; then echo "synabun: an evaluation shell runs one command (-c)." >&2; exit 126; fi',
+    `exec /usr/bin/env -i ${keep} HOME=${sh(home)} TMPDIR=${sh(`${temp}/`)} CLAUDE_CODE_TMPDIR=${sh(temp)} SHELL=${sh(inner.path)} ${gitEnv} ${sh(SANDBOX_EXEC)} -f ${sh(profilePath)} ${sh(inner.path)}${inner.args.map((a) => ` ${a}`).join('')} -c "$cmd"`,
+    '',
+  ].join('\n');
+}
+
+/** The folders on PATH that sit in a denied tree, and the toolchains under each home: read back for the shell. */
+export function evalShellReadPaths({ path = process.env.PATH, homes = [], claudeConfigDir = null } = {}) {
+  const out = new Set();
+  for (const dir of String(path || '').split(':')) if (dir && isAbsolute(dir) && existsSync(dir)) out.add(canon(dir));
+  for (const h of homes.filter(Boolean)) for (const rel of EVAL_SHELL_TOOLCHAINS) { const p = join(h, rel); if (existsSync(p)) out.add(canon(p)); }
+  if (claudeConfigDir) out.add(canon(join(claudeConfigDir, 'shell-snapshots')));
+  return [...out];
+}
+
+/** The trial home's .gitconfig: a commit's identity (EVAL_SHELL_GIT_IDENTITY) and git's default-branch hint off. */
+export function evalShellGitConfig() {
+  return [
+    `# SynaBun (${EVAL_SHELL_VERSION}): git's settings for a skill evaluation's shell. Written per trial.`,
+    '[user]', `\tname = ${EVAL_SHELL_GIT_IDENTITY.name}`, `\temail = ${EVAL_SHELL_GIT_IDENTITY.email}`,
+    '[advice]', '\tdefaultBranchName = false', '',
+  ].join('\n');
+}
+
+/**
+ * Set up one trial's shell: its temp, home and git folders (the home with its .gitconfig), its profile and its wrapper
+ * (in the trial's own sandbox folder, outside the job folder, so nothing the shell does can change them).
+ * → { wrapperPath, profilePath, shell, env, paths }
+ * `env` is what the runtime gets (TMPDIR and CLAUDE_CODE_TMPDIR: its own temp files go into the trial's).
+ * @param {object} o
+ * @param {string} o.cwd the job folder
+ * @param {string[]} [o.denyReadTrees] SynaBun's data home, the home folders
+ * @param {string[]} [o.readOnly] evalShellReadPaths()
+ * @param {string} [o.shell] the runtime's shell (its family is the inner shell)
+ * @param {string} [o.wrapperName] the wrapper's file name (OpenCode accepts a shell by its name: zsh, bash or sh)
+ */
+export function prepareEvalShell({ cwd, denyReadTrees = [], readOnly = [], shell = process.env.SHELL, wrapperName = null, note = '' } = {}) {
+  const paths = evalShellPaths(cwd);
+  for (const dir of [paths.temp, paths.home, paths.git, paths.dir]) mkdirSync(dir, { recursive: true, mode: 0o700 });
+  writeFileSync(join(paths.home, '.gitconfig'), evalShellGitConfig(), { mode: 0o600 });
+  const inner = evalInnerShell(shell);
+  const profile = evalShellProfile({ root: paths.root, temp: paths.temp, home: paths.home, git: paths.git, denyReadTrees, readOnly, note });
+  const profilePath = join(paths.dir, 'shell.sb');
+  writeFileSync(profilePath, profile, { mode: 0o600 });
+  const name = wrapperName && /^(?:zsh|bash|sh)$/.test(wrapperName) ? wrapperName : inner;
+  const wrapperPath = join(paths.dir, name);
+  writeFileSync(wrapperPath, evalShellWrapper({ profilePath, shell: inner, home: canon(paths.home), temp: canon(paths.temp), root: paths.root, git: canon(paths.git) }), { mode: 0o700 });
+  chmodSync(wrapperPath, 0o700);
+  return { wrapperPath, profilePath, shell: inner, paths, env: { TMPDIR: `${canon(paths.temp)}/`, CLAUDE_CODE_TMPDIR: canon(paths.temp) } };
+}

package/neural-interface/lib/skills/eval-trace.js

@@ -0,0 +1,119 @@
+// ── What an evaluation run did: its tool calls, and whether the runtime picked the skill (P4, SK-085) ──
+// The dispatcher keeps the tool calls of an evaluation run (assistant-dispatch.js, transcript format `tools`), read
+// here from each provider's own events; a trigger trial is read from them:
+//
+//   Claude Code   a `Skill` call naming the skill: method `native_skill_tool` (seen). A Read of its SKILL.md without
+//                 one: `successful_skill_file_read` (an inference).
+//   OpenCode      a `skill` tool call naming it (`{ id }`): `native_skill_tool`;
+//                 a read of its SKILL.md: the inference.
+//   Codex         Codex has no skill tool: it lists its skills and the model reads the file with its shell. A command
+//                 that ran (exit 0) and names the skill's SKILL.md: `successful_skill_file_read` (an inference).
+//
+// A skill named that is not the one under test is evidence too (the runtime picked another), never a "yes". The
+// runtime's own listing at start (Claude's init `skills`) is kept as the catalog the run saw: a baseline that lists
+// the skill under test was not a baseline. Startup discovery is never counted as a selection. A call's input is kept with
+// its secrets redacted, the way an answer and a written file are (lib/log-redact.js). Pure.
+
+import { redactSecrets } from '../../../lib/log-redact.js';
+
+const clipJson = (value, max = 600) => {
+  let text;
+  try { text = typeof value === 'string' ? value : JSON.stringify(value ?? null); } catch { text = String(value); }
+  return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+};
+const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+
+/**
+ * The tool calls one provider event carries: [{ id, tool, input, phase?, ok? }]. Claude: the tool_use blocks of a
+ * whole assistant message; Codex: a command (started, then done with its exit code), an MCP call, a file change;
+ * OpenCode: a tool part (each update of a call has its id: the caller keeps one entry per id).
+ */
+export function toolCallsOf(payload = {}) {
+  const event = payload?.event || {};
+  const eventType = String(payload?.eventType || event.type || '');
+  const out = [];
+  if (payload.provider === 'claude-code') {
+    if (event.type === 'assistant' && Array.isArray(event.message?.content)) {
+      for (const block of event.message.content) {
+        if (block?.type === 'tool_use') out.push({ id: String(block.id || ''), tool: String(block.name || ''), input: isObject(block.input) ? block.input : {}, sub: !!event.parent_tool_use_id });
+      }
+    }
+    return out;
+  }
+  if (payload.provider === 'codex') {
+    if (event.type === 'item.started' || event.type === 'item.completed') {
+      const item = event.item || {};
+      const done = event.type === 'item.completed';
+      if (item.type === 'command_execution') {
+        const command = typeof item.command === 'string' ? item.command : Array.isArray(item.command) ? item.command.join(' ') : '';
+        out.push({ id: String(item.id || ''), tool: 'shell', input: { command }, phase: done ? 'done' : 'started', ok: done ? Number(item.exit_code) === 0 : null });
+      } else if (item.type === 'mcp_tool_call') out.push({ id: String(item.id || ''), tool: `${item.server || ''}.${item.tool || ''}`, input: {}, phase: done ? 'done' : 'started' });
+      else if (item.type === 'file_change') out.push({ id: String(item.id || ''), tool: 'file_change', input: { paths: (item.changes || []).map((c) => c?.path).filter(Boolean) }, phase: done ? 'done' : 'started' });
+      else if (item.type === 'web_search') out.push({ id: String(item.id || ''), tool: 'web_search', input: { query: item.query || '' }, phase: done ? 'done' : 'started' });
+    }
+    return out;
+  }
+  if (payload.provider === 'opencode') {
+    if (/^message[.:]part[.:]updated$/i.test(eventType)) {
+      const part = event.part || {};
+      if (part.type === 'tool') out.push({ id: String(part.callID || part.id || ''), tool: String(part.tool || part.name || ''), input: isObject(part.state?.input) ? part.state.input : {}, phase: String(part.state?.status || ''), ok: part.state?.status === 'completed' ? true : part.state?.status === 'error' ? false : null });
+    }
+    return out;
+  }
+  return out;
+}
+
+/** Claude's init event, as the catalog the run saw: { model, skills, tools, cwd } (null for any other event). */
+export function initOf(payload = {}) {
+  const event = payload?.event || {};
+  if (payload.provider !== 'claude-code' || event.type !== 'system' || event.subtype !== 'init') return null;
+  const names = (list) => (Array.isArray(list) ? list.map((x) => (typeof x === 'string' ? x : x?.name)).filter((x) => typeof x === 'string').slice(0, 200) : null);
+  return { model: typeof event.model === 'string' ? event.model : null, skills: names(event.skills), tools: names(event.tools), cwd: typeof event.cwd === 'string' ? event.cwd : null };
+}
+
+/** A value with a secret in any of its texts redacted (each string as it is, before it is written as JSON). Bounded. */
+function redactValue(value, depth = 0) {
+  if (typeof value === 'string') return redactSecrets(value.length > 20_000 ? value.slice(0, 20_000) : value);
+  if (depth > 8 || value === null || typeof value !== 'object') return value;
+  if (Array.isArray(value)) return value.slice(0, 200).map((v) => redactValue(v, depth + 1));
+  return Object.fromEntries(Object.entries(value).slice(0, 200).map(([k, v]) => [k, redactValue(v, depth + 1)]));
+}
+
+/** A tool call kept for the record: its tool, its input as bounded JSON text, every secret in it redacted first. */
+export const callRecord = (c) => ({ tool: c.tool, input: clipJson(redactValue(c.input), 400), ...(c.phase ? { phase: c.phase } : {}), ...(c.ok === true || c.ok === false ? { ok: c.ok } : {}) });
+
+const skillFilePattern = (name) => new RegExp(`(?:^|[\\\\/\\s'"])skills[\\\\/]${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}[\\\\/]SKILL\\.md\\b`, 'i');
+const SKILL_MD = /(?:^|[\\/\s'"])skills[\\/]([a-z0-9][a-z0-9._-]{0,63})[\\/]SKILL\.md\b/gi;
+
+/**
+ * Whether the run picked the skill: { observed: true|false, method, evidence: [{ tool, input }], others: [names] }.
+ * `observed` is false when nothing named it (the caller decides whether the run got far enough to say so).
+ */
+export function detectSelection({ provider, calls = [], skillName }) {
+  const name = String(skillName || '');
+  const evidence = [];
+  const others = new Set();
+  let native = false;
+  let read = false;
+  const file = name ? skillFilePattern(name) : null;
+  for (const c of calls) {
+    const tool = String(c.tool || '');
+    const input = isObject(c.input) ? c.input : {};
+    if ((provider === 'claude-code' && tool === 'Skill') || (provider === 'opencode' && tool === 'skill')) {
+      // Claude Code's Skill sends { skill }, OpenCode's skill tool { id } (seen in a live run).
+      const asked = String(input.skill ?? input.id ?? input.command ?? '').replace(/^\//, '');
+      const bare = asked.includes(':') ? asked.split(':').pop() : asked;
+      if (name && (asked === name || bare === name)) { native = true; evidence.push(callRecord(c)); } else if (asked) others.add(asked.slice(0, 64));
+      continue;
+    }
+    const text = provider === 'codex' ? String(input.command || '') : String(input.file_path ?? input.filePath ?? input.path ?? '');
+    if (!text) continue;
+    if (provider === 'codex' && c.phase === 'started') continue;   // a command that ran (its done item, exit 0) is a read
+    if (provider === 'codex' && c.ok === false) continue;
+    const readTool = provider === 'codex' ? tool === 'shell' : /^(?:read|Read)$/.test(tool);
+    if (!readTool) continue;
+    if (file && file.test(text)) { read = true; evidence.push(callRecord(c)); continue; }
+    for (const m of text.matchAll(SKILL_MD)) if (m[1] !== name) others.add(m[1]);
+  }
+  return { observed: native || read, method: native ? 'native_skill_tool' : read ? 'successful_skill_file_read' : null, evidence: evidence.slice(0, 5), others: [...others].slice(0, 10) };
+}

package/neural-interface/lib/skills/evals-api.js

@@ -0,0 +1,212 @@
+// ── /api/skills-studio/v2/evals: skill evaluations over HTTP (P4) ──
+// The routes of lib/skills/evals.js, beside the Skills Service's other v2 routes. Who may do what (as AI creation,
+// lib/skills/creation-api.js: an evaluation runs workers and spends money):
+//
+//   the owner        from SynaBun's own page: every route, a read included, needs this boot's page secret cookie
+//                    (lib/page-session.js: never handed to, nor accepted from, SynaBun's automation browser or a peer
+//                    process the owner-authority lookup refuses) and the owner route guard (its Host, no guest); a
+//                    write also its X-SynaBun-UI: 1 header (and the guard its Origin). Estimates, starts, reads, waits
+//                    on and cancels every evaluation, writes review notes on its trials, exports and compares runs,
+//                    converts another tool's suite, confirms Codex for a skill version. Nothing starts without the
+//                    owner's Run (or the Assistant's request below). Where the peer lookup cannot run (not macOS, no
+//                    `ps`), the page secret and the guard are what is asked, as before it (docs/skills-studio.md).
+//   the Assistant    a request carrying the credential its brain's launch was given (X-Synabun-Brain-Token, resolved
+//                    to that brain's session): estimates, starts (under its session's budget), reads, waits on,
+//                    cancels and reports the evaluations of its own session, and reads whether Codex is confirmed for
+//                    a version (it never confirms it). Its results are the owner's too (the Studio lists every
+//                    evaluation). X-Synabun-Role and X-Synabun-Terminal prove nothing; a dispatched worker is
+//                    refused whatever it sends.
+//   anyone else      403 (a guest, an agent without a brain's credential).
+//
+// A trial's output file is answered as JSON text (never as a page of this origin), bounded, never through a link.
+
+import AdmZip from 'adm-zip';
+import { sendGuardError } from '../http-guards.js';
+import { PathConfineError } from '../path-confine.js';
+import { V2_BASE, principalOf } from './api-v2.js';
+import { isTrialId } from './evals.js';
+import { isEvalRunId } from '../../public/shared/skills/ss-eval-plan.js';
+
+const IDEMPOTENCY_TTL_MS = 10 * 60 * 1000;
+const FORMATS = new Set(['native', 'skill-creator', 'agent-skills']);
+
+function fail(res, err) {
+  const status = err instanceof PathConfineError ? err.status : (Number.isInteger(err?.status) && err.status >= 400 && err.status < 600 ? err.status : 500);
+  const body = { ok: false, error: status >= 500 && !(err instanceof PathConfineError) ? `The evaluation failed: ${err?.message || err}` : (err?.message || String(err)) };
+  if (typeof err?.code === 'string' && !/^E[A-Z]+$/.test(err.code)) body.code = err.code;
+  for (const key of ['problems', 'estimate', 'skipped', 'codex']) if (err?.[key] !== undefined) body[key] = err[key];
+  return res.status(status).json(body);
+}
+
+/**
+ * @param {object} app express app
+ * @param {object} o
+ * @param {object} o.evals lib/skills/evals.js
+ * @param {(req) => boolean} [o.isGuestRequest]
+ * @param {Function} [o.ownerGuard] lib/http-guards.js ownerRouteGuard(…) of this server
+ * @param {(req) => ({ kind: 'assistant', assistantSessionId } | { kind: 'worker' } | null)} [o.assistantCaller]
+ * @param {{ verify(req), require(req, res, next), attach(res) }} [o.pageSession] lib/page-session.js
+ */
+export function registerSkillEvalRoutes(app, { evals, isGuestRequest = null, ownerGuard = null, assistantCaller = null, pageSession = null } = {}) {
+  const idem = new Map();
+  const route = (method, path, ...handlers) => app[method](`${V2_BASE}${path}`, ...handlers);
+  const safe = (fn) => async (req, res) => { try { await fn(req, res); } catch (err) { fail(res, err); } };
+
+  const json = (req, res, next) => {
+    if (String(req.headers['sec-fetch-site'] || '').toLowerCase() === 'cross-site') return sendGuardError(res, 403, 'CROSS_SITE', 'Cross-site requests are refused.');
+    const hasBody = Number(req.headers['content-length'] || 0) > 0 || req.headers['transfer-encoding'] !== undefined;
+    if (hasBody && typeof req.is === 'function' && !req.is('application/json')) return sendGuardError(res, 415, 'UNSUPPORTED_MEDIA_TYPE', 'Send JSON (Content-Type: application/json).');
+    if (req.body !== undefined && req.body !== null && (typeof req.body !== 'object' || Array.isArray(req.body))) return res.status(400).json({ ok: false, error: 'The body is a JSON object.', code: 'invalid_body' });
+    next();
+  };
+
+  /**
+   * Who may evaluate: the owner from the page (a read too: this boot's page secret and the owner route guard; a write
+   * also the page header), or an Assistant brain by its credential.
+   */
+  const caller = (write, { ownerOnly = false } = {}) => (req, res, next) => {
+    const principal = principalOf(req, isGuestRequest);
+    if (principal.kind === 'guest') return sendGuardError(res, 403, 'GUEST_FORBIDDEN', 'Available to the owner only.');
+    if (principal.kind === 'owner') {
+      req.evaluator = { kind: 'owner', assistantSessionId: null };
+      if (write && req.headers['x-synabun-ui'] !== '1') return sendGuardError(res, 403, 'UI_HEADER_REQUIRED', 'Send the X-SynaBun-UI: 1 header.');
+      // The page secret (lib/page-session.js verify: absent, stale, carried by the automation browser or by a peer the
+      // owner-authority lookup refuses, it is refused): what a local process that merely left the agent headers out lacks.
+      if (pageSession && !pageSession.verify(req)) {
+        if (write) return pageSession.require(req, res, next);
+        return sendGuardError(res, 403, 'PAGE_SECRET_REQUIRED', 'Evaluations are read from SynaBun\'s own page. Reload the page and try again.');
+      }
+      return ownerGuard ? ownerGuard(req, res, next) : next();
+    }
+    let who = null;
+    try { who = typeof assistantCaller === 'function' ? assistantCaller(req) : null; } catch { who = null; }
+    if (who?.kind === 'worker') return sendGuardError(res, 403, 'WORKERS_NOT_PERMITTED', 'Dispatched workers cannot run skill evaluations: the Assistant or the owner does.');
+    if (who?.kind !== 'assistant' || typeof who.assistantSessionId !== 'string' || !who.assistantSessionId) {
+      return sendGuardError(res, 403, 'ASSISTANT_CREDENTIAL_REQUIRED', 'Skill evaluations are the owner\'s (Skills Studio) and the SynaBun Assistant\'s own brain\'s (its launch credential): a role header is not enough.');
+    }
+    if (ownerOnly) return sendGuardError(res, 403, 'UI_ONLY', 'Review notes and the Codex confirmation are the owner\'s, from Skills Studio.');
+    req.evaluator = { kind: 'assistant', assistantSessionId: who.assistantSessionId };
+    next();
+  };
+  /** The owner sees every evaluation; an Assistant brain those its own session started. */
+  const ownOnly = (run, req) => req.evaluator.kind === 'owner' || (run.by?.kind === 'assistant' && typeof run.assistantSessionId === 'string' && run.assistantSessionId !== '' && run.assistantSessionId === req.evaluator.assistantSessionId);
+  const idCheck = (req, res, next) => {
+    if (!isEvalRunId(req.params.evalRunId)) return res.status(400).json({ ok: false, error: 'Not an evaluation id.', code: 'invalid_id' });
+    if (req.params.trialId !== undefined && !isTrialId(req.params.trialId)) return res.status(400).json({ ok: false, error: 'Not a trial id.', code: 'invalid_trial' });
+    next();
+  };
+  const mine = (req, res) => {
+    const run = evals.get(req.params.evalRunId);
+    if (!ownOnly(run, req)) { sendGuardError(res, 403, 'NOT_YOURS', 'This evaluation was started by someone else.'); return null; }
+    return run;
+  };
+  /** Same caller, same route, same key within ten minutes: the first answer again (one evaluation, not two). */
+  const once = (handler) => async (req, res) => {
+    const key = typeof req.headers['idempotency-key'] === 'string' ? req.headers['idempotency-key'] : (typeof req.body?.idempotencyKey === 'string' ? req.body.idempotencyKey : null);
+    if (!key) return handler(req, res);
+    if (key.length > 200) return res.status(400).json({ ok: false, error: 'An idempotency key is at most 200 characters.', code: 'invalid_idempotency_key' });
+    const slot = `${req.evaluator?.kind}\0${req.evaluator?.assistantSessionId || ''}\0${req.method}\0${req.originalUrl || req.url}\0${key}`;
+    const hit = idem.get(slot);
+    if (hit && Date.now() - hit.at < IDEMPOTENCY_TTL_MS) return res.status(hit.status).json({ ...hit.body, replayed: true });
+    const original = res.json.bind(res);
+    res.json = (body) => {
+      if (res.statusCode < 300) { idem.set(slot, { at: Date.now(), status: res.statusCode, body }); if (idem.size > 300) idem.delete(idem.keys().next().value); }
+      return original(body);
+    };
+    return handler(req, res);
+  };
+  const bodyOf = (req) => { const b = { ...(req.body || {}) }; delete b.idempotencyKey; delete b.assistantSessionId; delete b.assistant_session_id; return b; };
+
+  // The fixed paths first: they are not evaluation ids.
+  route('get', '/evals/options', caller(false), safe(async (req, res) => res.json({ ok: true, ...(await evals.options()) })));
+  route('get', '/evals/queue', caller(false), safe(async (req, res) => {
+    const q = evals.queue();
+    if (req.evaluator.kind === 'owner') return res.json({ ok: true, ...q });
+    // The Assistant sees the queue's length and its own place in it, not what the owner runs.
+    const own = (r) => r && evals.get(r.evalRunId)?.assistantSessionId === req.evaluator.assistantSessionId;
+    res.json({ ok: true, active: q.active.filter(own), waiting: q.waiting.filter(own), activeCount: q.active.length, waitingCount: q.waiting.length, maxActive: q.maxActive, queueMax: q.queueMax });
+  }));
+  route('get', '/evals/compare', caller(false), safe(async (req, res) => {
+    const a = String(req.query?.a || ''); const b = String(req.query?.b || '');
+    if (!isEvalRunId(a) || !isEvalRunId(b)) return res.status(400).json({ ok: false, error: 'Name two evaluations (a, b).', code: 'invalid_id' });
+    for (const id of [a, b]) { const run = evals.get(id); if (!ownOnly(run, req)) return sendGuardError(res, 403, 'NOT_YOURS', 'This evaluation was started by someone else.'); }
+    res.json({ ok: true, comparison: evals.compare(a, b) });
+  }));
+  // Codex is the owner's opt-in per skill version (lib/skills/evals.js): whether it is confirmed, and the confirmation.
+  route('get', '/evals/codex-trust', caller(false), safe(async (req, res) => {
+    const ids = String(req.query?.revisionIds || '').split(',').map((x) => x.trim()).filter(Boolean).slice(0, 10);
+    if (!ids.length || ids.some((x) => !/^rev_[0-9a-f]{20}$/.test(x))) return res.status(400).json({ ok: false, error: 'Name the version(s) (revisionIds).', code: 'invalid_request' });
+    res.json({ ok: true, ...evals.codexTrust(ids) });
+  }));
+  route('post', '/evals/codex-trust', json, caller(true, { ownerOnly: true }), safe(async (req, res) => {
+    const b = req.body || {};
+    res.json({ ok: true, ...evals.setCodexTrust({ revisionIds: Array.isArray(b.revisionIds) ? b.revisionIds : [], trusted: b.trusted !== false }) });
+  }));
+  route('post', '/evals/estimate', json, caller(true), safe(async (req, res) => res.json({ ok: true, ...(await evals.estimate(bodyOf(req))) })));
+  route('post', '/evals/import', json, caller(true), safe(async (req, res) => {
+    const b = req.body || {};
+    const tooBig = (v) => typeof v === 'string' ? v.length > 1024 * 1024 : JSON.stringify(v ?? null).length > 1024 * 1024;
+    if (tooBig(b.tasks) || tooBig(b.triggers)) return res.status(413).json({ ok: false, error: 'A suite to import is at most 1 MB.', code: 'too_large' });
+    const out = evals.importSuite({ tasks: b.tasks ?? null, triggers: b.triggers ?? null, skillName: typeof b.skillName === 'string' ? b.skillName.slice(0, 64) : '', seed: Number.isInteger(b.seed) ? b.seed : 42 });
+    res.status(out.ok ? 200 : 422).json({ ok: out.ok, ...out, ...(out.ok ? {} : { error: out.problems[0]?.message || 'The suite cannot be imported.', code: out.problems[0]?.code || 'invalid_suite' }) });
+  }));
+  route('get', '/evals', caller(false), safe(async (req, res) => {
+    const q = req.query || {};
+    const skillId = typeof q.skillId === 'string' && /^skl_[0-9a-f]{20}$/.test(q.skillId) ? q.skillId : null;
+    const revisionId = typeof q.revisionId === 'string' && /^rev_[0-9a-f]{20}$/.test(q.revisionId) ? q.revisionId : null;
+    const list = evals.list({ skillId, revisionId, limit: Number(q.limit) || 30, ...(req.evaluator.kind === 'assistant' ? { assistantSessionId: req.evaluator.assistantSessionId } : {}) });
+    res.json({ ok: true, evals: list, queue: req.evaluator.kind === 'owner' ? evals.queue() : null });
+  }));
+  route('post', '/evals', json, caller(true), once(safe(async (req, res) => {
+    const run = await evals.start(bodyOf(req), { by: { kind: req.evaluator.kind }, assistantSessionId: req.evaluator.assistantSessionId });
+    res.status(202).json({ ok: true, eval: run });
+  })));
+  route('get', '/evals/:evalRunId', caller(false), idCheck, safe(async (req, res) => {
+    const run = mine(req, res); if (!run) return;
+    res.json({ ok: true, eval: run });
+  }));
+  route('get', '/evals/:evalRunId/wait', caller(false), idCheck, safe(async (req, res) => {
+    if (!mine(req, res)) return;
+    const since = Number.isInteger(Number(req.query?.since)) ? Number(req.query.since) : -1;
+    const run = await evals.wait(req.params.evalRunId, { sinceVersion: since, timeoutMs: Math.min(50, Math.max(1, Number(req.query?.seconds) || 25)) * 1000, compact: req.query?.full !== '1' });
+    res.json({ ok: true, eval: run });
+  }));
+  route('post', '/evals/:evalRunId/cancel', json, caller(true), idCheck, safe(async (req, res) => {
+    if (!mine(req, res)) return;
+    res.json({ ok: true, eval: await evals.cancel(req.params.evalRunId) });
+  }));
+  route('post', '/evals/:evalRunId/feedback', json, caller(true, { ownerOnly: true }), idCheck, safe(async (req, res) => {
+    const b = req.body || {};
+    res.json({ ok: true, feedback: evals.feedback(req.params.evalRunId, { trialId: typeof b.trialId === 'string' ? b.trialId : '', text: typeof b.text === 'string' ? b.text : '', reviewed: typeof b.reviewed === 'boolean' ? b.reviewed : null }) });
+  }));
+  route('get', '/evals/:evalRunId/trials/:trialId', caller(false), idCheck, safe(async (req, res) => {
+    if (!mine(req, res)) return;
+    res.json({ ok: true, trial: evals.trial(req.params.evalRunId, req.params.trialId) });
+  }));
+  route('get', '/evals/:evalRunId/trials/:trialId/file', caller(false), idCheck, safe(async (req, res) => {
+    if (!mine(req, res)) return;
+    const path = typeof req.query?.path === 'string' ? req.query.path : '';
+    if (!path || path.length > 300 || path.split('/').includes('..') || path.startsWith('/')) return res.status(400).json({ ok: false, error: 'Name a file the trial wrote (path).', code: 'invalid_path' });
+    res.set('Cache-Control', 'no-store');
+    res.json({ ok: true, file: evals.trialFile(req.params.evalRunId, req.params.trialId, path) });
+  }));
+  route('get', '/evals/:evalRunId/export', caller(false), idCheck, safe(async (req, res) => {
+    if (!mine(req, res)) return;
+    const format = typeof req.query?.format === 'string' && FORMATS.has(req.query.format) ? req.query.format : 'native';
+    const out = evals.exportRun(req.params.evalRunId, format);
+    const safeName = out.name.replace(/[^A-Za-z0-9._-]/g, '_');
+    res.set('Cache-Control', 'no-store');
+    res.set('X-Content-Type-Options', 'nosniff');
+    if (out.name.endsWith('.zip')) {
+      const zip = new AdmZip();
+      const folder = safeName.replace(/\.skill-creator\.zip$/, '');
+      for (const e of out.entries) zip.addFile(`${folder}/${e.path}`, Buffer.from(e.text, 'utf-8'));
+      res.set('Content-Type', 'application/zip');
+      res.set('Content-Disposition', `attachment; filename="${safeName}"`);
+      return res.send(zip.toBuffer());
+    }
+    res.set('Content-Type', 'application/json; charset=utf-8');
+    res.set('Content-Disposition', `attachment; filename="${safeName}"`);
+    res.send(out.entries[0].text);
+  }));
+}