skilltrigger 0.0.0-stage → 0.0.2

package/bin/lib/environment.mjs

@@ -0,0 +1,78 @@
+// What the inherited environment contributes to every run, counted. Each `claude -p`
+// inherits the user's whole configuration minus CLAUDECODE: memory files, hooks and MCP
+// servers all reach the model, and a memory file naming the skill moves the rate without
+// changing any other recorded field. So the report records how many of each there were
+// — counts only: never a file's contents, a path, a hook command or a server name (ST-20).
+//
+// Reading only. Nothing under the configuration directory is ever written.
+//
+//   memory files  project: CLAUDE.md, CLAUDE.local.md and .claude/CLAUDE.md in the run's
+//                 temporary project and each of its ancestors, plus .claude/rules/**/*.md
+//                 in the project itself; user: CLAUDE.md and rules/**/*.md in the
+//                 configuration directory ($CLAUDE_CONFIG_DIR, else ~/.claude)
+//   hooks         hook commands in the user and project settings files and in each
+//                 enabled plugin's hooks/hooks.json (managed settings are not read)
+//   mcpServers    the length of the init event's mcp_servers, null when it has none
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { dirname, join } from 'node:path'
+
+const isFile = (f) => {
+  try {
+    return statSync(f).isFile()
+  } catch {
+    return false
+  }
+}
+
+function markdownUnder(dir) {
+  if (!existsSync(dir)) return 0
+  let n = 0
+  for (const d of readdirSync(dir, { withFileTypes: true })) {
+    if (d.isDirectory()) n += markdownUnder(join(dir, d.name))
+    else if (d.isFile() && d.name.endsWith('.md')) n++
+  }
+  return n
+}
+
+export const configDirOf = (env = process.env) => env.CLAUDE_CONFIG_DIR || join(env.HOME || homedir(), '.claude')
+
+export function memoryFiles({ projectDir, configDir }) {
+  let project = markdownUnder(join(projectDir, '.claude', 'rules'))
+  for (let dir = projectDir; ; dir = dirname(dir)) {
+    for (const f of ['CLAUDE.md', 'CLAUDE.local.md', join('.claude', 'CLAUDE.md')]) if (isFile(join(dir, f))) project++
+    if (dirname(dir) === dir) break
+  }
+  const user = (isFile(join(configDir, 'CLAUDE.md')) ? 1 : 0) + markdownUnder(join(configDir, 'rules'))
+  return { project, user }
+}
+
+function hooksIn(file) {
+  if (!isFile(file)) return 0
+  let hooks
+  try {
+    hooks = JSON.parse(readFileSync(file, 'utf8'))?.hooks
+  } catch {
+    return 0
+  }
+  if (!hooks || typeof hooks !== 'object') return 0
+  let n = 0
+  for (const groups of Object.values(hooks)) if (Array.isArray(groups)) for (const grp of groups) n += Array.isArray(grp?.hooks) ? grp.hooks.length : 0
+  return n
+}
+
+export function countHooks({ configDir, projectDir = null, plugins = [] }) {
+  const files = [join(configDir, 'settings.json'), ...(projectDir ? [join(projectDir, '.claude', 'settings.json'), join(projectDir, '.claude', 'settings.local.json')] : [])]
+  for (const p of plugins) if (p.enabled && p.installPath) files.push(join(p.installPath, 'hooks', 'hooks.json'))
+  return files.reduce((n, f) => n + hooksIn(f), 0)
+}
+
+// `memory` is passed when it was counted earlier, from a directory since removed.
+export function environmentSummary({ env = process.env, projectDir = null, memory = null, plugins = [], init = null }) {
+  const configDir = configDirOf(env)
+  return {
+    memoryFiles: memory ?? memoryFiles({ projectDir, configDir }),
+    hooks: countHooks({ configDir, projectDir, plugins }),
+    mcpServers: Array.isArray(init?.mcp_servers) ? init.mcp_servers.length : null,
+  }
+}

package/bin/lib/gates.mjs

@@ -0,0 +1,186 @@
+// The gates. Each one is a way the measurement used to return a clean-looking zero
+// that meant nothing; each returns { id, status, detail, fix } where status is ok,
+// warn, fail or skip. Any fail refuses the run.
+//
+//   cli         claude is on PATH and answers --version (the version is recorded)
+//   auth        `claude auth status` says logged in
+//   round-trip  `claude -p "Reply with exactly: pong"` answers pong, with the chosen
+//               model — catches the expired OAuth that auth status can miss, and the
+//               400 an outdated CLI answers for a model it does not know
+//   conflict    no enabled plugin (or other visible skill) shadows the stub
+//   sleep       the machine cannot sleep mid-run (macOS: caffeinate), else a warning
+//   roster      how many slash commands and skills the model sees, from the init event
+import { spawn } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { execClaude, streamClaude } from './claude.mjs'
+import { configDirOf, environmentSummary, memoryFiles } from './environment.mjs'
+import { findConflicts, parsePluginList, toggleCommands } from './plugins.mjs'
+
+export const PONG_PROMPT = 'Reply with exactly: pong'
+const g = (id, status, detail, fix) => ({ id, status, detail, ...(fix ? { fix } : {}) })
+const firstLine = (s) => String(s ?? '').trim().split('\n')[0].slice(0, 200)
+const truncate = (s, n) => (s.length > n ? `${s.slice(0, n - 1)}…` : s)
+
+async function cliGate(env) {
+  const r = await execClaude(['--version'], { env, timeoutMs: 15000 })
+  if (r.notFound) return g('cli', 'fail', `claude not found on PATH`, 'install Claude Code (https://code.claude.com/docs) or put `claude` on PATH; SKILLTRIGGER_CLAUDE names another binary')
+  const version = r.stdout.trim().split('\n')[0]
+  if (r.code !== 0 || !/\d+\.\d+\.\d+/.test(version)) return g('cli', 'fail', `claude --version did not answer a version (exit ${r.code}${r.timedOut ? ', timed out' : ''}): ${firstLine(r.stderr || r.stdout)}`, 'reinstall or update Claude Code: `claude update`')
+  return { ...g('cli', 'ok', `claude ${version}`), version }
+}
+
+async function authGate(env) {
+  const r = await execClaude(['auth', 'status'], { env, timeoutMs: 20000 })
+  let loggedIn = null
+  try {
+    loggedIn = JSON.parse(r.stdout).loggedIn === true
+  } catch {
+    if (/not logged in|logged out/i.test(r.stdout)) loggedIn = false
+    else if (/logged in/i.test(r.stdout)) loggedIn = true
+  }
+  if (r.code === 0 && loggedIn) return g('auth', 'ok', 'claude auth status: logged in')
+  return g('auth', 'fail', `claude auth status: not logged in (exit ${r.code}${r.timedOut ? ', timed out' : ''})`, 'run `claude auth login` (or `/login` in an interactive session), then preflight again')
+}
+
+// One stream-json round trip serves two gates: the answer is the round trip, the
+// init event is the roster. It runs in an empty temporary project so the roster is
+// what every run will see, minus the stub.
+async function roundTrip({ env, model }) {
+  const dir = mkdtempSync(join(tmpdir(), 'skilltrigger-preflight-'))
+  try {
+    let init = null
+    let text = ''
+    let result = null
+    const args = ['-p', PONG_PROMPT, '--output-format', 'stream-json', '--verbose', '--no-session-persistence', ...(model ? ['--model', model] : [])]
+    const r = await streamClaude(args, {
+      cwd: dir,
+      env,
+      timeoutMs: 120000,
+      onEvent: (ev) => {
+        if (ev.type === 'system' && ev.subtype === 'init') init = ev
+        if (ev.type === 'assistant') for (const c of ev.message?.content ?? []) if (c.type === 'text') text += c.text ?? ''
+        if (ev.type === 'result') {
+          result = ev
+          return true
+        }
+        return false
+      },
+    })
+    const answer = (typeof result?.result === 'string' ? result.result : text).trim()
+    let gate
+    if (r.timedOut) gate = g('round-trip', 'fail', 'claude -p did not answer within 120 s', 'check the network and the CLI by hand: `claude -p "Reply with exactly: pong"`')
+    else if (result?.is_error || /Failed to authenticate|API Error|Invalid API key|OAuth token has expired/i.test(answer)) {
+      gate = g('round-trip', 'fail', `claude -p answered an error: ${firstLine(answer)}`, /authenticat|oauth|api key|401/i.test(answer) ? 'log in again: `claude auth login` — auth status can say logged in while the token is expired' : 'update the CLI (`claude update`) or pass a --model this CLI knows')
+    } else if (!/^\W*pong\W*$/i.test(answer)) gate = g('round-trip', 'fail', `claude -p answered "${firstLine(answer || r.stderr) || '(nothing)'}", not pong (exit ${r.code})`, 'run `claude -p "Reply with exactly: pong"` by hand and fix what it prints')
+    else gate = g('round-trip', 'ok', `claude -p answered pong${model ? ` (model ${model})` : ''}`)
+    // The memory files are counted from this directory, before it is removed: the runs'
+    // projects are made the same way, beside it, so they share its ancestors.
+    const memory = memoryFiles({ projectDir: dir, configDir: configDirOf(env) })
+    return { gate, init, memory }
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+function rosterGate(init) {
+  if (!init) return g('roster', 'fail', 'no init event in the stream — the roster cannot be recorded', 'update the CLI: `claude update`; the init event of `claude -p --output-format stream-json --verbose` lists the roster')
+  if (!Array.isArray(init.slash_commands)) return g('roster', 'fail', 'the init event lists no slash_commands — the roster cannot be recorded', 'update the CLI: `claude update`')
+  // Members beside the counts: two rosters of one size can differ (ST-20). Hashed, not
+  // named — the first 12 hex digits of each name's SHA-256, sorted — because the roster
+  // is every command and skill on the machine, private ones included, and a report is
+  // meant to be pasted. The preflight project holds no stub, so no stub is among them.
+  const tag = (name) => createHash('sha256').update(name).digest('hex').slice(0, 12)
+  const hashes = (list) => [...new Set(list.map((s) => (typeof s === 'string' ? s : s?.name)).filter(Boolean).map(tag))].sort()
+  const skillList = Array.isArray(init.skills) ? init.skills : null
+  const roster = { slashCommands: init.slash_commands.length, skills: skillList ? skillList.length : null, commandHashes: hashes(init.slash_commands), skillHashes: skillList ? hashes(skillList) : null }
+  const skills = roster.skills === null ? '' : `, ${roster.skills} skills`
+  return { ...g('roster', 'ok', `${roster.slashCommands} slash commands${skills} visible to the model`), roster, model: init.model ?? null, entries: [...init.slash_commands, ...(Array.isArray(init.skills) ? init.skills.map((s) => (typeof s === 'string' ? s : s?.name)) : [])].filter(Boolean) }
+}
+
+async function conflictGate({ env, skillName, allowConflict, rosterEntries }) {
+  const r = await execClaude(['plugin', 'list', '--json'], { env, timeoutMs: 30000 })
+  let plugins = parsePluginList(r.stdout)
+  let asked = { cmd: 'claude plugin list --json', r }
+  if (r.code !== 0 && !plugins?.length) {
+    // An older CLI without --json: the human form.
+    const plain = await execClaude(['plugin', 'list'], { env, timeoutMs: 30000 })
+    plugins = parsePluginList(plain.stdout)
+    asked = { cmd: 'claude plugin list', r: plain }
+  }
+  if (plugins === null) {
+    // Unreadable is not empty: a list in a new shape would otherwise read as "no plugins"
+    // and the gate would pass with nothing checked (ST-18).
+    const got = truncate(firstLine(asked.r.stdout || asked.r.stderr), 80) || '(nothing)'
+    const detail = `cannot read the plugin list: \`${asked.cmd}\` exited ${asked.r.code}${asked.r.timedOut ? ' (timed out)' : ''} and printed "${got}" — which plugins are enabled is unknown`
+    const fix = ['run `claude plugin list --json` by hand; if the CLI changed its output, update skilltrigger or report the shape', '--allow-conflict measures anyway and records that it did']
+    return { ...g('conflict', allowConflict ? 'warn' : 'fail', allowConflict ? `${detail} (allowed by --allow-conflict, recorded in the report)` : detail, fix.join('\n')), unreadable: true }
+  }
+  const enabled = plugins.filter((p) => p.enabled)
+  if (!skillName) return { ...g('conflict', 'warn', `no --skill given, so nothing to compare against ${enabled.length} enabled plugin(s)${enabled.length ? `: ${enabled.map((p) => p.id).join(', ')}` : ''}`, 'pass --skill <dir> to check this gate'), plugins }
+  const conflicts = findConflicts({ plugins, skillName, roster: rosterEntries })
+  if (!conflicts.length) return { ...g('conflict', 'ok', `no enabled plugin or visible skill named ${skillName} (${enabled.length} enabled plugin(s) checked)`), plugins }
+  const { disable, enable } = toggleCommands(conflicts)
+  const unaccounted = conflicts.filter((c) => !c.id).map((c) => c.via)
+  const lines = [
+    ...(disable.length ? ['disable before the run:', ...disable.map((c) => `  ${c}`), 'and re-enable after it:', ...enable.map((c) => `  ${c}`)] : []),
+    ...(unaccounted.length ? [`move aside the skill the roster shows (${unaccounted.join(', ')}) — a user or project skill of the same name`] : []),
+    'skilltrigger runs none of these itself; --allow-conflict measures anyway and records that it did',
+  ]
+  const detail = `${conflicts.map((c) => c.via).join('; ')} — the model would load it, not the stub`
+  return { ...g('conflict', allowConflict ? 'warn' : 'fail', allowConflict ? `${detail} (allowed by --allow-conflict, recorded in the report)` : detail, lines.join('\n')), conflicts, plugins }
+}
+
+// macOS: `caffeinate -i -s -w <pid>` holds off idle and system sleep for as long as
+// this process lives and exits with it. Elsewhere there is no one portable switch.
+export function sleepGate({ platform = process.platform, caffeinate = '/usr/bin/caffeinate', pid = process.pid } = {}) {
+  if (platform !== 'darwin') {
+    return Promise.resolve(g('sleep', 'warn', `cannot hold the machine awake on ${platform}: a sleep mid-run turns runs into timeouts`, 'keep the machine awake for the run, e.g. `systemd-inhibit --what=idle:sleep skilltrigger run …` on Linux'))
+  }
+  return new Promise((resolve) => {
+    const child = spawn(caffeinate, ['-i', '-s', '-w', String(pid)], { stdio: 'ignore', detached: false })
+    child.on('error', (e) => resolve(g('sleep', 'fail', `could not start caffeinate: ${e.code ?? e.message}`, 'run under `caffeinate -i -s skilltrigger run …` by hand')))
+    child.on('spawn', () => {
+      child.unref()
+      resolve(g('sleep', 'ok', `caffeinate -i -s -w ${pid} holds the machine awake until this process exits`))
+    })
+  })
+}
+
+// → { gates, ok, facts: { cliVersion, model, roster, conflictAllowed, environment } }
+export async function preflight({ env = process.env, model = null, skillName = null, allowConflict = false, platform = process.platform } = {}) {
+  const gates = []
+  const cli = await cliGate(env)
+  gates.push(cli)
+  const facts = { cliVersion: cli.version ?? null, model, roster: null, conflictAllowed: false, environment: null }
+  if (cli.status !== 'ok') {
+    for (const id of ['auth', 'round-trip', 'conflict']) gates.push(g(id, 'skip', 'not run: the cli gate failed'))
+    gates.push(await sleepGate({ platform }))
+    gates.push(g('roster', 'skip', 'not run: the cli gate failed'))
+  } else {
+    gates.push(await authGate(env))
+    const { gate, init, memory } = await roundTrip({ env, model })
+    gates.push(gate)
+    const roster = rosterGate(init)
+    const conflict = await conflictGate({ env, skillName, allowConflict, rosterEntries: roster.entries ?? [] })
+    gates.push(conflict)
+    gates.push(await sleepGate({ platform }))
+    gates.push(roster)
+    facts.roster = roster.roster ?? null
+    facts.model = model ?? roster.model ?? null
+    facts.conflictAllowed = conflict.status === 'warn' && Boolean(conflict.conflicts?.length || conflict.unreadable)
+    facts.environment = environmentSummary({ env, memory, plugins: conflict.plugins ?? [], init })
+  }
+  return { gates, ok: !gates.some((x) => x.status === 'fail'), facts }
+}
+
+export function formatGates(gates) {
+  const out = []
+  for (const x of gates) {
+    out.push(`  ${x.status.padEnd(5)} ${x.id.padEnd(11)} ${x.detail}`)
+    if (x.fix && x.status !== 'ok') for (const [i, l] of x.fix.split('\n').entries()) out.push(`${' '.repeat(20)}${i === 0 ? 'fix: ' : '     '}${l}`)
+  }
+  return out.join('\n')
+}

package/bin/lib/plugins.mjs

@@ -0,0 +1,106 @@
+// The shadowing trap: an installed plugin that carries a skill of the same name. The
+// model loads the real one, the stub is never invoked, and the rate reads as zero.
+//
+// Two sources, because neither sees everything:
+//   - `claude plugin list --json` names every plugin, whether it is enabled and where
+//     it is installed; the skills are read off <installPath>/skills/*/SKILL.md (and
+//     commands/*.md). Reading only — nothing under the plugin cache is ever written.
+//   - the roster in the `init` event of a stub-less `claude -p`, which lists what the
+//     model actually sees: it also catches a user-level skill or a plugin the list
+//     printed in a form this parser could not read.
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { parseFrontmatter } from './skill.mjs'
+
+const split = (id) => {
+  const at = id.lastIndexOf('@')
+  return at > 0 ? { name: id.slice(0, at), marketplace: id.slice(at + 1) } : { name: id, marketplace: null }
+}
+
+// → the plugins, [] when the output says there are none, or null when it is in no shape
+// this parser knows. The two answers are kept apart on purpose: "zero plugins" lets the
+// conflict gate pass, "could not read" must not (ST-18).
+export function parsePluginList(stdout) {
+  const text = String(stdout ?? '').trim()
+  if (!text) return null
+  if (text.startsWith('[')) {
+    let list
+    try {
+      list = JSON.parse(text)
+    } catch {
+      return null
+    }
+    if (!Array.isArray(list) || !list.every((p) => typeof p?.id === 'string' && p.id)) return null
+    return list.map((p) => ({ id: p.id, ...split(p.id), enabled: p.enabled !== false, scope: p.scope ?? null, installPath: p.installPath ?? null }))
+  }
+  // The human form:   ❯ name@marketplace / Version: / Scope: / Status: ✔ enabled
+  const out = []
+  let cur = null
+  for (const line of text.split('\n')) {
+    const head = line.match(/^\s*(?:❯|>|\*|-)?\s*([\w.-]+@[\w.-]+)\s*$/)
+    if (head) {
+      cur = { id: head[1], ...split(head[1]), enabled: true, scope: null, installPath: null }
+      out.push(cur)
+      continue
+    }
+    if (!cur) continue
+    const scope = line.match(/^\s*Scope:\s*(\S+)/i)
+    if (scope) cur.scope = scope[1]
+    const status = line.match(/^\s*Status:\s*(.*)$/i)
+    if (status) cur.enabled = !/disabled/i.test(status[1])
+  }
+  if (out.length) return out
+  // A human form that says so: "No plugins installed.", or a heading over "(none)".
+  if (/\bno plugins\b/i.test(text) || /^installed plugins:\s*(\(none\))?\s*$/i.test(text.replace(/\s+/g, ' ').trim())) return []
+  return null
+}
+
+export function pluginSkills(installPath) {
+  if (!installPath || !existsSync(installPath)) return []
+  const names = new Set()
+  const skills = join(installPath, 'skills')
+  if (existsSync(skills)) {
+    for (const d of readdirSync(skills, { withFileTypes: true })) {
+      if (!d.isDirectory()) continue
+      const f = join(skills, d.name, 'SKILL.md')
+      if (!existsSync(f)) continue
+      let name = d.name
+      try {
+        name = parseFrontmatter(readFileSync(f, 'utf8'))?.name || d.name
+      } catch {}
+      names.add(name)
+    }
+  }
+  const commands = join(installPath, 'commands')
+  if (existsSync(commands)) for (const f of readdirSync(commands)) if (f.endsWith('.md')) names.add(f.slice(0, -3))
+  return [...names].sort()
+}
+
+// → [{ id, scope, via }] — id null when the roster shows a same-named skill no plugin
+// accounts for (a user or project skill).
+export function findConflicts({ plugins, skillName, roster = [] }) {
+  const found = new Map()
+  for (const p of plugins) {
+    if (!p.enabled) continue
+    if (pluginSkills(p.installPath).includes(skillName)) found.set(p.id, { id: p.id, scope: p.scope, via: `${p.id} carries a skill named ${skillName}` })
+  }
+  for (const entry of roster) {
+    const e = String(entry).replace(/^\//, '')
+    const [prefix, rest] = e.includes(':') ? [e.slice(0, e.lastIndexOf(':')), e.slice(e.lastIndexOf(':') + 1)] : [null, e]
+    if (rest !== skillName) continue
+    if (prefix) {
+      const p = plugins.find((x) => x.enabled && x.name === prefix)
+      if (p && !found.has(p.id)) found.set(p.id, { id: p.id, scope: p.scope, via: `roster: ${e}` })
+      else if (!p && ![...found.values()].some((f) => f.via === `roster: ${e}`)) found.set(`roster:${e}`, { id: null, scope: null, via: `roster: ${e}` })
+    } else if (!found.has(`roster:${e}`)) found.set(`roster:${e}`, { id: null, scope: null, via: `roster: ${e}` })
+  }
+  return [...found.values()]
+}
+
+// The commands are printed, never run: disabling a plugin is the user's decision, and so
+// is re-enabling it — bin/lib/toggles.mjs only reminds them of it (ST-21).
+export function toggleCommands(conflicts) {
+  const withScope = (verb, c) => `claude plugin ${verb} ${c.id}${c.scope ? ` --scope ${c.scope}` : ''}`
+  const plugins = conflicts.filter((c) => c.id)
+  return { disable: plugins.map((c) => withScope('disable', c)), enable: plugins.map((c) => withScope('enable', c)) }
+}

package/bin/lib/report.mjs

@@ -0,0 +1,160 @@
+// The report: what was measured, against what, and whether it may be read at all.
+//
+// It carries the eval queries (and any extra fields they came with), the counts, the
+// versions, the roster's member names and the model each run reported — nothing else.
+// No description text (a hash and a length stand in for it, so compare can tell two
+// texts apart), no paths, no stub names, no stderr; what the inherited environment
+// contributed is counted, never quoted.
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export const TRIGGER_THRESHOLD = 0.5
+export const NO_VERDICT_SHARE = 0.1
+
+export function localDate(d = new Date()) {
+  const p = (n) => String(n).padStart(2, '0')
+  return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`
+}
+
+export function summarise({ items, outcomes, models = [], skillName, description, overridden, facts, runsPerQuery, timeoutSeconds, planned, aborted, toolVersion, date = localDate() }) {
+  const queries = items.map((item, i) => {
+    const o = outcomes[i] ?? []
+    const hits = o.filter((x) => x === 'triggered').length
+    const runs = o.filter((x) => x === 'triggered' || x === 'not-triggered').length
+    const rate = runs ? hits / runs : null
+    const pass = rate === null ? null : item.should_trigger ? rate >= TRIGGER_THRESHOLD : rate < TRIGGER_THRESHOLD
+    // partial: some of its executed runs measured nothing, but not all of them.
+    return { ...item, outcomes: o, models: models[i] ?? [], hits, runs, timeouts: o.filter((x) => x === 'timeout').length, errors: o.filter((x) => x === 'error').length, pass, partial: runs > 0 && runs < o.length }
+  })
+  const sum = (list, k) => list.reduce((a, q) => a + q[k], 0)
+  const pos = queries.filter((q) => q.should_trigger)
+  const neg = queries.filter((q) => !q.should_trigger)
+  const executed = queries.reduce((a, q) => a + q.outcomes.length, 0)
+  const timeouts = sum(queries, 'timeouts')
+  const errors = sum(queries, 'errors')
+  const badShare = executed ? (timeouts + errors) / executed : 1
+  // The share counts runs, not queries: two positives can lose every run inside 10% and
+  // leave a positives total that silently omits them. A query that was run and measured
+  // nothing makes the verdict impossible to read query by query, so it is no verdict (ST-19).
+  const lostQueries = queries.filter((q) => q.outcomes.length > 0 && q.runs === 0).map((q) => q.query)
+  const partialQueries = queries.filter((q) => q.partial).map((q) => q.query)
+  const runModels = {}
+  for (const q of queries) for (const m of q.models) if (m) runModels[m] = (runModels[m] ?? 0) + 1
+  const verdict = !aborted && executed > 0 && badShare <= NO_VERDICT_SHARE && !lostQueries.length ? 'ok' : 'no-verdict'
+  return {
+    tool: 'skilltrigger',
+    toolVersion,
+    date,
+    skill: skillName,
+    cliVersion: facts.cliVersion,
+    model: facts.model ?? 'default',
+    roster: facts.roster,
+    runModels,
+    environment: facts.environment ?? null,
+    runsPerQuery,
+    timeoutSeconds,
+    triggerThreshold: TRIGGER_THRESHOLD,
+    noVerdictThreshold: NO_VERDICT_SHARE,
+    conflictAllowed: Boolean(facts.conflictAllowed),
+    description: { bytes: Buffer.byteLength(description), sha256: createHash('sha256').update(description).digest('hex'), overridden: Boolean(overridden) },
+    verdict,
+    aborted: Boolean(aborted),
+    lostQueries,
+    partialQueries,
+    totals: {
+      planned,
+      executed,
+      positives: { triggered: sum(pos, 'hits'), runs: sum(pos, 'runs') },
+      negatives: { fired: sum(neg, 'hits'), runs: sum(neg, 'runs') },
+      timeouts,
+      errors,
+      passed: queries.filter((q) => q.pass === true).length,
+      queries: queries.length,
+    },
+    queries,
+  }
+}
+
+const pct = (n) => `${Math.round(n * 100)}%`
+const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`
+
+export function formatEnvironment(e) {
+  if (!e) return 'unknown'
+  const mem = e.memoryFiles ? `${plural(e.memoryFiles.project, 'project memory file')}, ${plural(e.memoryFiles.user, 'user memory file')}` : 'memory files unknown'
+  return `${mem} · ${plural(e.hooks ?? 0, 'hook')} · ${e.mcpServers === null || e.mcpServers === undefined ? 'MCP servers not listed' : plural(e.mcpServers, 'MCP server')}`
+}
+export const formatRunModels = (m) => (m && Object.keys(m).length ? Object.entries(m).map(([k, v]) => `${k} ×${v}`).join(', ') : 'none reported')
+const cell = (s) => String(s).replace(/\|/g, '\\|').replace(/\n/g, ' ')
+
+const quoted = (list) => list.map((q) => `"${q.length > 60 ? `${q.slice(0, 59)}…` : q}"`).join(', ')
+const queriesWord = (n) => `${n} ${n === 1 ? 'query' : 'queries'}`
+
+export function headline(rep) {
+  const t = rep.totals
+  if (rep.verdict !== 'ok') {
+    const share = (t.timeouts + t.errors) / (t.executed || 1) > rep.noVerdictThreshold || !t.executed || rep.aborted
+    const lost = rep.lostQueries ?? []
+    const parts = []
+    if (share) parts.push(`${t.timeouts} timeout(s) and ${t.errors} error(s) in ${t.executed} run(s)${rep.aborted ? ` (stopped after ${t.executed} of ${t.planned})` : ''} — more than ${pct(rep.noVerdictThreshold)} of the runs did not measure anything`)
+    if (lost.length) parts.push(`${queriesWord(lost.length)} lost every run to timeouts or errors — ${quoted(lost)} — and the totals would leave ${lost.length === 1 ? 'it' : 'them'} out`)
+    if (!parts.length) parts.push(`${t.timeouts} timeout(s) and ${t.errors} error(s) in ${t.executed} run(s)`)
+    return `no verdict: ${parts.join('; ')}`
+  }
+  return `positives triggered ${t.positives.triggered}/${t.positives.runs} · negatives fired ${t.negatives.fired}/${t.negatives.runs} · ${t.timeouts} timeout(s), ${t.errors} error(s) counted apart`
+}
+
+// A line naming the queries that kept the verdict on fewer measured runs than planned.
+export function partialLine(rep) {
+  const p = rep.partialQueries ?? []
+  return p.length ? `measured on fewer runs than planned: ${queriesWord(p.length)} — ${quoted(p)}` : null
+}
+
+export function toMarkdown(rep) {
+  const t = rep.totals
+  const roster = rep.roster ? `${rep.roster.slashCommands} slash commands${rep.roster.skills === null ? '' : `, ${rep.roster.skills} skills`}` : 'unknown'
+  const out = [
+    `# skilltrigger — ${rep.skill}, ${rep.date}`,
+    '',
+    rep.verdict === 'ok' ? `**${headline(rep)}**` : `**No verdict.** ${headline(rep).replace(/^no verdict: /, '')}. The per-query counts below are not a measurement.`,
+    '',
+    '| | |',
+    '|---|---|',
+    `| CLI | ${cell(rep.cliVersion)} |`,
+    `| Model | ${cell(rep.model)} |`,
+    `| Models the runs reported | ${cell(formatRunModels(rep.runModels))} |`,
+    `| Roster | ${roster} |`,
+    `| Inherited environment | ${formatEnvironment(rep.environment)} |`,
+    `| Runs per query | ${rep.runsPerQuery} |`,
+    `| Timeout | ${rep.timeoutSeconds} s |`,
+    `| Pass threshold | trigger rate ≥ ${rep.triggerThreshold} for a positive, < ${rep.triggerThreshold} for a negative |`,
+    `| No verdict when | timeouts + errors > ${pct(rep.noVerdictThreshold)} of runs, or any query lost every run |`,
+    `| Description | ${rep.description.bytes} bytes, sha256 ${rep.description.sha256.slice(0, 12)}${rep.description.overridden ? ', overridden with --description' : ''} |`,
+    ...(rep.conflictAllowed ? ['| Plugin conflict | **allowed with --allow-conflict** — a same-named skill was visible during the runs |'] : []),
+    `| Version | skilltrigger ${rep.toolVersion} |`,
+    '',
+    '| Query | Should trigger | Hits/runs | Timeouts | Errors | Pass |',
+    '|---|---|---:|---:|---:|---|',
+    ...rep.queries.map((q) => {
+      const pass = q.pass === null ? (q.outcomes.length ? 'lost every run' : '—') : q.pass ? 'pass' : 'fail'
+      return `| ${cell(q.query)} | ${q.should_trigger ? 'yes' : 'no'} | ${q.hits}/${q.runs} | ${q.timeouts} | ${q.errors} | ${pass}${q.partial ? ` (${q.runs} of ${q.outcomes.length} runs)` : ''} |`
+    }),
+    '',
+    ...(partialLine(rep) ? [`${cell(partialLine(rep).replace(/^m/, 'M'))}.`, ''] : []),
+    `${t.passed} of ${t.queries} queries pass. Two runs per query resolve to ±1 per query: read a difference of one hit as noise.`,
+    '',
+  ]
+  return out.join('\n')
+}
+
+// <out>/<date>-<skill>.json and .md; a second report the same day gets -2, -3…
+export function writeReport(rep, outDir) {
+  mkdirSync(outDir, { recursive: true })
+  let base = `${rep.date}-${rep.skill}`
+  for (let n = 2; existsSync(join(outDir, `${base}.json`)) || existsSync(join(outDir, `${base}.md`)); n++) base = `${rep.date}-${rep.skill}-${n}`
+  const json = join(outDir, `${base}.json`)
+  const md = join(outDir, `${base}.md`)
+  writeFileSync(json, JSON.stringify(rep, null, 2) + '\n')
+  writeFileSync(md, toMarkdown(rep))
+  return { json, md }
+}

package/bin/lib/runner.mjs

@@ -0,0 +1,79 @@
+// The runner. Strictly serial — there is no workers option, on purpose: with N
+// parallel runs the model sees N stubs with the same description, invokes whichever
+// it likes, and only the run whose stub was picked counts the hit, so the measured
+// rate is about 1/N of the true one. skill-creator's default of ten workers measured
+// a tenth of the truth.
+//
+// One run: a fresh temporary project holding one stub command whose frontmatter
+// description is the text under test, a `claude -p` in it, the stream read until the
+// first decisive message, the process killed, the directory removed.
+import { randomBytes } from 'node:crypto'
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { streamClaude } from './claude.mjs'
+import { createDetector } from './stream.mjs'
+
+export const OUTCOMES = ['triggered', 'not-triggered', 'timeout', 'error']
+
+// A YAML block scalar, so quotes and colons in the description cannot break it.
+export function stubText(skillName, description) {
+  const body = String(description).split('\n').map((l) => (l ? `  ${l}` : '')).join('\n')
+  return `---\ndescription: |\n${body}\n---\n\n# ${skillName}\n\nThis command stands in for the ${skillName} skill while its description is measured.\n`
+}
+
+export function makeProject(skillName, description) {
+  const dir = mkdtempSync(join(tmpdir(), 'skilltrigger-run-'))
+  const stubName = `${skillName}-stub-${randomBytes(4).toString('hex')}`
+  const commands = join(dir, '.claude', 'commands')
+  mkdirSync(commands, { recursive: true })
+  const stubPath = join(commands, `${stubName}.md`)
+  writeFileSync(stubPath, stubText(skillName, description))
+  return { dir, stubName, stubPath, remove: () => rmSync(dir, { recursive: true, force: true }) }
+}
+
+// → { outcome, reason, ms, model }
+export async function runOnce({ query, skillName, description, model, timeoutMs, env }) {
+  const project = makeProject(skillName, description)
+  try {
+    const detector = createDetector({ stubName: project.stubName, stubPath: project.stubPath })
+    const args = ['-p', query, '--output-format', 'stream-json', '--verbose', '--include-partial-messages', '--no-session-persistence', ...(model ? ['--model', model] : [])]
+    const r = await streamClaude(args, { cwd: project.dir, env, timeoutMs, onEvent: (ev) => detector.feed(ev) })
+    const used = detector.init?.model ?? null
+    if (r.stopped) return { ...r.stopped, ms: r.ms, model: used }
+    if (r.timedOut) return { outcome: 'timeout', reason: `no decision within ${timeoutMs / 1000} s`, ms: r.ms, model: used }
+    if (r.notFound) return { outcome: 'error', reason: 'claude not found on PATH', ms: r.ms, model: used }
+    return { ...detector.end({ code: r.code, stderr: r.stderr }), ms: r.ms, model: used }
+  } finally {
+    project.remove()
+  }
+}
+
+// Serial, query by query, run by run. Stops as soon as the timeouts and errors pass
+// the no-verdict share of the whole plan: from there no verdict is possible, and every
+// further run is spent on a number that will not be reported.
+export async function runAll({ items, runs, skillName, description, model, timeoutMs, env, noVerdictShare, onRun = () => {}, runOnceFn = runOnce }) {
+  const planned = items.length * runs
+  const outcomes = items.map(() => [])
+  // The model each run's init event reported, beside its outcome: an alias can resolve to
+  // a different model than the preflight saw (ST-20).
+  const models = items.map(() => [])
+  let bad = 0
+  let done = 0
+  let aborted = false
+  outer: for (let r = 0; r < runs; r++) {
+    for (const [i, item] of items.entries()) {
+      const res = await runOnceFn({ query: item.query, skillName, description, model, timeoutMs, env })
+      outcomes[i].push(res.outcome)
+      models[i].push(res.model ?? null)
+      done++
+      if (res.outcome === 'timeout' || res.outcome === 'error') bad++
+      onRun({ item, run: r + 1, done, planned, ...res })
+      if (bad > noVerdictShare * planned) {
+        aborted = done < planned
+        break outer
+      }
+    }
+  }
+  return { outcomes, models, planned, done, aborted }
+}