skillgauntlet 1.1.1 → 1.3.0

package/lib/trial.js

@@ -8,7 +8,7 @@
 //   answers without calling any tool                            -> not_triggered, counted apart
 //
 // NOT YET VERIFIED against a live host: written from the Python runner, exercised only in dry-run.
-import { execFileSync, spawn } from 'node:child_process';
+import { execFileSync, spawn, spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
@@ -31,12 +31,19 @@ export const HOSTS = {
 export function backendFor(host, model) {
   if (host === 'codewhale') return 'codewhale:api.deepseek.com';
   if (host !== 'codebuddy') return backendHost();
-  return isDeepseekModel(model) ? 'codebuddy:api.deepseek.com' : 'codebuddy';
+  return usesOwnDeepseek('codebuddy', model) ? 'codebuddy:api.deepseek.com' : 'codebuddy';
 }
 
 // A model id starting with "deepseek" means the DeepSeek API, whichever host drives it.
 export const isDeepseekModel = (model) => /^deepseek/i.test(String(model || ''));
 
+// A DeepSeek model that the host lists itself (CodeBuddy's own deepseek-v4.1-flash) is served by the host's account
+// and needs no key of ours. Only a deepseek-* name the host does not list takes the package's own DeepSeek route
+// (a models.json entry in the trial directory and DEEPSEEK_API_KEY from the environment).
+const nativeModels = new Map();
+export const setNativeModels = (host, list) => nativeModels.set(host, new Set(list || []));
+export const usesOwnDeepseek = (host, model) => isDeepseekModel(model) && !nativeModels.get(host)?.has(model);
+
 // DeepSeek's integration guide for WorkBuddy/CodeBuddy: a models.json entry whose key is read from the
 // environment (${DEEPSEEK_API_KEY}), so the file holds no secret. It is written into the trial's own
 // directory (project level), never into the user's ~/.codebuddy.
@@ -66,7 +73,12 @@ const LINE_BREAK = /\r?\n/;
 
 // On Windows `where` returns the extension-less shim first (a shell script Node cannot exec), then the
 // .cmd that it can -- with shell: true. Prefer a real executable, then a cmd shim, then whatever is left.
+// A host CLI found somewhere off PATH (or just installed) and accepted by the user, keyed by its command name.
+const binOverride = new Map();
+export const setHostBin = (name, file) => binOverride.set(name, file);
+
 export function resolveBin(name) {
+  if (binOverride.has(name)) return binOverride.get(name);
   const finder = process.platform === 'win32' ? 'where' : 'which';
   let found = [];
   try {
@@ -83,6 +95,7 @@ export function resolveBin(name) {
 // Whether a host CLI can be found on PATH at all. resolveBin falls back to the bare name, which on Windows
 // ends in cmd.exe's "not recognized" (or a spawn ENOENT) deep inside a trial; this asks up front.
 export function binInstalled(name) {
+  if (binOverride.has(name)) return true;
   try {
     execFileSync(process.platform === 'win32' ? 'where' : 'which', [name], { stdio: 'ignore' });
     return true;
@@ -103,6 +116,44 @@ export function requireHost(host, probe = binInstalled) {
   throw err;
 }
 
+// One tiny call before the real work: a host that is not signed in retries silently for minutes, so the user
+// would stare at a frozen screen. A short timeout turns that into an immediate, actionable message.
+// The first non-empty line of some CLI output, for an error message.
+const firstLine = (text) => String(text || '').split(LINE_BREAK).map((l) => l.trim()).find(Boolean) || '';
+
+// What a host CLI answered, as an error reason: the CLI's own words when it gave any. A CLI can exit 0 with nothing on
+// stdout and its complaint on stderr (CodeBuddy does for a custom model without an endpoint url), so a clean exit
+// status proves nothing; an empty or unreadable stdout is the failure.
+export function hostAnswerProblem(r, timeout) {
+  if (r.error) return r.error.code === 'ETIMEDOUT' ? `no answer within ${timeout}s` : String(r.error.message).split(LINE_BREAK)[0];
+  if (r.signal) return `no answer within ${timeout}s`;
+  const out = String(r.stdout || '');
+  if (r.status !== 0) return firstLine(r.stderr) || firstLine(out) || `exit status ${r.status}`;
+  if (!out.trim()) return firstLine(r.stderr) || 'the CLI answered with nothing';
+  try {
+    const parsed = JSON.parse(out);
+    const last = Array.isArray(parsed) ? parsed.filter((m) => m?.type === 'result').pop() : parsed;
+    if (last?.is_error) return firstLine(last.result) || 'the CLI reported an error';
+  } catch { return firstLine(out); }
+  return null;
+}
+
+export function pingHost(host, { bin, model = null, timeout = 90, spawn = spawnSync } = {}) {
+  const spec = HOSTS[host];
+  if (!spec || host === 'codewhale') return;
+  const args = ['-p', 'Reply with the single word OK.', '--output-format', 'json', '--tools', '', '--no-session-persistence'];
+  if (model && !usesOwnDeepseek(host, model)) args.push('--model', model); // the package's own DeepSeek route has no host-side name
+  const launch = launchSpec(bin, args, spec.bin);
+  const r = spawn(launch.cmd, launch.args, { encoding: 'utf8', timeout: timeout * 1000, shell: launch.shell, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
+  const why = hostAnswerProblem(r, timeout);
+  if (why) {
+    const e = new Error(`host ${host} did not answer: ${why}`);
+    e.messageKey = 'hostNotReady';
+    e.vars = { host: HOST_NAME[host] || host, why: why.slice(0, 200) };
+    throw e;
+  }
+}
+
 // Variables a host (the Claude desktop app, the Agent SDK, an outer Claude Code session) sets for its own
 // child processes. A trial inherits them otherwise, and they change what the model under test sees: under
 // the desktop app the CLI adds Artifact, ArtifactComments and ArtifactData to the tool list, and the model
@@ -189,7 +240,7 @@ export function runTrial({ bin, skillDir, skillName, query, model, timeout = 180
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sg-'));
     const spec = HOSTS[host];
     copyDir(skillDir, path.join(tmp, ...spec.skillsDir, skillName));
-    if (host === 'codebuddy' && isDeepseekModel(model)) writeModelsJson(tmp, model);
+    if (host === 'codebuddy' && usesOwnDeepseek('codebuddy', model)) writeModelsJson(tmp, model);
     const args = host === 'codebuddy'
       ? ['-p', query, '--output-format', 'stream-json', '--verbose', '--setting-sources', 'project', '--strict-mcp-config',
         '--no-session-persistence', '--tools', spec.tools]
@@ -362,23 +413,24 @@ ${text}`;
     const launch = launchSpec(bin, args, HOSTS[host].bin);
     // CodeBuddy reads a custom model from the working directory's .codebuddy/models.json, so a DeepSeek draft
     // runs in a scratch directory that has one.
-    const scratch = host === 'codebuddy' && isDeepseekModel(model) ? fs.mkdtempSync(path.join(os.tmpdir(), 'sg-draft-')) : null;
+    const scratch = host === 'codebuddy' && usesOwnDeepseek('codebuddy', model) ? fs.mkdtempSync(path.join(os.tmpdir(), 'sg-draft-')) : null;
     if (scratch) writeModelsJson(scratch, model);
-    let out;
+    let r;
     try {
-      out = execFileSync(launch.cmd, launch.args, {
+      r = spawnSync(launch.cmd, launch.args, {
         encoding: 'utf8', timeout: timeout * 1000, maxBuffer: 32 * 1024 * 1024,
-        shell: launch.shell, windowsHide: true, ...(scratch ? { cwd: scratch } : {}),
+        shell: launch.shell, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'], ...(scratch ? { cwd: scratch } : {}),
       });
-    } catch (err) {
-      // The default message of a failed execFileSync is the whole command line, which here is the drafting prompt.
-      const why = err.code === 'ETIMEDOUT' || err.killed
-        ? `no answer within ${timeout}s (a rejected key or an unreachable endpoint retries silently until then)`
-        : String(err.stderr || err.message).split(/\r?\n/).map((l) => l.trim()).find(Boolean) || 'exit status ' + err.status;
-      throw new Error(`the drafting call failed: ${why}`);
     } finally {
       if (scratch) { try { fs.rmSync(scratch, { recursive: true, force: true }); } catch { /* best effort */ } }
     }
+    // The CLI's own words, not "Unexpected end of JSON input": an empty answer with a complaint on stderr is common.
+    const problem = hostAnswerProblem(r, timeout);
+    if (problem) {
+      const hint = r.error?.code === 'ETIMEDOUT' || r.signal ? ' (a rejected key or an unreachable endpoint retries silently until then)' : '';
+      throw new Error(`the drafting call failed: ${problem}${hint}`);
+    }
+    const out = r.stdout;
     const parsed = JSON.parse(out);
     // Claude Code answers with one result object; CodeBuddy Code with the whole message list, whose last
     // element is the result.

package/lib/ui.js

@@ -6,7 +6,14 @@
 // the run mid-flow.
 import readline, { emitKeypressEvents } from 'node:readline';
 import { stdin as input, stdout as output } from 'node:process';
-import { words } from './i18n.js';
+import { words, t } from './i18n.js';
+
+// Thrown by a list when the user presses ← or Esc: the caller goes back one question.
+export class Back extends Error {
+  constructor() { super('back'); this.name = 'Back'; }
+}
+let backEnabled = false;
+export const setBack = (on) => { backEnabled = on; };
 
 let rl = null;
 const queue = [];
@@ -37,10 +44,22 @@ export function pauseUi() {
   }
 }
 
+// On Windows a terminal stdin that was ever put in raw mode can keep the process alive after the last prompt (the
+// program handled Ctrl+C, then sat there with no prompt back). Let go of it explicitly.
 export function closeUi() {
   pauseUi();
+  if (input.isTTY) {
+    try {
+      if (input.setRawMode) input.setRawMode(false);
+      input.pause();
+      input.unref?.();
+      input.destroy?.();
+    } catch { /* already closed */ }
+  }
 }
 
+export const hasTerminal = () => Boolean(input.isTTY);
+
 export function say(line = '') {
   output.write(`${line}\n`);
 }
@@ -61,6 +80,9 @@ export async function ask(question, def = '') {
 }
 
 export async function confirm(question, def = false) {
+  if (input.isTTY) {
+    return select(question, [{ label: t('ui.no'), value: false }, { label: t('ui.yes'), value: true }], def ? 1 : 0, { hotkeys: { y: 1, n: 0 } });
+  }
   const hint = def ? 'Y/n' : 'y/N';
   for (;;) {
     const a = (await ask(`${question} (${hint})`, '')).toLowerCase();
@@ -86,7 +108,9 @@ export async function choose(title, options, def = 1) {
   }
 }
 
-export async function select(title, options, defIndex = 0) {
+// opts.clear: erase the list once an entry is chosen, so a caller that shows the next level right away redraws
+// in the same place instead of stacking lists down the terminal.
+export async function select(title, options, defIndex = 0, opts = {}) {
   if (!input.isTTY) {
     say('');
     say(title);
@@ -108,6 +132,7 @@ export async function select(title, options, defIndex = 0) {
   const WINDOW = 8;
   const width = Math.min(48, Math.max(...options.map((o) => o.label.length)));
   let first = true;
+  let height = 0;
   const render = () => {
     const size = Math.min(WINDOW, options.length);
     const start = Math.min(Math.max(0, idx - Math.floor(size / 2)), Math.max(0, options.length - size));
@@ -118,6 +143,8 @@ export async function select(title, options, defIndex = 0) {
       const body = i === idx ? `\x1b[7m▶ ${label} \x1b[0m` : `  ${label} `;
       lines.push(`${body}${o.hint ? `  ${o.hint}` : ''}`);
     }
+    if (backEnabled && !opts.noBack) lines.push(`\x1b[2m${t('backHint')}\x1b[0m`);
+    height = lines.length;
     if (first) {
       output.write(`\n${lines.join('\n')}\n`);
       first = false;
@@ -142,10 +169,21 @@ export async function select(title, options, defIndex = 0) {
         render();
       } else if (key.name === 'return' || key.name === 'enter') {
         cleanup();
+        if (opts.clear) output.write(`\x1b[${height + 1}A\x1b[J`);
         resolve(options[idx].value);
+      } else if ((key.name === 'escape' || key.name === 'left') && backEnabled && !opts.noBack) {
+        cleanup();
+        output.write(`\x1b[${height + 1}A\x1b[J`);
+        reject(new Back());
       } else if (key.ctrl && key.name === 'c') {
         cleanup();
         reject(new Error('interrupted'));
+      } else if (ch && opts.hotkeys && Object.hasOwn(opts.hotkeys, ch.toLowerCase())) {
+        // a hotkey (y / n) chooses at once
+        idx = opts.hotkeys[ch.toLowerCase()];
+        render();
+        cleanup();
+        resolve(options[idx].value);
       } else if (ch && /^[1-9]$/.test(ch) && Number(ch) <= options.length) {
         idx = Number(ch) - 1;
         render();

package/lib/update.js

@@ -0,0 +1,73 @@
+// A check for a newer published version before an interactive session. It must never get in the way: any failure
+// (offline, slow registry, odd reply, unwritable cache) means "no update known" and the session goes on with the
+// version that is running.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REGISTRY = 'https://registry.npmjs.org/skillgauntlet/latest';
+const CHECK_EVERY_MS = 12 * 60 * 60 * 1000;
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export function currentVersion() {
+  try { return JSON.parse(fs.readFileSync(path.join(here, '..', 'package.json'), 'utf8')).version; } catch { return null; }
+}
+
+// a.b.c with an optional pre-release tail; a pre-release is older than its release.
+export function isNewer(latest, current) {
+  const parse = (v) => {
+    const m = String(v).trim().match(/^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/);
+    return m ? { n: [Number(m[1]), Number(m[2]), Number(m[3])], pre: m[4] || null } : null;
+  };
+  const a = parse(latest);
+  const b = parse(current);
+  if (!a || !b) return false;
+  for (let i = 0; i < 3; i += 1) if (a.n[i] !== b.n[i]) return a.n[i] > b.n[i];
+  return Boolean(b.pre) && !a.pre;
+}
+
+// A copy that runs from a git checkout is a developer's own; it is not nagged about releases.
+export function isCheckout() {
+  return fs.existsSync(path.join(here, '..', '.git'));
+}
+
+function cachePath() {
+  return path.join(process.env.SKILLGAUNTLET_CACHE || path.join(os.homedir(), '.cache', 'skillgauntlet'), 'update-check.json');
+}
+
+function readCache(file) {
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return {}; }
+}
+
+function writeCache(file, data) {
+  try {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, `${JSON.stringify(data)}\n`);
+  } catch { /* the cache is an optimisation */ }
+}
+
+// { current, latest } when a newer version is known and has not been declined, else null. Never throws.
+export async function checkForUpdate({ fetchLatest, now = Date.now(), file = cachePath(), current = currentVersion(), env = process.env } = {}) {
+  try {
+    if (!current || env.SG_NO_UPDATE_CHECK || env.CI) return null;
+    const cache = readCache(file);
+    let latest = cache.latest;
+    if (!latest || !cache.checkedAt || now - cache.checkedAt > CHECK_EVERY_MS) {
+      const get = fetchLatest || (async () => {
+        const res = await fetch(REGISTRY, { signal: AbortSignal.timeout(3000), headers: { accept: 'application/json' } });
+        if (!res.ok) throw new Error(`registry ${res.status}`);
+        return (await res.json()).version;
+      });
+      latest = await get();
+      writeCache(file, { ...cache, latest, checkedAt: now });
+    }
+    if (!isNewer(latest, current) || cache.declined === latest) return null;
+    return { current, latest };
+  } catch { return null; }
+}
+
+// Remember that the user said no to this version, so the question is not asked again until a newer one appears.
+export function declineUpdate(latest, file = cachePath()) {
+  writeCache(file, { ...readCache(file), declined: latest });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skillgauntlet",
-  "version": "1.1.1",
+  "version": "1.3.0",
   "description": "Check an agent skill before you install it: context cost, permission surface, code risk, instruction hijacking, description quality, trigger rate and gain.",
   "type": "module",
   "bin": {
@@ -10,6 +10,7 @@
     "bin",
     "lib",
     "contracts",
+    "schemas",
     "README.md",
     "LICENSE",
     "LICENSE-DATA"
@@ -20,7 +21,8 @@
   "scripts": {
     "start": "node ./bin/skillgauntlet.js",
     "static": "node ./bin/skillgauntlet.js static",
-    "test": "node --test tests/*.test.mjs"
+    "test": "node --test tests/*.test.mjs",
+    "leaderboard": "node scripts/build-leaderboard.mjs"
   },
   "keywords": [
     "agent",

package/schemas/submission-1.json

@@ -0,0 +1,210 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/SkillGauntlet/SkillGauntlet/blob/main/schemas/submission-1.json",
+  "title": "SkillGauntlet community submission, version 1",
+  "description": "One submission directory holds meta.json, raw.jsonl, summary.json and queries.json. Definitions: meta, rawLine, summary, queries.",
+  "$defs": {
+    "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+    "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+    "isoTime": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z$" },
+    "shortText": { "type": "string", "maxLength": 300 },
+    "rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
+    "interval": {
+      "type": "array",
+      "minItems": 2,
+      "maxItems": 2,
+      "items": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }
+    },
+    "meta": {
+      "type": "object",
+      "required": [
+        "schema", "runId", "runnerVersion", "skill", "host", "backend", "model", "runs", "maxExplore",
+        "querySets", "platform", "startedAt", "finishedAt", "trials", "interrupted", "submission"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "schema": { "const": "skillgauntlet/run/1" },
+        "runId": { "type": "string", "pattern": "^[0-9]{8}-[0-9]{6}(-[0-9]+)?$" },
+        "runnerVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+([-+][0-9A-Za-z.-]+)?$" },
+        "skill": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$" },
+        "host": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+        "backend": { "type": "string", "maxLength": 100 },
+        "model": { "type": ["string", "null"], "maxLength": 100 },
+        "runs": { "type": "integer", "minimum": 1, "maximum": 20 },
+        "maxExplore": { "type": "integer", "minimum": 0, "maximum": 20 },
+        "depth": { "type": "object", "maxProperties": 8 },
+        "querySets": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 4,
+          "items": {
+            "type": "object",
+            "required": ["id", "queries", "sha256"],
+            "properties": {
+              "id": { "type": "string", "maxLength": 40 },
+              "source": { "type": "string", "maxLength": 40 },
+              "file": { "type": "string", "maxLength": 120 },
+              "queries": { "type": "integer", "minimum": 1 },
+              "clauses": { "type": "integer", "minimum": 0 },
+              "sha256": { "type": "string", "pattern": "^[0-9a-f]{16}$" },
+              "negativesQuality": { "type": "object", "maxProperties": 4 }
+            },
+            "additionalProperties": false
+          }
+        },
+        "skillFingerprint": { "type": ["string", "null"], "pattern": "^[0-9a-f]{16}$" },
+        "platform": { "$ref": "#/$defs/shortText" },
+        "startedAt": { "$ref": "#/$defs/isoTime" },
+        "finishedAt": { "$ref": "#/$defs/isoTime" },
+        "trials": { "type": "integer", "minimum": 1 },
+        "interrupted": { "const": false },
+        "resumedAt": { "type": "array", "maxItems": 50, "items": { "$ref": "#/$defs/isoTime" } },
+        "modelResolved": { "type": ["string", "null"], "maxLength": 100 },
+        "apiKeySource": { "type": ["string", "null"], "maxLength": 40 },
+        "claudeCodeVersion": { "type": ["string", "null"], "maxLength": 100 },
+        "loadedSkills": { "type": "array", "maxItems": 300, "items": { "type": "string", "maxLength": 100 } },
+        "tools": { "type": "array", "maxItems": 300, "items": { "type": "string", "maxLength": 100 } },
+        "incompleteTrials": { "type": "integer", "minimum": 0 },
+        "submission": { "$ref": "#/$defs/submission" }
+      }
+    },
+    "submission": {
+      "type": "object",
+      "required": ["schema", "contributor", "skill", "host", "model", "queryPack", "environment", "submittedAt"],
+      "additionalProperties": false,
+      "properties": {
+        "schema": { "const": "skillgauntlet/submission/1" },
+        "contributor": { "type": "string", "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$" },
+        "skill": {
+          "type": "object",
+          "required": ["name", "repo", "commit", "subdir", "skillMdSha256"],
+          "additionalProperties": false,
+          "properties": {
+            "name": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$" },
+            "repo": { "type": "string", "pattern": "^https://github\\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" },
+            "commit": { "$ref": "#/$defs/commit" },
+            "subdir": { "type": "string", "maxLength": 200, "pattern": "^[A-Za-z0-9_./-]*$" },
+            "skillMdSha256": { "$ref": "#/$defs/sha256" }
+          }
+        },
+        "host": {
+          "type": "object",
+          "required": ["id", "version"],
+          "additionalProperties": false,
+          "properties": {
+            "id": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+            "version": { "type": ["string", "null"], "maxLength": 100 }
+          }
+        },
+        "model": {
+          "type": "object",
+          "required": ["requested", "resolved"],
+          "additionalProperties": false,
+          "properties": {
+            "requested": { "type": ["string", "null"], "maxLength": 100 },
+            "resolved": { "type": ["string", "null"], "maxLength": 100 }
+          }
+        },
+        "queryPack": {
+          "type": "object",
+          "required": ["id", "sha256", "queries"],
+          "additionalProperties": false,
+          "properties": {
+            "id": { "type": "string", "maxLength": 40 },
+            "sha256": { "$ref": "#/$defs/sha256" },
+            "queries": { "type": "integer", "minimum": 1 }
+          }
+        },
+        "environment": {
+          "type": "object",
+          "description": "Used to group and explain results, never to prove anything: every field is self-reported.",
+          "required": ["id", "platform", "hostVersion", "backend", "loadedSkills", "toolCount"],
+          "additionalProperties": false,
+          "properties": {
+            "id": { "type": "string", "pattern": "^[0-9a-f]{12}$" },
+            "platform": { "$ref": "#/$defs/shortText" },
+            "hostVersion": { "type": ["string", "null"], "maxLength": 100 },
+            "backend": { "type": "string", "maxLength": 100 },
+            "loadedSkills": { "type": "array", "maxItems": 300, "items": { "type": "string", "maxLength": 100 } },
+            "toolCount": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "submittedAt": { "$ref": "#/$defs/isoTime" }
+      }
+    },
+    "rawLine": {
+      "type": "object",
+      "required": ["host", "verdict", "queryId", "run", "query", "shouldTrigger"],
+      "properties": {
+        "host": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+        "verdict": { "enum": ["triggered", "not_triggered", "other_skill", "error", "timeout"] },
+        "invokedSkill": { "type": ["string", "null"], "maxLength": 200 },
+        "firstTool": { "type": ["string", "null"], "maxLength": 200 },
+        "tools": { "type": "array", "maxItems": 100, "items": { "type": "string", "maxLength": 200 } },
+        "exploreSteps": { "type": "integer", "minimum": 0 },
+        "error": { "type": ["string", "null"], "maxLength": 400 },
+        "events": { "type": "object", "maxProperties": 10 },
+        "queryId": { "type": "string", "maxLength": 40 },
+        "run": { "type": "integer", "minimum": 0, "maximum": 50 },
+        "query": { "type": "string", "minLength": 1, "maxLength": 4000 },
+        "shouldTrigger": { "type": "boolean" }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "required": ["trials", "recall", "fpr", "questions", "host", "runs", "recallCi", "fprCi"],
+      "properties": {
+        "trials": { "type": "integer", "minimum": 1 },
+        "recall": { "$ref": "#/$defs/rate" },
+        "fpr": { "$ref": "#/$defs/rate" },
+        "noToolRate": { "$ref": "#/$defs/rate" },
+        "failedTrials": { "type": "integer", "minimum": 0 },
+        "incomplete": { "type": "integer", "minimum": 0 },
+        "model": { "type": ["string", "null"], "maxLength": 100 },
+        "backend": { "type": "string", "maxLength": 100 },
+        "host": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+        "runs": { "type": "integer", "minimum": 1, "maximum": 20 },
+        "recallCi": { "$ref": "#/$defs/interval" },
+        "fprCi": { "$ref": "#/$defs/interval" },
+        "questions": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 2000,
+          "items": {
+            "type": "object",
+            "required": ["query", "shouldTrigger", "hits", "n"],
+            "properties": {
+              "query": { "type": "string", "maxLength": 4000 },
+              "shouldTrigger": { "type": "boolean" },
+              "hits": { "type": "integer", "minimum": 0 },
+              "n": { "type": "integer", "minimum": 1 }
+            }
+          }
+        }
+      }
+    },
+    "queries": {
+      "type": "object",
+      "required": ["schema", "queries"],
+      "properties": {
+        "schema": { "const": "skillgauntlet/queries/1" },
+        "queries": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 2000,
+          "items": {
+            "type": "object",
+            "required": ["id", "query", "shouldTrigger"],
+            "properties": {
+              "id": { "type": "string", "maxLength": 40 },
+              "query": { "type": "string", "minLength": 1, "maxLength": 4000 },
+              "shouldTrigger": { "type": "boolean" },
+              "clause": { "type": ["string", "null"], "maxLength": 300 },
+              "note": { "type": ["string", "null"], "maxLength": 600 }
+            }
+          }
+        }
+      }
+    }
+  }
+}