skillgauntlet 1.1.0 → 1.2.0

package/lib/ui.js

@@ -6,7 +6,14 @@
 // the run mid-flow.
 import readline, { emitKeypressEvents } from 'node:readline';
 import { stdin as input, stdout as output } from 'node:process';
-import { words } from './i18n.js';
+import { words, t } from './i18n.js';
+
+// Thrown by a list when the user presses ← or Esc: the caller goes back one question.
+export class Back extends Error {
+  constructor() { super('back'); this.name = 'Back'; }
+}
+let backEnabled = false;
+export const setBack = (on) => { backEnabled = on; };
 
 let rl = null;
 const queue = [];
@@ -37,10 +44,22 @@ export function pauseUi() {
   }
 }
 
+// On Windows a terminal stdin that was ever put in raw mode can keep the process alive after the last prompt (the
+// program handled Ctrl+C, then sat there with no prompt back). Let go of it explicitly.
 export function closeUi() {
   pauseUi();
+  if (input.isTTY) {
+    try {
+      if (input.setRawMode) input.setRawMode(false);
+      input.pause();
+      input.unref?.();
+      input.destroy?.();
+    } catch { /* already closed */ }
+  }
 }
 
+export const hasTerminal = () => Boolean(input.isTTY);
+
 export function say(line = '') {
   output.write(`${line}\n`);
 }
@@ -61,6 +80,9 @@ export async function ask(question, def = '') {
 }
 
 export async function confirm(question, def = false) {
+  if (input.isTTY) {
+    return select(question, [{ label: t('ui.no'), value: false }, { label: t('ui.yes'), value: true }], def ? 1 : 0, { hotkeys: { y: 1, n: 0 } });
+  }
   const hint = def ? 'Y/n' : 'y/N';
   for (;;) {
     const a = (await ask(`${question} (${hint})`, '')).toLowerCase();
@@ -86,7 +108,9 @@ export async function choose(title, options, def = 1) {
   }
 }
 
-export async function select(title, options, defIndex = 0) {
+// opts.clear: erase the list once an entry is chosen, so a caller that shows the next level right away redraws
+// in the same place instead of stacking lists down the terminal.
+export async function select(title, options, defIndex = 0, opts = {}) {
   if (!input.isTTY) {
     say('');
     say(title);
@@ -108,6 +132,7 @@ export async function select(title, options, defIndex = 0) {
   const WINDOW = 8;
   const width = Math.min(48, Math.max(...options.map((o) => o.label.length)));
   let first = true;
+  let height = 0;
   const render = () => {
     const size = Math.min(WINDOW, options.length);
     const start = Math.min(Math.max(0, idx - Math.floor(size / 2)), Math.max(0, options.length - size));
@@ -118,6 +143,8 @@ export async function select(title, options, defIndex = 0) {
       const body = i === idx ? `\x1b[7m▶ ${label} \x1b[0m` : `  ${label} `;
       lines.push(`${body}${o.hint ? `  ${o.hint}` : ''}`);
     }
+    if (backEnabled && !opts.noBack) lines.push(`\x1b[2m${t('backHint')}\x1b[0m`);
+    height = lines.length;
     if (first) {
       output.write(`\n${lines.join('\n')}\n`);
       first = false;
@@ -142,10 +169,21 @@ export async function select(title, options, defIndex = 0) {
         render();
       } else if (key.name === 'return' || key.name === 'enter') {
         cleanup();
+        if (opts.clear) output.write(`\x1b[${height + 1}A\x1b[J`);
         resolve(options[idx].value);
+      } else if ((key.name === 'escape' || key.name === 'left') && backEnabled && !opts.noBack) {
+        cleanup();
+        output.write(`\x1b[${height + 1}A\x1b[J`);
+        reject(new Back());
       } else if (key.ctrl && key.name === 'c') {
         cleanup();
         reject(new Error('interrupted'));
+      } else if (ch && opts.hotkeys && Object.hasOwn(opts.hotkeys, ch.toLowerCase())) {
+        // a hotkey (y / n) chooses at once
+        idx = opts.hotkeys[ch.toLowerCase()];
+        render();
+        cleanup();
+        resolve(options[idx].value);
       } else if (ch && /^[1-9]$/.test(ch) && Number(ch) <= options.length) {
         idx = Number(ch) - 1;
         render();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skillgauntlet",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Check an agent skill before you install it: context cost, permission surface, code risk, instruction hijacking, description quality, trigger rate and gain.",
   "type": "module",
   "bin": {
@@ -10,6 +10,7 @@
     "bin",
     "lib",
     "contracts",
+    "schemas",
     "README.md",
     "LICENSE",
     "LICENSE-DATA"
@@ -20,7 +21,8 @@
   "scripts": {
     "start": "node ./bin/skillgauntlet.js",
     "static": "node ./bin/skillgauntlet.js static",
-    "test": "node --test tests/*.test.mjs"
+    "test": "node --test tests/*.test.mjs",
+    "leaderboard": "node scripts/build-leaderboard.mjs"
   },
   "keywords": [
     "agent",

package/schemas/submission-1.json

@@ -0,0 +1,210 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/SkillGauntlet/SkillGauntlet/blob/main/schemas/submission-1.json",
+  "title": "SkillGauntlet community submission, version 1",
+  "description": "One submission directory holds meta.json, raw.jsonl, summary.json and queries.json. Definitions: meta, rawLine, summary, queries.",
+  "$defs": {
+    "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+    "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+    "isoTime": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?Z$" },
+    "shortText": { "type": "string", "maxLength": 300 },
+    "rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
+    "interval": {
+      "type": "array",
+      "minItems": 2,
+      "maxItems": 2,
+      "items": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }
+    },
+    "meta": {
+      "type": "object",
+      "required": [
+        "schema", "runId", "runnerVersion", "skill", "host", "backend", "model", "runs", "maxExplore",
+        "querySets", "platform", "startedAt", "finishedAt", "trials", "interrupted", "submission"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "schema": { "const": "skillgauntlet/run/1" },
+        "runId": { "type": "string", "pattern": "^[0-9]{8}-[0-9]{6}(-[0-9]+)?$" },
+        "runnerVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+([-+][0-9A-Za-z.-]+)?$" },
+        "skill": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$" },
+        "host": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+        "backend": { "type": "string", "maxLength": 100 },
+        "model": { "type": ["string", "null"], "maxLength": 100 },
+        "runs": { "type": "integer", "minimum": 1, "maximum": 20 },
+        "maxExplore": { "type": "integer", "minimum": 0, "maximum": 20 },
+        "depth": { "type": "object", "maxProperties": 8 },
+        "querySets": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 4,
+          "items": {
+            "type": "object",
+            "required": ["id", "queries", "sha256"],
+            "properties": {
+              "id": { "type": "string", "maxLength": 40 },
+              "source": { "type": "string", "maxLength": 40 },
+              "file": { "type": "string", "maxLength": 120 },
+              "queries": { "type": "integer", "minimum": 1 },
+              "clauses": { "type": "integer", "minimum": 0 },
+              "sha256": { "type": "string", "pattern": "^[0-9a-f]{16}$" },
+              "negativesQuality": { "type": "object", "maxProperties": 4 }
+            },
+            "additionalProperties": false
+          }
+        },
+        "skillFingerprint": { "type": ["string", "null"], "pattern": "^[0-9a-f]{16}$" },
+        "platform": { "$ref": "#/$defs/shortText" },
+        "startedAt": { "$ref": "#/$defs/isoTime" },
+        "finishedAt": { "$ref": "#/$defs/isoTime" },
+        "trials": { "type": "integer", "minimum": 1 },
+        "interrupted": { "const": false },
+        "resumedAt": { "type": "array", "maxItems": 50, "items": { "$ref": "#/$defs/isoTime" } },
+        "modelResolved": { "type": ["string", "null"], "maxLength": 100 },
+        "apiKeySource": { "type": ["string", "null"], "maxLength": 40 },
+        "claudeCodeVersion": { "type": ["string", "null"], "maxLength": 100 },
+        "loadedSkills": { "type": "array", "maxItems": 300, "items": { "type": "string", "maxLength": 100 } },
+        "tools": { "type": "array", "maxItems": 300, "items": { "type": "string", "maxLength": 100 } },
+        "incompleteTrials": { "type": "integer", "minimum": 0 },
+        "submission": { "$ref": "#/$defs/submission" }
+      }
+    },
+    "submission": {
+      "type": "object",
+      "required": ["schema", "contributor", "skill", "host", "model", "queryPack", "environment", "submittedAt"],
+      "additionalProperties": false,
+      "properties": {
+        "schema": { "const": "skillgauntlet/submission/1" },
+        "contributor": { "type": "string", "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$" },
+        "skill": {
+          "type": "object",
+          "required": ["name", "repo", "commit", "subdir", "skillMdSha256"],
+          "additionalProperties": false,
+          "properties": {
+            "name": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$" },
+            "repo": { "type": "string", "pattern": "^https://github\\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" },
+            "commit": { "$ref": "#/$defs/commit" },
+            "subdir": { "type": "string", "maxLength": 200, "pattern": "^[A-Za-z0-9_./-]*$" },
+            "skillMdSha256": { "$ref": "#/$defs/sha256" }
+          }
+        },
+        "host": {
+          "type": "object",
+          "required": ["id", "version"],
+          "additionalProperties": false,
+          "properties": {
+            "id": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+            "version": { "type": ["string", "null"], "maxLength": 100 }
+          }
+        },
+        "model": {
+          "type": "object",
+          "required": ["requested", "resolved"],
+          "additionalProperties": false,
+          "properties": {
+            "requested": { "type": ["string", "null"], "maxLength": 100 },
+            "resolved": { "type": ["string", "null"], "maxLength": 100 }
+          }
+        },
+        "queryPack": {
+          "type": "object",
+          "required": ["id", "sha256", "queries"],
+          "additionalProperties": false,
+          "properties": {
+            "id": { "type": "string", "maxLength": 40 },
+            "sha256": { "$ref": "#/$defs/sha256" },
+            "queries": { "type": "integer", "minimum": 1 }
+          }
+        },
+        "environment": {
+          "type": "object",
+          "description": "Used to group and explain results, never to prove anything: every field is self-reported.",
+          "required": ["id", "platform", "hostVersion", "backend", "loadedSkills", "toolCount"],
+          "additionalProperties": false,
+          "properties": {
+            "id": { "type": "string", "pattern": "^[0-9a-f]{12}$" },
+            "platform": { "$ref": "#/$defs/shortText" },
+            "hostVersion": { "type": ["string", "null"], "maxLength": 100 },
+            "backend": { "type": "string", "maxLength": 100 },
+            "loadedSkills": { "type": "array", "maxItems": 300, "items": { "type": "string", "maxLength": 100 } },
+            "toolCount": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "submittedAt": { "$ref": "#/$defs/isoTime" }
+      }
+    },
+    "rawLine": {
+      "type": "object",
+      "required": ["host", "verdict", "queryId", "run", "query", "shouldTrigger"],
+      "properties": {
+        "host": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+        "verdict": { "enum": ["triggered", "not_triggered", "other_skill", "error", "timeout"] },
+        "invokedSkill": { "type": ["string", "null"], "maxLength": 200 },
+        "firstTool": { "type": ["string", "null"], "maxLength": 200 },
+        "tools": { "type": "array", "maxItems": 100, "items": { "type": "string", "maxLength": 200 } },
+        "exploreSteps": { "type": "integer", "minimum": 0 },
+        "error": { "type": ["string", "null"], "maxLength": 400 },
+        "events": { "type": "object", "maxProperties": 10 },
+        "queryId": { "type": "string", "maxLength": 40 },
+        "run": { "type": "integer", "minimum": 0, "maximum": 50 },
+        "query": { "type": "string", "minLength": 1, "maxLength": 4000 },
+        "shouldTrigger": { "type": "boolean" }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "required": ["trials", "recall", "fpr", "questions", "host", "runs", "recallCi", "fprCi"],
+      "properties": {
+        "trials": { "type": "integer", "minimum": 1 },
+        "recall": { "$ref": "#/$defs/rate" },
+        "fpr": { "$ref": "#/$defs/rate" },
+        "noToolRate": { "$ref": "#/$defs/rate" },
+        "failedTrials": { "type": "integer", "minimum": 0 },
+        "incomplete": { "type": "integer", "minimum": 0 },
+        "model": { "type": ["string", "null"], "maxLength": 100 },
+        "backend": { "type": "string", "maxLength": 100 },
+        "host": { "enum": ["claude-code", "codebuddy", "codewhale"] },
+        "runs": { "type": "integer", "minimum": 1, "maximum": 20 },
+        "recallCi": { "$ref": "#/$defs/interval" },
+        "fprCi": { "$ref": "#/$defs/interval" },
+        "questions": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 2000,
+          "items": {
+            "type": "object",
+            "required": ["query", "shouldTrigger", "hits", "n"],
+            "properties": {
+              "query": { "type": "string", "maxLength": 4000 },
+              "shouldTrigger": { "type": "boolean" },
+              "hits": { "type": "integer", "minimum": 0 },
+              "n": { "type": "integer", "minimum": 1 }
+            }
+          }
+        }
+      }
+    },
+    "queries": {
+      "type": "object",
+      "required": ["schema", "queries"],
+      "properties": {
+        "schema": { "const": "skillgauntlet/queries/1" },
+        "queries": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 2000,
+          "items": {
+            "type": "object",
+            "required": ["id", "query", "shouldTrigger"],
+            "properties": {
+              "id": { "type": "string", "maxLength": 40 },
+              "query": { "type": "string", "minLength": 1, "maxLength": 4000 },
+              "shouldTrigger": { "type": "boolean" },
+              "clause": { "type": ["string", "null"], "maxLength": 300 },
+              "note": { "type": ["string", "null"], "maxLength": 600 }
+            }
+          }
+        }
+      }
+    }
+  }
+}