skillgauntlet 1.1.1 → 1.3.0

package/lib/models.js

@@ -0,0 +1,46 @@
+// Model names a host will accept, read from the host itself so the list is the user's own (it depends on the
+// account and the custom models they set up) and never goes stale inside this package. Any failure gives [].
+import { execFileSync, execSync } from 'node:child_process';
+import { launchSpec } from './spawn.js';
+
+const OPTS = { encoding: 'utf8', timeout: 20000, windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 16 * 1024 * 1024 };
+// The arguments here are fixed words, so a shell launch (a .cmd shim) is one command string; passing an argument
+// list together with shell: true makes Node print a deprecation warning into the user's terminal.
+const run = (cmd, args, { shell } = {}) => (shell ? execSync(`"${cmd}" ${args.join(' ')}`, OPTS) : execFileSync(cmd, args, OPTS));
+
+// CodeBuddy prints "Currently supported: (a, b, c)" in --help.
+export function parseCodebuddyModels(help) {
+  const m = String(help).match(/Currently supported:\s*\(([^)]*)\)/);
+  return m ? m[1].split(',').map((s) => s.trim()).filter(Boolean) : [];
+}
+
+// `codex debug models` renders its catalog as JSON; entries marked visibility "list" are the ones its own picker shows.
+export function parseCodexModels(json) {
+  try {
+    const parsed = JSON.parse(json);
+    const list = Array.isArray(parsed) ? parsed : parsed.models || [];
+    return list.filter((m) => m?.slug && m.visibility === 'list').map((m) => m.slug);
+  } catch { return []; }
+}
+
+export function hostModels(host, bin, exec = run) {
+  try {
+    if (host === 'codebuddy') {
+      const l = launchSpec(bin, ['--help'], 'codebuddy');
+      return parseCodebuddyModels(exec(l.cmd, l.args, { shell: l.shell }));
+    }
+    if (host === 'codex') {
+      const l = launchSpec(bin, ['debug', 'models'], 'codex');
+      return parseCodexModels(exec(l.cmd, l.args, { shell: l.shell }));
+    }
+  } catch { /* the caller falls back to typing */ }
+  return [];
+}
+
+// A model the user typed that the host does not list would fail only after the question-drafting call started. Names the
+// package turns into a DeepSeek model of its own (deepseek-*) are exempt: they are not in the host's list by design.
+// Returns the host's current list when the name is not in it, else null (also null when the list could not be read).
+export function unknownModel(model, known) {
+  if (!model || !known?.length || known.includes(model) || /^deepseek/i.test(model)) return null;
+  return known;
+}

package/lib/report.js

@@ -31,7 +31,7 @@ export function staticMarkdown(result) {
     L.push('');
     for (const c of g.checks) {
       L.push(`- ${ICON[c.status]} **${c.id} ${t(c.titleKey)}** — ${t(c.detailKey, c.detailVars)}`);
-      for (const e of c.evidence.slice(0, 3)) L.push(`  - \`L${e.line}\` ${e.match}`);
+      for (const e of c.evidence.slice(0, 3)) L.push(`  - \`${e.file ? `${e.file}:` : ''}L${e.line}\` ${e.match}`);
     }
     L.push('');
   }

package/lib/schema.js

@@ -0,0 +1,90 @@
+// A small JSON Schema checker: enough of the vocabulary to validate schemas/submission-1.json
+// (type, enum, const, pattern, length and size bounds, required, properties, additionalProperties,
+// items, local $ref) and nothing else. The package has no dependencies, and CI must not install any
+// to look at a stranger's file. A keyword this checker does not know is an error, not silently ignored,
+// so the schema cannot drift into promising checks nobody runs.
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const KNOWN = new Set([
+  '$schema', '$id', '$defs', '$ref', 'title', 'description', 'type', 'enum', 'const', 'pattern', 'minLength',
+  'maxLength', 'minimum', 'maximum', 'required', 'properties', 'additionalProperties', 'items', 'minItems',
+  'maxItems', 'maxProperties',
+]);
+
+export function loadSchema() {
+  return JSON.parse(fs.readFileSync(path.join(here, '..', 'schemas', 'submission-1.json'), 'utf8'));
+}
+
+function typeOf(v) {
+  if (v === null) return 'null';
+  if (Array.isArray(v)) return 'array';
+  if (Number.isInteger(v)) return 'integer';
+  return typeof v;
+}
+
+function typeMatches(want, v) {
+  const got = typeOf(v);
+  return want === got || (want === 'number' && got === 'integer');
+}
+
+// Depth guard: a hostile file must not be able to blow the stack of the checker.
+export function maxDepth(v, limit = 12, d = 0) {
+  if (d > limit) return d;
+  if (v && typeof v === 'object') {
+    let m = d;
+    for (const x of Object.values(v)) m = Math.max(m, maxDepth(x, limit, d + 1));
+    return m;
+  }
+  return d;
+}
+
+// Returns a list of { path, message }. Empty means valid.
+export function validate(root, value, defName) {
+  const errors = [];
+  const resolve = (ref) => {
+    const m = /^#\/\$defs\/(\w+)$/.exec(ref);
+    if (!m || !root.$defs?.[m[1]]) throw new Error(`unsupported $ref ${ref}`);
+    return root.$defs[m[1]];
+  };
+  const walk = (schema, v, at) => {
+    for (const k of Object.keys(schema)) if (!KNOWN.has(k)) throw new Error(`schema keyword not supported by the checker: ${k}`);
+    if (schema.$ref) { walk(resolve(schema.$ref), v, at); return; }
+    const fail = (message) => errors.push({ path: at || '(root)', message });
+    if (schema.const !== undefined && v !== schema.const) { fail(`must be ${JSON.stringify(schema.const)}`); return; }
+    if (schema.enum && !schema.enum.includes(v)) { fail(`must be one of ${schema.enum.map((x) => JSON.stringify(x)).join(', ')}`); return; }
+    if (schema.type) {
+      const want = [].concat(schema.type);
+      if (!want.some((w) => typeMatches(w, v))) { fail(`must be ${want.join(' or ')}, got ${typeOf(v)}`); return; }
+    }
+    if (typeof v === 'string') {
+      if (schema.minLength !== undefined && v.length < schema.minLength) fail(`shorter than ${schema.minLength}`);
+      if (schema.maxLength !== undefined && v.length > schema.maxLength) fail(`longer than ${schema.maxLength}`);
+      if (schema.pattern && !new RegExp(schema.pattern).test(v)) fail(`does not match ${schema.pattern}`);
+    }
+    if (typeof v === 'number') {
+      if (schema.minimum !== undefined && v < schema.minimum) fail(`below ${schema.minimum}`);
+      if (schema.maximum !== undefined && v > schema.maximum) fail(`above ${schema.maximum}`);
+    }
+    if (Array.isArray(v)) {
+      if (schema.minItems !== undefined && v.length < schema.minItems) fail(`fewer than ${schema.minItems} items`);
+      if (schema.maxItems !== undefined && v.length > schema.maxItems) fail(`more than ${schema.maxItems} items`);
+      if (schema.items) v.forEach((x, i) => walk(schema.items, x, `${at}[${i}]`));
+    }
+    if (v && typeof v === 'object' && !Array.isArray(v)) {
+      const keys = Object.keys(v);
+      if (schema.maxProperties !== undefined && keys.length > schema.maxProperties) fail(`more than ${schema.maxProperties} properties`);
+      for (const r of schema.required || []) if (!(r in v)) errors.push({ path: at ? `${at}.${r}` : r, message: 'is required' });
+      const props = schema.properties || {};
+      for (const k of keys) {
+        const sub = at ? `${at}.${k}` : k;
+        if (props[k]) walk(props[k], v[k], sub);
+        else if (schema.additionalProperties === false) errors.push({ path: sub, message: 'is not allowed here' });
+      }
+    }
+  };
+  walk(resolve(`#/$defs/${defName}`), value, '');
+  return errors;
+}

package/lib/scrub.js

@@ -0,0 +1,186 @@
+// Nothing leaves the machine before it has been through here.
+//
+// Run records are written by tools that capture error text, stderr and tool names, so they can hold the
+// user's home path, login name, host name, an environment value or a key that was echoed in an error.
+// A submission is public and permanent (it becomes a pull request), so the rule is: strip what is known
+// to be personal, redact what is shaped like a secret, then scan the result again with a wider net and
+// refuse to package anything the scan still dislikes. Stripping is best effort; the refusal is the guarantee.
+//
+// The question text is the one thing that must not be rewritten: the query-pack hash covers it. It is
+// scanned but never edited, and a hit there stops the submission with a message naming the question.
+import os from 'node:os';
+
+// Patterns that are redacted when found, and reported when they survive the redaction.
+export const SECRET_PATTERNS = [
+  ['anthropic-key', /sk-ant-[A-Za-z0-9_-]{16,}/g],
+  ['openai-style-key', /\bsk-[A-Za-z0-9_-]{16,}/g],
+  ['github-token', /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})/g],
+  ['aws-access-key', /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g],
+  ['google-api-key', /\bAIza[0-9A-Za-z_-]{30,}/g],
+  ['slack-token', /\bxox[abprs]-[A-Za-z0-9-]{10,}/g],
+  ['jwt', /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g],
+  ['bearer-token', /\bBearer\s+[A-Za-z0-9._~+/=-]{20,}/gi],
+  ['private-key', /-----BEGIN [A-Z ]*PRIVATE KEY-----/g],
+  ['url-credentials', /\bhttps?:\/\/[^/\s:@]+:[^/\s@]+@/gi],
+  ['assignment', /\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password|passwd|authorization)(["']?\s*[:=]\s*["']?)[^\s"',;]{8,}/gi],
+];
+
+const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
+const SENSITIVE_ENV = /(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|AUTH|COOKIE|SESSION)/i;
+
+const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+// What this machine knows about its user: replaced wherever it shows up, in any slash style.
+export function localContext({ env = process.env, extraPaths = [], user = null } = {}) {
+  const home = os.homedir();
+  const names = new Set();
+  for (const n of [user, env.USERNAME, env.USER, env.LOGNAME, safeUser()]) if (n && n.length >= 4) names.add(n);
+  const hosts = new Set([os.hostname(), env.COMPUTERNAME, env.HOSTNAME].filter((h) => h && h.length >= 3));
+  const paths = new Set([home, os.tmpdir(), ...extraPaths].filter(Boolean));
+  const secrets = new Set();
+  for (const [k, v] of Object.entries(env)) {
+    if (SENSITIVE_ENV.test(k) && v && v.length >= 6) secrets.add(v);
+  }
+  return { home, names: [...names], hosts: [...hosts], paths: [...paths], secrets: [...secrets] };
+}
+
+// For CI: it knows nothing about the submitter's machine, so only the shape-based patterns apply.
+export function emptyContext() {
+  return { home: '', names: [], hosts: [], paths: [], secrets: [] };
+}
+
+function safeUser() {
+  try { return os.userInfo().username; } catch { return null; }
+}
+
+// Both slash styles of one path, longest first so a path inside another is replaced as a whole.
+function pathVariants(p) {
+  const fwd = p.replace(/\\/g, '/').replace(/\/+$/, '');
+  const back = fwd.replace(/\//g, '\\');
+  return [...new Set([p.replace(/[\\/]+$/, ''), fwd, back])].filter((x) => x.length >= 4);
+}
+
+function compileContext(ctx) {
+  const pathRes = ctx.paths
+    .flatMap(pathVariants)
+    .sort((a, b) => b.length - a.length)
+    .map((p) => [new RegExp(escapeRe(p), 'gi'), p === ctx.home || pathVariants(ctx.home).includes(p) ? '<home>' : '<path>']);
+  const nameRe = ctx.names.length ? new RegExp(`(?<![A-Za-z0-9])(?:${ctx.names.map(escapeRe).join('|')})(?![A-Za-z0-9])`, 'gi') : null;
+  const hostRe = ctx.hosts.length ? new RegExp(`(?<![A-Za-z0-9])(?:${ctx.hosts.map(escapeRe).join('|')})(?![A-Za-z0-9])`, 'gi') : null;
+  const secretRes = ctx.secrets.map((s) => new RegExp(escapeRe(s), 'g'));
+  return { pathRes, nameRe, hostRe, secretRes };
+}
+
+export const GENERIC_PATHS = [
+  [/[A-Za-z]:[\\/]+Users[\\/]+[^\\/\s"']+/gi, '<home>'],
+  [/(?<![A-Za-z0-9_.-])\/(?:home|Users)\/[^/\s"']+/g, '<home>'],
+  [/\b[A-Za-z]:[\\/][^\s"'<>|*?]+/g, '<path>'],
+  [/(?<![A-Za-z0-9_.:-])\/(?:tmp|var|private|opt|mnt|root|usr|etc|proc|run|snap|srv|Volumes)\/[^\s"'<>|*?]*/g, '<path>'],
+];
+
+// Returns { text, counts }. counts says how many of each kind were replaced, for the preview.
+export function scrubString(s, compiled, counts = {}) {
+  let out = s;
+  const bump = (k, n) => { if (n) counts[k] = (counts[k] || 0) + n; };
+  const sub = (re, repl, kind) => {
+    let n = 0;
+    out = out.replace(re, (...m) => { n += 1; return typeof repl === 'function' ? repl(...m) : repl; });
+    bump(kind, n);
+  };
+  for (const re of compiled.secretRes) sub(re, '<redacted>', 'env-value');
+  for (const [name, re] of SECRET_PATTERNS) {
+    if (name === 'assignment') sub(re, (_m, sep) => `<redacted-key>${sep}<redacted>`, 'secret');
+    else sub(re, '<redacted>', 'secret');
+  }
+  for (const [re, repl] of compiled.pathRes) sub(re, repl, 'path');
+  for (const [re, repl] of GENERIC_PATHS) sub(re, repl, 'path');
+  // host name first: a login name is often a piece of it (ALICE-PC)
+  if (compiled.hostRe) sub(compiled.hostRe, '<host>', 'hostname');
+  if (compiled.nameRe) sub(compiled.nameRe, '<user>', 'username');
+  sub(EMAIL, '<email>', 'email');
+  return { text: out, counts };
+}
+
+// The wider net, run on already-scrubbed data. Returns a list of { where, kind, sample }.
+// Hex strings are exempt (hashes and commits are meant to be there); everything else long, mixed
+// letters-and-digits and random-looking is treated as a probable credential.
+export function findSecrets(value, ctx = localContext(), where = '') {
+  const compiled = compileContext(ctx);
+  const hits = [];
+  const visit = (v, at) => {
+    if (typeof v === 'string') {
+      for (const [name, re] of SECRET_PATTERNS) {
+        re.lastIndex = 0;
+        // an already-redacted assignment still matches the assignment pattern; the placeholder is not a secret
+        const m = [...v.matchAll(re)].find((x) => !/<redacted/.test(x[0]));
+        if (m) hits.push({ where: at, kind: name, sample: mask(m[0]) });
+      }
+      for (const re of compiled.secretRes) {
+        if (re.test(v)) hits.push({ where: at, kind: 'env-value', sample: '(a value from one of your environment variables)' });
+        re.lastIndex = 0;
+      }
+      for (const tok of v.split(/[^A-Za-z0-9_\-+=]+/)) {
+        if (tok.length >= 32 && !/^[0-9a-f]+$/i.test(tok) && /[A-Za-z]/.test(tok) && /\d/.test(tok) && entropy(tok) >= 4.2) {
+          hits.push({ where: at, kind: 'high-entropy-token', sample: mask(tok) });
+        }
+      }
+      for (const re of [...compiled.pathRes.map((x) => x[0]), ...GENERIC_PATHS.slice(0, 2).map((x) => x[0])]) {
+        re.lastIndex = 0;
+        const m = re.exec(v);
+        re.lastIndex = 0;
+        if (m) hits.push({ where: at, kind: 'local-path', sample: mask(m[0]) });
+      }
+      if (compiled.nameRe) {
+        compiled.nameRe.lastIndex = 0;
+        if (compiled.nameRe.test(v)) hits.push({ where: at, kind: 'username', sample: '(your login name)' });
+        compiled.nameRe.lastIndex = 0;
+      }
+    } else if (Array.isArray(v)) v.forEach((x, i) => visit(x, `${at}[${i}]`));
+    else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) visit(x, at ? `${at}.${k}` : k);
+  };
+  visit(value, where);
+  return hits;
+}
+
+function mask(s) {
+  return s.length <= 8 ? '****' : `${s.slice(0, 4)}…(${s.length} chars)`;
+}
+
+function entropy(s) {
+  const f = new Map();
+  for (const c of s) f.set(c, (f.get(c) || 0) + 1);
+  let h = 0;
+  for (const n of f.values()) h -= (n / s.length) * Math.log2(n / s.length);
+  return h;
+}
+
+// Scrub a parsed JSON value. keepKeys names string fields that are copied as they are (the question
+// text); those are only scanned. Returns { value, counts }.
+export function scrubValue(value, ctx = localContext(), { keepKeys = [] } = {}) {
+  const compiled = compileContext(ctx);
+  const counts = {};
+  const keep = new Set(keepKeys);
+  const walk = (v, key) => {
+    if (typeof v === 'string') return keep.has(key) ? v : scrubString(v, compiled, counts).text;
+    if (Array.isArray(v)) return v.map((x) => walk(x, key));
+    if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x, k)]));
+    return v;
+  };
+  return { value: walk(value, ''), counts };
+}
+
+// The whole gate for one package: scrub every file, then scan. files is { name: parsed value | array of
+// parsed lines }. Returns { files, counts, problems }; a non-empty problems list means do not package.
+// `verbatim` names files that are scanned but never edited (the question pack: its hash covers every field).
+export function scrubPackage(files, ctx = localContext(), { verbatim = [] } = {}) {
+  const out = {};
+  const counts = {};
+  const problems = [];
+  for (const [name, content] of Object.entries(files)) {
+    const { value, counts: c } = verbatim.includes(name) ? { value: content, counts: {} } : scrubValue(content, ctx, { keepKeys: ['query'] });
+    out[name] = value;
+    for (const [k, n] of Object.entries(c)) counts[k] = (counts[k] || 0) + n;
+    for (const h of findSecrets(value, ctx, name)) problems.push(h);
+  }
+  return { files: out, counts, problems };
+}

package/lib/static.js

@@ -29,12 +29,23 @@ function lineOf(text, index) {
   return text.slice(0, index).split(/\r?\n/).length;
 }
 
-function hits(text, re, limit = 5) {
+function hits(text, re, limit = 5, file = null) {
   const out = [];
   re.lastIndex = 0;
   let m;
   while ((m = re.exec(text)) && out.length < limit) {
-    out.push({ line: lineOf(text, m.index), match: m[0].trim().slice(0, 60) });
+    out.push({ ...(file ? { file } : {}), line: lineOf(text, m.index), match: m[0].trim().slice(0, 60) });
+  }
+  return out;
+}
+
+// The same search across several files, each hit carrying its own file and the line inside that file. (Searching
+// one joined text gave line numbers that matched no file.)
+function hitsIn(files, re, limit = 5) {
+  const out = [];
+  for (const f of files) {
+    if (out.length >= limit) break;
+    out.push(...hits(f.text, re, limit - out.length, f.rel));
   }
   return out;
 }
@@ -88,9 +99,9 @@ export function analyze(skillDir) {
   const { fm, text } = readFrontmatter(dir);
   const files = collect(dir);
   const codeFiles = files.filter((f) => CODE_EXT.has(f.ext));
-  const codeText = codeFiles.map((f) => {
-    try { return fs.readFileSync(f.path, 'utf8'); } catch { return ''; }
-  }).join('\n');
+  const codeTexts = codeFiles.map((f) => {
+    try { return { rel: f.rel, text: fs.readFileSync(f.path, 'utf8') }; } catch { return { rel: f.rel, text: '' }; }
+  });
   const body = text.replace(/^---[\s\S]*?---/, '');
   const totalBytes = total(files);
   const size = humanSize(totalBytes);
@@ -126,30 +137,30 @@ export function analyze(skillDir) {
   const undeclared = !allowed && codeFiles.length > 0;
   const C = [];
   for (const [id, re] of [['C1', RE.network], ['C2', RE.exec], ['C3', RE.secrets], ['C4', RE.obfusc], ['C5', RE.install]]) {
-    const h = hits(codeText, re);
+    const h = hitsIn(codeTexts, re);
     const detailKey = h.length ? (undeclared ? 'check.C.escalated' : 'check.C.hits') : 'check.C.none';
     C.push(check(id, h.length ? (undeclared ? 'warn' : 'info') : 'pass', detailKey, { n: h.length }, h));
   }
 
   // D. instruction hijacking -- aimed at the reader, not the shell
   const D = [];
-  const dh = hits(text, RE.hijack);
+  const dh = hits(text, RE.hijack, 5, 'SKILL.md');
   D.push(check('D1', dh.length ? 'fail' : 'pass', dh.length ? 'check.D1.hit' : 'check.D.none', {}, dh));
-  const sec = hits(text, RE.secrets);
+  const sec = hits(text, RE.secrets, 5, 'SKILL.md');
   D.push(check('D2', sec.length ? 'warn' : 'pass', sec.length ? 'check.D2.hit' : 'check.D.none', {}, sec));
-  const ex = hits(text, RE.exfil);
+  const ex = hits(text, RE.exfil, 5, 'SKILL.md');
   D.push(check('D3', ex.length ? 'warn' : 'pass', ex.length ? 'check.D3.hit' : 'check.D.none', {}, ex));
-  const zw = hits(text, RE.zeroWidth);
+  const zw = hits(text, RE.zeroWidth, 5, 'SKILL.md');
   D.push(check('D4', zw.length ? 'fail' : 'pass', zw.length ? 'check.D4.hit' : 'check.D.none', {}, zw));
-  const img = hits(text, RE.extImage);
+  const img = hits(text, RE.extImage, 5, 'SKILL.md');
   D.push(check('D5', img.length ? 'warn' : 'pass', img.length ? 'check.D5.hit' : 'check.D.none', {}, img));
 
   // E. description quality -- the only layer that is also the trigger layer's input
   const E = [];
   E.push(check('E1', desc ? 'pass' : 'fail', desc ? 'check.E1.detail' : 'check.E1.missing', { chars: desc.length }));
-  const tw = hits(desc, RE.triggerWords, 1);
+  const tw = hits(desc, RE.triggerWords, 1, 'SKILL.md (description)');
   E.push(check('E2', tw.length ? 'pass' : 'warn', tw.length ? 'check.E2.hit' : 'check.E2.miss'));
-  const wide = hits(desc, RE.wide, 1);
+  const wide = hits(desc, RE.wide, 1, 'SKILL.md (description)');
   E.push(check('E3', wide.length ? 'warn' : 'pass', wide.length ? 'check.E3.hit' : 'check.E3.none', {}, wide));
   const clauses = (desc.match(/\bor\b|、|，|,|\//g) || []).length + 1;
   E.push(check('E4', 'info', 'check.E4.detail', { n: clauses }));