skillgauntlet 1.2.0 → 1.3.1

package/lib/runs.js

@@ -118,6 +118,11 @@ function handle(dir, meta) {
       meta.trials += 1;
       if (meta.trials % 25 === 0) flushMeta();
     },
+    // What the model said and did first, per trial. Local only: submissions copy a fixed list of files, and this
+    // is not one of them.
+    appendAnswer(row) {
+      fs.appendFileSync(path.join(dir, 'answers.jsonl'), `${JSON.stringify(row)}\n`);
+    },
     close(summary, { interrupted = false } = {}) {
       meta.finishedAt = new Date().toISOString();
       meta.interrupted = interrupted;
@@ -165,3 +170,15 @@ export function readRun(dir) {
   try { summary = JSON.parse(fs.readFileSync(path.join(dir, 'summary.json'), 'utf8')); } catch { /* unfinished */ }
   return { meta, records: raw, summary };
 }
+
+// Answers kept beside a run, keyed "queryId#run". A run made before they were kept has none: an empty map.
+export function readAnswers(dir) {
+  const out = new Map();
+  try {
+    for (const line of fs.readFileSync(path.join(dir, 'answers.jsonl'), 'utf8').split(/\r?\n/).filter(Boolean)) {
+      const row = JSON.parse(line);
+      out.set(`${row.queryId}#${row.run}`, row);
+    }
+  } catch { /* none kept */ }
+  return out;
+}

package/lib/static.js

@@ -158,9 +158,9 @@ export function analyze(skillDir) {
   // E. description quality -- the only layer that is also the trigger layer's input
   const E = [];
   E.push(check('E1', desc ? 'pass' : 'fail', desc ? 'check.E1.detail' : 'check.E1.missing', { chars: desc.length }));
-  const tw = hits(desc, RE.triggerWords, 1);
+  const tw = hits(desc, RE.triggerWords, 1, 'SKILL.md (description)');
   E.push(check('E2', tw.length ? 'pass' : 'warn', tw.length ? 'check.E2.hit' : 'check.E2.miss'));
-  const wide = hits(desc, RE.wide, 1);
+  const wide = hits(desc, RE.wide, 1, 'SKILL.md (description)');
   E.push(check('E3', wide.length ? 'warn' : 'pass', wide.length ? 'check.E3.hit' : 'check.E3.none', {}, wide));
   const clauses = (desc.match(/\bor\b|、|，|,|\//g) || []).length + 1;
   E.push(check('E4', 'info', 'check.E4.detail', { n: clauses }));

package/lib/trial.js

@@ -8,7 +8,7 @@
 //   answers without calling any tool                            -> not_triggered, counted apart
 //
 // NOT YET VERIFIED against a live host: written from the Python runner, exercised only in dry-run.
-import { execFileSync, spawn } from 'node:child_process';
+import { execFileSync, spawn, spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
@@ -31,12 +31,19 @@ export const HOSTS = {
 export function backendFor(host, model) {
   if (host === 'codewhale') return 'codewhale:api.deepseek.com';
   if (host !== 'codebuddy') return backendHost();
-  return isDeepseekModel(model) ? 'codebuddy:api.deepseek.com' : 'codebuddy';
+  return usesOwnDeepseek('codebuddy', model) ? 'codebuddy:api.deepseek.com' : 'codebuddy';
 }
 
 // A model id starting with "deepseek" means the DeepSeek API, whichever host drives it.
 export const isDeepseekModel = (model) => /^deepseek/i.test(String(model || ''));
 
+// A DeepSeek model that the host lists itself (CodeBuddy's own deepseek-v4.1-flash) is served by the host's account
+// and needs no key of ours. Only a deepseek-* name the host does not list takes the package's own DeepSeek route
+// (a models.json entry in the trial directory and DEEPSEEK_API_KEY from the environment).
+const nativeModels = new Map();
+export const setNativeModels = (host, list) => nativeModels.set(host, new Set(list || []));
+export const usesOwnDeepseek = (host, model) => isDeepseekModel(model) && !nativeModels.get(host)?.has(model);
+
 // DeepSeek's integration guide for WorkBuddy/CodeBuddy: a models.json entry whose key is read from the
 // environment (${DEEPSEEK_API_KEY}), so the file holds no secret. It is written into the trial's own
 // directory (project level), never into the user's ~/.codebuddy.
@@ -111,18 +118,35 @@ export function requireHost(host, probe = binInstalled) {
 
 // One tiny call before the real work: a host that is not signed in retries silently for minutes, so the user
 // would stare at a frozen screen. A short timeout turns that into an immediate, actionable message.
-export function pingHost(host, { bin, model = null, timeout = 90, run = execFileSync } = {}) {
+// The first non-empty line of some CLI output, for an error message.
+const firstLine = (text) => String(text || '').split(LINE_BREAK).map((l) => l.trim()).find(Boolean) || '';
+
+// What a host CLI answered, as an error reason: the CLI's own words when it gave any. A CLI can exit 0 with nothing on
+// stdout and its complaint on stderr (CodeBuddy does for a custom model without an endpoint url), so a clean exit
+// status proves nothing; an empty or unreadable stdout is the failure.
+export function hostAnswerProblem(r, timeout) {
+  if (r.error) return r.error.code === 'ETIMEDOUT' ? `no answer within ${timeout}s` : String(r.error.message).split(LINE_BREAK)[0];
+  if (r.signal) return `no answer within ${timeout}s`;
+  const out = String(r.stdout || '');
+  if (r.status !== 0) return firstLine(r.stderr) || firstLine(out) || `exit status ${r.status}`;
+  if (!out.trim()) return firstLine(r.stderr) || 'the CLI answered with nothing';
+  try {
+    const parsed = JSON.parse(out);
+    const last = Array.isArray(parsed) ? parsed.filter((m) => m?.type === 'result').pop() : parsed;
+    if (last?.is_error) return firstLine(last.result) || 'the CLI reported an error';
+  } catch { return firstLine(out); }
+  return null;
+}
+
+export function pingHost(host, { bin, model = null, timeout = 90, spawn = spawnSync } = {}) {
   const spec = HOSTS[host];
   if (!spec || host === 'codewhale') return;
   const args = ['-p', 'Reply with the single word OK.', '--output-format', 'json', '--tools', '', '--no-session-persistence'];
-  if (model && !isDeepseekModel(model)) args.push('--model', model);
+  if (model && !usesOwnDeepseek(host, model)) args.push('--model', model); // the package's own DeepSeek route has no host-side name
   const launch = launchSpec(bin, args, spec.bin);
-  try {
-    run(launch.cmd, launch.args, { encoding: 'utf8', timeout: timeout * 1000, shell: launch.shell, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
-  } catch (err) {
-    const why = err.code === 'ETIMEDOUT' || err.killed
-      ? `no answer within ${timeout}s`
-      : String(err.stderr || err.message).split(LINE_BREAK).map((l) => l.trim()).find(Boolean) || `exit status ${err.status}`;
+  const r = spawn(launch.cmd, launch.args, { encoding: 'utf8', timeout: timeout * 1000, shell: launch.shell, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
+  const why = hostAnswerProblem(r, timeout);
+  if (why) {
     const e = new Error(`host ${host} did not answer: ${why}`);
     e.messageKey = 'hostNotReady';
     e.vars = { host: HOST_NAME[host] || host, why: why.slice(0, 200) };
@@ -130,6 +154,21 @@ export function pingHost(host, { bin, model = null, timeout = 90, run = execFile
   }
 }
 
+// What a trial shows of the model, kept apart from the verdict: the words it said before its first action and what that
+// action was. A trial ends at the first action (the verdict is known then), so this is "up to the first step", never a
+// complete answer. It goes to a local file, not into the run's raw records, which are what gets submitted.
+export const SAID_LIMIT = 2000;
+export function inputSummary(name, input) {
+  if (!input || typeof input !== 'object' || !Object.keys(input).length) return '';
+  const pick = input.command ?? input.file_path ?? input.path ?? input.pattern ?? input.skill ?? input.name ?? input.query;
+  const text = typeof pick === 'string' ? pick : JSON.stringify(input);
+  return `${text}`.slice(0, 300);
+}
+export function addSaid(rec, text) {
+  if (typeof text !== 'string' || !text) return;
+  rec.said = `${rec.said || ''}${text}`.slice(0, SAID_LIMIT);
+}
+
 // Variables a host (the Claude desktop app, the Agent SDK, an outer Claude Code session) sets for its own
 // child processes. A trial inherits them otherwise, and they change what the model under test sees: under
 // the desktop app the CLI adds Artifact, ArtifactComments and ArtifactData to the tool list, and the model
@@ -216,7 +255,7 @@ export function runTrial({ bin, skillDir, skillName, query, model, timeout = 180
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sg-'));
     const spec = HOSTS[host];
     copyDir(skillDir, path.join(tmp, ...spec.skillsDir, skillName));
-    if (host === 'codebuddy' && isDeepseekModel(model)) writeModelsJson(tmp, model);
+    if (host === 'codebuddy' && usesOwnDeepseek('codebuddy', model)) writeModelsJson(tmp, model);
     const args = host === 'codebuddy'
       ? ['-p', query, '--output-format', 'stream-json', '--verbose', '--setting-sources', 'project', '--strict-mcp-config',
         '--no-session-persistence', '--tools', spec.tools]
@@ -275,6 +314,7 @@ export function runTrial({ bin, skillDir, skillName, query, model, timeout = 180
           rec.init = { model: ev.model, apiKeySource: ev.apiKeySource, claudeCodeVersion: ev.claude_code_version,
             skills: ev.skills || [], tools: ev.tools || [] };
         } else if (type === 'stream_event') {
+          if (ev.event?.delta?.type === 'text_delta') addSaid(rec, ev.event.delta.text);
           const block = ev.event?.content_block;
           if (block?.type === 'tool_use') {
             rec.events.toolUse += 1;
@@ -287,9 +327,11 @@ export function runTrial({ bin, skillDir, skillName, query, model, timeout = 180
           }
         } else if (type === 'assistant') {
           for (const c of ev.message?.content || []) {
+            if (c.type === 'text') addSaid(rec, c.text);
             if (c.type !== 'tool_use') continue;
             rec.tools.push(c.name);
             rec.firstTool = rec.firstTool || c.name;
+            rec.firstInput = rec.firstInput || inputSummary(c.name, c.input);
             if (c.name === 'Skill') {
               const inv = String(c.input?.skill || '');
               if (inv === skillName || inv.endsWith(`:${skillName}`)) finish('triggered', { invokedSkill: inv });
@@ -389,23 +431,24 @@ ${text}`;
     const launch = launchSpec(bin, args, HOSTS[host].bin);
     // CodeBuddy reads a custom model from the working directory's .codebuddy/models.json, so a DeepSeek draft
     // runs in a scratch directory that has one.
-    const scratch = host === 'codebuddy' && isDeepseekModel(model) ? fs.mkdtempSync(path.join(os.tmpdir(), 'sg-draft-')) : null;
+    const scratch = host === 'codebuddy' && usesOwnDeepseek('codebuddy', model) ? fs.mkdtempSync(path.join(os.tmpdir(), 'sg-draft-')) : null;
     if (scratch) writeModelsJson(scratch, model);
-    let out;
+    let r;
     try {
-      out = execFileSync(launch.cmd, launch.args, {
+      r = spawnSync(launch.cmd, launch.args, {
         encoding: 'utf8', timeout: timeout * 1000, maxBuffer: 32 * 1024 * 1024,
-        shell: launch.shell, windowsHide: true, ...(scratch ? { cwd: scratch } : {}),
+        shell: launch.shell, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'], ...(scratch ? { cwd: scratch } : {}),
       });
-    } catch (err) {
-      // The default message of a failed execFileSync is the whole command line, which here is the drafting prompt.
-      const why = err.code === 'ETIMEDOUT' || err.killed
-        ? `no answer within ${timeout}s (a rejected key or an unreachable endpoint retries silently until then)`
-        : String(err.stderr || err.message).split(/\r?\n/).map((l) => l.trim()).find(Boolean) || 'exit status ' + err.status;
-      throw new Error(`the drafting call failed: ${why}`);
     } finally {
       if (scratch) { try { fs.rmSync(scratch, { recursive: true, force: true }); } catch { /* best effort */ } }
     }
+    // The CLI's own words, not "Unexpected end of JSON input": an empty answer with a complaint on stderr is common.
+    const problem = hostAnswerProblem(r, timeout);
+    if (problem) {
+      const hint = r.error?.code === 'ETIMEDOUT' || r.signal ? ' (a rejected key or an unreachable endpoint retries silently until then)' : '';
+      throw new Error(`the drafting call failed: ${problem}${hint}`);
+    }
+    const out = r.stdout;
     const parsed = JSON.parse(out);
     // Claude Code answers with one result object; CodeBuddy Code with the whole message list, whose last
     // element is the result.

package/lib/update.js

@@ -0,0 +1,73 @@
+// A check for a newer published version before an interactive session. It must never get in the way: any failure
+// (offline, slow registry, odd reply, unwritable cache) means "no update known" and the session goes on with the
+// version that is running.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REGISTRY = 'https://registry.npmjs.org/skillgauntlet/latest';
+const CHECK_EVERY_MS = 12 * 60 * 60 * 1000;
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export function currentVersion() {
+  try { return JSON.parse(fs.readFileSync(path.join(here, '..', 'package.json'), 'utf8')).version; } catch { return null; }
+}
+
+// a.b.c with an optional pre-release tail; a pre-release is older than its release.
+export function isNewer(latest, current) {
+  const parse = (v) => {
+    const m = String(v).trim().match(/^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/);
+    return m ? { n: [Number(m[1]), Number(m[2]), Number(m[3])], pre: m[4] || null } : null;
+  };
+  const a = parse(latest);
+  const b = parse(current);
+  if (!a || !b) return false;
+  for (let i = 0; i < 3; i += 1) if (a.n[i] !== b.n[i]) return a.n[i] > b.n[i];
+  return Boolean(b.pre) && !a.pre;
+}
+
+// A copy that runs from a git checkout is a developer's own; it is not nagged about releases.
+export function isCheckout() {
+  return fs.existsSync(path.join(here, '..', '.git'));
+}
+
+function cachePath() {
+  return path.join(process.env.SKILLGAUNTLET_CACHE || path.join(os.homedir(), '.cache', 'skillgauntlet'), 'update-check.json');
+}
+
+function readCache(file) {
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return {}; }
+}
+
+function writeCache(file, data) {
+  try {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, `${JSON.stringify(data)}\n`);
+  } catch { /* the cache is an optimisation */ }
+}
+
+// { current, latest } when a newer version is known and has not been declined, else null. Never throws.
+export async function checkForUpdate({ fetchLatest, now = Date.now(), file = cachePath(), current = currentVersion(), env = process.env } = {}) {
+  try {
+    if (!current || env.SG_NO_UPDATE_CHECK || env.CI) return null;
+    const cache = readCache(file);
+    let latest = cache.latest;
+    if (!latest || !cache.checkedAt || now - cache.checkedAt > CHECK_EVERY_MS) {
+      const get = fetchLatest || (async () => {
+        const res = await fetch(REGISTRY, { signal: AbortSignal.timeout(3000), headers: { accept: 'application/json' } });
+        if (!res.ok) throw new Error(`registry ${res.status}`);
+        return (await res.json()).version;
+      });
+      latest = await get();
+      writeCache(file, { ...cache, latest, checkedAt: now });
+    }
+    if (!isNewer(latest, current) || cache.declined === latest) return null;
+    return { current, latest };
+  } catch { return null; }
+}
+
+// Remember that the user said no to this version, so the question is not asked again until a newer one appears.
+export function declineUpdate(latest, file = cachePath()) {
+  writeCache(file, { ...readCache(file), declined: latest });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skillgauntlet",
-  "version": "1.2.0",
+  "version": "1.3.1",
   "description": "Check an agent skill before you install it: context cost, permission surface, code risk, instruction hijacking, description quality, trigger rate and gain.",
   "type": "module",
   "bin": {