skilldrop-cli 0.16.31 → 0.16.32

package/bin/skilldrop.js

@@ -66,7 +66,7 @@ Usage:
   skilldrop uninstall --agent <name...>   remove subagents
   skilldrop uninstall --loop <name...>    remove loops (stage skills are left in place)
   skilldrop validate [--from <src>]       structural check of a catalog (for catalog authors)
-  skilldrop scan [<skill...>] [--from <src>]  supply-chain scan — flag network/exec/credential
+  skilldrop scan [<skill...>] [--agent] [--from <src>]  supply-chain scan of skills and agents — flag network/exec/credential
                                           patterns in scripts and injection-shaped instructions
                                           in SKILL.md before you trust a catalog (RFC-0022)
   skilldrop new-skill <name> --pack <p>   scaffold a skill in the catalog you're in (for catalog authors;
@@ -440,11 +440,11 @@ function gate(cat, names) {
 
 /* RFC-0050: the worst scan finding per skill, for a policy's scan.max. Eval fixtures and
    example inputs (⚪) never count. */
-function worstFindings(cat, names) {
+function worstFindings(cat, names, scanner = scanSkill) {
   const out = {};
   for (const s of names) {
     let w;
-    for (const f of scanSkill(cat, s)) if (!f.fixture && (!w || gov.SEVERITY[f.sev] > gov.SEVERITY[w])) w = f.sev;
+    for (const f of scanner(cat, s)) if (!f.fixture && (!w || gov.SEVERITY[f.sev] > gov.SEVERITY[w])) w = f.sev;
     if (w) out[s] = w;
   }
   return out;
@@ -455,7 +455,8 @@ function enforcePolicy(flags, cat, names, kind) {
   const policy = gov.load(flags);
   gov.prepare(policy);  // an audit log that can't be written stops the install before it starts
   if (!policy || !names.length) return policy;
-  const worst = kind === "skill" && policy.scan && policy.scan.max !== undefined ? worstFindings(cat, names) : {};
+  const worst = policy.scan && policy.scan.max !== undefined && (kind === "skill" || kind === "agent")
+    ? worstFindings(cat, names, kind === "agent" ? scanAgent : scanSkill) : {};
   const problems = gov.check(policy, { source: cat.source, pinned: cat.pinned, names, worst, kind });
   if (problems.length) {
     for (const pr of problems) console.error(`refused ${pr.name}: ${pr.why}`);
@@ -614,10 +615,73 @@ function permissionsOf(cat, s) {
   if (cat.shape === "apm") return null;
   try { return manifestOf(cat, s).permissions || null; } catch (e) { return null; }
 }
+/* RFC-0054: what a skill's instructions tell the agent to run. A skill with no scripts still
+   reaches the shell and the network through the agent, so its fenced shell blocks are read the
+   way a reviewer would: the program each command starts, and the hosts it names. Shell builtins,
+   keywords and the text tools every agent already has are left out, as are interpreters running
+   the skill's own scripts/ (those are covered by the script rules). Prose outside a fenced shell
+   block is not parsed: an instruction to run something belongs in one, where a reader sees it. */
+const SHELL_LANGS = new Set(["bash", "sh", "shell", "console", "zsh"]);
+const NOT_A_PROGRAM = new Set(("cd echo export set unset true false test printf read source exit return then fi do done " +
+  "for if elif else while until case esac in function local shift wait time eval exec alias type command " +
+  "mkdir ls cat cp mv rm touch head tail grep egrep sed awk sort uniq wc tr cut xargs tee diff find jq less more " +
+  "pwd basename dirname date sleep env which chmod ln realpath mktemp yes seq").split(" "));
+const INTERPRETERS = new Set(["python", "python3", "node", "bash", "sh", "zsh", "deno", "bun"]);
+function instructedRuns(dir) {
+  const files = ["SKILL.md", "reference.md"].map((f) => path.join(dir, f)).filter((f) => fs.existsSync(f));
+  const refs = path.join(dir, "references");
+  if (fs.existsSync(refs)) for (const f of fs.readdirSync(refs)) if (f.endsWith(".md")) files.push(path.join(refs, f));
+  const commands = new Map(), hosts = new Map();
+  for (const f of files) {
+    const lines = fs.readFileSync(f, "utf8").split(/\r?\n/);
+    let lang = null;
+    lines.forEach((raw, i) => {
+      const fence = raw.match(/^\s*```+\s*([\w-]*)/);
+      if (fence) { lang = lang === null ? fence[1].toLowerCase() : null; return; }
+      if (lang === null || !SHELL_LANGS.has(lang)) return;
+      const where = { file: path.relative(dir, f), line: i + 1, text: raw.trim().slice(0, 120) };
+      const line = raw.replace(/(^|\s)#.*$/, "").replace(/^\s*\$\s+/, "").trim();
+      if (!line) return;
+      for (const h of line.matchAll(/https?:\/\/([a-z0-9][a-z0-9.-]*\.[a-z]{2,})/gi))
+        if (!hosts.has(h[1].toLowerCase())) hosts.set(h[1].toLowerCase(), where);
+      // Quoted text is an argument (a grep pattern, a message), never a command, so blank it
+      // before splitting on | ; && — otherwise `grep 'a|b'` reads as a command named b.
+      const bare = line.replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, (q) => (/scripts\/|CLAUDE_SKILL_DIR/.test(q) ? q.slice(1, -1) : " X "));
+      for (let part of bare.replace(/\$\(/g, " ; ").replace(/[`()]/g, " ").split(/&&|\|\||\||;/)) {
+        let w = part.trim().split(/\s+/).filter(Boolean);
+        while (w.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(w[0])) w.shift();
+        if (w[0] === "sudo" || w[0] === "nohup") w.shift();
+        const c = w[0];
+        if (!c || !/^[a-z][\w.+-]*$/.test(c) || NOT_A_PROGRAM.has(c)) continue;
+        if (INTERPRETERS.has(c) && w.slice(1).some((x) => /(^|\/)scripts\//.test(x) || /CLAUDE_SKILL_DIR/.test(x))) continue;
+        const prog = INTERPRETERS.has(c) && w[1] === "-m" && /^[a-z][\w.-]*$/.test(w[2] || "") ? w[2] : c;  // python3 -m pip -> pip
+        if (!commands.has(prog)) commands.set(prog, where);
+      }
+    });
+  }
+  return { commands, hosts };
+}
+
 function permissionGaps(cat, s, findings) {
   const dir = skillDir(cat, s);
-  if (!dir || !fs.existsSync(path.join(dir, "scripts"))) return [];
+  if (!dir) return [];
+  const scripted = fs.existsSync(path.join(dir, "scripts"));
   const perm = permissionsOf(cat, s);
+  const told = cat.shape === "apm" ? { commands: new Map(), hosts: new Map() } : instructedRuns(dir);
+  const instructionGaps = () => {
+    const out = [];
+    const cmds = (perm && perm.commands) || [], net = (perm && perm.network) || [];
+    for (const [c, w] of told.commands)
+      if (!cmds.includes("*") && !cmds.includes(c))
+        out.push({ sev: "🟨", id: "undeclared-instructed-command", owasp: ["AST03", "AST10"], ...w,
+                   why: `the instructions tell the agent to run \`${c}\`, but the manifest's permissions.commands doesn't list it` });
+    for (const [h, w] of told.hosts)
+      if (!net.includes(h))
+        out.push({ sev: "🟨", id: "undeclared-instructed-network", owasp: ["AST03", "AST10"], ...w,
+                   why: `the instructions tell the agent to reach ${h}, but the manifest's permissions.network doesn't list it` });
+    return out;
+  };
+  if (!scripted) return instructionGaps();
   if (!perm)
     return [{ sev: "🟨", id: "no-permissions", owasp: ["AST03", "AST10"], file: "manifest.json", line: 0,
               why: "ships scripts but declares no permissions (network, commands, files) — nothing to check them against",
@@ -633,7 +697,7 @@ function permissionGaps(cat, s, findings) {
     if (f.id === "broad-fs" && perm.files !== "anywhere")
       gap("undeclared-files", f, `writes outside its own paths, but the manifest declares files: "${perm.files}"`);
   }
-  return gaps;
+  return gaps.concat(instructionGaps());
 }
 function permissionSummary(perm) {
   if (!perm) return "no permissions declared";
@@ -643,35 +707,82 @@ function permissionSummary(perm) {
   return `${net} · ${cmds} · ${files}`;
 }
 
-function printScan(cat, names, { compact = false } = {}) {
+/* RFC-0054: an agent is a system prompt the tool adopts whole, with the tools its frontmatter
+   grants, so it gets the prose rules a SKILL.md gets plus the checks only an agent needs. */
+function scanAgent(cat, a) {
+  const file = agentPath(cat, a);
+  const findings = [];
+  if (!file || !fs.existsSync(file)) return findings;
+  const rel = path.basename(file);
+  const text = fs.readFileSync(file, "utf8");
+  text.split(/\r?\n/).forEach((line, i) => {
+    if (line.length > 400) return;
+    for (const r of PROSE_RULES)
+      if (r.re.test(line)) findings.push({ sev: r.sev, id: r.id, why: r.why, owasp: r.owasp, file: rel, line: i + 1, text: line.trim().slice(0, 120) });
+  });
+  const fmEnd = text.startsWith("---") ? text.indexOf("\n---", 3) : -1;
+  const fm = fmEnd > 0 ? text.slice(0, fmEnd) : "";
+  const lineOf = (re) => { const i = fm.split("\n").findIndex((l) => re.test(l)); return i < 0 ? 1 : i + 1; };
+  const at = (re) => ({ file: rel, line: lineOf(re), text: (fm.split("\n")[lineOf(re) - 1] || "").trim().slice(0, 120) });
+  const tools = (fm.match(/^tools:\s*(.*)$/m) || [null, null])[1];
+  const list = tools === null ? null : tools.replace(/[[\]"']/g, "").split(",").map((t) => t.trim()).filter(Boolean);
+  if (list === null)
+    findings.push({ sev: "🟧", id: "agent-all-tools", owasp: ["AST03", "LLM06"], ...at(/^name:/),
+                    why: "has no `tools:` list, so it inherits every tool the session has, including any MCP server's" });
+  else {
+    const reads = list.filter((t) => /^(WebFetch|WebSearch)$/.test(t) || /^mcp__/.test(t));
+    const acts = list.filter((t) => /^(Bash|Write|Edit|NotebookEdit)$/.test(t));
+    if (reads.length && acts.length)
+      findings.push({ sev: "🟧", id: "agent-untrusted-and-act", owasp: ["AST03", "LLM01", "LLM06"], ...at(/^tools:/),
+                      why: `reads content from outside (${reads.join(", ")}) and can act on it (${acts.join(", ")}) in the same context: an injected instruction in what it reads can run` });
+  }
+  if (/^permissionMode:\s*(bypassPermissions|acceptEdits)/m.test(fm))
+    findings.push({ sev: "🟥", id: "agent-skips-approval", owasp: ["AST03", "LLM06"], ...at(/^permissionMode:/),
+                    why: "sets a permission mode that skips the user's approval for its actions" });
+  if (/^hooks:/m.test(fm))
+    findings.push({ sev: "🟧", id: "agent-hooks", owasp: ["AST03", "AST01"], ...at(/^hooks:/),
+                    why: "defines hooks: commands that run on events, outside the conversation the user reads" });
+  const order = { "🟥": 0, "🟧": 1, "🟨": 2, "⚪": 3 };
+  return findings.sort((x, y) => order[x.sev] - order[y.sev]);
+}
+
+function printScan(cat, names, { compact = false, scanner = scanSkill, kind = "" } = {}) {
   let total = 0, worst = null;
   for (const s of names) {
-    const f = scanSkill(cat, s);
+    const f = scanner(cat, s);
     if (!f.length) continue;
     total += f.length;
     if (!worst || f[0].sev === "🟥") worst = worst === "🟥" ? worst : f[0].sev;
-    console.log(`\n${s}`);
+    console.log(`\n${kind === "agent" ? "agent " : ""}${s}`);
     for (const x of (compact ? f.slice(0, 3) : f))
       console.log(`  ${x.sev} ${x.id} — ${x.why}  [${x.owasp.join(", ")}]\n     ${x.file}:${x.line}  ${x.text}`);
-    if (compact && f.length > 3) console.log(`     … ${f.length - 3} more (skilldrop scan ${s} --from ${cat.source})`);
+    if (compact && f.length > 3) console.log(`     … ${f.length - 3} more (skilldrop scan ${kind === "agent" ? "--agent " : ""}${s} --from ${cat.source})`);
   }
   return { total, worst };
 }
 
 function scan(args) {
   const cat = resolveCatalog(args.flags.from);
-  const names = args._.length ? args._.slice() : skillsIn(cat);
+  // `scan` alone covers every skill and agent; `scan <skills>` the named skills;
+  // `scan --agent [names]` agents only (--agents works too) (an agent may share a skill's name).
+  const agentsOnly = !!(args.flags.agent || args.flags.agents);
+  const names = agentsOnly ? [] : (args._.length ? args._.slice() : skillsIn(cat));
+  const agents = agentsOnly ? (args._.length ? args._.slice() : agentsIn(cat)) : (args._.length ? [] : agentsIn(cat));
   for (const s of names) if (!skillExists(cat, s)) die(`unknown skill '${s}' in catalog '${cat.source}'`);
+  for (const a of agents) if (!agentsIn(cat).includes(a)) die(`unknown agent '${a}' in catalog '${cat.source}'`);
 
   if (args.flags.json) {
     const out = names.map((s) => ({ skill: s, findings: scanSkill(cat, s) })).filter((r) => r.findings.length);
+    const aout = agents.map((a) => ({ agent: a, findings: scanAgent(cat, a) })).filter((r) => r.findings.length);
     return emitJSON({
-      catalog: cat.source, scanned: names.length,
-      flagged: out.length, findingCount: out.reduce((n, r) => n + r.findings.length, 0), skills: out,
+      catalog: cat.source, scanned: names.length, agentsScanned: agents.length,
+      flagged: out.length + aout.length,
+      findingCount: out.reduce((n, r) => n + r.findings.length, 0) + aout.reduce((n, r) => n + r.findings.length, 0),
+      skills: out, agents: aout,
     });
   }
-  console.log(`Scanning ${names.length} skill(s) in catalog '${cat.source}' for supply-chain patterns…`);
-  const { total } = printScan(cat, names);
+  console.log(`Scanning ${names.length} skill(s)${agents.length ? ` and ${agents.length} agent(s)` : ""} in catalog '${cat.source}' for supply-chain patterns…`);
+  const total = printScan(cat, names).total + printScan(cat, agents, { scanner: scanAgent, kind: "agent" }).total;
   if (!total) {
     console.log("\nNo flagged patterns found.");
   } else {
@@ -1373,9 +1484,9 @@ function install(args) {
     console.log(`\nWARNING: third-party catalog '${cat.source}'. Skills are instructions your AI agent will follow — review each installed SKILL.md under ${dest} before first use. Install copied files only; nothing was executed.`);
     // Supply-chain scan (RFC-0022): surface risky patterns at the moment of trust. Reports only —
     // the files are already copied, and a match is a prompt to read, not a verdict.
-    const scripted = names.filter((s) => fs.existsSync(path.join(skillDir(cat, s), "scripts")));
+    const scripted = names.filter((s) => fs.existsSync(path.join(skillDir(cat, s), "scripts")) || permissionsOf(cat, s));
     if (scripted.length) {
-      console.log(`\nWhat the scripts declare they do (RFC-0039):`);
+      console.log(`\nWhat each skill declares it runs and reaches (RFC-0039, RFC-0054):`);
       for (const s of scripted) console.log(`  ${s}: ${permissionSummary(permissionsOf(cat, s))}`);
     }
     const { total } = printScan(cat, names, { compact: true });
@@ -2255,8 +2366,12 @@ function installAgents(args) {
   if (ide === "copilot") console.log("Invoke with: copilot --agent <name>");
   if (ide === "codex") console.log("sandbox_mode is left unset, so each agent inherits the parent session's permissions.");
   if (ide === "antigravity") console.log("Emitted with `subagent: true` so invoke_subagent can reach them.");
-  if (cat.source !== BUNDLED)
+  if (cat.source !== BUNDLED) {
     console.log(`\nWARNING: third-party catalog '${cat.source}'. An agent is a system prompt your tool will adopt — read it before first use. Install copied files only; nothing was executed.`);
+    const { total } = printScan(cat, names, { compact: true, scanner: scanAgent, kind: "agent" });
+    console.log(total ? `\n${total} pattern(s) worth reading before you trust these agents. Full detail: skilldrop scan --agent --from ${cat.source}`
+                      : "\nSupply-chain scan: no flagged patterns.");
+  }
 }
 
 function uninstallAgents(args) {

package/contracts/skill.schema.json

@@ -42,12 +42,12 @@
       "type": "object",
       "additionalProperties": false,
       "required": ["network", "commands", "files"],
-      "description": "RFC-0039 (OWASP AST03, AST10). What the skill's scripts do, declared so a reader and `skilldrop scan` can compare it with what the code does. Required for every skill that ships scripts/.",
+      "description": "RFC-0039, RFC-0054 (OWASP AST03, AST10). What the skill's scripts do and what its instructions tell the agent to run, declared so a reader and `skilldrop scan` can compare it with the code and with every fenced shell block in SKILL.md and its reference files. Required for every skill that ships scripts/, and for any skill whose instructions run a program or reach a host.",
       "properties": {
         "network": { "type": "array", "items": { "type": "string" },
-                     "description": "Hosts the scripts contact, e.g. api.figma.com. Empty means none." },
+                     "description": "Hosts the scripts contact or the instructed commands reach, e.g. api.figma.com. Empty means none." },
         "commands": { "type": "array", "items": { "type": "string" },
-                      "description": "External programs the scripts run, e.g. pdftotext. \"*\" means commands the user configures. Empty means none." },
+                      "description": "External programs the scripts run or the instructions tell the agent to run, e.g. pdftotext, terraform. \"*\" means commands the user configures. Empty means none." },
         "files": { "type": "string", "enum": ["none", "named-paths", "project", "anywhere"],
                    "description": "What the scripts write: nothing; only paths the user names; inside the current project; or anywhere." },
         "notes": { "type": "string", "description": "Anything a reviewer should know that the fields can't say." }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skilldrop-cli",
-  "version": "0.16.31",
+  "version": "0.16.32",
   "description": "Portable AI-agent skills, measured against the agent without them. ADRs, PRDs, runbooks, decks and reviews, installable into Claude Code, Cursor, Kiro, and any IDE that takes custom instructions. One command per skill or per role-based pack.",
   "author": {
     "name": "Sanjay Ananthanarayan",

package/packs/converters/skills/mermaid-render/manifest.json

@@ -32,7 +32,17 @@
     "png",
     "render"
   ],
-  "permissions": {"network": [], "commands": ["mmdc"], "files": "named-paths", "notes": "mmdc (mermaid-cli) renders SVG, PNG and PDF if it is installed. Without it the script writes an HTML page that loads Mermaid from cdn.jsdelivr.net when opened; the script itself makes no network calls."},
+  "permissions": {
+    "network": [
+      "registry.npmjs.org"
+    ],
+    "commands": [
+      "mmdc",
+      "npm"
+    ],
+    "files": "named-paths",
+    "notes": "mmdc (mermaid-cli) renders SVG, PNG and PDF if it is installed. Without it the script writes an HTML page that loads Mermaid from cdn.jsdelivr.net when opened; the script itself makes no network calls. Its instructions suggest installing mermaid-cli with `npm install -g @mermaid-js/mermaid-cli`, which reaches the npm registry."
+  },
   "model": {
     "tier": "standard",
     "rationale": "The script lints and renders deterministically, but fixing a broken diagram means reading the author's intent and rewriting syntax without changing nodes, edges or labels, and knowing when to ask instead."

package/packs/dev-team/skills/codebase-tour/evals/files/3/.github/workflows/ci.yml

@@ -7,7 +7,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.25"
       - run: make lint
       - run: make test
   integration:
@@ -24,5 +24,5 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.25"
       - run: make test-integration

package/packs/dev-team/skills/codebase-tour/evals/files/3/go.mod

@@ -1,5 +1,5 @@
 module example.com/tidepool
 
-go 1.22
+go 1.25.0
 
-require github.com/jackc/pgx/v5 v5.5.5
+require github.com/jackc/pgx/v5 v5.9.2

package/packs/dev-team/skills/dependency-upgrade-review/SKILL.md

@@ -1,11 +1,11 @@
 ---
 name: dependency-upgrade-review
-description: Give a merge verdict on a dependency-upgrade pull request (Dependabot, Renovate or a hand-made bump) across npm, pip, Go, Cargo and Maven lockfiles and GitHub Actions pins. Reads the release notes for every skipped version, lists what the lockfile changes beyond the PR title (transitive bumps, new and removed packages, install scripts, registry changes), checks that each SHA pin is the commit its release tag points to, says whether CI ran the changed code, finds the repo's uses of changed APIs and writes the rollback, ending in MERGE, MERGE AFTER CHECKS or DO NOT MERGE. Use when the user says "review this Dependabot PR", "is this Renovate bump safe to merge", "can I merge this dependency upgrade", "check this lockfile diff", or "is this action SHA pin right".
+description: Give a merge verdict on a dependency-upgrade pull request (Dependabot, Renovate or a hand-made bump) across npm, pip, Go, Cargo and Maven lockfiles and GitHub Actions pins. Reads the release notes for every skipped version, lists what the lockfile changes beyond the PR title (transitive bumps, new and removed packages, install scripts, registry changes), checks each SHA pin against its release tag and against the PR's other pins, checks raised runtime floors against the images and runners that install it, says whether CI ran the changed code, traces what each caller of a changed API now does and writes the rollback. The reply leads with MERGE, MERGE AFTER CHECKS or DO NOT MERGE and the reasons that decide it. Use when the user says "review this Dependabot PR", "is this Renovate bump safe to merge", "can I merge this dependency upgrade", "check this lockfile diff", or "is this action SHA pin right".
 ---
 
 # dependency-upgrade-review
 
-A dependency bump is someone else's code entering the repo under a one-line title. This skill reads it the way a careful maintainer does: every release between the old and new version, every line of the lockfile, every pin, and the tests that would have caught a break. The verdict follows from those checks, not from a green badge. Reviewing the repo's own code changes is `pre-merge-review`; writing the notes for a release of this project is `release-notes`; planning a framework migration across many PRs is `migration-plan`.
+A dependency bump is someone else's code entering the repo under a one-line title. This skill reads it the way a careful maintainer does: every release between the old and new version, every line of the lockfile, every pin, every place the code installs and runs, and the tests that would have caught a break. The verdict follows from those checks, not from a green badge. The reply is short: the verdict and what decides it come first, and the checks that found nothing take one line. Reviewing the repo's own code changes is `pre-merge-review`; writing the notes for a release of this project is `release-notes`; planning a framework migration across many PRs is `migration-plan`.
 
 ## How to respond
 
@@ -15,11 +15,13 @@ A dependency bump is someone else's code entering the repo under a one-line titl
 - **Use only what you were given.** Don't add facts, names, numbers, incidents, history, steps or sections the input doesn't contain. What you need and don't have is left out of the artifact and listed once at the end under "To confirm".
 - **Deliver from what you have.** When the request gives you something to work on, state your assumptions in a line and produce the result. When it gives you nothing to work on, ask for it in one or two plain sentences and say what you will do once you have it.
 - **Write for someone who has never heard of this skill.** No skill names, no paths or scripts from this folder, no internal terms, and nothing about how the run was set up. A next step is one plain sentence at the end that describes the work.
-- **For this skill:** A check nobody ran is an open check, written as the command that runs it. It is never reported as passed, and "CI is green" closes only the checks CI actually ran.
+- **For this skill:** A check nobody ran is an open check, written as the command that runs it. It is never reported as passed, and "CI is green" closes only the checks CI actually ran. Say what was not checked and the command that checks it; never narrate the session (what was blocked, denied or unavailable).
 
-1. **Get the diff, the notes and the code, asking once.** Three inputs: the PR diff covering manifests, lockfiles and workflow files (`gh pr diff 212` or `git diff origin/main...HEAD`); the release notes or changelog for every version between old and new; the repo, or at least the files that import the package and the CI workflow files. Ask for what is missing in one message. With network access, fetch missing notes from the package's releases page or changelog; without it, each missing release becomes an open check.
+Steps 1 to 10 are the review. Step 11 is the reply, and most of what the review turns up stays out of it.
 
-2. **Run the diff script first.** It lists every changed dependency with old → new, semver class, direct or transitive, and flags added and removed packages, install hooks, native-code hints, registry changes, licence changes and pins, then prints a floor verdict.
+1. **Get the diff, the notes and the code, asking once.** Three inputs: the PR diff covering manifests, lockfiles and workflow files (`gh pr diff 212` or `git diff origin/main...HEAD`); the release notes or changelog for every version between old and new; the repo, or at least the files that import the package and the CI workflow files. Also read what decides where the code installs and runs: the Dockerfile or base image, every workflow that installs dependencies (release, deploy and scheduled ones too, not only the PR's), `.nvmrc` / `.python-version` / `engines`, and any note in the PR body of the runtime the lockfile was generated on. When the PR has more than one commit, note who made each; a hand-made commit on a bot's PR is read first. Ask for what is missing in one message. With network access, fetch missing notes; without it, each missing release becomes an open check.
+
+2. **Run the diff script first.** It lists every changed dependency with old → new, semver class, direct or transitive, and flags added and removed packages, install hooks, native-code hints, registry changes, licence changes, runtime requirement changes, pins, and one tag pinned to different commits, then prints a floor verdict.
    ```bash
    # Claude Code
    python3 "${CLAUDE_SKILL_DIR}/scripts/upgrade_diff.py" pr.diff --title "Bump quillmark from 3.4.1 to 4.1.0"
@@ -28,50 +30,65 @@ A dependency bump is someone else's code entering the repo under a one-line titl
    ```
    It reads stdin with `-` (`gh pr diff 212 | python3 scripts/upgrade_diff.py -`). If it reports version lines out of context, re-run on `git diff -U12`. If no script can run, read every lockfile hunk by hand with the per-ecosystem notes in [`reference.md`](reference.md).
 
-3. **List everything the lockfile changes beyond the title.** The title names one package; the lockfile is the truth. Report each of: transitive bumps (with class), packages added, packages removed, new install scripts (`hasInstallScript: true`, a `preinstall` / `install` / `postinstall` / `prepare` hook, pnpm `requiresBuild: true`), new native or binary dependencies (`prebuild-install`, `node-gyp`, a Cargo `-sys` crate), a changed `resolved` / `source` / index URL, and a maintainer or publisher change when the input shows one. ✅ *"Also changes: `entity-codec` 2.1.0 → 3.0.2 (major, licence MIT → MPL-2.0), adds `quill-native-highlight` 1.0.0 with an install script, removes `html-escape-lite`."* ❌ *"Bumps quillmark to 4.1.0."*
+3. **List everything the lockfile changes beyond the title.** The title names one package; the lockfile is the truth. Find each of: transitive bumps (with class), packages added, packages removed, new install scripts (`hasInstallScript: true`, a `preinstall` / `install` / `postinstall` / `prepare` hook, pnpm `requiresBuild: true`), new native or binary dependencies (`prebuild-install`, `node-gyp`, a Cargo `-sys` crate, wheels for only some platforms), a changed `resolved` / `source` / index URL, a raised `engines` or `python-versions`, and a maintainer or publisher change when the input shows one. ✅ *"Also changes: `entity-codec` 2.1.0 → 3.0.2 (major, licence MIT → MPL-2.0), adds `quill-native-highlight` 1.0.0 with an install script, removes `html-escape-lite`."* ❌ *"Bumps quillmark to 4.1.0."*
 
 4. **Class each jump.** Major, minor or patch from the version numbers. Below 1.0 the minor is the breaking number: 0.8.3 → 0.9.0 is a breaking jump, and below 0.1 so is the patch. A Go module path gaining `/v2` is a new module, not a bump. A prerelease target (`-rc.1`, `-beta`) or a downgrade is never a plain MERGE. A moving major tag (`@v2` → `@v3`) spans every v2.x after the one in use plus v3.0.0 to the current v3.
 
-5. **Read every release between old and new, not only the target.** For each direct dependency, list every release after the old version up to and including the new one, and pull four things from each: breaking changes, deprecations, security fixes, licence changes. Transitive majors, 0.x breaking jumps and new packages get the same reading; transitive patch and minor bumps are listed, not read. Name each release whose notes are missing. ✅ *"3.5.0 fixes an attribute-injection bug in link titles and deprecates `setOptions()`; 4.0.0 removes `renderInline()` and turns `sanitize` off by default; 4.0.1 and 4.1.0 are fixes and an optional highlighter."* ❌ *"4.1.0 adds a native highlighter. Looks fine."* A security fix is a reason to take the upgrade; when the full jump is blocked, name the smallest version that carries the fix as the alternative.
+5. **Read every release between old and new, not only the target.** For each direct dependency, read every release after the old version up to and including the new one, and pull five things from each: breaking changes, dropped runtime versions, deprecations, security fixes, licence changes. Transitive majors, 0.x breaking jumps and new packages get the same reading; transitive patch and minor bumps are listed, not read. Name each release whose notes are missing. Copy version numbers exactly as the notes give them: "deprecated since 2.2, warns from 2.5.0" is two facts. Note any migration aid an intermediate release offers (an opt-in flag, a compatibility mode); it is the staged path. A security fix is a reason to take the upgrade; when the full jump is blocked, name the smallest version that carries the fix as the alternative.
+
+6. **Search for every changed API, then trace each caller.** For each removed or renamed function, changed signature, changed default, changed return value and changed error, search and report `file:line` for each hit, or "no hits". Give the search (`git grep -n -E "renderInline|setOptions|render\(" -- '*.js' '*.ts'`); with no repo in hand, give that command as an open check and claim neither hits nor no hits. Then read the calling function and say what it does under the new behaviour, and the fix in this repo's code:
+   - A return value that became an exception skips every `if result is None` / `== -1` branch and escapes the retry or fallback built on it.
+   - A tuple or dict that became an object breaks the unpacking or the key access.
+   - A changed default hits every call that doesn't pass the option.
+   - An import inside a function fails only when that function runs, never at import time or test collection.
+   ✅ *"`load_profile` (`app/cache.py:31`) falls back to the database when `cache.get()` returns `None`; from 5.0.0 `get()` raises `CacheMiss`, so every miss escapes as an error and the fallback never runs. Catch `CacheMiss` there, or move to 4.6 first with `strict_miss=True`."* ❌ *"`get()` behaviour changed in 5.0.0; check usages."*
 
-6. **Search the repo for every API the notes change.** For each removed or renamed function, changed signature and changed default, search and report `file:line` for each hit, or "no hits". A changed default hits every call that doesn't pass the option. ✅ *"`src/render/note.js:7` calls `quillmark.render(markdown)` with no `sanitize`, so from 4.0.0 note bodies pass raw HTML through."* ❌ *"Check whether the code uses `renderInline`."* Give the search you ran (`git grep -n -E "renderInline|setOptions|render\(" -- '*.js' '*.ts'`); with no repo in hand, give that command as an open check and claim neither hits nor no hits.
+7. **Check every runtime floor against where the code runs.** A raised minimum (Python `>=3.10`, Node `engines >=20`, a Go directive, an MSRV, wheels built only for some interpreter versions, an action needing a newer runner) is compared with the Dockerfile `FROM`, each workflow's `setup-*` version and matrix, `.nvmrc` / `.python-version`, self-hosted runner versions, and the runtime the lockfile was generated on. CI and the lock tool running a newer runtime than production is how a green PR breaks the image build. ✅ *"`deploy/Dockerfile:1` builds on `node:18-alpine` and 6.0 needs Node >=20; CI runs Node 22 (`ci.yml:14`), so the PR passed and the image build will fail at `npm ci`."*
 
-7. **Verify every pin; an unverified SHA is an open check.** For a SHA-pinned action (or any dependency pinned to a commit), the pinned SHA must be the commit the release tag points to:
+8. **Verify every pin the PR changes; an unverified SHA is an open check.** A SHA-pinned action (or any dependency pinned to a commit) must be the commit its release tag points to:
    ```bash
    git ls-remote https://github.com/<owner>/<repo> refs/tags/<tag> 'refs/tags/<tag>^{}'
    ```
-   For an annotated tag the `^{}` line is the commit; the other line is the tag object. For a lightweight tag the single line is the commit. Not run means `UNVERIFIED`, never "looks right": a SHA from a fork of the repo also resolves under the upstream path. A SHA with no version comment is an unknown release. A tag-only pin (`@v3`) moved to another tag is weaker than a SHA, because the tag can be moved after review; say so and give the command that finds the SHA to pin. Say what the job hands the action: a `secrets.*` token, `id-token: write` or `contents: write` raises the stakes of every pin in that job.
+   For an annotated tag the `^{}` line is the commit; for a lightweight tag the single line is. Not run means unverified, never "looks right": a SHA from a fork also resolves under the upstream path. Compare the PR's pins with each other: one action and tag pinned to two different SHAs means at most one is the tag's commit, and the suspect is the one changed by hand or in the job with secrets. A SHA with no version comment is an unknown release. A tag-only pin (`@v3`) is weaker than a SHA, because the tag can be moved after review. Say what the job hands the action: a `secrets.*` token, `id-token: write` or `contents: write` raises the stakes. Pins the PR does not change are not part of this review.
 
-8. **Say whether CI ran the changed code.** A green check proves what it ran and nothing else. For each changed package or action, name the workflow job and test file that exercise it, or say nothing does. The usual gaps: an action or tool used only in a workflow triggered by a tag push, a release, a schedule or `workflow_dispatch`, which the PR never ran; a library used only in a file no test imports (a step 6 hit with no test); a test that runs the code but not the changed behaviour. ✅ *"`ci / test` runs `test/note.test.js`, which renders Markdown with no raw HTML, so the `sanitize` change is untested; nothing imports `src/render/comment.js`."* ❌ *"All checks pass."*
+9. **Say whether CI ran the changed code.** A green check proves what it ran and nothing else. For each changed package or action, name the workflow job and test that exercise it, or say nothing does. The usual gaps: a workflow triggered only by a tag push, a release, a schedule or `workflow_dispatch`; a file no test imports (a step 6 hit with no test); a mock that returns the old behaviour (a fake client that never times out tests the old contract); a PR job that installs differently from the release job (`npm ci --ignore-scripts` on the PR, plain `npm ci` in release means a new install script first runs in the job holding the secrets). ✅ *"`ci / test` runs `test/note.test.js`, which renders Markdown with no raw HTML, so the `sanitize` change is untested; nothing imports `src/render/comment.js`."* ❌ *"All checks pass."*
 
-9. **Write the rollback.** The exact revert: `git revert` of the merge commit, or the pin-back for the ecosystem (`npm install quillmark@3.4.1`, `poetry add tidyparse@2.2.0`, `go get <module>@v0.8.3 && go mod tidy`, `cargo update -p <crate> --precise <old>`, the old `uses:` line) with the lockfile regenerated and committed. Then name what a revert does not undo: a lockfile format migration, a data, cache or schema format the new version wrote, an artifact built or published with it (a registry version cannot be republished), and a security fix that rolling back reopens. When the old version sits inside a fixed advisory's range, the pin-back target is the first fixed version, not the old one: ✅ *"pin back to 3.5.0"* ❌ *"pin back to 3.4.1"* when 3.5.0 fixed a bug affecting 3.0.0 to 3.4.x. Per-ecosystem commands are in [`reference.md`](reference.md).
-
-10. **Derive the verdict from the checks.** Exactly one of:
-    - **DO NOT MERGE** when any holds: a removed or changed API is still used and the PR does not fix it; a new install script or native binary download nobody has confirmed is expected; a dependency source or registry moved to an unexplained host; a SHA that does not match its tag; a pin weakened (SHA to tag, tag to branch); a licence the project cannot ship; a target version an advisory names as vulnerable; a red CI check.
-    - **MERGE AFTER CHECKS** when none of those holds but any of these does: an unverified SHA; a major or 0.x breaking jump; a release whose notes were not read; changed code no CI job runs; a deprecated API still used; a new package or transitive major; a licence change that needs sign-off; a moved tag-only pin; no repo to search. Each check is a command or a concrete action.
+10. **Write the rollback, then derive the verdict.** Rollback: `git revert` of the merge commit, or the pin-back for the ecosystem (`npm install quillmark@3.4.1`, `poetry add tidyparse@2.2.0`, `go get <module>@v0.8.3 && go mod tidy`, the old `uses:` line and SHA) with the lockfile regenerated, plus what a revert does not undo (a lockfile format migration, data or cache the new version wrote, an artifact already published or deployed, a security fix that rolling back reopens; then pin back to the first fixed version). Per-ecosystem commands are in [`reference.md`](reference.md). Then exactly one verdict per PR:
+    - **DO NOT MERGE** when any holds: a removed or changed API is still used and the PR does not fix it; a runtime floor above what production or the release job runs; a new install script or native binary download nobody has confirmed is expected; a dependency source moved to an unexplained host; a SHA that does not match its tag, or two SHAs for one tag; a pin weakened (SHA to tag, tag to branch); a licence the project cannot ship; a target version an advisory names as vulnerable; a red CI check.
+    - **MERGE AFTER CHECKS** when none of those holds but any of these does: an unverified SHA; a major or 0.x breaking jump; a release whose notes were not read; changed code no CI job runs; a deprecated API still used; a new package or transitive major; a licence change that needs sign-off; a moved tag-only pin; no repo to search.
     - **MERGE** only when every check above is closed, with the evidence in the reply.
-    The script's floor is a minimum. Raise it on what steps 5 to 8 find. Lower DO NOT MERGE only when the user confirms the install script or new source is expected, and say so in the reply.
-
-11. **Reply in this order, in one message:** the verdict and the one reason that decides it; 🟥 blockers, 🟧 checks before merge and 🟨 notes, each with `file:line` or the command; a short table of what the PR changes, including everything beyond the title; the rollback; "To confirm". A routine PR fits in about 300 words. Lead with what blocks, not with the package history.
+    The script's floor is a minimum; raise it on what steps 5 to 9 find. Lower DO NOT MERGE only when the user confirms the install script or new source is expected, and say so.
+
+11. **Reply short, verdict first, in one message.** Only what changes the decision or the fix goes in.
+    - **First line:** the verdict and the one to three reasons that decide it, in plain words. With two PRs, one line each. ✅ *"DO NOT MERGE. 6.0 needs Node 20 and the production image runs 18, and two call sites break on removed APIs."* ❌ *"Here is my review of the quillmark upgrade."*
+    - **🟥 Blockers, numbered, the one that breaks first on top.** Each: what breaks, `file:line`, why the green check missed it, and the fix in this repo, in two to four lines. When an intermediate release offers a migration aid, one line after the blockers gives the staged path. The count in the first line matches the list.
+    - **🟧 Before merge:** each open check as the command or action that closes it. An unverified SHA is written as the command and the SHA it must equal.
+    - **Also in the lockfile:** one line naming every package added, removed or moved beyond the title, with install scripts and source changes called out. Not a table.
+    - **Rollback:** one or two lines, the command and what a revert leaves behind.
+    - **To confirm** only for what only the user can answer (is this install script expected?); omit it when empty.
+    - **Last line:** one sentence offering the next piece of work (the code fixes, the pin change).
+    No tables, no recited changelog, no pins or files the PR does not touch, no empty sections. A clean MERGE fits in about 120 words; a blocked PR with three blockers in about 300.
 
 **Non-interactive runs** (subagent, CI, headless): with no diff, emit `BLOCKED: need the PR diff covering the manifests, lockfiles and workflow files (gh pr diff <number>)`. Missing release notes, repo or CI files do not block: each becomes a named open check, the verdict is at best MERGE AFTER CHECKS, and the gap is stated first as `[assumption] notes for <versions> not read` or `[assumption] CI coverage unknown, treated as none`.
 
 ## Useful references in this skill
 
-- [`reference.md`](reference.md): reading each ecosystem's lockfile diff, where install scripts and sources show up, how to list the releases between two versions, SHA verification details, and per-ecosystem rollback commands with what a revert leaves behind
+- [`reference.md`](reference.md): reading each ecosystem's lockfile diff, where install scripts, sources and runtime floors show up, how to list the releases between two versions, SHA verification details, tracing behaviour changes, and per-ecosystem rollback commands with what a revert leaves behind
 - [`scripts/upgrade_diff.py`](scripts/upgrade_diff.py): stdlib parser for a unified diff of manifests, lockfiles and workflow files; prints changes, flags, pin checks and a floor verdict (exit 2 on bad input)
-- [`examples/ledgerline-tidyparse.md`](examples/ledgerline-tidyparse.md): worked example, a Poetry patch bump with a transitive bump, real script output and the review that reaches MERGE
+- [`examples/ledgerline-tidyparse.md`](examples/ledgerline-tidyparse.md): worked example, a Poetry patch bump that earns MERGE, and a blocked reply in the same shape
 
 ## Quality bar
 
-- **Exactly one verdict, first, and it follows from the checks.** Any unverified SHA, added install script, or removed API still in use rules out MERGE.
-- **Every release between old and new is accounted for:** read, with its breaking changes, deprecations, security fixes and licence changes named, or listed as unread.
-- **Everything the lockfile changes beyond the title is listed:** transitive bumps, added and removed packages, install scripts, native code, source changes.
-- **Every changed API has a search result:** `file:line` hits, "no hits", or the command to run when there is no repo.
-- **Every SHA pin is verified or marked UNVERIFIED with the `git ls-remote` command;** tag-only pins are called weaker.
-- **CI coverage is named per changed package:** the job and test that exercise it, or "nothing does".
+- **The first line is the verdict and the reasons that decide it,** one verdict per PR, following from the checks. Any unverified SHA, added install script, or removed API still in use rules out MERGE.
+- **Every release between old and new is accounted for:** read, with its breaking changes, runtime drops, deprecations, security fixes and licence changes, or named as unread.
+- **Everything the lockfile changes beyond the title is listed,** in one line: transitive bumps, added and removed packages, install scripts, native code, source and runtime changes.
+- **Every changed API has a search result and every hit is traced:** what the caller does now and the fix, or the search command when there is no repo.
+- **Every raised runtime floor is compared** with the base image, the CI runtime, the runners and the lockfile tool's runtime.
+- **Every changed SHA pin is verified or given as the `git ls-remote` command,** and the PR's pins are compared with each other.
+- **CI coverage is named per changed package:** the job and test that exercise it, or "nothing does", and how each workflow installs.
 - **A rollback with the exact command** and what a revert does not undo.
-- **No invented versions, advisories, APIs, files or test names.** Everything cited appears in the diff, the notes or the repo supplied.
+- **Nothing reads as checked that was not run,** and nothing describes the session's tools or network.
+- **No invented versions, advisories, APIs, files or test names;** version numbers are copied from the notes.
 
 ## When to use this skill
 
@@ -91,11 +108,12 @@ A dependency bump is someone else's code entering the repo under a one-line titl
 
 ## Anti-patterns to avoid
 
+- ❌ **The report reply.** A package table, a rollback essay and an empty "To confirm" around a verdict the reader has to dig for. Verdict and reasons first; the rest in a line each.
 - ❌ **Reading only the target version's notes.** The breaking change usually lives in a release the PR skipped.
-- ❌ **Trusting the title.** "Bump X from 1 to 2" says nothing about the twelve transitive packages that moved with it.
-- ❌ **Calling a SHA pin fine because it is a SHA.** A SHA proves nothing until it matches the release tag.
-- ❌ **"CI is green, ship it."** Release workflows, publish steps and untested code paths never ran on the PR.
-- ❌ **"No breaking changes affect us" without a search.** Show the grep and its hits, or say it was not run.
-- ❌ **A rollback that is only "revert the PR".** Name the lockfile regeneration and what a revert leaves behind.
-- ❌ **Reciting the changelog.** Report the releases that change the verdict; one line covers the rest.
+- ❌ **Trusting the title.** "Bump X from 1 to 2" says nothing about the transitive packages that moved with it.
+- ❌ **Finding the change but not the caller.** "`fetch()` now raises" is half a finding; "so the retry loop never retries" is the finding.
+- ❌ **Trusting CI's runtime.** CI on Python 3.11 says nothing about an image built on 3.9.
+- ❌ **Calling a SHA pin fine because it is a SHA.** A SHA proves nothing until it matches the release tag, and two SHAs for one tag cannot both match.
+- ❌ **"CI is green, ship it."** Release workflows, publish steps, lazy imports and mocked paths never ran on the PR.
+- ❌ **Explaining the session.** "The network was blocked, so I couldn't check" is about the run, not the PR; give the command.
 - ❌ **Hedged verdicts.** "Probably safe" is not a verdict; pick one of the three and give the reason.

package/packs/dev-team/skills/dependency-upgrade-review/evals/evals.json

@@ -12,6 +12,7 @@
         "Says the green CI does not cover the change: the only test renders Markdown with no raw HTML, and no test imports src/render/comment.js",
         "Accounts for the skipped releases, not only 4.1.0: names the 3.5.0 link-title security fix and the setOptions() deprecation that src/render/note.js still uses",
         "Gives a rollback that pins quillmark back and regenerates package-lock.json (or reverts the PR), and names taking 3.5.x as the way to get the security fix without the breaking changes",
+        "Opens with the verdict and the decisive reasons (the removed renderInline() call and the sanitize default) in plain words before any list, keeps the lockfile changes to a line or a short list rather than a table, and does not describe the session's network or tool access",
         "Does not invent versions, advisory IDs, APIs, files or tests that are not in the supplied input"
       ]
     },
@@ -26,6 +27,7 @@
         "Reports that release.yml still passes the deprecated password input (release.yml line 22) and that verify-metadata now defaults to true, so the next release can fail on an invalid long description",
         "Classes the jumps: checkout-lite v3.1.0 to v3.2.0 as minor, pypi-publish v2 to v3 as major",
         "Gives a rollback (revert the PR or restore the old uses: lines) and notes a revert cannot undo a release already published to PyPI with v3",
+        "Does not report either SHA or the tag move as verified, and does not explain the gap by describing the session's network or tool access; it gives the command and the SHA it must equal instead",
         "Does not invent action versions, SHAs, inputs or workflow steps that are not in the supplied files"
       ]
     },
@@ -40,6 +42,7 @@
         "Flags github.com/lanternhq/clock v1.2.0 as a new indirect module the PR title does not mention",
         "Says the passing CI only covers this if a test exercises the burst-0 Wait path, and lists that as a check rather than assuming it",
         "Gives a rollback command such as go get github.com/lanternhq/ratelimit@v0.8.3 followed by go mod tidy, or reverting the PR",
+        "Stays short, as the user asked for a quick verdict: the verdict and its reasons come first, and there is no package table and no empty section",
         "Does not invent file names, call sites, tests or release versions that are not in the prompt"
       ]
     },
@@ -72,6 +75,36 @@
         "Explains that the green checks never ran the changed code. ci.yml runs on pull_request with `npm ci --ignore-scripts` and never runs scripts/release.mjs. release.yml runs plain `npm ci`, so notary-telemetry's install script would first execute in the production environment, with RELEASE_SIGNING_KEY and NPM_TOKEN in the job and contents: write / id-token: write permissions. Mentions NOTARY_TELEMETRY=0 and/or keeping 1.8.x as mitigations.",
         "Gives a rollback for each PR: revert #412 to restore the c8aee2ae521bdb1ef1b97761673ebbe92c953b0f (v3.2.0) pins; revert #413, or pin release-notary back to 1.8.2 and regenerate package-lock.json. Does not invent versions, flags or APIs that are not in the files, and does not claim to have run npm install, the workflows, or the release script."
       ]
+    },
+    {
+      "id": 6,
+      "held_out": "2026-10-20",
+      "prompt": "Dependabot opened PR #412 on our payments-gateway service bumping github.com/fernwire/httpkit from 1.8.1 to 1.8.4. It's only a patch bump and every check is green, so I'd like to merge it before the release cut this afternoon. Can I merge this dependency upgrade? The PR body is in PR.md, the diff is pr.diff, and I pasted the upstream changelog into vendor-notes/httpkit-CHANGELOG.md. The repo source is checked out here.",
+      "assertions": [
+        "Picks out the v1.8.3 entry in vendor-notes/httpkit-CHANGELOG.md (a version the Dependabot release notes in PR.md leave out): NewClient no longer applies a default 30-second Timeout, and the zero value now means no client-level timeout. It treats this as the key finding even though the bump looks like a patch.",
+        "Points to internal/payments/client.go NewClient, which calls httpkit.NewClient with only WithBaseURL and WithRetries and whose comment says it relies on the 30s default (INC-2291), and to internal/payments/settle.go, which calls Charge with context.Background() and no deadline. So after the upgrade a hung gateway can block the settlement batch indefinitely. It also notes that internal/notify/client.go sets WithTimeout(5*time.Second) explicitly and is unaffected.",
+        "Explains that the green CI does not test this change: client_test.go injects fakeDoer, so the real httpkit client is never exercised, and ci.yml runs `go test -race -short ./...` without `-tags integration`, so client_integration_test.go (TestChargeHungGatewayTimesOut) did not run on this PR. It runs only nightly or on manual dispatch via integration.yml.",
+        "Gives a verdict of DO NOT MERGE as-is, or MERGE AFTER CHECKS on the specific condition that payments NewClient gains an explicit httpkit.WithTimeout(30*time.Second) (or a per-call context deadline in SettleBatch) and the integration test passes. It is never plain MERGE.",
+        "Lists the go.mod/go.sum changes the PR title does not mention: indirect github.com/fernwire/backoff v0.3.0 -> v0.4.0 and a newly added indirect github.com/fernwire/otelshim v0.1.0. It ties them to the v1.8.2 (backoff, full-jitter) and v1.8.4 (WithTracer) changelog entries rather than calling them unexplained.",
+        "Gives a concrete rollback: revert the merge commit, or `go get github.com/fernwire/httpkit@v1.8.1 && go mod tidy`, which also returns backoff to v0.3.0 and drops otelshim. It notes that adding the explicit WithTimeout is safe to keep on either version.",
+        "Does not claim to have run tests or verified the go.sum hashes or upstream tags. It gives the network or execution steps as commands to run (for example `go mod verify`, `go test -tags integration -run TestChargeHungGatewayTimesOut ./internal/payments/`, or triggering integration.yml via workflow_dispatch).",
+        "Invents no httpkit versions, options or behaviours beyond those in the files (for example no made-up v1.8.5, no claimed retry-on-POST change, no options other than WithBaseURL, WithRetries, WithTimeout and WithTracer)."
+      ]
+    },
+    {
+      "id": 7,
+      "held_out": "2026-10-20",
+      "prompt": "Renovate grouped this week's dev dependency updates into PR #88 for our invoice-widgets package: five npm packages in one go (details in PR.md, diff in pr.diff). After last quarter's incident with a grouped bump I'm nervous about these. Is this Renovate bump safe to merge, or should I split it up? Repo files are here.",
+      "assertions": [
+        "Ends with a verdict of MERGE. It does not downgrade to MERGE AFTER CHECKS or DO NOT MERGE over problems the files do not contain (for example an install script, a registry change, a maintainer change, or a breaking change that affects this repo), and it does not insist on splitting the PR without a concrete reason from the files.",
+        "Checks bundlecrate 6.1.0's new Node >=18.18 minimum against the repo and finds it harmless: .nvmrc is 20.11.1, package.json engines is node >=20, and ci.yml uses node-version-file: .nvmrc.",
+        "Checks the repo's uses of the changed APIs and finds nothing that needs action. vellum-test 2.8.0 deprecates vt.legacyTimers(), but tests/due-date.test.ts uses only vt.fakeTimers/vt.restoreTimers. lintwright 4.3.0's new no-floating-promise rule is only in lintwright:strict, while .lintwright.json extends lintwright:recommended.",
+        "Lists the lockfile changes beyond the five headline bumps: @bundlecrate/resolver-core 1.0.0 added, bundlecrate-legacy-resolver 2.3.4 removed, and the transitive picolabel 0.9.1 -> 0.9.2. It ties each to a changelog entry (bundlecrate 6.1.0 resolver extraction, lintwright 4.3.0 picolabel update).",
+        "States that no lockfile entry in pr.diff has \"hasInstallScript\" and that every resolved URL is on https://registry.npmjs.org/, which matches .npmrc. So there are no new install-time scripts and no registry switch.",
+        "Says CI did exercise the changed code, mapping the packages to jobs in .github/workflows/ci.yml: lintwright -> lint, prettyquill -> format, tsforge-types -> typecheck, vellum-test -> test, bundlecrate -> build-and-smoke (which runs test:dist against the built bundle). It says the pass/fail status comes from PR.md and was not observed directly.",
+        "Gives a rollback: revert the merge commit (restoring package.json and package-lock.json together) and run `npm ci`. It notes these are all devDependencies, so only the bundlecrate-built dist/ could reach users, and that build is covered by the smoke step.",
+        "Does not claim to have verified integrity hashes, provenance or upstream tags offline. It invents no versions, rules or APIs not in the files, and frames anything network-dependent (for example `npm audit signatures`, `npm ci` locally) as an optional command, not a blocker."
+      ]
     }
   ]
 }

package/packs/dev-team/skills/dependency-upgrade-review/evals/files/6/.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: ci
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d1f0a9c7e5b2846f0c1a7d9e3b5f8c2a6d4e017 # v4.2.2
+      - uses: actions/setup-go@8e2c4a6f0b1d3957a2c8e4f6b0d1a3c5e7f9b204 # v5.1.0
+        with:
+          go-version: "1.23"
+      - run: go vet ./...
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d1f0a9c7e5b2846f0c1a7d9e3b5f8c2a6d4e017 # v4.2.2
+      - uses: actions/setup-go@8e2c4a6f0b1d3957a2c8e4f6b0d1a3c5e7f9b204 # v5.1.0
+        with:
+          go-version: "1.23"
+      # Unit tests only. Integration tests (-tags integration) run nightly in
+      # integration.yml because they need a long timeout.
+      - run: go test -race -short ./...
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d1f0a9c7e5b2846f0c1a7d9e3b5f8c2a6d4e017 # v4.2.2
+      - uses: actions/setup-go@8e2c4a6f0b1d3957a2c8e4f6b0d1a3c5e7f9b204 # v5.1.0
+        with:
+          go-version: "1.23"
+      - run: go build ./...