vigiles 27.1.5 → 27.1.7

package/dist/adapters/codex/mock-model.js

@@ -109,6 +109,33 @@ function renderResponsesSSE(text, opts = {}) {
         `data: ${JSON.stringify({ type: e.type, ...e.data })}\n\n`)
         .join("");
 }
+/**
+ * 🔴 KNOWN BLIND SPOT — NARROW ON PURPOSE, AND THAT IS NOT THE SAME AS COMPLETE.
+ * Roadmapped P1 2026-09-10; do not read a `requestContains` miss on a Codex trace
+ * as "not delivered" until this is closed.
+ *
+ * This reads `.text` and nothing else, and its caller takes only the LAST input
+ * item with `role:"user", type:"message"`. Measured against openai@7.13.0:
+ * `ResponseInputItem` has THIRTY-TWO variants and we look at one. The ignored
+ * ones include `FunctionCallOutput`, `ShellCallOutput` and `LocalShellCallOutput`
+ * — the Responses-API analogues of an Anthropic `tool_result`, i.e. exactly where
+ * Claude Code relocated a hook's additionalContext in 2.1.228 and where the same
+ * payload would land here.
+ *
+ * WHY THIS IS A COMMENT AND NOT A FIX. `driver.ts` adapts this into a
+ * `ModelRequest`, so it feeds `requestContains` — the same predicate whose
+ * `.text`-only twin in src/mock-model.ts produced a false "not delivered", a
+ * false issue (zernie/vigiles#231) and a near-miss upstream report. The trap is
+ * armed; nobody has stepped on it only because both delivery harnesses are
+ * Claude-Code-only today. Closing it properly means deciding what `prompt` MEANS
+ * across 32 item types — a semantic change, not a bug fix, and too large to ride
+ * along with the mock-model repair.
+ *
+ * The Claude Code side now takes `ContentBlockParam` from @anthropic-ai/sdk and
+ * fails the BUILD on an unhandled variant (see `flattenBlock` in
+ * src/mock-model.ts). openai ships the matching union under Apache-2.0; applying
+ * the same construction here is the fix, and it is the roadmapped work.
+ */
 function joinInputText(content) {
     if (!Array.isArray(content))
         return "";

package/dist/eval.js

@@ -1412,6 +1412,37 @@ exports.claudeEvalDriver = {
     parse: parseClaudeRun,
     harness: "claude-code",
 };
+/**
+ * Is this directory entry a skill DIRECTORY — following a symlink to one?
+ *
+ * 🔴 `Dirent.isDirectory()` describes the ENTRY, not its target: for a symlink
+ * pointing at a directory it is FALSE. Skipping on it therefore skips symlinked
+ * skills entirely, and the failure is silent and misreads as a finding — the skill
+ * never enters the temp install, so nothing fires on any prompt and the run reports
+ * 0% recall plus a competing-skill count short by however many were linked. The
+ * tool then says "usually SETUP, not the description", which is true and useless,
+ * because the setup it means is its own.
+ *
+ * A symlinked skills tree is not exotic: it is what a consumer gets when skills are
+ * distributed as an npm package and linked into `.claude/skills/`, which is exactly
+ * how a package publishes them.
+ *
+ * `statSync` follows the link, so it answers about the TARGET. A dangling link
+ * throws, and that is not-a-skill-dir — the same answer the `SKILL.md` check
+ * downstream would give it, one step earlier and without an exception escaping.
+ */
+function isSkillDirEntry(parent, entry) {
+    if (entry.isDirectory())
+        return true;
+    if (!entry.isSymbolicLink())
+        return false;
+    try {
+        return (0, node_fs_1.statSync)((0, node_path_1.join)(parent, entry.name)).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
  * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`)
@@ -1432,7 +1463,7 @@ function packageSkillsDir(skillsDir, opts = {}) {
     (0, node_fs_1.mkdirSync)(skillsOut, { recursive: true });
     let copied = 0;
     for (const entry of (0, node_fs_1.readdirSync)(abs, { withFileTypes: true })) {
-        if (!entry.isDirectory())
+        if (!isSkillDirEntry(abs, entry))
             continue;
         const srcSkill = (0, node_path_1.join)(abs, entry.name, "SKILL.md");
         if (!(0, node_fs_1.existsSync)(srcSkill))
@@ -1589,7 +1620,7 @@ function copySkillsInto(src, skillsOut, stub, present) {
         throw new Error(`installSet source not found: ${src} (resolved ${abs})`);
     let copied = 0;
     for (const entry of (0, node_fs_1.readdirSync)(abs, { withFileTypes: true })) {
-        if (!entry.isDirectory() || present.has(entry.name))
+        if (!isSkillDirEntry(abs, entry) || present.has(entry.name))
             continue;
         const srcSkill = (0, node_path_1.join)(abs, entry.name, "SKILL.md");
         if (!(0, node_fs_1.existsSync)(srcSkill))
@@ -1656,7 +1687,7 @@ function countSkills(pluginDir) {
         return 0;
     let n = 0;
     for (const e of (0, node_fs_1.readdirSync)(dir, { withFileTypes: true }))
-        if (e.isDirectory() && (0, node_fs_1.existsSync)((0, node_path_1.join)(dir, e.name, "SKILL.md")))
+        if (isSkillDirEntry(dir, e) && (0, node_fs_1.existsSync)((0, node_path_1.join)(dir, e.name, "SKILL.md")))
             n++;
     return n;
 }

package/dist/harness-assert.d.ts

@@ -156,7 +156,17 @@ export declare function outputContains(trace: Trace, needle: string | RegExp): b
  * Did ANY request the model received contain `needle` — searching the system
  * prompt and every message across all requests? The predicate that proves
  * injected context *reached the model*: a SessionStart hook's `additionalContext`
- * or a slash command's expansion. Harness tier only — the eval tier drives the
+ * or a slash command's expansion.
+ *
+ * THIS IS A PROJECTION OF THE REQUEST, NOT THE REQUEST — state what it omits
+ * before building a claim on it. Until 2026-09-10 this sentence was false: the
+ * flattener read only `.text`, so the eight `ContentBlockParam` variants that
+ * carry `content` were invisible, and a payload Claude Code had delivered inside
+ * a `tool_result` read as "never arrived" (zernie/vigiles#231, a false finding).
+ * It now covers every block type the pinned SDK union names, and serialises any
+ * it does not. STILL OMITTED BY DESIGN: `tool_use` / `server_tool_use` inputs —
+ * those are what the model SAID, not what it was TOLD, so a needle in a tool
+ * argument must not read as delivery. See `flattenBlock` in mock-model.ts. Harness tier only — the eval tier drives the
  * real API, so its `modelRequests` (and this) is empty. Behind `assertRequestContains`.
  */
 export declare function requestContains(trace: Trace, needle: string | RegExp): boolean;

package/dist/harness-assert.js

@@ -423,7 +423,17 @@ function requestText(trace) {
  * Did ANY request the model received contain `needle` — searching the system
  * prompt and every message across all requests? The predicate that proves
  * injected context *reached the model*: a SessionStart hook's `additionalContext`
- * or a slash command's expansion. Harness tier only — the eval tier drives the
+ * or a slash command's expansion.
+ *
+ * THIS IS A PROJECTION OF THE REQUEST, NOT THE REQUEST — state what it omits
+ * before building a claim on it. Until 2026-09-10 this sentence was false: the
+ * flattener read only `.text`, so the eight `ContentBlockParam` variants that
+ * carry `content` were invisible, and a payload Claude Code had delivered inside
+ * a `tool_result` read as "never arrived" (zernie/vigiles#231, a false finding).
+ * It now covers every block type the pinned SDK union names, and serialises any
+ * it does not. STILL OMITTED BY DESIGN: `tool_use` / `server_tool_use` inputs —
+ * those are what the model SAID, not what it was TOLD, so a needle in a tool
+ * argument must not read as delivery. See `flattenBlock` in mock-model.ts. Harness tier only — the eval tier drives the
  * real API, so its `modelRequests` (and this) is empty. Behind `assertRequestContains`.
  */
 function requestContains(trace, needle) {

package/dist/mock-model.js

@@ -232,19 +232,143 @@ function splitRequestCounts(requests) {
             sideChannelCount++;
     return { count: requests.length - sideChannelCount, sideChannelCount };
 }
-/** Flatten Anthropic content (string, or an array of text/other blocks) to text. */
+/**
+ * Flatten one Anthropic content block to the text the MODEL actually received.
+ *
+ * WHY THIS IS A TYPED, EXHAUSTIVE SWITCH AND NOT A `.text` LOOKUP — the whole
+ * point of the file, and it was paid for. Until 2026-09-10 this read `b.text`
+ * and returned `""` for anything else. Eight of the sixteen `ContentBlockParam`
+ * variants carry their payload in `content`, not `text`, so half the union was
+ * invisible to every instrument built on `extractRequest` — `requestContains`,
+ * `refs-nudge.harness.mjs`, `injectable-events-delivery.harness.mjs`.
+ *
+ * WHAT THAT COST. Claude Code <= 2.1.227 delivered a `PostToolUse` hook's
+ * `additionalContext` as its own `text` block. From 2.1.228 (a PATCH release,
+ * 2026-08-11) it arrives appended to the `tool_result` block's `content`, inside
+ * a `<system-reminder>`. Nothing broke: the model received the payload on both
+ * versions. Our probe went blind, every test above reported "not delivered", and
+ * that false reading was written up as zernie/vigiles#231 and very nearly filed
+ * upstream as a regression in somebody else's product. MEASURED both ways on one
+ * machine, claude 2.1.267, changing only this function: blind = "landed=false",
+ * typed = "landed=true".
+ *
+ * WHAT THE TYPE BUYS, precisely — it is NOT a change detector:
+ *   - it does NOT notice a payload moving between fields the type already allows
+ *     (`ToolResultBlockParam.content` predates the relocation; nothing changed);
+ *   - it DOES make a silently-unhandled variant impossible: the `never` binding
+ *     below fails `tsc` until every case is written out, so the seventeenth
+ *     block type Anthropic ships breaks the BUILD instead of quietly emptying a
+ *     measurement.
+ *
+ * WHY THE DEFAULT SERIALISES INSTEAD OF RETURNING `""`. This is a measurement
+ * instrument, and its proven failure mode is the FALSE NEGATIVE — a payload that
+ * was there, reported missing. So an unrecognised block is over-included (its
+ * JSON) rather than dropped: a stale pin then costs a noisy match, never a
+ * silent hole. That asymmetry is deliberate; do not "tidy" it to `""`.
+ *
+ * The repo's `assertNever` is deliberately NOT used: it throws, and this parses
+ * untrusted wire JSON where an unknown block must degrade, not crash.
+ */
+function flattenBlock(b) {
+    switch (b.type) {
+        // WALKED — every string field is readable text the model was shown, and
+        // reading any one of them by name is what this function keeps getting
+        // wrong. `text` also carries `citations[].cited_text` / `document_title`
+        // (quoted source text); `search_result` carries `title` and `source`
+        // BESIDE its `content`; `document` spreads its text across `title`,
+        // `context` and `source` (`PlainTextSource.data`,
+        // `ContentBlockSource.content`). A base64 source is skipped inside
+        // `flattenUnknown` — bytes, not text.
+        case "text":
+        case "search_result":
+        case "document":
+            return flattenUnknown(b);
+        // READ NARROWLY, and the dropped field is named so the next reader can
+        // check the claim instead of trusting it: `signature` is an opaque
+        // attestation blob, not context.
+        case "thinking":
+            return b.thinking;
+        // The families whose payload hangs off `content`. Their only other field is
+        // `tool_use_id` — a correlation identifier, not text the model was shown.
+        // `content` is optional on `tool_result` alone; on the rest it is required
+        // and is an OBJECT, which `flattenContent` hands to `flattenUnknown`.
+        case "tool_result":
+        case "web_search_tool_result":
+        case "web_fetch_tool_result":
+        case "code_execution_tool_result":
+        case "bash_code_execution_tool_result":
+        case "text_editor_code_execution_tool_result":
+        case "tool_search_tool_result":
+            return b.content === undefined ? "" : flattenContent(b.content);
+        // The model's own call, not context delivered TO it — kept out of
+        // `requestContains` on purpose so a needle in a tool ARGUMENT is never read
+        // as "the model was told this". Their `id` / `name` are identifiers.
+        case "tool_use":
+        case "server_tool_use":
+            return "";
+        // No readable text by construction: `redacted_thinking.data` is encrypted,
+        // `container_upload.file_id` is an identifier, an image is pixels.
+        case "image":
+        case "redacted_thinking":
+        case "container_upload":
+            return "";
+        default: {
+            // Compile-time: unreachable, and that is the guard — a new variant makes
+            // this assignment fail. Run-time: reachable via wire JSON from a newer
+            // API than the pinned types, so it degrades loudly instead of throwing.
+            const unhandled = b;
+            return JSON.stringify(unhandled);
+        }
+    }
+}
+/**
+ * Flatten an arbitrary wire payload to text by walking every string leaf.
+ *
+ * WHY A GENERIC WALK AND NOT ONE MORE NAMED FIELD. The first version of
+ * `flattenBlock` grouped eight variants as "the ones that carry `content`" and
+ * handed each to `flattenContent`, which accepts only a string or an array.
+ * Six of those eight carry an OBJECT there — `web_fetch_tool_result`,
+ * `web_search_tool_result`, `code_execution_tool_result`,
+ * `bash_code_execution_tool_result`, `text_editor_code_execution_tool_result`,
+ * `tool_search_tool_result` — so they still flattened to "". The grouping was
+ * made on the field's NAME while the defect lives in its TYPE, which is the
+ * same mistake, one level up, as the `.text`-only read it replaced. Found by
+ * review on this PR, not by a run (zernie/vigiles#233).
+ *
+ * And no single field would have fixed it: the payload's text sits at a
+ * different key in each shape — `stdout`/`stderr` on a bash result, a nested
+ * `content` document on a fetch result, `data` on a plain-text source. Keying
+ * on any one of them re-commits the shape assumption. Walking commits to none.
+ */
+function flattenUnknown(v) {
+    if (typeof v === "string")
+        return v;
+    if (Array.isArray(v))
+        return v.map(flattenUnknown).join("");
+    if (typeof v !== "object" || v === null)
+        return "";
+    const o = v;
+    // A base64 source is bytes, not text. Including it would bury every real
+    // match under megabytes of encoding — the one over-inclusion that costs more
+    // than the false negative it avoids.
+    if (o.type === "base64")
+        return "";
+    return Object.entries(o)
+        .filter(([k]) => k !== "type" && k !== "media_type") // discriminators
+        .map(([, val]) => flattenUnknown(val))
+        .join("");
+}
+/**
+ * Flatten Anthropic content to text: a string, an array of blocks, or the
+ * OBJECT a server-tool result carries (see `flattenUnknown`).
+ */
 function flattenContent(content) {
     if (typeof content === "string")
         return content;
     if (!Array.isArray(content))
-        return "";
+        return flattenUnknown(content);
     return content
-        .map((b) => {
-        if (typeof b === "string")
-            return b;
-        const t = b.text;
-        return typeof t === "string" ? t : "";
-    })
+        .map((b) => typeof b === "string" ? b : flattenBlock(b))
         .join("");
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "27.1.5",
+  "version": "27.1.7",
   "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
   "keywords": [
     "claude-code",
@@ -95,6 +95,7 @@
     "docs:api": "typedoc"
   },
   "devDependencies": {
+    "@anthropic-ai/sdk": "^0.124.0",
     "@eslint/js": "^10.0.1",
     "@jackchuka/mdschema": "^0.12.8",
     "@microsoft/api-extractor": "^7.58.9",