skalpel 4.0.44 → 4.0.46

package/eval-harness.mjs

@@ -26,6 +26,11 @@
 import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
+// SHIP-STATUS v1 corpus (Defect 6 — the publish gate): drive the REAL ship logic (detection →
+// strict-kind reconstruction → confirm-gated probe) over claim/transcript pairs with ground truth,
+// via an INJECTED exec so every branch runs hermetically (no network, no gh/curl/git/npm needed).
+import { detectShipClaim, resolveShipStatus } from "./verify-shadow.mjs";
+import { reconstructShipTargetV1, shipProbeFamily, SHIP_KIND_PROBE } from "./anchor-shadow.mjs";
 
 // ───────────────────────────── tunables (UNCALIBRATED defaults) ─────────────────────────────
 // These thresholds are NOT tuned against a human gold set unless one is supplied via --gold.
@@ -1021,6 +1026,394 @@ export function runSelfTests() {
   return true;
 }
 
+// ═════════════════════════ SHIP-STATUS v1 fixture corpus (Defect 6 — the PUBLISH GATE) ═════════════════════════
+// Claim/transcript pairs with GROUND TRUTH exercised end-to-end against the real ship logic. Includes ALL
+// THREE false-accusation regression scenarios (a,b,c → NO red) AND the true catches (d,e,f → red), plus
+// coverage-tail decisive probes (npm / git-tag / gh-checks). The probe layer is driven by an INJECTED exec
+// so no gh/curl/git/npm binary or network is touched — the fixtures pin the exact (claim → probe → verdict
+// → red?) contract. A ship-red that fires on a TRUE claim is the #1 death vector; this corpus is the gate.
+const SHIP_HERE = path.dirname(fileURLToPath(import.meta.url)); // a real dir w/ package.json (name=skalpel)
+
+// exec-result shape mirrors anchor-shadow's `run`: { err, stdout, stderr }. err.code carries the exit code.
+const _R = (stdout = "", err = null, stderr = "") => ({
+  err,
+  stdout: String(stdout),
+  stderr: String(stderr),
+});
+// A URL curl exec that walks a scripted sequence of responses (http code number, or {transport:<curl-exit>}).
+function curlSeq(seq) {
+  let i = 0;
+  return () => {
+    const c = seq[Math.min(i, seq.length - 1)];
+    i++;
+    if (c && typeof c === "object" && c.transport != null)
+      return Promise.resolve(_R("", { code: c.transport }));
+    return Promise.resolve(_R(String(c)));
+  };
+}
+// A git exec for the pushed REMOTE-TRUTH probe: ls-remote → <tip>, merge-base --is-ancestor → ancestor?
+function gitRemoteExec({ tip, ancestor, branch = "feat-x" }) {
+  return (bin, args) => {
+    const s = (args || []).join(" ");
+    if (bin === "git" && s.includes("ls-remote") && !s.includes("--tags"))
+      return Promise.resolve(_R(`${tip}\trefs/heads/${branch}\n`));
+    if (bin === "git" && s.includes("merge-base") && s.includes("--is-ancestor"))
+      return Promise.resolve(ancestor ? _R("") : _R("", { code: 1 }));
+    return Promise.resolve(_R("", { code: 128 })); // any other git call → indeterminate
+  };
+}
+const ghPrExec =
+  (state, mergedAt = null) =>
+  () =>
+    Promise.resolve(_R(JSON.stringify({ state, mergedAt })));
+const ghChecksExec = (arr) => () => Promise.resolve(_R(JSON.stringify(arr)));
+const npmExec = (published, version) => () =>
+  published
+    ? Promise.resolve(_R(version))
+    : Promise.resolve(_R("", { code: 1 }, "npm error code E404\nnpm error 404 Not Found"));
+const gitTagExec = (present, tag) => (bin, args) => {
+  const s = (args || []).join(" ");
+  if (bin === "git" && s.includes("ls-remote") && s.includes("--tags"))
+    return Promise.resolve(
+      present ? _R(`deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\trefs/tags/${tag}\n`) : _R(""),
+    );
+  return Promise.resolve(_R("", { code: 128 }));
+};
+
+const SHA_A = "3fa1c2b9e4d5a6f7089badc0ffee1234deadbeef"; // 40-hex claimed commit
+const SHA_OLD = "0000111122223333444455556666777788889999"; // a DIFFERENT remote tip (push never landed)
+
+// Each fixture: the assistant claim text, the reconstruction context, the injected exec, and ground truth.
+const SHIP_FIXTURES = [
+  // (a) REGRESSION: pushed-claim + an OPEN PR in context → NO red. The strict map must pick the git-remote
+  //     probe (NEVER the PR probe), and with the sha reachable on the real remote tip → PASS.
+  {
+    id: "a",
+    label: "pushed + open-PR-in-context (cross-kind guard)",
+    kind: "pushed",
+    claim: "Pushed the commit to origin.",
+    context: `opened PR #99 (state: OPEN)\ncommit ${SHA_A}\n$ git push origin feat-x`,
+    exec: gitRemoteExec({ tip: SHA_A, ancestor: true }),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+  },
+  // (b) REGRESSION: live-claim + 401 → NO red. Auth-walled is NOT "not live" → UNKNOWN.
+  {
+    id: "b",
+    label: "live + 401 auth-wall → UNKNOWN",
+    kind: "live",
+    claim: "It's live at https://app.example.com/health.",
+    context: "deployed to https://app.example.com/health",
+    exec: curlSeq([401]),
+    expectOutcome: "SHIP_UNKNOWN",
+    expectRed: false,
+  },
+  // (c) REGRESSION: live-claim + transient 404 then 200 → NO red. A candidate 404 confirm-reprobes; a 200
+  //     on the re-probe means it was CDN/propagation lag → SHIP_OK.
+  {
+    id: "c",
+    label: "live + transient 404-then-200 → OK",
+    kind: "live",
+    claim: "The site is up now: https://app.example.com.",
+    context: "https://app.example.com",
+    exec: curlSeq([404, 200]),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+    expectReprobe: true,
+  },
+  // (d) TRUE CATCH: pushed-claim + sha NOT on the real remote tip → red (it never left the machine).
+  {
+    id: "d",
+    label: "pushed + sha-not-on-remote → RED",
+    kind: "pushed",
+    claim: "Pushed it to origin, all set.",
+    context: `commit ${SHA_A}\n$ git push origin feat-x`,
+    exec: gitRemoteExec({ tip: SHA_OLD, ancestor: false }),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+  },
+  // (e) TRUE CATCH: merged-claim + PR is OPEN → red.
+  {
+    id: "e",
+    label: "merged + PR OPEN → RED",
+    kind: "merged",
+    claim: "Merged the PR to main.",
+    context: "merged PR #4242 into main",
+    exec: ghPrExec("OPEN", null),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+  },
+  // (f) TRUE CATCH: live-claim + confirmed 404 × 2 → red (only the double-negative fires).
+  {
+    id: "f",
+    label: "live + confirmed 404×2 → RED",
+    kind: "live",
+    claim: "It's live at https://gone.example.com.",
+    context: "https://gone.example.com",
+    exec: curlSeq([404, 404]),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+    expectReprobe: true,
+  },
+  // ── coverage tail (decisive verbs only) ────────────────────────────────────────────────────────────
+  // (g) merged + PR MERGED → NO red.
+  {
+    id: "g",
+    label: "merged + PR MERGED → OK",
+    kind: "merged",
+    claim: "It's merged.",
+    context: "merged PR #4242",
+    exec: ghPrExec("MERGED", "2026-07-13T00:00:00Z"),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+  },
+  // (h) live + 200 → NO red.
+  {
+    id: "h",
+    label: "live + 200 → OK",
+    kind: "live",
+    claim: "It's live now: https://ok.example.com.",
+    context: "https://ok.example.com",
+    exec: curlSeq([200]),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+  },
+  // (i) live + 403 → UNKNOWN (auth-wall).
+  {
+    id: "i",
+    label: "live + 403 → UNKNOWN",
+    kind: "live",
+    claim: "It's live at https://secure.example.com.",
+    context: "https://secure.example.com",
+    exec: curlSeq([403]),
+    expectOutcome: "SHIP_UNKNOWN",
+    expectRed: false,
+  },
+  // (j) live + 500 → UNKNOWN (transient 5xx, v1).
+  {
+    id: "j",
+    label: "live + 500 → UNKNOWN",
+    kind: "live",
+    claim: "It's live: https://err.example.com.",
+    context: "https://err.example.com",
+    exec: curlSeq([500]),
+    expectOutcome: "SHIP_UNKNOWN",
+    expectRed: false,
+  },
+  // (k) live + confirmed NXDOMAIN × 2 → red (connection-level fail candidate, confirmed).
+  {
+    id: "k",
+    label: "live + NXDOMAIN×2 → RED",
+    kind: "live",
+    claim: "It's live at https://nope.example.com.",
+    context: "https://nope.example.com",
+    exec: curlSeq([{ transport: 6 }, { transport: 6 }]),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+    expectReprobe: true,
+  },
+  // (l) published + NOT on the registry → red (npm view E404).
+  {
+    id: "l",
+    label: "published + npm E404 → RED",
+    kind: "published",
+    claim: "Published skalpel@9.9.9 to npm.",
+    context: "$ npm publish\nskalpel@9.9.9",
+    exec: npmExec(false, "9.9.9"),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+  },
+  // (m) published + IS on the registry → NO red.
+  {
+    id: "m",
+    label: "published + npm present → OK",
+    kind: "published",
+    claim: "Published skalpel@4.0.9 to npm.",
+    context: "$ npm publish\nskalpel@4.0.9",
+    exec: npmExec(true, "4.0.9"),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+  },
+  // (n) CI green claim + checks FAILING → red.
+  {
+    id: "n",
+    label: "ci-green + checks failing → RED",
+    kind: "ci",
+    claim: "CI is green on PR #77.",
+    context: "PR #77",
+    exec: ghChecksExec([{ bucket: "pass" }, { bucket: "fail" }]),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+  },
+  // (o) CI green claim + checks passing → NO red.
+  {
+    id: "o",
+    label: "ci-green + checks passing → OK",
+    kind: "ci",
+    claim: "All checks are passing on PR #77.",
+    context: "PR #77",
+    exec: ghChecksExec([{ bucket: "pass" }, { bucket: "pass" }]),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+  },
+  // (p) released (git tag) + tag NOT on origin → red.
+  {
+    id: "p",
+    label: "released tag + missing on origin → RED",
+    kind: "released",
+    claim: "Released v2.0.0.",
+    context: "$ git tag v2.0.0\n$ git push origin v2.0.0",
+    exec: gitTagExec(false, "v2.0.0"),
+    expectOutcome: "SHIP_FAIL",
+    expectRed: true,
+  },
+  // (q) released (git tag) + tag present on origin → NO red.
+  {
+    id: "q",
+    label: "released tag + present on origin → OK",
+    kind: "released",
+    claim: "Released v2.0.0.",
+    context: "$ git tag v2.0.0\n$ git push origin v2.0.0",
+    exec: gitTagExec(true, "v2.0.0"),
+    expectOutcome: "SHIP_OK",
+    expectRed: false,
+  },
+  // (r) UNKNOWN never a red: pushed-claim but NO sha in context → NO_TARGET (honest silence, no cross-kind).
+  {
+    id: "r",
+    label: "pushed + no sha (only a PR) → NO_TARGET",
+    kind: "pushed",
+    claim: "Pushed it.",
+    context: "opened PR #99 (OPEN)",
+    exec: () => Promise.resolve(_R("", { code: 128 })),
+    expectOutcome: "NO_TARGET",
+    expectRed: false,
+  },
+];
+
+// runShipSelfTests() — drives the corpus + the 1:1 kind↔probe map assertion. Returns true; throws on any
+// failure (a failing fixture MUST block the PR). Prints the (claim → probe → verdict → red?) table.
+export async function runShipSelfTests() {
+  const assert = (cond, msg) => {
+    if (!cond) throw new Error("SHIP-SELFTEST FAIL: " + msg);
+  };
+  console.log("\n" + "═".repeat(96));
+  console.log(
+    "SHIP-STATUS v1 FIXTURE CORPUS — the publish gate (all 6 regression cases + coverage tail)",
+  );
+  console.log("═".repeat(96));
+
+  // ── PART 1: strict kind↔probe map is 1:1 (ZERO cross-kind selections) ──────────────────────────────
+  // A mega-context that simultaneously offers EVERY target type. Each ship kind must reconstruct ONLY its
+  // own probe family — a `pushed` claim must NEVER select the PR probe, etc.
+  const mega = [
+    "opened PR #123 (state OPEN)",
+    "deployed to https://app.example.com/health",
+    `commit ${SHA_A}`,
+    "$ git push origin feat-x",
+    "$ git tag v3.1.4",
+    "$ npm publish",
+    "skalpel@3.1.4",
+  ].join("\n");
+  const payloadMega = { cwd: SHIP_HERE, prompt: mega };
+  for (const kind of Object.keys(SHIP_KIND_PROBE)) {
+    const t = reconstructShipTargetV1(kind, mega, payloadMega);
+    assert(t, `1:1 map — ${kind} reconstructs a target from the mega-context`);
+    const fam = shipProbeFamily(t.kind);
+    assert(
+      fam === SHIP_KIND_PROBE[kind],
+      `1:1 map — ${kind} → ${fam} but expected ${SHIP_KIND_PROBE[kind]} (target.kind=${t.kind})`,
+    );
+  }
+  // Explicit anti-cross-kind: pushed never becomes a PR; merged never a URL/git; live never a PR.
+  assert(
+    reconstructShipTargetV1("pushed", mega, payloadMega).kind === "git-remote",
+    "pushed → git-remote only",
+  );
+  assert(reconstructShipTargetV1("merged", mega, payloadMega).kind === "pr", "merged → pr only");
+  assert(
+    ["url"].includes(reconstructShipTargetV1("live", mega, payloadMega).kind),
+    "live → url only",
+  );
+  // No cross-kind FALLBACK when the own-kind target is absent → NO_TARGET (null), never a wrong-kind probe.
+  assert(
+    reconstructShipTargetV1("pushed", "opened PR #99 (OPEN)", { cwd: SHIP_HERE }) === null,
+    "pushed with only a PR (no sha) → null (NOT a PR probe)",
+  );
+  assert(
+    reconstructShipTargetV1("merged", "deployed to https://x.example.com", { cwd: SHIP_HERE }) ===
+      null,
+    "merged with only a URL (no PR) → null (NOT a URL probe)",
+  );
+  assert(
+    reconstructShipTargetV1("live", "merged PR #5 into main", { cwd: SHIP_HERE }) === null,
+    "live with only a PR (no URL) → null (NOT a PR probe)",
+  );
+  console.log(
+    `  ✓ kind↔probe map is 1:1 across ${Object.keys(SHIP_KIND_PROBE).length} kinds — ZERO cross-kind selections`,
+  );
+  console.log("");
+
+  // ── PART 2: the fixture corpus (claim → probe → verdict → red?) ────────────────────────────────────
+  const rows = [];
+  let pass = 0;
+  for (const f of SHIP_FIXTURES) {
+    const det = detectShipClaim(f.claim);
+    assert(det.claim, `[${f.id}] detects a ship claim in: ${f.claim}`);
+    assert(det.kind === f.kind, `[${f.id}] claim kind ${det.kind} !== expected ${f.kind}`);
+    const res = await resolveShipStatus({
+      kind: det.kind,
+      contextText: f.context,
+      payload: { cwd: SHIP_HERE, prompt: f.claim },
+      exec: f.exec,
+      confirmDelayMs: 0, // never actually sleep 60s in the eval
+    });
+    const red = res.mismatch === true; // a red fires iff the logic decides SHIP_FAIL (armed gates aside)
+    assert(
+      res.outcome === f.expectOutcome,
+      `[${f.id}] outcome ${res.outcome} !== expected ${f.expectOutcome} (${f.label})`,
+    );
+    assert(red === f.expectRed, `[${f.id}] red=${red} !== expected ${f.expectRed} (${f.label})`);
+    if ("expectReprobe" in f)
+      assert(
+        res.confirm_reprobe === f.expectReprobe,
+        `[${f.id}] confirm_reprobe=${res.confirm_reprobe} !== expected ${f.expectReprobe}`,
+      );
+    pass++;
+    rows.push({
+      id: f.id,
+      probe: res.probe_kind || SHIP_KIND_PROBE[det.kind] || "-",
+      verdict: res.outcome,
+      red: red ? "RED" : "—",
+      reprobe: res.confirm_reprobe ? "2/2" : "",
+      label: f.label,
+    });
+  }
+
+  // pretty table
+  const pad = (s, n) => String(s).padEnd(n);
+  console.log(
+    `  ${pad("#", 3)}${pad("kind", 11)}${pad("probe", 12)}${pad("verdict", 14)}${pad("red?", 6)}${pad("reprobe", 9)}case`,
+  );
+  console.log("  " + "─".repeat(92));
+  for (let i = 0; i < SHIP_FIXTURES.length; i++) {
+    const f = SHIP_FIXTURES[i];
+    const r = rows[i];
+    console.log(
+      `  ${pad(r.id, 3)}${pad(f.kind, 11)}${pad(r.probe, 12)}${pad(r.verdict, 14)}${pad(r.red, 6)}${pad(r.reprobe, 9)}${r.label}`,
+    );
+  }
+  console.log("  " + "─".repeat(92));
+  console.log(
+    `\n  SHIP CORPUS: ${pass}/${SHIP_FIXTURES.length} fixtures pass (incl. regression a,b,c → NO red; catches d,e,f → RED).`,
+  );
+  if (pass !== SHIP_FIXTURES.length) throw new Error("SHIP-SELFTEST FAIL: not all fixtures passed");
+  console.log(
+    "  ALL SHIP FIXTURES PASSED — strict kind↔probe, remote-truth, HTTP precision + confirm-gate verified.",
+  );
+  return true;
+}
+
 // ───────────────────────────── CLI ─────────────────────────
 function fmt(obj) {
   return JSON.stringify(obj, null, 2);
@@ -1134,6 +1527,33 @@ const isMain = (() => {
     return false;
   }
 })();
-if (isMain) main();
+if (isMain) {
+  const a0 = process.argv[2];
+  if (a0 === "--ship-selftest") {
+    // The SHIP-STATUS v1 publish gate (Defect 6) — async, so it gets its own entry.
+    runShipSelfTests()
+      .then((ok) => process.exit(ok ? 0 : 1))
+      .catch((e) => {
+        console.error(String((e && e.stack) || e));
+        process.exit(1);
+      });
+  } else if (a0 === "--selftest") {
+    // `--selftest` now covers BOTH the intervention machinery AND the ship-status v1 corpus.
+    try {
+      runSelfTests();
+    } catch (e) {
+      console.error(String((e && e.stack) || e));
+      process.exit(1);
+    }
+    runShipSelfTests()
+      .then((ok) => process.exit(ok ? 0 : 1))
+      .catch((e) => {
+        console.error(String((e && e.stack) || e));
+        process.exit(1);
+      });
+  } else {
+    main();
+  }
+}
 
 export { V1_onset, V2_cue, V3_repeat, FIRE_ALWAYS, FIRE_NEVER, calibrate };