skalpel 4.0.44 → 4.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/anchor-shadow.mjs +428 -0
- package/card.mjs +64 -12
- package/eval-harness.mjs +421 -1
- package/first-run.mjs +722 -0
- package/package.json +1 -1
- package/skalpel-setup.mjs +8 -0
- package/verify-shadow.mjs +231 -75
package/package.json
CHANGED
package/skalpel-setup.mjs
CHANGED
|
@@ -23,6 +23,7 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
23
23
|
import { identity as authIdentity, cliAuthPath } from "./auth.mjs";
|
|
24
24
|
import { isGenuineUserTurn } from "./session-turns.mjs";
|
|
25
25
|
import { appendAdjudication, suppressProof, lastReveal, parseArmToken } from "./insights.mjs";
|
|
26
|
+
import { maybeFirstRunCatch } from "./first-run.mjs";
|
|
26
27
|
|
|
27
28
|
// CLI (the `skalpel` bin): `skalpel setup [--api https://graph.skalpel.ai]
|
|
28
29
|
// [--user <id>]`, plus `skalpel uninstall` and `skalpel login`. A bare invocation
|
|
@@ -1119,6 +1120,13 @@ async function main() {
|
|
|
1119
1120
|
// never blocks (stays default OFF). Reversible anytime with `skalpel verify off`.
|
|
1120
1121
|
await promptVerifyConsent();
|
|
1121
1122
|
|
|
1123
|
+
// THE FIRST CATCH — first-run magic. Immediately after consent-arming completes, if this is a TTY and
|
|
1124
|
+
// the user consented, scan their OWN coding history (read-only, ~3s hard budget) and reveal ONE real
|
|
1125
|
+
// "done over a failing test" catch from it — collapsing time-to-first-aha to seconds. Fail-open and
|
|
1126
|
+
// byte-identical-off: non-TTY / CI / declined-consent never scans or reads a transcript, and any error
|
|
1127
|
+
// skips silently to the normal setup ending. Never throws; never blocks setup.
|
|
1128
|
+
await maybeFirstRunCatch({ isTTY, env: process.env });
|
|
1129
|
+
|
|
1122
1130
|
// 2) find the real coding sessions in the last 30 days (the client reads; the server builds)
|
|
1123
1131
|
const s1 = spinner(`Looking through your last ${LOOKBACK_DAYS} days of Claude Code + Codex…`);
|
|
1124
1132
|
await sleep(500);
|
package/verify-shadow.mjs
CHANGED
|
@@ -8,15 +8,17 @@
|
|
|
8
8
|
// invented), RE-RUN it out-of-band, and record the real PASS/FAIL the agent cannot forge. The money
|
|
9
9
|
// metric is the MISMATCH rate: how often the agent claimed success but its own proof, re-run, FAILS.
|
|
10
10
|
//
|
|
11
|
-
// SHIP-STATUS catch (
|
|
12
|
-
// tests to the DEPLOY/GTM claim — "it's live / deployed / merged / pushed /
|
|
13
|
-
// turn ASSERTS a ship status, we reconstruct the
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
11
|
+
// SHIP-STATUS v1 catch ("Remote-Truth", same DARK gates): the SAME "the agent is its own reporter"
|
|
12
|
+
// problem extends past tests to the DEPLOY/GTM claim — "it's live / deployed / merged / pushed / released
|
|
13
|
+
// / CI green". When the last turn ASSERTS a ship status, we reconstruct EXACTLY the target of that claim's
|
|
14
|
+
// own family (STRICT kind↔probe, NO cross-kind fallback) and run ONE READ-ONLY probe: pushed → REMOTE-TRUTH
|
|
15
|
+
// (`git ls-remote origin <branch>` for the real tip, then a local `git merge-base --is-ancestor` — a push
|
|
16
|
+
// that never left the machine is caught, a real feature-branch push is never false-accused); merged → `gh
|
|
17
|
+
// pr view`; live/deployed → `curl` (with HTTP precision: 401/403/405/5xx → UNKNOWN, only a CONFIRMED 404/
|
|
18
|
+
// gone fires); released/published → `npm view` / `git ls-remote --tags`; CI green → `gh pr checks`. On a
|
|
19
|
+
// DECISIVE negative we surface the SAME honest reveal the test catch uses — from REAL probe output, gated
|
|
20
|
+
// on the SAME opt-in flag. The read-only probe primitives (execFile arg-arrays, strict whitelist, validated
|
|
21
|
+
// args, bounded cap) are REUSED from the security-reviewed anchor-shadow.mjs (PR #586) — never re-implemented.
|
|
20
22
|
//
|
|
21
23
|
// SHADOW: this LOGS ONLY. It never injects into the model, never blocks or slows a turn, and never
|
|
22
24
|
// deploys anything. It re-runs the session's OWN test/build proof, which is normally read-only; a proof
|
|
@@ -30,9 +32,9 @@
|
|
|
30
32
|
// npx tsc|jest|vitest|eslint|biome, pytest, cargo test|build|check|clippy, go test|build|vet,
|
|
31
33
|
// tsc, jest, vitest, eslint, biome, ruff, mypy, pyright, python -m <allowlisted>). Anything else
|
|
32
34
|
// is REFUSED. The command never comes from prompt/claim CONTENT — only from the session's own
|
|
33
|
-
// prior Bash tool_use commands. The SHIP-STATUS probe is likewise allowlisted to a fixed set of
|
|
34
|
-
// READ-ONLY reachability/status commands (gh pr view, curl reachability, git
|
|
35
|
-
//
|
|
35
|
+
// prior Bash tool_use commands. The SHIP-STATUS v1 probe is likewise allowlisted to a fixed set of
|
|
36
|
+
// READ-ONLY reachability/status commands (gh pr view / pr checks, curl reachability, git ls-remote +
|
|
37
|
+
// merge-base --is-ancestor, npm view) — see anchor-shadow.mjs runShipProbeV1/validateShipTargetV1.
|
|
36
38
|
// 2. execFile, NEVER a shell. Args are passed as a literal ARGUMENT ARRAY; there is no shell to
|
|
37
39
|
// interpret metacharacters, so a crafted string can never be executed as a command. shell:false.
|
|
38
40
|
// The ship probe validates every reconstructed value (pr=digits, url=well-formed http(s), sha=hex)
|
|
@@ -82,12 +84,20 @@ import {
|
|
|
82
84
|
AUTO_DARKEN_MIN_ADJ,
|
|
83
85
|
AUTO_DARKEN_MIN_PRECISION,
|
|
84
86
|
} from "./insights.mjs";
|
|
85
|
-
// SHIP-STATUS catch: REUSE the security-reviewed, READ-ONLY probe primitives from anchor-shadow.mjs
|
|
86
|
-
// (PR #586)
|
|
87
|
-
//
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
|
|
87
|
+
// SHIP-STATUS v1 catch: REUSE the security-reviewed, READ-ONLY probe primitives from anchor-shadow.mjs
|
|
88
|
+
// (PR #586) — never re-implemented here. The v1 layer is STRICT kind↔probe (no cross-kind fallback):
|
|
89
|
+
// reconstructShipTargetV1(claim, ctx, payload) mines EXACTLY the target of the claim's own family (or
|
|
90
|
+
// null → honest silence); runShipProbeV1(target) runs that family's single READ-ONLY command via execFile
|
|
91
|
+
// arg-arrays with a bounded cap and re-validated args; SHIP_KIND_PROBE/shipProbeFamily are the 1:1 map;
|
|
92
|
+
// recentContextText is the bounded-tail context reader. Importing (not copying) keeps ONE copy of the
|
|
93
|
+
// safety-critical probe logic under one review.
|
|
94
|
+
import {
|
|
95
|
+
recentContextText,
|
|
96
|
+
reconstructShipTargetV1,
|
|
97
|
+
runShipProbeV1,
|
|
98
|
+
shipProbeFamily,
|
|
99
|
+
SHIP_KIND_PROBE,
|
|
100
|
+
} from "./anchor-shadow.mjs";
|
|
91
101
|
|
|
92
102
|
const DIR = join(homedir(), ".skalpel");
|
|
93
103
|
export const VERIFY_LOG_PATH = join(DIR, "verify-shadow.log");
|
|
@@ -889,10 +899,10 @@ export function buildRevealLine(row) {
|
|
|
889
899
|
// sha on main?), not a re-run. We look at the agent's LAST assistant prose (same source as the test
|
|
890
900
|
// claim) and fire ONLY on an ASSERTION of ship status — never a question, never a stated intention.
|
|
891
901
|
//
|
|
892
|
-
// Each pattern maps to a canonical claim
|
|
893
|
-
// (live | deployed | merged | pushed | released
|
|
894
|
-
//
|
|
895
|
-
//
|
|
902
|
+
// Each pattern maps to a canonical claim kind that anchor-shadow.mjs `reconstructShipTargetV1` understands
|
|
903
|
+
// (live | deployed | merged | pushed | released | published | ci). Each kind has EXACTLY ONE probe family
|
|
904
|
+
// (SHIP_KIND_PROBE) — a missed claim just means no catch, but a false claim would probe a target the agent
|
|
905
|
+
// never actually shipped, so precision over recall. First matching pattern per line wins.
|
|
896
906
|
const SHIP_CLAIM_PATTERNS = [
|
|
897
907
|
// deployed / deploy complete / pushed the deploy
|
|
898
908
|
[
|
|
@@ -919,10 +929,20 @@ const SHIP_CLAIM_PATTERNS = [
|
|
|
919
929
|
"pushed",
|
|
920
930
|
/\b(?:it'?s\s+|now\s+|has\s+been\s+|is\s+)?pushed\b|\bpushed\s+(?:it|the\s+(?:commit|branch|changes?)|to\s+(?:main|origin|remote))\b/i,
|
|
921
931
|
],
|
|
932
|
+
// published (to npm) — the registry claim, distinct from a git-tag release
|
|
933
|
+
[
|
|
934
|
+
"published",
|
|
935
|
+
/\bpublished\b|\bpush(?:ed)?\s+to\s+npm\b|\bnpm\s+publish(?:ed)?\b|\bon\s+(?:the\s+)?(?:npm\s+)?registry\b/i,
|
|
936
|
+
],
|
|
922
937
|
// released / shipped / shipped to prod
|
|
923
938
|
[
|
|
924
939
|
"released",
|
|
925
|
-
/\b(?:it'?s\s+|now\s+|has\s+been\s+)?released\b|\bshipp?ed\s+(?:it|to\s+prod(?:uction)?)\b/i,
|
|
940
|
+
/\b(?:it'?s\s+|now\s+|has\s+been\s+)?released\b|\bshipp?ed\s+(?:it|to\s+prod(?:uction)?)\b|\btagged\s+(?:the\s+)?release\b/i,
|
|
941
|
+
],
|
|
942
|
+
// CI green / checks passing / pipeline green — the REMOTE checks claim (distinct from a local test claim)
|
|
943
|
+
[
|
|
944
|
+
"ci",
|
|
945
|
+
/\bci\s+(?:is\s+|are\s+|now\s+|all\s+)?(?:green|passing|pass(?:ed|es)?)\b|\b(?:the\s+)?checks?\s+(?:are\s+|is\s+|all\s+|now\s+)?(?:green|passing|pass(?:ed|es)?)\b|\ball\s+checks?\s+(?:are\s+)?(?:green|passing|passed)\b|\bpipeline\s+(?:is\s+)?(?:green|passing)\b/i,
|
|
926
946
|
],
|
|
927
947
|
];
|
|
928
948
|
|
|
@@ -931,7 +951,7 @@ const SHIP_FUTURE_OR_QUESTION =
|
|
|
931
951
|
/\b(?:will|i'?ll|we'?ll|going\s+to|gonna|about\s+to|let\s+me|planning\s+to|need\s+to|shall\s+i|should\s+i|can\s+i|once\s+|after\s+i|before\s+i|ready\s+to)\b/i;
|
|
932
952
|
|
|
933
953
|
// detectShipClaim(text) → { claim:bool, kind, text:<the claim line, <=120c> }. Scans lines; returns the
|
|
934
|
-
// first ASSERTIVE ship-status line. `kind` is the canonical claim keyword
|
|
954
|
+
// first ASSERTIVE ship-status line. `kind` is the canonical claim keyword reconstructShipTargetV1 consumes.
|
|
935
955
|
export function detectShipClaim(text) {
|
|
936
956
|
const s = typeof text === "string" ? text : "";
|
|
937
957
|
if (!s.trim()) return { claim: false, kind: null, text: "" };
|
|
@@ -955,19 +975,34 @@ function safeShipTarget(t) {
|
|
|
955
975
|
if (!t || typeof t !== "object") return null;
|
|
956
976
|
if (t.kind === "url") return { kind: "url", url: clipText(t.url, 160) };
|
|
957
977
|
if (t.kind === "pr") return { kind: "pr", n: t.n };
|
|
958
|
-
if (t.kind === "git")
|
|
959
|
-
|
|
978
|
+
if (t.kind === "git-remote")
|
|
979
|
+
return {
|
|
980
|
+
kind: "git-remote",
|
|
981
|
+
sha: String(t.sha || "").toLowerCase(),
|
|
982
|
+
remote: t.remote || "origin",
|
|
983
|
+
branch: t.branch || null,
|
|
984
|
+
};
|
|
985
|
+
if (t.kind === "npm") return { kind: "npm", name: t.name, version: t.version };
|
|
986
|
+
if (t.kind === "git-tag") return { kind: "git-tag", tag: t.tag, remote: t.remote || "origin" };
|
|
987
|
+
if (t.kind === "gh-checks")
|
|
988
|
+
return { kind: "gh-checks", n: t.n ?? null, sha: t.sha ? String(t.sha).slice(0, 12) : null };
|
|
960
989
|
return null;
|
|
961
990
|
}
|
|
962
991
|
|
|
963
|
-
// probeCommandString(t) — a human receipt of the
|
|
964
|
-
// It mirrors anchor-shadow.mjs
|
|
992
|
+
// probeCommandString(t) — a human receipt of the READ-ONLY command(s) that ran (for the log + report).
|
|
993
|
+
// It mirrors anchor-shadow.mjs runShipProbeV1's argv; it is never executed (execFile ran the arg-array).
|
|
965
994
|
function probeCommandString(t) {
|
|
966
995
|
if (!t || typeof t !== "object") return null;
|
|
967
996
|
if (t.kind === "url") return `curl -sS -m 5 -o /dev/null -w %{http_code} ${clipText(t.url, 120)}`;
|
|
968
997
|
if (t.kind === "pr") return `gh pr view ${t.n} --json state,mergedAt`;
|
|
969
|
-
if (t.kind === "git")
|
|
970
|
-
|
|
998
|
+
if (t.kind === "git-remote")
|
|
999
|
+
return `git ls-remote ${t.remote || "origin"} ${t.branch || "<branch>"} ; git merge-base --is-ancestor ${String(t.sha || "").slice(0, 12)} <remoteTip>`;
|
|
1000
|
+
if (t.kind === "npm") return `npm view ${t.name}@${t.version} version`;
|
|
1001
|
+
if (t.kind === "git-tag") return `git ls-remote --tags ${t.remote || "origin"} ${t.tag}`;
|
|
1002
|
+
if (t.kind === "gh-checks")
|
|
1003
|
+
return t.n != null
|
|
1004
|
+
? `gh pr checks ${t.n} --json state,bucket`
|
|
1005
|
+
: `gh run list --commit ${String(t.sha || "").slice(0, 12)} --json conclusion,status`;
|
|
971
1006
|
return null;
|
|
972
1007
|
}
|
|
973
1008
|
|
|
@@ -1002,7 +1037,7 @@ export function buildShipRevealLine(row) {
|
|
|
1002
1037
|
const ev = String(row.evidence || "");
|
|
1003
1038
|
if (t.kind === "url") {
|
|
1004
1039
|
const m = ev.match(/http_code=(\d+)/);
|
|
1005
|
-
const tail = m ? `returns HTTP ${m[1]}` : "is
|
|
1040
|
+
const tail = m ? `returns HTTP ${m[1]}` : "is unreachable (no response)";
|
|
1006
1041
|
return `🔬 skalpel caught it — your agent said "${claim}" but \`${clipText(t.url, 48)}\` ${tail}. It's not live.`;
|
|
1007
1042
|
}
|
|
1008
1043
|
if (t.kind === "pr") {
|
|
@@ -1010,12 +1045,22 @@ export function buildShipRevealLine(row) {
|
|
|
1010
1045
|
const st = m ? m[1].toUpperCase() : "OPEN";
|
|
1011
1046
|
return `🔬 skalpel caught it — your agent said "${claim}" but PR #${t.n} is ${st}, not merged. It's not done.`;
|
|
1012
1047
|
}
|
|
1013
|
-
if (t.kind === "git") {
|
|
1048
|
+
if (t.kind === "git-remote") {
|
|
1014
1049
|
const sha = String(t.sha || "").slice(0, 7);
|
|
1015
|
-
|
|
1050
|
+
const branch = `${t.remote || "origin"}/${t.branch || "?"}`;
|
|
1051
|
+
const tm = ev.match(/remoteTip=([0-9a-f]{7,12})/);
|
|
1052
|
+
const at = tm ? ` at ${tm[1]}` : "";
|
|
1053
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but ${sha} isn't in ${branch}${at} — it never left your machine. It's not pushed.`;
|
|
1054
|
+
}
|
|
1055
|
+
if (t.kind === "npm") {
|
|
1056
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but ${t.name}@${t.version} isn't on the npm registry. It's not published.`;
|
|
1016
1057
|
}
|
|
1017
|
-
if (t.kind === "
|
|
1018
|
-
return `🔬 skalpel caught it — your agent said "${claim}" but
|
|
1058
|
+
if (t.kind === "git-tag") {
|
|
1059
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but tag ${t.tag} isn't on ${t.remote || "origin"}. It's not released.`;
|
|
1060
|
+
}
|
|
1061
|
+
if (t.kind === "gh-checks") {
|
|
1062
|
+
const ref = t.n != null ? `PR #${t.n}` : `commit ${String(t.sha || "").slice(0, 7)}`;
|
|
1063
|
+
return `🔬 skalpel caught it — your agent said "${claim}" but checks on ${ref} are failing, not green.`;
|
|
1019
1064
|
}
|
|
1020
1065
|
return `🔬 skalpel caught it — your agent said "${claim}" but the ship-status probe came back negative.`;
|
|
1021
1066
|
}
|
|
@@ -1152,11 +1197,13 @@ function maybeSurfaceReveal(row) {
|
|
|
1152
1197
|
line: revealLineFor(row),
|
|
1153
1198
|
});
|
|
1154
1199
|
// MEASURABLE: one reveal_shown insight per fire → fire-rate + (later) retention, no fabricated data.
|
|
1200
|
+
// A ship reveal carries its ship-claim kind (pushed/merged/live/…) so per-kind precision is derivable.
|
|
1155
1201
|
recordInsight({
|
|
1156
1202
|
kind: "reveal_shown",
|
|
1157
1203
|
display: revealLineFor(row),
|
|
1158
1204
|
proof: sig,
|
|
1159
1205
|
session: row.session || null,
|
|
1206
|
+
...(row.category === "ship" ? { ship_kind: row.ship_claim || null } : {}),
|
|
1160
1207
|
});
|
|
1161
1208
|
} else if (isPass) {
|
|
1162
1209
|
clearReveal(row.session || null); // resolved — don't wallpaper a catch the agent already fixed
|
|
@@ -1179,12 +1226,98 @@ function maybeSurfaceReveal(row) {
|
|
|
1179
1226
|
}
|
|
1180
1227
|
}
|
|
1181
1228
|
|
|
1182
|
-
//
|
|
1183
|
-
//
|
|
1184
|
-
//
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1229
|
+
// SHIP-STATUS v1 confirm-gate: the wait between a URL FAIL_CANDIDATE (404/410/NXDOMAIN/refused) and its
|
|
1230
|
+
// ONE confirming re-probe. Default ~60s; overridable via env (SKALPEL_SHIP_CONFIRM_MS, 0 for tests). The
|
|
1231
|
+
// wait happens ONLY inside the DETACHED, unref'd `--run` worker — never on the per-turn hook deadline.
|
|
1232
|
+
const SHIP_CONFIRM_DELAY_MS = (() => {
|
|
1233
|
+
const v = parseInt(process.env.SKALPEL_SHIP_CONFIRM_MS ?? "", 10);
|
|
1234
|
+
return Number.isFinite(v) && v >= 0 ? v : 60_000;
|
|
1235
|
+
})();
|
|
1236
|
+
function shipSleep(ms) {
|
|
1237
|
+
return new Promise((r) => {
|
|
1238
|
+
const t = setTimeout(r, ms);
|
|
1239
|
+
if (t && typeof t.unref === "function") t.unref(); // never keep the worker alive on our account
|
|
1240
|
+
});
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// resolveShipStatus({kind, contextText, payload, target?, exec?, ...}) — the TESTABLE core of the ship
|
|
1244
|
+
// catch: reconstruct the strict single-family target (or accept a prebuilt one), run its ONE read-only
|
|
1245
|
+
// probe, and (only for a URL FAIL_CANDIDATE) confirm-gate before ever calling it a red. Returns a plain
|
|
1246
|
+
// result object — no I/O, no lock, no reveal. `exec` (the read-only command runner) and `sleep` are
|
|
1247
|
+
// injectable purely so the eval harness can drive every branch hermetically. Never throws to the caller.
|
|
1248
|
+
// outcome ∈ { NO_TARGET, SHIP_OK, SHIP_FAIL, SHIP_UNKNOWN }; mismatch === (outcome === "SHIP_FAIL").
|
|
1249
|
+
export async function resolveShipStatus({
|
|
1250
|
+
kind,
|
|
1251
|
+
contextText,
|
|
1252
|
+
payload,
|
|
1253
|
+
target,
|
|
1254
|
+
exec,
|
|
1255
|
+
confirmDelayMs,
|
|
1256
|
+
sleep = shipSleep,
|
|
1257
|
+
beforeReprobe,
|
|
1258
|
+
}) {
|
|
1259
|
+
const tgt = target !== undefined ? target : reconstructShipTargetV1(kind, contextText, payload);
|
|
1260
|
+
if (!tgt) {
|
|
1261
|
+
return {
|
|
1262
|
+
kind,
|
|
1263
|
+
target: null,
|
|
1264
|
+
probe_kind: null,
|
|
1265
|
+
outcome: "NO_TARGET",
|
|
1266
|
+
mismatch: false,
|
|
1267
|
+
evidence: "",
|
|
1268
|
+
confirm_reprobe: false,
|
|
1269
|
+
confirm_outcome: null,
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
const probe_kind = shipProbeFamily(tgt.kind);
|
|
1273
|
+
const first = await runShipProbeV1(tgt, exec);
|
|
1274
|
+
let verdict = first.verdict;
|
|
1275
|
+
let evidence = first.evidence;
|
|
1276
|
+
let confirm_reprobe = false;
|
|
1277
|
+
let confirm_outcome = null;
|
|
1278
|
+
// Defect 3 CONFIRM-GATE: a URL fail candidate (a 404/410/gone or a NXDOMAIN/refused, which can be pure
|
|
1279
|
+
// CDN/propagation lag) NEVER reveals immediately — wait ~60s and re-probe ONCE; only a CONFIRMED
|
|
1280
|
+
// double-negative (2/2 fail candidate) fires. A PASS on the re-probe means it was transient → SHIP_OK.
|
|
1281
|
+
if (verdict === "FAIL_CANDIDATE" && tgt.kind === "url") {
|
|
1282
|
+
if (typeof beforeReprobe === "function") {
|
|
1283
|
+
try {
|
|
1284
|
+
beforeReprobe();
|
|
1285
|
+
} catch {
|
|
1286
|
+
/* the confirm re-probe proceeds even if the lock refresh fails */
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
const wait = confirmDelayMs != null ? confirmDelayMs : SHIP_CONFIRM_DELAY_MS;
|
|
1290
|
+
if (wait > 0) await sleep(wait);
|
|
1291
|
+
const second = await runShipProbeV1(tgt, exec);
|
|
1292
|
+
confirm_reprobe = true;
|
|
1293
|
+
confirm_outcome = second.verdict;
|
|
1294
|
+
evidence = second.evidence || evidence;
|
|
1295
|
+
verdict =
|
|
1296
|
+
second.verdict === "FAIL_CANDIDATE" ? "FAIL" : second.verdict === "PASS" ? "PASS" : "UNKNOWN";
|
|
1297
|
+
} else if (verdict === "FAIL_CANDIDATE") {
|
|
1298
|
+
// A non-URL probe should never yield a lone candidate; treat as UNKNOWN so it can never single-red.
|
|
1299
|
+
verdict = "UNKNOWN";
|
|
1300
|
+
}
|
|
1301
|
+
const outcome =
|
|
1302
|
+
verdict === "PASS" ? "SHIP_OK" : verdict === "FAIL" ? "SHIP_FAIL" : "SHIP_UNKNOWN";
|
|
1303
|
+
return {
|
|
1304
|
+
kind,
|
|
1305
|
+
target: tgt,
|
|
1306
|
+
probe_kind,
|
|
1307
|
+
outcome,
|
|
1308
|
+
mismatch: outcome === "SHIP_FAIL",
|
|
1309
|
+
evidence,
|
|
1310
|
+
confirm_reprobe,
|
|
1311
|
+
confirm_outcome,
|
|
1312
|
+
};
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
// maybeShipStatusCatch(...) — the SHIP-STATUS v1 proof-category. When the agent's last turn ASSERTS a ship
|
|
1316
|
+
// status, reconstruct EXACTLY the target of that claim's own family (STRICT kind↔probe, no cross-kind
|
|
1317
|
+
// fallback) and run ONE READ-ONLY probe (reused from anchor-shadow.mjs). Appends one `category:"ship"`
|
|
1318
|
+
// shadow row and, on a DECISIVE negative, surfaces the SAME reveal (gated on SKALPEL_VERIFY_REVEAL).
|
|
1319
|
+
// Returns { surfaced } — true ONLY when a decisive ship FAIL was revealed, so the caller can skip the test
|
|
1320
|
+
// path (the concrete ship catch is the stronger signal). LOG ONLY otherwise; never injects, never throws.
|
|
1188
1321
|
async function maybeShipStatusCatch({ text, session, cwd, transcriptPath }) {
|
|
1189
1322
|
try {
|
|
1190
1323
|
const { claim, kind, text: claimText } = detectShipClaim(text);
|
|
@@ -1193,16 +1326,18 @@ async function maybeShipStatusCatch({ text, session, cwd, transcriptPath }) {
|
|
|
1193
1326
|
const sig = `ship|${session || ""}|${djb2(claimText)}`;
|
|
1194
1327
|
if (alreadyVerified(sig, SHIP_LAST_PATH)) return { surfaced: false };
|
|
1195
1328
|
|
|
1196
|
-
// Reconstruct the
|
|
1197
|
-
//
|
|
1198
|
-
//
|
|
1329
|
+
// Reconstruct the STRICT single-family target. The context is the bounded recent transcript tail
|
|
1330
|
+
// (where the agent's own `gh`/`git`/deploy output printed the PR/URL/sha); the claim line is appended
|
|
1331
|
+
// so a target named only in the claim ("deployed to https://x") is seen too.
|
|
1199
1332
|
const payload = {
|
|
1200
1333
|
cwd: cwd || process.cwd(),
|
|
1201
1334
|
transcript_path: transcriptPath,
|
|
1202
1335
|
prompt: claimText,
|
|
1203
1336
|
};
|
|
1204
|
-
const target =
|
|
1337
|
+
const target = reconstructShipTargetV1(kind, recentContextText(payload), payload);
|
|
1205
1338
|
if (!target) {
|
|
1339
|
+
// No target of the RIGHT kind → NO_TARGET → honest silence (no red). Logged so the instrument can
|
|
1340
|
+
// count the reconstruction miss. (Cheap, no probe, no lock.)
|
|
1206
1341
|
markVerified(sig, SHIP_LAST_PATH);
|
|
1207
1342
|
appendShadowLog({
|
|
1208
1343
|
ts: new Date().toISOString(),
|
|
@@ -1210,59 +1345,53 @@ async function maybeShipStatusCatch({ text, session, cwd, transcriptPath }) {
|
|
|
1210
1345
|
category: "ship",
|
|
1211
1346
|
claim_text: claimText,
|
|
1212
1347
|
ship_claim: kind,
|
|
1348
|
+
probe_kind: SHIP_KIND_PROBE[kind] || null,
|
|
1213
1349
|
target: null,
|
|
1214
1350
|
probe_command: null,
|
|
1215
1351
|
pass_fail: null,
|
|
1216
1352
|
outcome: "NO_TARGET",
|
|
1217
1353
|
mismatch: false,
|
|
1354
|
+
confirm_reprobe: false,
|
|
1355
|
+
confirm_outcome: null,
|
|
1218
1356
|
evidence: "",
|
|
1219
1357
|
});
|
|
1220
1358
|
return { surfaced: false };
|
|
1221
1359
|
}
|
|
1222
1360
|
|
|
1223
1361
|
// One read-only probe, single-flight (shares the lock with the test re-run so a burst never piles up).
|
|
1362
|
+
// The lock is held across the (possibly ~60s) URL confirm-gate; touchLock refreshes our own mtime so
|
|
1363
|
+
// it isn't read as stale mid-confirm. All of this runs in the DETACHED worker — off the hook deadline.
|
|
1224
1364
|
if (!acquireLock()) return { surfaced: false };
|
|
1225
|
-
let
|
|
1226
|
-
let ev1 = null;
|
|
1227
|
-
let ev2 = null; // set only when a URL no-response triggers a confirming re-probe (2/2 rule)
|
|
1365
|
+
let resolved;
|
|
1228
1366
|
try {
|
|
1229
|
-
|
|
1230
|
-
ev1 = res && res.evidence;
|
|
1231
|
-
// URL FALSE-RED SAFEGUARD: any HTTP status = reachable/live. Only a genuine no-response may be a
|
|
1232
|
-
// red, and only after a CONFIRMING re-probe. Held under the same single-flight lock (mtime refreshed).
|
|
1233
|
-
if (target.kind === "url" && !urlResponded(ev1)) {
|
|
1234
|
-
touchLock();
|
|
1235
|
-
const res2 = await runProbe(target);
|
|
1236
|
-
ev2 = res2 && res2.evidence;
|
|
1237
|
-
if (urlResponded(ev2)) res = res2; // came back on the 2nd probe → surface the LIVE result
|
|
1238
|
-
}
|
|
1367
|
+
resolved = await resolveShipStatus({ kind, target, payload, beforeReprobe: touchLock });
|
|
1239
1368
|
} finally {
|
|
1240
1369
|
releaseLock();
|
|
1241
1370
|
}
|
|
1242
1371
|
markVerified(sig, SHIP_LAST_PATH);
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
const pass = target.kind === "url" ? shipUrlPass(ev1, ev2) : res && res.pass;
|
|
1247
|
-
const outcome = pass === true ? "SHIP_OK" : pass === false ? "SHIP_FAIL" : "SHIP_UNKNOWN";
|
|
1248
|
-
const pass_fail = pass === true ? "PASS" : pass === false ? "FAIL" : "UNKNOWN";
|
|
1249
|
-
const compact = safeShipTarget(target);
|
|
1372
|
+
|
|
1373
|
+
const outcome = resolved.outcome;
|
|
1374
|
+
const pass_fail = outcome === "SHIP_OK" ? "PASS" : outcome === "SHIP_FAIL" ? "FAIL" : "UNKNOWN"; // NO_TARGET handled above
|
|
1250
1375
|
const row = {
|
|
1251
1376
|
ts: new Date().toISOString(),
|
|
1252
1377
|
session: session || null,
|
|
1253
1378
|
category: "ship",
|
|
1254
1379
|
claim_text: claimText,
|
|
1255
1380
|
ship_claim: kind,
|
|
1256
|
-
|
|
1381
|
+
probe_kind: resolved.probe_kind,
|
|
1382
|
+
target: safeShipTarget(target),
|
|
1257
1383
|
probe_command: probeCommandString(target),
|
|
1258
1384
|
pass_fail,
|
|
1259
1385
|
outcome,
|
|
1260
|
-
mismatch:
|
|
1261
|
-
|
|
1386
|
+
mismatch: outcome === "SHIP_FAIL", // claimed shipped, the read-only probe DECISIVELY says it is NOT
|
|
1387
|
+
// The instrument records the confirm re-probe (URL fail-candidate) and its second-probe verdict.
|
|
1388
|
+
confirm_reprobe: resolved.confirm_reprobe,
|
|
1389
|
+
confirm_outcome: resolved.confirm_outcome,
|
|
1390
|
+
evidence: String(resolved.evidence || "").slice(0, 200),
|
|
1262
1391
|
};
|
|
1263
1392
|
appendShadowLog(row);
|
|
1264
1393
|
maybeSurfaceReveal(row); // gated on SKALPEL_VERIFY_REVEAL — dark by default
|
|
1265
|
-
return { surfaced:
|
|
1394
|
+
return { surfaced: outcome === "SHIP_FAIL" && revealEnabled(process.env) };
|
|
1266
1395
|
} catch {
|
|
1267
1396
|
return { surfaced: false }; // SHADOW: swallow everything — never affect the hook or the turn
|
|
1268
1397
|
}
|
|
@@ -1456,10 +1585,11 @@ export function renderVerifyReport() {
|
|
|
1456
1585
|
}
|
|
1457
1586
|
}
|
|
1458
1587
|
|
|
1459
|
-
// SHIP-STATUS proof-category — the deploy/GTM claim ("it's live / merged / pushed /
|
|
1460
|
-
// checked by a READ-ONLY probe. Its own instrument: how many ship claims
|
|
1461
|
-
// decisive,
|
|
1462
|
-
//
|
|
1588
|
+
// SHIP-STATUS v1 proof-category — the deploy/GTM claim ("it's live / merged / pushed / released / CI
|
|
1589
|
+
// green") checked by a STRICT single-family READ-ONLY probe. Its own instrument: how many ship claims
|
|
1590
|
+
// were probed, how many were decisive, how many were caught (claimed shipped, the probe DECISIVELY said
|
|
1591
|
+
// NO), plus a PER-KIND S1/S2 breakdown and the URL confirm-reprobe count. Every number is real probe
|
|
1592
|
+
// output — an UNKNOWN probe (auth-wall/5xx/timeout/missing gh|curl|npm/DNS) is excluded, never a lie.
|
|
1463
1593
|
if (shipRows.length) {
|
|
1464
1594
|
const shipProbed = shipRows.filter((r) => r.target).length;
|
|
1465
1595
|
const shipDecisive = shipRows.filter(
|
|
@@ -1467,9 +1597,13 @@ export function renderVerifyReport() {
|
|
|
1467
1597
|
).length;
|
|
1468
1598
|
const shipCaught = shipRows.filter((r) => r.mismatch === true).length;
|
|
1469
1599
|
const shipUnknown = shipRows.filter((r) => r.outcome === "SHIP_UNKNOWN").length;
|
|
1600
|
+
const shipNoTarget = shipRows.filter((r) => r.outcome === "NO_TARGET").length;
|
|
1601
|
+
const shipReprobes = shipRows.filter((r) => r.confirm_reprobe === true).length;
|
|
1470
1602
|
L.push("");
|
|
1471
1603
|
L.push("─".repeat(72));
|
|
1472
|
-
L.push(
|
|
1604
|
+
L.push(
|
|
1605
|
+
" SHIP-STATUS v1 catch (claimed live/merged/pushed/released/CI → STRICT read-only probe)",
|
|
1606
|
+
);
|
|
1473
1607
|
L.push(` ship claims logged ${shipRows.length}`);
|
|
1474
1608
|
L.push(
|
|
1475
1609
|
` (S1) target-reconstruction rate ${pct(shipProbed, shipRows.length)} ${shipProbed}/${shipRows.length} ship claims yielded a probeable target`,
|
|
@@ -1478,8 +1612,30 @@ export function renderVerifyReport() {
|
|
|
1478
1612
|
` (S2) SHIP-MISMATCH rate ${pct(shipCaught, shipDecisive)} ${shipCaught}/${shipDecisive} decisive probes said NOT shipped after a ship claim`,
|
|
1479
1613
|
);
|
|
1480
1614
|
L.push(
|
|
1481
|
-
` (excluded: ${shipUnknown} unknown probe(s) — timeout/missing gh|curl/DNS, not counted as a lie)`,
|
|
1615
|
+
` (excluded: ${shipUnknown} unknown probe(s) — auth-wall/5xx/timeout/missing gh|curl|npm/DNS, not counted as a lie)`,
|
|
1616
|
+
);
|
|
1617
|
+
L.push(
|
|
1618
|
+
` (no-target: ${shipNoTarget} claim(s) with no probeable target of the right kind — honest silence, no red)`,
|
|
1619
|
+
);
|
|
1620
|
+
L.push(
|
|
1621
|
+
` (confirm-reprobes: ${shipReprobes} URL fail-candidate(s) re-probed after ~60s — only a 2/2 double-negative fires)`,
|
|
1482
1622
|
);
|
|
1623
|
+
// PER-KIND S1(fires)/S2(catches) + reprobes. Each kind maps to exactly one probe family.
|
|
1624
|
+
const kinds = [...new Set(shipRows.map((r) => r.ship_claim).filter(Boolean))].sort();
|
|
1625
|
+
if (kinds.length) {
|
|
1626
|
+
L.push("");
|
|
1627
|
+
L.push(" per-kind (probe · S1 fires · S2 catches · reprobes):");
|
|
1628
|
+
for (const k of kinds) {
|
|
1629
|
+
const kr = shipRows.filter((r) => r.ship_claim === k);
|
|
1630
|
+
const s1 = kr.length;
|
|
1631
|
+
const s2 = kr.filter((r) => r.mismatch === true).length;
|
|
1632
|
+
const rp = kr.filter((r) => r.confirm_reprobe === true).length;
|
|
1633
|
+
const pk = (kr.find((r) => r.probe_kind) || {}).probe_kind || SHIP_KIND_PROBE[k] || "?";
|
|
1634
|
+
L.push(
|
|
1635
|
+
` ${String(k).padEnd(10)} → ${String(pk).padEnd(11)} S1 ${s1} · S2 ${s2} · reprobe ${rp}`,
|
|
1636
|
+
);
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1483
1639
|
const recentShip = shipRows
|
|
1484
1640
|
.filter((r) => r.mismatch)
|
|
1485
1641
|
.slice(-3)
|