portable-agent-layer 0.85.6 → 0.85.7

package/assets/schema/pal-settings.schema.json

@@ -82,6 +82,14 @@
           "type": "boolean",
           "description": "Keep a rolling local sample (about 230 messages) of user message text, its reaction label and the end of the reply it answered, in memory/state/reaction-samples.json, so the reaction rules can be audited. Stores message text; set false to keep none."
         },
+        "claimCheck": {
+          "type": "boolean",
+          "description": "At each stop, check whether the final reply claims a result (tests pass, CI green, works now) that no command in that turn could have shown, and log the verdict to memory/signals/claim-checks/. Stores the claiming sentence only when nothing backed it."
+        },
+        "claimCheckBlocks": {
+          "type": "boolean",
+          "description": "Opt-in, off by default. Send a reply with an unbacked result claim back to the agent once, asking it to show the output or say the result is unverified. Needs claimCheck. Not on Cursor, whose stop payload carries no reply text."
+        },
         "steeringTestReport": { "type": "boolean", "description": "When true, the assistant notes to the user which steering self-check fired. For the dual-live test period; ships false." }
       }
     },

package/assets/templates/pal-settings.json

@@ -29,6 +29,8 @@
     "dueReminders": false,
     "interactionAwareness": true,
     "reactionSampling": true,
+    "claimCheck": true,
+    "claimCheckBlocks": false,
     "steeringTestReport": false
   },
   "steering": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "portable-agent-layer",
-  "version": "0.85.6",
+  "version": "0.85.7",
   "description": "PAL — Portable Agent Layer: persistent personal context for AI coding assistants",
   "type": "module",
   "bin": {

package/src/hooks/StopOrchestrator.ts

@@ -6,7 +6,8 @@
  */
 
 import { checkReadmeSync } from "./handlers/readme-sync";
-import { blockResponse, isCodex, isCursor } from "./lib/agent";
+import { stopBlockResponse } from "./lib/agent";
+import { watchClaims } from "./lib/claim-log";
 import { duplicatesCursorHooks } from "./lib/cursor-shadow";
 import { logError } from "./lib/log";
 import { isPalSpawnedInference } from "./lib/spawn-guard";
@@ -24,21 +25,21 @@ try {
   // is worth less than not blocking at all — require the reason to raise one.
   const decision = checkReadmeSync();
   if (decision.decision === "block" && decision.reason) {
-    if (isCursor()) {
-      // Cursor stop hook: followup_message auto-sends to the agent
-      process.stdout.write(JSON.stringify({ followup_message: decision.reason }));
-    } else if (isCodex()) {
-      // Codex stop hook: additionalContext re-queues as next prompt
-      process.stdout.write(JSON.stringify({ additionalContext: decision.reason }));
-    } else {
-      // Claude Code, the Copilot CLI and VS Code's own Copilot each read a
-      // different stop-block shape; VS Code ignores the top-level keys entirely.
-      process.stdout.write(blockResponse(decision.reason, "Stop"));
-    }
+    process.stdout.write(stopBlockResponse(decision.reason));
     process.exit(0);
   }
 } catch (err) {
   logError("StopOrchestrator:readme-sync", err);
 }
 
-await stopTurn(await readStdinJSON<StopTurnPayload>());
+const payload = await readStdinJSON<StopTurnPayload>();
+try {
+  const sendBack = watchClaims(payload);
+  if (sendBack) {
+    process.stdout.write(stopBlockResponse(sendBack));
+    process.exit(0);
+  }
+} catch (err) {
+  logError("StopOrchestrator:claim-check", err);
+}
+await stopTurn(payload);

package/src/hooks/lib/agent.ts

@@ -267,6 +267,13 @@ export function promptContextResponse(context: string): string | null {
   return `${context}\n`;
 }
 
+/** Stops the agent from ending its turn and hands it the reason as its next input. */
+export function stopBlockResponse(reason: string): string {
+  if (isCursor()) return JSON.stringify({ followup_message: reason });
+  if (isCodex()) return JSON.stringify({ additionalContext: reason });
+  return blockResponse(reason, "Stop");
+}
+
 export function blockResponse(reason: string, hookEventName?: string): string {
   if (isCursor()) {
     return JSON.stringify({ permission: "deny", user_message: reason });

package/src/hooks/lib/claim-check.ts

@@ -0,0 +1,122 @@
+/**
+ * Whether a reply claims a result (tests pass, CI green, it works now) that no
+ * command in the same turn could have shown. Precision over recall: a hedged or
+ * negated sentence is never a claim, and an unreadable turn is never unbacked.
+ */
+
+import { isSystemText } from "./prompt-text";
+import { isCodexInjectedContext } from "./transcript";
+
+const CLAIM_RE =
+  /\b(?:tests?|suite|ci|checks?|builds?|gates?|pipeline)\b.*\b(?:pass(?:es|ed)?|green|succeed(?:s|ed)?)\b|\bverified\b|\b(?:works|working) now\b|\bnow works\b/i;
+const HEDGED_RE =
+  /\b(?:not|no longer|fail(?:s|ed|ing)?|until|when|if|unless|should|would|will|might|could|may)\b|n't\b/i;
+
+const COMMAND_TOOLS = new Set([
+  "Bash",
+  "BashOutput",
+  "Monitor",
+  "PowerShell",
+  "exec",
+  "shell",
+  "exec_command",
+]);
+
+export type ClaimVerdict = "none" | "backed" | "unbacked" | "unknown";
+
+export interface ClaimCheck {
+  verdict: ClaimVerdict;
+  claims: string[];
+}
+
+function sentences(reply: string): string[] {
+  return reply
+    .split(/(?<=[.!?])\s+|\n+/)
+    .map((s) => s.trim().replace(/[.!?]+$/, ""))
+    .filter(Boolean);
+}
+
+/** A quoted claim is being talked about, not made. */
+function withoutQuotes(sentence: string): string {
+  return sentence.replace(/"[^"]*"|“[^”]*”|`[^`]*`/g, "");
+}
+
+export function resultClaims(reply: string): string[] {
+  return sentences(reply).filter((s) => {
+    const own = withoutQuotes(s);
+    return CLAIM_RE.test(own) && !HEDGED_RE.test(own);
+  });
+}
+
+interface TranscriptLine {
+  type?: string;
+  message?: { content?: unknown };
+  payload?: { type?: string; role?: string; name?: string; content?: unknown };
+}
+
+function parsed(line: string): TranscriptLine | null {
+  try {
+    return JSON.parse(line);
+  } catch {
+    return null;
+  }
+}
+
+function blocks(content: unknown): { type?: string; name?: string; text?: string }[] {
+  return Array.isArray(content) ? content : [];
+}
+
+function promptText(entry: TranscriptLine): string | null {
+  if (entry.type === "user") {
+    const content = entry.message?.content;
+    if (typeof content === "string") return content;
+    const texts = blocks(content).filter((b) => b.type === "text");
+    return texts.length ? texts.map((b) => b.text ?? "").join(" ") : null;
+  }
+  if (entry.type === "response_item" && entry.payload?.role === "user") {
+    return blocks(entry.payload.content)
+      .map((b) => b.text ?? "")
+      .join(" ");
+  }
+  return null;
+}
+
+function isUserPrompt(entry: TranscriptLine): boolean {
+  const text = promptText(entry);
+  return text !== null && !isSystemText(text) && !isCodexInjectedContext(text);
+}
+
+function commandsIn(entry: TranscriptLine): number {
+  if (entry.type === "assistant")
+    return blocks(entry.message?.content).filter(
+      (b) => b.type === "tool_use" && COMMAND_TOOLS.has(b.name ?? "")
+    ).length;
+  const call = entry.payload;
+  if (entry.type !== "response_item" || !call) return 0;
+  if (call.type === "local_shell_call") return 1;
+  const isCall = call.type === "function_call" || call.type === "custom_tool_call";
+  return isCall && COMMAND_TOOLS.has(call.name ?? "") ? 1 : 0;
+}
+
+function isTurnEntry(entry: TranscriptLine): boolean {
+  return ["user", "assistant", "response_item"].includes(entry.type ?? "");
+}
+
+/** Null when no line reads as a turn, so an unknown transcript format never counts as zero. */
+export function commandsThisTurn(lines: string[]): number | null {
+  const entries = lines.map(parsed).filter((e): e is TranscriptLine => e !== null);
+  if (!entries.some(isTurnEntry)) return null;
+  let commands = 0;
+  for (const entry of entries) {
+    if (isUserPrompt(entry)) commands = 0;
+    commands += commandsIn(entry);
+  }
+  return commands;
+}
+
+export function checkClaims(reply: string, commands: number | null): ClaimCheck {
+  const claims = resultClaims(reply);
+  if (claims.length === 0) return { verdict: "none", claims };
+  if (commands === null) return { verdict: "unknown", claims };
+  return { verdict: commands > 0 ? "backed" : "unbacked", claims };
+}

package/src/hooks/lib/claim-log.ts

@@ -0,0 +1,131 @@
+/**
+ * The claim check at stop: every reply that makes a result claim is logged to
+ * memory/signals/claim-checks/ with its verdict. The claiming sentence is kept
+ * only when no command backed it, so the log can be audited. Sending the reply
+ * back is opt-in (dynamicContext.claimCheckBlocks).
+ */
+
+import { appendFileSync, existsSync, readdirSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { currentAttribution } from "./actor";
+import {
+  type ClaimCheck,
+  type ClaimVerdict,
+  checkClaims,
+  commandsThisTurn,
+} from "./claim-check";
+import { hookFinalReply, hookSessionId } from "./hook-turn";
+import { ensureDir, paths } from "./paths";
+import { isEnabled, isOptedIn } from "./settings";
+import type { StopTurnPayload } from "./stop";
+
+const CLAIM_MAX = 200;
+
+export interface ClaimRecord {
+  ts: string;
+  session: string;
+  runtime?: string;
+  verdict: ClaimVerdict;
+  commands: number | null;
+  claims?: string[];
+  blocked?: true;
+}
+
+function claimsDir(): string {
+  return ensureDir(resolve(paths.signals(), "claim-checks"));
+}
+
+function monthFile(date: Date): string {
+  return resolve(claimsDir(), `${date.toISOString().slice(0, 7)}.jsonl`);
+}
+
+function transcriptLines(payload: StopTurnPayload): string[] {
+  const path = payload.transcript_path ?? payload.transcriptPath;
+  if (!path || !existsSync(path)) return [];
+  return readFileSync(path, "utf-8").split("\n").filter(Boolean);
+}
+
+function needsAudit(verdict: ClaimVerdict): boolean {
+  return verdict === "unbacked" || verdict === "unknown";
+}
+
+interface Watched {
+  check: ClaimCheck;
+  commands: number | null;
+  session: string;
+  blocked: boolean;
+}
+
+function record({ check, commands, session, blocked }: Watched, now: Date) {
+  const line: ClaimRecord = {
+    ts: now.toISOString(),
+    session,
+    runtime: currentAttribution().runtime,
+    verdict: check.verdict,
+    commands,
+    ...(needsAudit(check.verdict) && {
+      claims: check.claims.map((c) => c.slice(0, CLAIM_MAX)),
+    }),
+    ...(blocked && { blocked }),
+  };
+  appendFileSync(monthFile(now), `${JSON.stringify(line)}\n`);
+}
+
+function alreadySentBack(payload: StopTurnPayload): boolean {
+  return payload.stop_hook_active === true || (payload.loop_count ?? 0) > 0;
+}
+
+function sendsBack(payload: StopTurnPayload, check: ClaimCheck): boolean {
+  return (
+    check.verdict === "unbacked" &&
+    isOptedIn("claimCheckBlocks") &&
+    !alreadySentBack(payload)
+  );
+}
+
+function sendBackReason(claims: string[]): string {
+  const quoted = claims.map((c) => `"${c.slice(0, CLAIM_MAX)}"`).join(", ");
+  return `Your reply claims a result no command in this turn showed: ${quoted}. Run the check and show its output, or say it is unverified.`;
+}
+
+/**
+ * Checks the final reply against the turn's commands and logs any result claim.
+ * Returns the reason to send the reply back with, when that is opted in.
+ */
+export function watchClaims(
+  payload: StopTurnPayload | null,
+  now: Date = new Date()
+): string | null {
+  if (!payload || !isEnabled("claimCheck")) return null;
+  const reply = hookFinalReply(payload);
+  if (!reply) return null;
+  const commands = commandsThisTurn(transcriptLines(payload));
+  const check = checkClaims(reply, commands);
+  if (check.verdict === "none") return null;
+  const blocked = sendsBack(payload, check);
+  record({ check, commands, session: hookSessionId(payload) ?? "unknown", blocked }, now);
+  return blocked ? sendBackReason(check.claims) : null;
+}
+
+function readRecords(file: string): ClaimRecord[] {
+  return readFileSync(file, "utf-8")
+    .split("\n")
+    .filter(Boolean)
+    .flatMap((line) => {
+      try {
+        return [JSON.parse(line) as ClaimRecord];
+      } catch {
+        return [];
+      }
+    });
+}
+
+export function claimChecksSince(since: Date): ClaimRecord[] {
+  const fromMonth = since.toISOString().slice(0, 7);
+  const sinceTs = since.toISOString();
+  return readdirSync(claimsDir())
+    .filter((f) => f.endsWith(".jsonl") && f.slice(0, 7) >= fromMonth)
+    .sort()
+    .flatMap((f) => readRecords(resolve(claimsDir(), f)))
+    .filter((r) => r.ts >= sinceTs);
+}

package/src/hooks/lib/stop.ts

@@ -41,6 +41,8 @@ import {
 export interface StopTurnPayload extends HookTurnPayload {
   transcript_path?: string | null;
   transcriptPath?: string | null;
+  stop_hook_active?: boolean;
+  loop_count?: number;
 }
 
 type Transcript = ReturnType<typeof readTranscriptFile>;

package/src/hooks/lib/transcript.ts

@@ -38,7 +38,7 @@ interface CodexPayload {
 
 const CODEX_INJECTED_CONTEXT = ["# AGENTS.md instructions", "<environment_context>"];
 
-function isCodexInjectedContext(text: string): boolean {
+export function isCodexInjectedContext(text: string): boolean {
   return CODEX_INJECTED_CONTEXT.some((prefix) => text.startsWith(prefix));
 }
 

package/src/tools/agent/interaction.ts

@@ -7,8 +7,9 @@
  */
 
 import { parseArgs } from "node:util";
+import { claimChecksSince } from "../../hooks/lib/claim-log";
 import { turnsSince } from "../../hooks/lib/interaction";
-import { reportLines, summarize } from "../lib/interaction-report";
+import { claimCheckLines, reportLines, summarize } from "../lib/interaction-report";
 import { scriptArgs } from "../lib/script-args";
 
 const HELP = `
@@ -18,7 +19,9 @@ const HELP = `
   text) and prints turns, channels, reactions, the hints sent, per agent the
   turns, replies filed and hints sent, and for each
   label (short, long, fast, skimming, friction) how the replies written while
-  it was active compare with the replies written while none was.
+  it was active compare with the replies written while none was. Last, the
+  replies that claimed a result (memory/signals/claim-checks/), and which of
+  them had no command behind the claim or were sent back.
 
   Usage: pal cli interaction report [--days N]   (default 7)
 `;
@@ -39,8 +42,11 @@ export function run(argv: string[] = scriptArgs()) {
     console.log(HELP);
     return;
   }
-  const events = turnsSince(new Date(Date.now() - days * DAY_MS));
-  for (const line of reportLines(summarize(events), days)) console.log(line);
+  const since = new Date(Date.now() - days * DAY_MS);
+  const events = turnsSince(since);
+  const claims = claimCheckLines(claimChecksSince(since));
+  for (const line of [...reportLines(summarize(events), days), ...claims])
+    console.log(line);
 }
 
 if (import.meta.main) run();

package/src/tools/lib/interaction-report.ts

@@ -3,6 +3,7 @@
  * the agent's replies changed while a measured label was in its context.
  */
 
+import type { ClaimRecord } from "../../hooks/lib/claim-log";
 import type { TurnEvent } from "../../hooks/lib/interaction";
 import { median } from "../../hooks/lib/interaction-mood";
 import {
@@ -198,6 +199,22 @@ function labelLine(label: string, s: ReplyStats, usual: ReplyStats): string {
   return parts.join(" · ");
 }
 
+const UNBACKED_LISTED = 5;
+
+export function claimCheckLines(records: ClaimRecord[]): string[] {
+  if (records.length === 0) return [];
+  const unbacked = records.filter((r) => r.verdict === "unbacked");
+  const unknown = records.filter((r) => r.verdict === "unknown").length;
+  const sentBack = records.filter((r) => r.blocked).length;
+  return [
+    "",
+    `Result claims: ${plural(records.length, "reply", "replies")} · ${unbacked.length} with no command behind it · ${sentBack} sent back · ${unknown} unreadable`,
+    ...unbacked
+      .slice(-UNBACKED_LISTED)
+      .flatMap((r) => (r.claims ?? []).map((c) => `  no command: ${c}`)),
+  ];
+}
+
 function plural(n: number, noun: string, nouns = `${noun}s`): string {
   return `${n} ${n === 1 ? noun : nouns}`;
 }