skalpel 3.4.16 → 3.4.18

package/prosumer-hooks/incremental-ingest.test.mjs

@@ -0,0 +1,320 @@
+import assert from "node:assert/strict";
+import {
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { gunzipSync } from "node:zlib";
+
+import {
+  drainIncrementalOutbox,
+  enqueueCompletedSession,
+  pendingItems,
+} from "./incremental-ingest.mjs";
+
+function fixture() {
+  const home = mkdtempSync(join(tmpdir(), "skalpel-session-end-"));
+  const dir = join(home, ".claude", "projects", "repo");
+  mkdirSync(dir, { recursive: true });
+  const transcriptPath = join(dir, "session-123.jsonl");
+  const transcript =
+    '{"type":"user","message":{"content":"implement it"}}\n' +
+    '{"type":"assistant","message":{"content":"done"}}\n';
+  writeFileSync(transcriptPath, transcript);
+  return {
+    home,
+    transcript,
+    payload: {
+      hook_event_name: "SessionEnd",
+      session_id: "session-123",
+      transcript_path: transcriptPath,
+      reason: "other",
+    },
+  };
+}
+
+function jsonResponse(body, status = 200) {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+const auth = async () => ({ uid: "user-123", token: "token-123" });
+const noInsight = () => {};
+
+test("SessionEnd snapshots one allowed transcript and deduplicates repeated events", () => {
+  const f = fixture();
+  try {
+    const first = enqueueCompletedSession(f.payload, { home: f.home });
+    const second = enqueueCompletedSession(f.payload, { home: f.home });
+    assert.equal(first.key, second.key);
+    assert.equal(first.session_id, "session-123");
+    assert.equal(pendingItems(f.home).length, 1);
+
+    const outside = join(f.home, "secret.jsonl");
+    writeFileSync(outside, "secret");
+    assert.equal(
+      enqueueCompletedSession({ ...f.payload, transcript_path: outside }, { home: f.home }),
+      null,
+      "a forged hook payload must not upload arbitrary local files",
+    );
+  } finally {
+    rmSync(f.home, { recursive: true, force: true });
+  }
+});
+
+test("accepted job sends exactly the completed transcript and records delivery", async () => {
+  const f = fixture();
+  try {
+    enqueueCompletedSession(f.payload, { home: f.home });
+    const calls = [];
+    const fetchFn = async (url, init) => {
+      calls.push({ url, init });
+      if (url.endsWith("/ingest")) return jsonResponse({ job_id: "job-1", replayed: false }, 202);
+      return jsonResponse({ state: "done", done: 1, total: 1, errors: 0, n_sessions: 1 });
+    };
+
+    const result = await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      fetchFn,
+      insightFn: noInsight,
+    });
+    assert.equal(result.pending, 0);
+    assert.equal(calls.length, 2);
+    const posted = JSON.parse(gunzipSync(calls[0].init.body).toString("utf8"));
+    assert.deepEqual(Object.keys(posted.raw_transcripts), ["session-123.jsonl"]);
+    assert.equal(posted.raw_transcripts["session-123.jsonl"], f.transcript);
+    assert.match(calls[0].init.headers["idempotency-key"], /^session-end:[a-f0-9]{64}:0$/);
+
+    const duplicate = enqueueCompletedSession(f.payload, { home: f.home });
+    assert.equal(duplicate.key.length, 64);
+    await drainIncrementalOutbox({
+      home: f.home,
+      identityFn: auth,
+      insightFn: noInsight,
+      fetchFn: async () => {
+        throw new Error("the account-scoped delivered ledger should suppress this replay");
+      },
+    });
+    assert.equal(pendingItems(f.home).length, 0);
+  } finally {
+    rmSync(f.home, { recursive: true, force: true });
+  }
+});
+
+test("accepted-response loss retries with the same persisted idempotency key", async () => {
+  const f = fixture();
+  try {
+    enqueueCompletedSession(f.payload, { home: f.home });
+    const keys = [];
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      fetchFn: async (_url, init) => {
+        keys.push(init.headers["idempotency-key"]);
+        throw new Error("response lost");
+      },
+    });
+    assert.equal(pendingItems(f.home).length, 1);
+
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      fetchFn: async (url, init) => {
+        if (url.endsWith("/ingest")) {
+          keys.push(init.headers["idempotency-key"]);
+          return jsonResponse({ job_id: "job-replayed", replayed: true }, 202);
+        }
+        return jsonResponse({ state: "done", done: 1, total: 1, n_sessions: 1 });
+      },
+    });
+    assert.equal(keys.length, 2);
+    assert.equal(keys[0], keys[1]);
+    assert.equal(pendingItems(f.home).length, 0);
+  } finally {
+    rmSync(f.home, { recursive: true, force: true });
+  }
+});
+
+test("active or quota-waiting jobs are polled by job id without reposting", async () => {
+  const f = fixture();
+  let clock = Date.parse("2026-07-12T05:00:00Z");
+  try {
+    const queued = enqueueCompletedSession(f.payload, { home: f.home, now: () => clock });
+    const snapshot = join(f.home, ".skalpel", "ingest-outbox", `${queued.key}.jsonl`);
+    const snapshotMtime = statSync(snapshot).mtimeMs;
+    let posts = 0;
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      now: () => clock,
+      workBudgetMs: 1,
+      pollIntervalMs: 1,
+      fetchFn: async (url) => {
+        if (url.endsWith("/ingest")) {
+          posts += 1;
+          return jsonResponse({ job_id: "job-waiting" }, 202);
+        }
+        return jsonResponse({
+          state: "waiting_quota",
+          phase: "waiting_quota",
+          done: 0,
+          total: 1,
+          errors: 0,
+          heartbeat_at: "2026-07-12T05:00:00Z",
+          terminal: false,
+          requires_replay: false,
+          retry_at: new Date(clock + 5 * 60 * 1000).toISOString(),
+        });
+      },
+    });
+    assert.equal(pendingItems(f.home)[0].job_id, "job-waiting");
+    assert.equal(pendingItems(f.home)[0].state, "waiting_quota");
+    assert.equal(pendingItems(f.home)[0].phase, "waiting_quota");
+    assert.equal(pendingItems(f.home)[0].terminal, false);
+    assert.equal(pendingItems(f.home)[0].requires_replay, false);
+    assert.equal(pendingItems(f.home)[0].next_poll_at, clock + 5 * 60 * 1000);
+    assert.equal(readFileSync(snapshot, "utf8"), f.transcript);
+    assert.equal(
+      statSync(snapshot).mtimeMs,
+      snapshotMtime,
+      "status heartbeats must update only tiny metadata, never rewrite the transcript snapshot",
+    );
+
+    let fetchedBeforeRetry = false;
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      now: () => clock,
+      fetchFn: async () => {
+        fetchedBeforeRetry = true;
+        throw new Error("must honor retry_at");
+      },
+    });
+    assert.equal(fetchedBeforeRetry, false, "nonterminal waiting_quota must pause until retry_at");
+
+    clock += 6 * 60 * 1000;
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      now: () => clock,
+      fetchFn: async (url) => {
+        assert.ok(url.includes("/ingest/status?job_id=job-waiting"));
+        return jsonResponse({ state: "done", done: 1, total: 1, n_sessions: 1 });
+      },
+    });
+    assert.equal(posts, 1, "the persisted job id must be resumed instead of creating another job");
+    assert.equal(pendingItems(f.home).length, 0);
+  } finally {
+    rmSync(f.home, { recursive: true, force: true });
+  }
+});
+
+test("terminal waiting_quota with requires_replay mints one new retry generation", async () => {
+  const f = fixture();
+  let clock = Date.parse("2026-07-12T05:00:00Z");
+  const keys = [];
+  try {
+    enqueueCompletedSession(f.payload, { home: f.home, now: () => clock });
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      now: () => clock,
+      fetchFn: async (url, init) => {
+        if (url.endsWith("/ingest")) {
+          keys.push(init.headers["idempotency-key"]);
+          return jsonResponse({ job_id: "job-failed" }, 202);
+        }
+        return jsonResponse({
+          state: "waiting_quota",
+          phase: "judge",
+          done: 0,
+          total: 1,
+          errors: 1,
+          terminal: true,
+          requires_replay: true,
+          error: "quota",
+        });
+      },
+    });
+    const pending = pendingItems(f.home)[0];
+    assert.equal(pending.state, "retry_queued");
+    assert.equal(pending.generation, 1);
+    assert.equal(pending.job_id, null);
+
+    clock += 6 * 60 * 1000;
+    await drainIncrementalOutbox({
+      home: f.home,
+      api: "https://graph.test",
+      identityFn: auth,
+      insightFn: noInsight,
+      now: () => clock,
+      fetchFn: async (url, init) => {
+        if (url.endsWith("/ingest")) {
+          keys.push(init.headers["idempotency-key"]);
+          return jsonResponse({ job_id: "job-retry" }, 202);
+        }
+        return jsonResponse({ state: "done", done: 1, total: 1, n_sessions: 1 });
+      },
+    });
+    assert.equal(keys.length, 2);
+    assert.notEqual(keys[0], keys[1]);
+    assert.match(keys[0], /:0$/);
+    assert.match(keys[1], /:1$/);
+    assert.equal(pendingItems(f.home).length, 0);
+  } finally {
+    rmSync(f.home, { recursive: true, force: true });
+  }
+});
+
+test("a bootstrap timestamp never falsely acknowledges an unknown per-session result", async () => {
+  const f = fixture();
+  try {
+    const item = enqueueCompletedSession(f.payload, { home: f.home });
+    const dir = join(f.home, ".skalpel");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(
+      join(dir, "bootstrap.json"),
+      JSON.stringify({
+        state: "done",
+        sessions: 1,
+        last_ingest_at: new Date(item.transcript_mtime_ms + 1_000).toISOString(),
+      }),
+    );
+    const result = await drainIncrementalOutbox({
+      home: f.home,
+      identityFn: async () => ({ uid: null, token: null }),
+      insightFn: noInsight,
+    });
+    assert.equal(result.pending, 1);
+    assert.equal(result.authRequired, true);
+    assert.equal(
+      pendingItems(f.home)[0].key,
+      item.key,
+      "bulk catch-up has no per-session receipt, so the durable item must remain until its job confirms",
+    );
+  } finally {
+    rmSync(f.home, { recursive: true, force: true });
+  }
+});

package/prosumer-hooks/install.mjs

@@ -15,7 +15,7 @@ const statuslineOnly = !uninstall && process.argv.includes("--statusline-only");
 
 // Where this installer's own files live — the globally-installed package, a source checkout, OR the
 // transient `npx` cache. All three must produce ONE working command, so we can't rely on a bin being
-// on PATH (only `npm i -g` puts it there; `npx` and a source run don't). Instead we copy the three
+// on PATH (only `npm i -g` puts it there; `npx` and a source run don't). Instead we copy the
 // self-contained hook files into a STABLE home and wire an absolute `node <path>` command that works
 // offline, survives npx cache GC, and needs no global install — so `npx @skalpelai/prosumer-hook` is
 // the single command that leaves genuinely-working hooks.
@@ -26,6 +26,7 @@ const HOOKS_DIR = join(homedir(), ".skalpel", "hooks");
 // gets healed to the absolute command on the next run.
 const HOOK_FILE = join(HOOKS_DIR, "skalpel-hook.mjs");
 const SESSION_FILE = join(HOOKS_DIR, "skalpel-hook-session.mjs");
+const SESSION_END_FILE = join(HOOKS_DIR, "skalpel-hook-session-end.mjs");
 const STATUSLINE_FILE = join(HOOKS_DIR, "skalpel-statusline.mjs");
 
 // Copy the hook runtime (the two entrypoints + their shared auth.mjs) into ~/.skalpel/hooks so the
@@ -41,6 +42,8 @@ function stageHookRuntime() {
   copyFileSync(join(PKG_DIR, "insights.mjs"), join(HOOKS_DIR, "insights.mjs")); // hooks import ./insights.mjs
   copyFileSync(join(PKG_DIR, "skalpel-hook.mjs"), HOOK_FILE);
   copyFileSync(join(PKG_DIR, "skalpel-hook-session.mjs"), SESSION_FILE);
+  copyFileSync(join(PKG_DIR, "incremental-ingest.mjs"), join(HOOKS_DIR, "incremental-ingest.mjs"));
+  copyFileSync(join(PKG_DIR, "skalpel-hook-session-end.mjs"), SESSION_END_FILE);
   copyFileSync(join(PKG_DIR, "skalpel-statusline.mjs"), STATUSLINE_FILE);
   copyFileSync(join(PKG_DIR, "auth.mjs"), join(HOOKS_DIR, "auth.mjs")); // hooks import ./auth.mjs
   copyFileSync(join(PKG_DIR, "metrics.mjs"), join(HOOKS_DIR, "metrics.mjs")); // hooks import ./metrics.mjs
@@ -65,6 +68,12 @@ if (!uninstall && !process.env.SKALPEL_HOOK_BIN) {
 // SKALPEL_HOOK_SESSION_BIN still override (dev, or a deliberate global-bin install).
 const CMD = process.env.SKALPEL_HOOK_BIN || `node ${HOOK_FILE}`;
 const SESSION_CMD = process.env.SKALPEL_HOOK_SESSION_BIN || `node ${SESSION_FILE}`;
+const sessionEndRuntime =
+  process.env.SKALPEL_HOOK_BIN && PKG_DIR !== HOOKS_DIR
+    ? join(PKG_DIR, "skalpel-hook-session-end.mjs")
+    : SESSION_END_FILE;
+const SESSION_END_CMD =
+  process.env.SKALPEL_HOOK_SESSION_END_BIN || `node ${sessionEndRuntime}`;
 const STATUSLINE_CMD = process.env.SKALPEL_STATUSLINE_BIN || `node ${STATUSLINE_FILE}`;
 const CLAUDE = join(homedir(), ".claude", "settings.json");
 const CODEX_DIR = join(homedir(), ".codex");
@@ -81,6 +90,13 @@ const HOOKS = [
   ["UserPromptSubmit", markerOf(CMD), CMD],
   ["SessionStart", markerOf(SESSION_CMD), SESSION_CMD],
 ];
+// Claude Code supports a once-per-session SessionEnd event and native background command hooks.
+// Codex currently supports neither SessionEnd nor async command hooks, so its honest fallback is
+// the existing next-launch history scan; do not write an inert Codex hook and claim parity.
+const CLAUDE_HOOKS = [
+  ...HOOKS,
+  ["SessionEnd", markerOf(SESSION_END_CMD), SESSION_END_CMD],
+];
 
 function readJson(p) {
   let raw;
@@ -113,9 +129,11 @@ function isOurs(c, marker) {
     marker,
     "skalpel-hook",
     "skalpel-hook-session",
+    "skalpel-hook-session-end",
     "skalpel-statusline",
     "skalpel-prosumer-hook",
     "skalpel-prosumer-hook-session",
+    "skalpel-prosumer-hook-session-end",
     "skalpel-prosumer-statusline",
   ];
   return markers.some((candidate) => {
@@ -138,7 +156,7 @@ function claude() {
   let removed = 0;
   if (!statuslineOnly) {
     d.hooks ??= {};
-    for (const [event, marker, command] of HOOKS) {
+    for (const [event, marker, command] of CLAUDE_HOOKS) {
       d.hooks[event] ??= [];
       const arr = d.hooks[event];
       const isMine = (e) => {
@@ -148,9 +166,14 @@ function claude() {
       };
       const had = arr.some(isMine);
       const kept = arr.filter((e) => !isMine(e)); // drop ours (both shapes) AND repair
-      // 8s, not 5s: the hosted backend can cold-start after idle, and the hook's own internal
-      // DEADLINE_MS (4.5s) needs headroom under this outer harness timeout or it gets killed first.
-      const group = { hooks: [{ type: "command", command, timeout: 8 }] };
+      // SessionEnd uses Claude Code's native async process: the transcript snapshot happens before
+      // networking inside that process, and session termination never waits for ingest/polling.
+      // Other hooks retain their tight interactive budget.
+      const handler =
+        event === "SessionEnd"
+          ? { type: "command", command, timeout: 55, async: true }
+          : { type: "command", command, timeout: 8 };
+      const group = { hooks: [handler] };
       d.hooks[event] = uninstall ? kept : [...kept, group];
       if (uninstall) {
         if (had) removed += 1;

package/prosumer-hooks/skalpel-hook-session-end.mjs

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+// Claude Code SessionEnd hook: snapshot the one completed transcript into a durable local outbox,
+// then submit/poll it in Claude's native async-hook process. It never writes stdout/stderr and
+// always exits 0, so graph maintenance cannot delay or break session termination.
+import { readFileSync } from "node:fs";
+
+import { captureAndDrainSession } from "./incremental-ingest.mjs";
+
+async function main() {
+  let payload;
+  try {
+    payload = JSON.parse(readFileSync(0, "utf8") || "{}");
+  } catch {
+    return;
+  }
+  await captureAndDrainSession(payload);
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch(() => process.exit(0));