skalpel 4.0.14 → 4.0.15

package/incremental-ingest.mjs

@@ -32,6 +32,7 @@ const ITEM_VERSION = 1;
 const MAX_TRANSCRIPT_BYTES = 90 * 1024 * 1024;
 const MAX_DELIVERED_KEYS = 2048;
 const MAX_TERMINAL_RETRIES = 3;
+const MAX_VANISHED_RESUBMITS = 1;
 const LOCK_STALE_MS = 2 * 60 * 1000;
 const DEFAULT_REQUEST_TIMEOUT_MS = 8_000;
 const DEFAULT_WORK_BUDGET_MS = 45_000;
@@ -125,8 +126,16 @@ function baseKey(sessionID, transcript) {
     .digest("hex");
 }
 
+// The idempotency key is a STABLE CONTENT ADDRESS: item.key is the sha256 of (session id + full
+// transcript), so the SAME content produces the SAME key across every retry and generation. A retry
+// of an already-accepted transcript therefore dedups server-side and can never mint a duplicate
+// job/session or re-judge at Gemini cost. The key rotates ONLY when the server EXPLICITLY discards
+// the payload and asks for a fresh operation (requires_replay -> replay_epoch bump); ordinary retries
+// (network, error, 404-after-accept) must keep the key fixed. Generation still drives retry backoff,
+// but no longer leaks into the key.
 function idempotencyKey(item) {
-  return `session-end:${item.key}:${item.generation || 0}`;
+  const epoch = item.replay_epoch || 0;
+  return epoch > 0 ? `session-end:${item.key}:replay${epoch}` : `session-end:${item.key}`;
 }
 
 function readLedger(home) {
@@ -227,6 +236,10 @@ function scheduleTerminalRetry(home, item, status, now, doctorFn) {
   item.server_state = status.state || "error";
   item.last_error = String(status.error || "graph job failed").slice(0, 240);
   item.generation = nextGeneration;
+  // Only a server that EXPLICITLY discarded the payload needs a fresh operation key. Every other
+  // failure re-sends the identical content and must reuse the stable content-addressed key so the
+  // server can dedup it — otherwise a retry duplicates the session and re-judges at full cost.
+  if (status.requires_replay === true) item.replay_epoch = (item.replay_epoch || 0) + 1;
   item.next_poll_at = null;
   item.updated_at = new Date(now()).toISOString();
   if (nextGeneration <= MAX_TERMINAL_RETRIES) {
@@ -254,6 +267,45 @@ function scheduleTerminalRetry(home, item, status, now, doctorFn) {
   );
 }
 
+// A job that was ACCEPTED (the server issued a job_id) whose /ingest/status now 404s. Per the server
+// contract a mid-poll 404 means the job most likely COMPLETED and was evicted from the ephemeral job
+// store — the acceptance already gave us at-least-once delivery upstream. Because the idempotency key
+// is now a stable content address, a bounded re-submit is guaranteed non-duplicating (the server
+// dedups it on the same key), so we try ONE safe re-submit and then treat the content as delivered
+// rather than re-judging forever at full Gemini cost or risking a duplicate. This path is reached
+// ONLY for content that already reached the server; genuinely-never-accepted content has no job_id
+// and never lands here, so at-least-once is preserved for it separately.
+function handleVanishedJob(home, item, auth, now, doctorFn) {
+  const attempt = (item.not_found_count || 0) + 1;
+  item.not_found_count = attempt;
+  item.last_job_id = item.job_id || item.last_job_id || null;
+  item.user_id = item.user_id || auth.uid;
+  item.updated_at = new Date(now()).toISOString();
+  if (attempt > MAX_VANISHED_RESUBMITS) {
+    item.state = "delivered_assumed";
+    saveItem(home, item);
+    doctorFn(
+      home,
+      `job ${item.last_job_id} for key ${item.key} 404'd ${attempt}x after accept; ` +
+        "acceptance = at-least-once delivered, so recording as delivered and NOT re-sending",
+    );
+    markDelivered(home, item, now);
+    return "done";
+  }
+  item.job_id = null;
+  item.state = "resubmit_queued";
+  item.next_poll_at = null;
+  item.next_attempt_at = now() + retryDelayMs(attempt);
+  saveItem(home, item);
+  doctorFn(
+    home,
+    `job ${item.last_job_id} for key ${item.key} 404'd after accept ` +
+      `(attempt ${attempt}/${MAX_VANISHED_RESUBMITS}); one safe re-submit queued with ` +
+      `stable idempotency-key ${idempotencyKey(item)}`,
+  );
+  return "retry";
+}
+
 async function pollAcceptedJob(home, item, auth, options, deadline) {
   const { api, fetchFn, doctorFn, now, pollIntervalMs, requestTimeoutMs, sleepFn } = options;
   const statusURL = `${api}/ingest/status?job_id=${encodeURIComponent(item.job_id)}`;
@@ -278,8 +330,7 @@ async function pollAcceptedJob(home, item, auth, options, deadline) {
       return "pending";
     }
     if (response.status === 404) {
-      scheduleTerminalRetry(home, item, { state: "error", error: "job not found" }, now, doctorFn);
-      return "retry";
+      return handleVanishedJob(home, item, auth, now, doctorFn);
     }
     if (!response.ok) {
       item.state = `status_http_${response.status}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.14",
+  "version": "4.0.15",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {