skalpel 4.0.0 → 4.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.0",
+  "version": "4.0.2",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {

package/skalpel-hook-session.mjs

@@ -121,6 +121,38 @@ const API = process.env.SKALPEL_API || CFG.api || "https://graph.skalpel.ai";
 // load with ~1s margin under the 8s backstop; warm it still returns in ~300ms, so no cost when hot.
 const DEADLINE_MS = 7000;
 
+// One-shot reveal: the detached background build drops ~/.skalpel/insights-ready.json when it
+// finishes. Surface it ONCE at the next session start, then clear it — so the user sees "your graph is
+// ready + here's what I learned" the moment it lands. This is the payoff of the async build.
+function graphReadyNote() {
+  const p = join(homedir(), ".skalpel", "insights-ready.json");
+  let m;
+  try {
+    m = JSON.parse(readFileSync(p, "utf8"));
+  } catch {
+    return null; // nothing pending
+  }
+  if (!m || !m.ready) return null;
+  try {
+    writeFileSync(p, JSON.stringify({ ready: false })); // one-shot — clear so it never repeats
+  } catch {
+    /* if we can't clear it we'd rather drop the reveal than repeat it, but proceed */
+  }
+  const h = (m.report && m.report.headline) || {};
+  const trap = m.report && (m.report.discriminative || [])[0] && m.report.discriminative[0].seq;
+  const n = m.n_sessions || h.sessions || 0;
+  const bits = [];
+  if (h.hours) bits.push(`~${h.hours}h with AI`);
+  if (h.rework_pct) bits.push(`~${h.rework_pct}% of it went to rework`);
+  if (trap) bits.push(`your signature trap is ${trap}`);
+  const learned = bits.length ? ` I learned: ${bits.join(", ")}.` : "";
+  return (
+    `[skalpel — the user's behavioral graph just FINISHED building in the background ` +
+    `(${n} session${n === 1 ? "" : "s"}). Open THIS reply with ONE natural line so they see it landed, ` +
+    `in skalpel's voice: "🔬 skalpel · your graph is ready —${learned}" Then continue normally.]`
+  );
+}
+
 async function main() {
   let payload = {};
   try {
@@ -169,9 +201,10 @@ async function main() {
     record("session", outcome, Date.now() - t0);
   }
 
-  // Recap (last session's felt wins) leads, then the standing profile. Either can fire alone — a first
-  // session has no recap; a session with no graph yet has no profile but can still recap.
-  const additionalContext = [recapNote, context].filter(Boolean).join("\n\n");
+  // A just-finished background build leads (the async payoff — surface it the instant it's ready),
+  // then last session's recap, then the standing profile. Any can fire alone.
+  const readyNote = graphReadyNote();
+  const additionalContext = [readyNote, recapNote, context].filter(Boolean).join("\n\n");
 
   // LOCAL INSIGHT RECORDS (~/.skalpel/insights.ndjson) — plain-English rows for the TUI, one per
   // note that fired. recordInsight swallows everything, so these never affect the hook output.

package/skalpel-setup.mjs

@@ -491,18 +491,24 @@ async function buildGraph(sp, files, id) {
     await sleep(600);
     return n;
   }
-  // Read the raw transcript files and upload them (the server filters + judges + builds). The upload
-  // must fit the server's single-call capacity — a TOTAL body-size cap and a session-count cap — so a
-  // heavy user with hundreds of sessions / hundreds of MB doesn't 413 and end up with no graph. Only
-  // the TOTAL is bounded (never per-file: big sessions are the richest data). Newest-first, so if the
-  // total cap is ever hit we keep the freshest history. Caps overridable for tuning.
+  // CHUNKED UPLOAD. The server folds sessions in idempotently (n+1 aggregate, dedup by session_id),
+  // so instead of one giant body we upload small BYTE-BOUNDED batches. A 150MB one-shot decompressed
+  // in-memory and OOM-killed the pod (the "async" build never even started — the blowup is in reading
+  // the body, before the job). Batches keep the server's per-request memory tiny and the ingest is
+  // naturally incremental. Batches run SEQUENTIALLY: a fold reads-then-writes the corpus, so two
+  // concurrent batches would race and lose sessions. All caps overridable for tuning.
+  const { gzipSync } = await import("node:zlib");
+  const MB = 1024 * 1024;
   const MAX_SESSIONS = Math.max(
     1,
-    Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "20", 10),
+    Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "40", 10),
+  );
+  const CHUNK_BYTES = Math.max(1, Number.parseInt(process.env.SKALPEL_CHUNK_MB || "12", 10)) * MB;
+  const CHUNK_SESSIONS = Math.max(
+    1,
+    Number.parseInt(process.env.SKALPEL_CHUNK_SESSIONS || "6", 10),
   );
-  const MAX_BYTES =
-    Math.max(1, Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_MB || "90", 10)) * 1024 * 1024;
-  // newest first, so when a cap is hit we keep the freshest history
+  // newest-first, so a capped history keeps the freshest sessions
   const ranked = files
     .map((f) => {
       try {
@@ -513,105 +519,101 @@ async function buildGraph(sp, files, id) {
       }
     })
     .filter(Boolean)
-    .sort((a, b) => b.m - a.m);
-  const raw = {};
-  let bytes = 0;
-  let dropped = 0;
+    .sort((a, b) => b.m - a.m)
+    .slice(0, MAX_SESSIONS);
+  // pack into byte-bounded batches; a lone session bigger than the cap gets its own batch (never split)
+  const batches = [];
+  let cur = {};
+  let curBytes = 0;
   for (const { f, size } of ranked) {
-    // Do NOT drop a session for being large — big sessions are the RICHEST data (the long ones
-    // where the real thrash happens). The only real limit is the total request body (413), so we
-    // only stop once the WHOLE upload would exceed MAX_BYTES / MAX_SESSIONS — never per-file.
-    if (Object.keys(raw).length >= MAX_SESSIONS || bytes + size > MAX_BYTES) {
-      dropped++;
-      continue;
+    if (
+      Object.keys(cur).length &&
+      (curBytes + size > CHUNK_BYTES || Object.keys(cur).length >= CHUNK_SESSIONS)
+    ) {
+      batches.push(cur);
+      cur = {};
+      curBytes = 0;
     }
     try {
-      raw[f.split("/").pop()] = readFileSync(f, "utf8");
-      bytes += size;
+      cur[f.split("/").pop()] = readFileSync(f, "utf8");
+      curBytes += size;
     } catch {
       /* skip unreadable */
     }
   }
-  // Final guard: JSON escaping inflates the body slightly, so trim the oldest kept sessions until the
-  // GZIP the body — transcripts are JSON text and compress ~3-4x, so 100MB+ of real sessions fits
-  // under the server cap without dropping ANY of them (big sessions are the richest data). The trim
-  // loop only fires if even the COMPRESSED payload exceeds the cap — for a normal user, never.
-  const { gzipSync } = await import("node:zlib");
-  let body = gzipSync(JSON.stringify({ user_id: id.uid, raw_transcripts: raw }));
-  const HARD_CAP = 90 * 1024 * 1024; // matches the server body cap; compressed size
-  while (body.length > HARD_CAP && Object.keys(raw).length > 1) {
-    const keys = Object.keys(raw); // insertion order = newest first → drop the oldest kept
-    delete raw[keys[keys.length - 1]];
-    dropped++;
-    body = gzipSync(JSON.stringify({ user_id: id.uid, raw_transcripts: raw }));
-  }
-  const kept = Object.keys(raw).length;
-  const plain = JSON.stringify({ user_id: id.uid, raw_transcripts: raw });
-  sp.text(`Building your graph from ${kept} session${kept === 1 ? "" : "s"}…`);
-  const post = (token, gzip) =>
+  if (Object.keys(cur).length) batches.push(cur);
+  if (!batches.length) return 0;
+
+  let token = id.access_token;
+  const postBatch = (raw, tok) =>
     fetch(`${API}/ingest`, {
       method: "POST",
       headers: {
         "content-type": "application/json",
-        ...(gzip ? { "content-encoding": "gzip" } : {}),
-        ...(token ? { authorization: `Bearer ${token}` } : {}),
+        "content-encoding": "gzip",
+        ...(tok ? { authorization: `Bearer ${tok}` } : {}),
       },
-      body: gzip ? body : plain,
+      body: gzipSync(JSON.stringify({ user_id: id.uid, raw_transcripts: raw })),
     });
 
-  let r = await post(id.access_token, true);
-  // SELF-HEAL on 401: token was stale AND refresh didn't recover it (refresh_token also expired).
-  // Re-run the Google login ONCE for a fresh session, then retry — instead of dead-ending the user.
-  if (r.status === 401 && !process.env.SKALPEL_USER) {
-    sp.text("Session expired — re-opening sign-in…");
-    spawnSync("node", [join(__dir, "login.mjs")], { stdio: "inherit" });
-    const fresh = await identity();
-    if (fresh?.access_token) r = await post(fresh.access_token, true);
-  }
-  // COMPAT: if a server that predates gzip decompression 5xx's on the gzipped body, retry it plain
-  // (uncompressed). Only when the plain body still fits the cap; otherwise gzip was load-bearing.
-  if (r.status >= 500 && plain.length < HARD_CAP) {
-    r = await post(id.access_token, false);
-  }
-  if (!r.ok) {
-    const hint =
-      r.status === 401
-        ? " — sign-in failed; run `skalpel-prosumer login` and retry"
-        : r.status === 413
-          ? " — history too large even after trimming"
-          : "";
-    throw new Error(`ingest ${r.status}${hint}`);
-  }
-  // ASYNC: /ingest now returns 202 {job_id} and judges in the BACKGROUND (so a monster 50MB session
-  // can take minutes without any request timeout). Poll /ingest/status and drive the spinner with the
-  // server's REAL progress ("Judging 8/15…"); resolve only when the job reaches state=done. A server
-  // that predates async returns 200 with n_sessions inline — handle that too (back-compat).
-  const res = await r.json().catch(() => ({}));
-  if (res && typeof res.n_sessions === "number") return res.n_sessions; // legacy synchronous server
-  const jobId = res && res.job_id;
-  if (!jobId) return res && typeof res.n_sessions === "number" ? res.n_sessions : 0;
-
-  const statusUrl = `${API}/ingest/status?job_id=${encodeURIComponent(jobId)}`;
-  const POLL_MS = 1500;
-  const MAX_POLLS = 800; // ~20 min ceiling — a monster history, never hit by a normal user
-  for (let i = 0; i < MAX_POLLS; i++) {
-    await sleep(POLL_MS);
-    let s;
-    try {
-      const sr = await fetch(statusUrl, {
-        headers: { authorization: `Bearer ${id.access_token}` },
-      });
-      s = await sr.json();
-    } catch {
-      continue; // transient network blip — keep polling, the job runs server-side regardless
+  let total = 0;
+  for (let bi = 0; bi < batches.length; bi++) {
+    const label = batches.length > 1 ? ` (batch ${bi + 1}/${batches.length})` : "";
+    sp.text(`Building your graph…${label}`);
+    let r = await postBatch(batches[bi], token);
+    // SELF-HEAL on 401: refresh the Google session once, then retry this batch.
+    if (r.status === 401 && !process.env.SKALPEL_USER) {
+      sp.text("Session expired — re-opening sign-in…");
+      spawnSync("node", [join(__dir, "login.mjs")], { stdio: "inherit" });
+      const fresh = await identity();
+      if (fresh?.access_token) {
+        token = fresh.access_token;
+        r = await postBatch(batches[bi], token);
+      }
+    }
+    if (!r.ok) {
+      const hint =
+        r.status === 401
+          ? " — sign-in failed; run `skalpel login` and retry"
+          : r.status === 413
+            ? " — a single session is too large"
+            : "";
+      throw new Error(`ingest ${r.status}${hint}`);
     }
-    if (s.state === "done") return s.n_sessions ?? 0;
-    if (s.state === "error") throw new Error("build failed");
-    if (s.state === "judging" && s.total) {
-      sp.text(`Judging your sessions… ${s.done}/${s.total}`);
+    const res = await r.json().catch(() => ({}));
+    if (typeof res.n_sessions === "number") {
+      total = res.n_sessions; // legacy synchronous server
+      continue;
     }
+    const jobId = res.job_id;
+    if (!jobId) continue;
+    // Poll THIS batch's job to completion before the next batch (folds must be sequential).
+    const statusUrl = `${API}/ingest/status?job_id=${encodeURIComponent(jobId)}`;
+    const POLL_MS = 1500;
+    const MAX_POLLS = 800; // ~20 min/batch ceiling — never hit by a normal batch
+    let done = false;
+    for (let i = 0; i < MAX_POLLS && !done; i++) {
+      await sleep(POLL_MS);
+      let s;
+      try {
+        s = await (
+          await fetch(statusUrl, { headers: { authorization: `Bearer ${token}` } })
+        ).json();
+      } catch {
+        continue; // transient blip — the job runs server-side regardless
+      }
+      if (s.state === "done") {
+        total = s.n_sessions ?? total;
+        done = true;
+      } else if (s.state === "error") {
+        throw new Error("build failed");
+      } else if (s.state === "judging" && s.total) {
+        sp.text(`Judging your sessions…${label} ${s.done}/${s.total}`);
+      }
+    }
+    if (!done) throw new Error("build timed out");
   }
-  throw new Error("build timed out");
+  return total;
 }
 
 function startBackgroundBuild() {
@@ -816,8 +818,12 @@ async function main() {
   //    so the user SEES skalpel already knows them before typing a prompt. Escape hatches:
   //    SKALPEL_SETUP_BACKGROUND=1 keeps the old fast "warming in the background" path; non-TTY (CI)
   //    always goes background/silent so a pipeline never blocks on the reveal.
+  // DEFAULT is now BACKGROUND: `skalpel` returns you to your terminal in seconds and the graph builds
+  // detached — the batches judge + fold server-side while you keep working, and a desktop notification
+  // + a "your graph is ready" line at your next Claude Code session surface the insights when it's done.
+  // Blocking in the foreground (watch all N batches) is opt-in for anyone who wants to see it live.
   const wantReveal =
-    (isTTY || process.env.SKALPEL_SETUP_WAIT_FOR_GRAPH === "1") &&
+    process.env.SKALPEL_SETUP_WAIT_FOR_GRAPH === "1" &&
     process.env.SKALPEL_SETUP_BACKGROUND !== "1";
 
   if (process.env.SKALPEL_DEMO) {
@@ -856,14 +862,20 @@ async function main() {
       );
     }
   } else if (files.length) {
-    // background/silent path (opt-out or CI): fast, no reveal
+    // DEFAULT async path: kick off the detached build and hand the terminal straight back.
     startBackgroundBuild();
     const capped = Math.min(
       files.length,
-      Math.max(1, Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "20", 10)),
+      Math.max(1, Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "40", 10)),
+    );
+    console.log(
+      `   ${G}✓${X} Building your graph from your newest ${capped} session${capped === 1 ? "" : "s"} — ${B}in the background${X}.`,
+    );
+    console.log(
+      `   ${D}Keep coding. I'll notify you and surface your insights the moment it's ready.${X}`,
     );
     await reportLiveAndLaunch(
-      `\n   ${B}${G}You're live.${X}  ${D}hooks are wired; your graph is warming from the newest ${capped} session${capped === 1 ? "" : "s"} in the background.${X}\n`,
+      `\n   ${B}${G}You're live.${X}  ${D}every prompt already pulls your history as it learns.${X}\n`,
     );
   } else {
     await reportLiveAndLaunch(
@@ -873,6 +885,8 @@ async function main() {
 }
 
 // The detached background build (spawned by main): does ONLY the upload+judge, no banner/hooks.
+// On success it stashes the fresh profile (for the next Claude Code SessionStart to reveal) and fires
+// a desktop notification, so the async build surfaces itself the moment it's ready.
 async function backgroundBuild() {
   const id = await identity();
   if (!id) return; // not signed in — nothing to build under
@@ -880,12 +894,54 @@ async function backgroundBuild() {
   if (!files.length) return;
   const sp = { text: () => {}, succeed: () => {} }; // silent — no TTY in the background
   try {
-    await buildGraph(sp, files, id);
+    const built = await buildGraph(sp, files, id);
+    if (!built) return;
+    let report = null;
+    try {
+      report = await fetchProfile(id);
+    } catch {
+      /* profile fetch is best-effort — the graph is still built */
+    }
+    try {
+      mkdirSync(join(homedir(), ".skalpel"), { recursive: true });
+      writeFileSync(
+        join(homedir(), ".skalpel", "insights-ready.json"),
+        JSON.stringify({ ready: true, ts: Date.now(), n_sessions: built, report }),
+      );
+    } catch {
+      /* the SessionStart reveal is a bonus; never block the build on it */
+    }
+    notifyDone(built);
   } catch {
     /* fail-open: a failed background build never surfaces; hooks fail open until it succeeds */
   }
 }
 
+// Desktop notification when the background build finishes — cross-platform, best-effort, never throws.
+function notifyDone(n) {
+  const title = "skalpel";
+  const msg = `Your graph is ready — learned from ${n} session${n === 1 ? "" : "s"}. Open Claude Code to see your insights.`;
+  try {
+    if (process.platform === "darwin") {
+      spawnSync(
+        "osascript",
+        ["-e", `display notification ${JSON.stringify(msg)} with title ${JSON.stringify(title)}`],
+        { stdio: "ignore" },
+      );
+    } else if (process.platform === "linux") {
+      spawnSync("notify-send", [title, msg], { stdio: "ignore" });
+    } else if (process.platform === "win32") {
+      spawnSync(
+        "powershell",
+        ["-NoProfile", "-Command", `New-BurntToastNotification -Text '${title}','${msg}'`],
+        { stdio: "ignore" },
+      );
+    }
+  } catch {
+    /* notifications are best-effort */
+  }
+}
+
 // `__build` = the detached background graph build spawned by setup; skips the banner/hooks entirely.
 if (isMain && sub === "__build") {
   backgroundBuild()