skalpel 4.0.0 → 4.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skalpel",
-  "version": "4.0.0",
+  "version": "4.0.1",
   "description": "Behavioral graph for AI-assisted coding — learns how you work and steers Claude Code + Codex in real time.",
   "type": "module",
   "bin": {

package/skalpel-setup.mjs

@@ -491,18 +491,24 @@ async function buildGraph(sp, files, id) {
     await sleep(600);
     return n;
   }
-  // Read the raw transcript files and upload them (the server filters + judges + builds). The upload
-  // must fit the server's single-call capacity — a TOTAL body-size cap and a session-count cap — so a
-  // heavy user with hundreds of sessions / hundreds of MB doesn't 413 and end up with no graph. Only
-  // the TOTAL is bounded (never per-file: big sessions are the richest data). Newest-first, so if the
-  // total cap is ever hit we keep the freshest history. Caps overridable for tuning.
+  // CHUNKED UPLOAD. The server folds sessions in idempotently (n+1 aggregate, dedup by session_id),
+  // so instead of one giant body we upload small BYTE-BOUNDED batches. A 150MB one-shot decompressed
+  // in-memory and OOM-killed the pod (the "async" build never even started — the blowup is in reading
+  // the body, before the job). Batches keep the server's per-request memory tiny and the ingest is
+  // naturally incremental. Batches run SEQUENTIALLY: a fold reads-then-writes the corpus, so two
+  // concurrent batches would race and lose sessions. All caps overridable for tuning.
+  const { gzipSync } = await import("node:zlib");
+  const MB = 1024 * 1024;
   const MAX_SESSIONS = Math.max(
     1,
-    Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "20", 10),
+    Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "40", 10),
+  );
+  const CHUNK_BYTES = Math.max(1, Number.parseInt(process.env.SKALPEL_CHUNK_MB || "12", 10)) * MB;
+  const CHUNK_SESSIONS = Math.max(
+    1,
+    Number.parseInt(process.env.SKALPEL_CHUNK_SESSIONS || "6", 10),
   );
-  const MAX_BYTES =
-    Math.max(1, Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_MB || "90", 10)) * 1024 * 1024;
-  // newest first, so when a cap is hit we keep the freshest history
+  // newest-first, so a capped history keeps the freshest sessions
   const ranked = files
     .map((f) => {
       try {
@@ -513,105 +519,101 @@ async function buildGraph(sp, files, id) {
       }
     })
     .filter(Boolean)
-    .sort((a, b) => b.m - a.m);
-  const raw = {};
-  let bytes = 0;
-  let dropped = 0;
+    .sort((a, b) => b.m - a.m)
+    .slice(0, MAX_SESSIONS);
+  // pack into byte-bounded batches; a lone session bigger than the cap gets its own batch (never split)
+  const batches = [];
+  let cur = {};
+  let curBytes = 0;
   for (const { f, size } of ranked) {
-    // Do NOT drop a session for being large — big sessions are the RICHEST data (the long ones
-    // where the real thrash happens). The only real limit is the total request body (413), so we
-    // only stop once the WHOLE upload would exceed MAX_BYTES / MAX_SESSIONS — never per-file.
-    if (Object.keys(raw).length >= MAX_SESSIONS || bytes + size > MAX_BYTES) {
-      dropped++;
-      continue;
+    if (
+      Object.keys(cur).length &&
+      (curBytes + size > CHUNK_BYTES || Object.keys(cur).length >= CHUNK_SESSIONS)
+    ) {
+      batches.push(cur);
+      cur = {};
+      curBytes = 0;
     }
     try {
-      raw[f.split("/").pop()] = readFileSync(f, "utf8");
-      bytes += size;
+      cur[f.split("/").pop()] = readFileSync(f, "utf8");
+      curBytes += size;
     } catch {
       /* skip unreadable */
     }
   }
-  // Final guard: JSON escaping inflates the body slightly, so trim the oldest kept sessions until the
-  // GZIP the body — transcripts are JSON text and compress ~3-4x, so 100MB+ of real sessions fits
-  // under the server cap without dropping ANY of them (big sessions are the richest data). The trim
-  // loop only fires if even the COMPRESSED payload exceeds the cap — for a normal user, never.
-  const { gzipSync } = await import("node:zlib");
-  let body = gzipSync(JSON.stringify({ user_id: id.uid, raw_transcripts: raw }));
-  const HARD_CAP = 90 * 1024 * 1024; // matches the server body cap; compressed size
-  while (body.length > HARD_CAP && Object.keys(raw).length > 1) {
-    const keys = Object.keys(raw); // insertion order = newest first → drop the oldest kept
-    delete raw[keys[keys.length - 1]];
-    dropped++;
-    body = gzipSync(JSON.stringify({ user_id: id.uid, raw_transcripts: raw }));
-  }
-  const kept = Object.keys(raw).length;
-  const plain = JSON.stringify({ user_id: id.uid, raw_transcripts: raw });
-  sp.text(`Building your graph from ${kept} session${kept === 1 ? "" : "s"}…`);
-  const post = (token, gzip) =>
+  if (Object.keys(cur).length) batches.push(cur);
+  if (!batches.length) return 0;
+
+  let token = id.access_token;
+  const postBatch = (raw, tok) =>
     fetch(`${API}/ingest`, {
       method: "POST",
       headers: {
         "content-type": "application/json",
-        ...(gzip ? { "content-encoding": "gzip" } : {}),
-        ...(token ? { authorization: `Bearer ${token}` } : {}),
+        "content-encoding": "gzip",
+        ...(tok ? { authorization: `Bearer ${tok}` } : {}),
       },
-      body: gzip ? body : plain,
+      body: gzipSync(JSON.stringify({ user_id: id.uid, raw_transcripts: raw })),
     });
 
-  let r = await post(id.access_token, true);
-  // SELF-HEAL on 401: token was stale AND refresh didn't recover it (refresh_token also expired).
-  // Re-run the Google login ONCE for a fresh session, then retry — instead of dead-ending the user.
-  if (r.status === 401 && !process.env.SKALPEL_USER) {
-    sp.text("Session expired — re-opening sign-in…");
-    spawnSync("node", [join(__dir, "login.mjs")], { stdio: "inherit" });
-    const fresh = await identity();
-    if (fresh?.access_token) r = await post(fresh.access_token, true);
-  }
-  // COMPAT: if a server that predates gzip decompression 5xx's on the gzipped body, retry it plain
-  // (uncompressed). Only when the plain body still fits the cap; otherwise gzip was load-bearing.
-  if (r.status >= 500 && plain.length < HARD_CAP) {
-    r = await post(id.access_token, false);
-  }
-  if (!r.ok) {
-    const hint =
-      r.status === 401
-        ? " — sign-in failed; run `skalpel-prosumer login` and retry"
-        : r.status === 413
-          ? " — history too large even after trimming"
-          : "";
-    throw new Error(`ingest ${r.status}${hint}`);
-  }
-  // ASYNC: /ingest now returns 202 {job_id} and judges in the BACKGROUND (so a monster 50MB session
-  // can take minutes without any request timeout). Poll /ingest/status and drive the spinner with the
-  // server's REAL progress ("Judging 8/15…"); resolve only when the job reaches state=done. A server
-  // that predates async returns 200 with n_sessions inline — handle that too (back-compat).
-  const res = await r.json().catch(() => ({}));
-  if (res && typeof res.n_sessions === "number") return res.n_sessions; // legacy synchronous server
-  const jobId = res && res.job_id;
-  if (!jobId) return res && typeof res.n_sessions === "number" ? res.n_sessions : 0;
-
-  const statusUrl = `${API}/ingest/status?job_id=${encodeURIComponent(jobId)}`;
-  const POLL_MS = 1500;
-  const MAX_POLLS = 800; // ~20 min ceiling — a monster history, never hit by a normal user
-  for (let i = 0; i < MAX_POLLS; i++) {
-    await sleep(POLL_MS);
-    let s;
-    try {
-      const sr = await fetch(statusUrl, {
-        headers: { authorization: `Bearer ${id.access_token}` },
-      });
-      s = await sr.json();
-    } catch {
-      continue; // transient network blip — keep polling, the job runs server-side regardless
+  let total = 0;
+  for (let bi = 0; bi < batches.length; bi++) {
+    const label = batches.length > 1 ? ` (batch ${bi + 1}/${batches.length})` : "";
+    sp.text(`Building your graph…${label}`);
+    let r = await postBatch(batches[bi], token);
+    // SELF-HEAL on 401: refresh the Google session once, then retry this batch.
+    if (r.status === 401 && !process.env.SKALPEL_USER) {
+      sp.text("Session expired — re-opening sign-in…");
+      spawnSync("node", [join(__dir, "login.mjs")], { stdio: "inherit" });
+      const fresh = await identity();
+      if (fresh?.access_token) {
+        token = fresh.access_token;
+        r = await postBatch(batches[bi], token);
+      }
+    }
+    if (!r.ok) {
+      const hint =
+        r.status === 401
+          ? " — sign-in failed; run `skalpel login` and retry"
+          : r.status === 413
+            ? " — a single session is too large"
+            : "";
+      throw new Error(`ingest ${r.status}${hint}`);
     }
-    if (s.state === "done") return s.n_sessions ?? 0;
-    if (s.state === "error") throw new Error("build failed");
-    if (s.state === "judging" && s.total) {
-      sp.text(`Judging your sessions… ${s.done}/${s.total}`);
+    const res = await r.json().catch(() => ({}));
+    if (typeof res.n_sessions === "number") {
+      total = res.n_sessions; // legacy synchronous server
+      continue;
+    }
+    const jobId = res.job_id;
+    if (!jobId) continue;
+    // Poll THIS batch's job to completion before the next batch (folds must be sequential).
+    const statusUrl = `${API}/ingest/status?job_id=${encodeURIComponent(jobId)}`;
+    const POLL_MS = 1500;
+    const MAX_POLLS = 800; // ~20 min/batch ceiling — never hit by a normal batch
+    let done = false;
+    for (let i = 0; i < MAX_POLLS && !done; i++) {
+      await sleep(POLL_MS);
+      let s;
+      try {
+        s = await (
+          await fetch(statusUrl, { headers: { authorization: `Bearer ${token}` } })
+        ).json();
+      } catch {
+        continue; // transient blip — the job runs server-side regardless
+      }
+      if (s.state === "done") {
+        total = s.n_sessions ?? total;
+        done = true;
+      } else if (s.state === "error") {
+        throw new Error("build failed");
+      } else if (s.state === "judging" && s.total) {
+        sp.text(`Judging your sessions…${label} ${s.done}/${s.total}`);
+      }
     }
+    if (!done) throw new Error("build timed out");
   }
-  throw new Error("build timed out");
+  return total;
 }
 
 function startBackgroundBuild() {