skalpel 3.4.6 → 3.4.7

package/prosumer-hooks/bootstrap.mjs

@@ -0,0 +1,382 @@
+#!/usr/bin/env node
+// First-run hosted graph bootstrap for the skalpel TUI. The worker emits one JSON object per
+// line so Bubble Tea can keep rendering while the server judges the user's coding history.
+// Authentication comes exclusively from the canonical `skalpel login` Cognito session.
+import {
+  accessSync,
+  closeSync,
+  constants,
+  createReadStream,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { createInterface } from "node:readline";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+import { gzipSync } from "node:zlib";
+import { identity } from "./auth.mjs";
+import { recordInsight } from "./insights.mjs";
+
+const DIR = join(homedir(), ".skalpel");
+const STATE_PATH = join(DIR, "bootstrap.json");
+const LOCK_PATH = join(DIR, "bootstrap.lock");
+const HERE = dirname(fileURLToPath(import.meta.url));
+const FORCE = process.argv.includes("--force");
+const LOOKBACK_DAYS = Math.max(1, Number.parseInt(process.env.SKALPEL_LOOKBACK_DAYS || "30", 10));
+const MIN_USER_TURNS = Math.max(
+  1,
+  Number.parseInt(process.env.SKALPEL_MIN_USER_TURNS || "5", 10),
+);
+const MAX_SESSIONS = Math.max(
+  1,
+  Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_SESSIONS || "20", 10),
+);
+const MAX_BYTES =
+  Math.max(1, Number.parseInt(process.env.SKALPEL_MAX_UPLOAD_MB || "90", 10)) * 1024 * 1024;
+const HARD_CAP = 90 * 1024 * 1024;
+const API = process.env.SKALPEL_API || readClientConfig().api || "https://graph.skalpel.ai";
+
+process.stdout.on("error", () => {});
+
+function emit(state, message, extra = {}) {
+  process.stdout.write(`${JSON.stringify({ state, message, ...extra })}\n`);
+}
+
+function readClientConfig() {
+  try {
+    return JSON.parse(readFileSync(join(DIR, "client.json"), "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+function readState() {
+  try {
+    return JSON.parse(readFileSync(STATE_PATH, "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+function writeState(state) {
+  mkdirSync(DIR, { recursive: true, mode: 0o700 });
+  const tmp = `${STATE_PATH}.tmp-${process.pid}`;
+  writeFileSync(tmp, JSON.stringify({ ...state, updated_at: new Date().toISOString() }, null, 2), {
+    mode: 0o600,
+  });
+  renameSync(tmp, STATE_PATH);
+}
+
+function acquireLock() {
+  mkdirSync(DIR, { recursive: true, mode: 0o700 });
+  try {
+    const fd = openSync(LOCK_PATH, "wx", 0o600);
+    writeFileSync(fd, String(process.pid));
+    closeSync(fd);
+    return true;
+  } catch {
+    try {
+      if (Date.now() - statSync(LOCK_PATH).mtimeMs > 25 * 60 * 1000) {
+        unlinkSync(LOCK_PATH);
+        return acquireLock();
+      }
+    } catch {
+      // A concurrent process may have released the lock between stat and read.
+    }
+    return false;
+  }
+}
+
+function releaseLock() {
+  try {
+    unlinkSync(LOCK_PATH);
+  } catch {
+    // Already released.
+  }
+}
+
+function walk(dir) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const entry of entries) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) out.push(...walk(path));
+    else if (path.endsWith(".jsonl")) out.push(path);
+  }
+  return out;
+}
+
+async function userTurnCount(path, cap = MIN_USER_TURNS) {
+  let count = 0;
+  const input = createReadStream(path, { encoding: "utf8" });
+  const lines = createInterface({ input, crlfDelay: Infinity });
+  try {
+    for await (const line of lines) {
+      if (!line.includes('"user"')) continue;
+      let event;
+      try {
+        event = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (event?.type === "user" || event?.role === "user") {
+        count += 1;
+        if (count >= cap) {
+          lines.close();
+          input.destroy();
+          return count;
+        }
+      }
+    }
+  } catch {
+    return 0;
+  }
+  return count;
+}
+
+async function scanHistory() {
+  const cutoff = Date.now() - LOOKBACK_DAYS * 864e5;
+  const candidates = [
+    ...walk(join(homedir(), ".claude", "projects")),
+    ...walk(join(homedir(), ".codex", "sessions")),
+  ].filter((path) => {
+    if (basename(path).startsWith("agent-")) return false;
+    try {
+      return statSync(path).mtimeMs >= cutoff;
+    } catch {
+      return false;
+    }
+  });
+  const sessions = [];
+  let cursor = 0;
+  const workers = Math.max(1, Number.parseInt(process.env.SKALPEL_SCAN_WORKERS || "16", 10));
+  async function worker() {
+    while (cursor < candidates.length) {
+      const path = candidates[cursor++];
+      if ((await userTurnCount(path)) >= MIN_USER_TURNS) sessions.push(path);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(workers, candidates.length || 1) }, worker));
+  return sessions;
+}
+
+function buildPayload(files, uid) {
+  const ranked = files
+    .map((path) => {
+      try {
+        const stat = statSync(path);
+        return { path, modified: stat.mtimeMs, size: stat.size };
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean)
+    .sort((a, b) => b.modified - a.modified);
+  const transcripts = {};
+  let bytes = 0;
+  for (const item of ranked) {
+    const count = Object.keys(transcripts).length;
+    if (count >= MAX_SESSIONS || (count > 0 && bytes + item.size > MAX_BYTES)) continue;
+    try {
+      let key = basename(item.path);
+      if (Object.hasOwn(transcripts, key)) key = `${count}-${key}`;
+      transcripts[key] = readFileSync(item.path, "utf8");
+      bytes += item.size;
+    } catch {
+      // Skip an unreadable transcript and continue with the next newest session.
+    }
+  }
+  let plain = JSON.stringify({ user_id: uid, raw_transcripts: transcripts });
+  let gzip = gzipSync(plain);
+  while (gzip.length > HARD_CAP && Object.keys(transcripts).length > 1) {
+    const keys = Object.keys(transcripts);
+    delete transcripts[keys[keys.length - 1]];
+    plain = JSON.stringify({ user_id: uid, raw_transcripts: transcripts });
+    gzip = gzipSync(plain);
+  }
+  if (gzip.length > HARD_CAP) throw new Error("coding history is too large to upload safely");
+  return { plain, gzip, sessions: Object.keys(transcripts).length };
+}
+
+async function postIngest(payload, token, compressed) {
+  return fetch(`${API}/ingest`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      ...(compressed ? { "content-encoding": "gzip" } : {}),
+      authorization: `Bearer ${token}`,
+    },
+    body: compressed ? payload.gzip : payload.plain,
+  });
+}
+
+async function pollJob(jobId, uid, token) {
+  const statusUrl = `${API}/ingest/status?job_id=${encodeURIComponent(jobId)}`;
+  let lastProgress = "";
+  for (let attempt = 0; attempt < 800; attempt += 1) {
+    await new Promise((resolve) => setTimeout(resolve, 1500));
+    let response;
+    try {
+      response = await fetch(statusUrl, { headers: { authorization: `Bearer ${token}` } });
+    } catch {
+      continue;
+    }
+    if (response.status === 401) throw new Error("session expired; run `skalpel login` and retry");
+    if (!response.ok) continue;
+    const status = await response.json().catch(() => ({}));
+    if (status.state === "done") return status.n_sessions ?? 0;
+    if (status.state === "error") {
+      writeState({ state: "failed", user_id: uid, error: "server graph build failed" });
+      throw new Error("server graph build failed");
+    }
+    const progress = `${status.done ?? 0}/${status.total ?? 0}`;
+    if (status.state === "judging" && progress !== lastProgress) {
+      lastProgress = progress;
+      writeState({ state: "judging", user_id: uid, job_id: jobId, ...status });
+      emit("judging", `Judging coding sessions ${progress}`, {
+        done: status.done ?? 0,
+        total: status.total ?? 0,
+      });
+    }
+  }
+  throw new Error("graph build is still running; retry to resume it");
+}
+
+async function fetchProfile(uid, token) {
+  try {
+    const response = await fetch(`${API}/profile?user_id=${encodeURIComponent(uid)}`, {
+      headers: { authorization: `Bearer ${token}` },
+    });
+    return response.ok ? response.json() : null;
+  } catch {
+    return null;
+  }
+}
+
+function recordProfile(profile, sessions) {
+  const headline = profile?.headline;
+  if (headline) {
+    recordInsight({
+      kind: "profile",
+      display: `learned ${headline.hours ?? 0}h across ${headline.sessions ?? sessions} sessions; ${headline.rework_pct ?? 0}% was rework`,
+    });
+  } else {
+    recordInsight({ kind: "profile", display: `standing profile built from ${sessions} sessions` });
+  }
+  const morphology = [...(profile?.morphology || [])].sort(
+    (a, b) => (b.hours_wasted || 0) - (a.hours_wasted || 0),
+  )[0];
+  if (morphology?.archetype) {
+    recordInsight({
+      kind: "profile",
+      display: `top recurring trap: ${morphology.archetype} (~${morphology.hours_wasted || 0}h)`,
+    });
+  }
+  const signature = profile?.discriminative?.[0];
+  if (signature?.seq) {
+    recordInsight({ kind: "profile", display: `signature loop: ${signature.seq}` });
+  }
+}
+
+async function main() {
+  const auth = await identity();
+  if (!auth.uid || !auth.token) throw new Error("sign in with `skalpel login` before learning history");
+
+  const prior = readState();
+  if (!FORCE && prior.state === "done" && prior.user_id === auth.uid) {
+    emit("skipped", "Coding history already learned", { terminal: true });
+    return;
+  }
+
+  emit("wiring", "Activating Claude Code and Codex hooks");
+  const wired = spawnSync(process.execPath, [join(HERE, "install.mjs")], { stdio: "ignore" });
+  if (wired.error || wired.status !== 0) throw new Error("could not activate coding-agent hooks");
+
+  if (prior.user_id === auth.uid && prior.job_id && prior.state !== "done") {
+    emit("resuming", "Resuming the existing graph build");
+    const sessions = await pollJob(prior.job_id, auth.uid, auth.token);
+    const profile = await fetchProfile(auth.uid, auth.token);
+    recordProfile(profile, sessions);
+    writeState({ state: "done", user_id: auth.uid, sessions });
+    emit("complete", `Learned from ${sessions} coding sessions`, { terminal: true, sessions });
+    return;
+  }
+
+  writeState({ state: "scanning", user_id: auth.uid });
+  emit("scanning", `Scanning the last ${LOOKBACK_DAYS} days of Claude Code and Codex`);
+  const files = await scanHistory();
+  if (!files.length) {
+    writeState({ state: "done", user_id: auth.uid, sessions: 0 });
+    emit("complete", "Hooks are active; your graph will grow as you work", {
+      terminal: true,
+      sessions: 0,
+    });
+    return;
+  }
+
+  const payload = buildPayload(files, auth.uid);
+  writeState({ state: "uploading", user_id: auth.uid, sessions: payload.sessions });
+  emit("uploading", `Uploading ${payload.sessions} coding sessions`);
+  let response = await postIngest(payload, auth.token, true);
+  if (response.status >= 500 && payload.plain.length < HARD_CAP) {
+    response = await postIngest(payload, auth.token, false);
+  }
+  if (!response.ok) {
+    if (response.status === 401) throw new Error("session expired; run `skalpel login` and retry");
+    throw new Error(`history ingest failed (${response.status})`);
+  }
+
+  const accepted = await response.json().catch(() => ({}));
+  let sessions;
+  if (typeof accepted.n_sessions === "number") {
+    sessions = accepted.n_sessions;
+  } else if (accepted.job_id) {
+    writeState({
+      state: "judging",
+      user_id: auth.uid,
+      job_id: accepted.job_id,
+      sessions: payload.sessions,
+    });
+    emit("judging", `Judging ${payload.sessions} coding sessions`, {
+      done: 0,
+      total: payload.sessions,
+    });
+    sessions = await pollJob(accepted.job_id, auth.uid, auth.token);
+  } else {
+    throw new Error("ingest response did not include a graph job");
+  }
+
+  const profile = await fetchProfile(auth.uid, auth.token);
+  recordProfile(profile, sessions);
+  writeState({ state: "done", user_id: auth.uid, sessions });
+  emit("complete", `Learned from ${sessions} coding sessions`, { terminal: true, sessions });
+}
+
+if (!acquireLock()) {
+  emit("skipped", "Coding history is already being learned in another skalpel window", {
+    terminal: true,
+  });
+} else {
+  main()
+    .catch((error) => {
+      const prior = readState();
+      const message = String(error?.message || error).replace(/[\r\n\t]+/g, " ").slice(0, 240);
+      writeState({ ...prior, state: "failed", error: message });
+      emit("failed", message, { terminal: true, retryable: true });
+      process.exitCode = 1;
+    })
+    .finally(releaseLock);
+}

package/prosumer-hooks/insights.mjs

@@ -0,0 +1,62 @@
+// insights.mjs — local insight records, shared by both hooks. One NDJSON line per user-visible
+// insight (steer/win/reset/ack/recap/profile), written in PLAIN ENGLISH for a local TUI to read —
+// never the bracketed [skalpel — …] model directive.
+// Append is atomic on POSIX for small writes, so concurrent hook double-fires never race or corrupt
+// the file (a live-mutated counter file WOULD lose increments under that race). recordInsight()
+// never throws and never blocks — a local record must not be able to break a fail-open hook.
+import {
+  appendFileSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const DIR = join(homedir(), ".skalpel");
+export const INSIGHTS_PATH = join(DIR, "insights.ndjson");
+
+// Unlike metrics.ndjson (write-only telemetry, uncapped), this file is READ by a TUI every render —
+// cap it so it never grows unbounded. Over the byte cap, keep only the newest lines via tmp-file +
+// atomic rename (same idiom as writeSteer), so a concurrent reader never sees a half-written file.
+const MAX_BYTES = 512 * 1024;
+const KEEP_LINES = 400;
+
+// Strip terminal-control characters (C0 incl. ESC, DEL, C1) and cap length. The reader renders
+// `display`/`label` into a live terminal; the server-provided steer_label flows in here, and the
+// TUI treats this file as untrusted anyway — sanitizing at write time is defense-in-depth so a
+// compromised upstream string can't smuggle OSC/CSI sequences (clipboard writes, title spoofing)
+// toward anyone's terminal. Newlines collapse to spaces (one record = one NDJSON line of prose).
+const MAX_TEXT = 2000;
+function cleanText(v) {
+  if (typeof v !== "string") return v;
+  // eslint-disable-next-line no-control-regex
+  const flat = v.replace(/[\r\n\t]+/g, " ").replace(/[\u0000-\u001f\u007f-\u009f]/g, "");
+  return flat.length > MAX_TEXT ? flat.slice(0, MAX_TEXT) + "…" : flat;
+}
+
+// rec: { kind, display, label?, mins?, harness?, session_id? } — `display` is the user-visible text.
+export function recordInsight(rec) {
+  try {
+    if (rec && typeof rec === "object") {
+      if ("display" in rec) rec = { ...rec, display: cleanText(rec.display) };
+      if (rec.label != null) rec = { ...rec, label: cleanText(rec.label) };
+    }
+    mkdirSync(DIR, { recursive: true });
+    try {
+      if (statSync(INSIGHTS_PATH).size > MAX_BYTES) {
+        const lines = readFileSync(INSIGHTS_PATH, "utf8").trim().split("\n").filter(Boolean);
+        const tmp = `${INSIGHTS_PATH}.tmp`;
+        writeFileSync(tmp, lines.slice(-KEEP_LINES).join("\n") + "\n");
+        renameSync(tmp, INSIGHTS_PATH); // atomic swap
+      }
+    } catch {
+      /* no file yet, or the trim failed — either way, still try the append below */
+    }
+    appendFileSync(INSIGHTS_PATH, JSON.stringify({ ts: new Date().toISOString(), ...rec }) + "\n");
+  } catch {
+    /* never block a hook on a local insight record */
+  }
+}