sphica 0.6.32 → 0.6.34

package/dist/deliver.js

@@ -11979,6 +11979,8 @@ var spoolDir = () => path6.join(sphicaHome(), "spool");
 var stateFile = () => path6.join(sphicaHome(), "capture.json");
 var rejectedDir = () => path6.join(spoolDir(), "rejected");
 var unregisteredDir = () => path6.join(spoolDir(), "unregistered");
+var callsDir = () => path6.join(spoolDir(), "calls");
+var callsRejectedDir = () => path6.join(callsDir(), "rejected");
 var HOLD_DAYS = 30;
 var HOLD_MAX = 1000;
 var baselineDir = () => path6.join(sphicaHome(), "worktree");
@@ -12027,7 +12029,7 @@ ${z}`,
     originalBytes: all
   };
 }
-function prune(held) {
+function prune(held, removed = () => {}) {
   let files;
   try {
     files = fs5.readdirSync(held).filter((f) => f.endsWith(".json") && !f.startsWith(".")).sort();
@@ -12037,16 +12039,21 @@ function prune(held) {
   const cutoff = Date.now() - HOLD_DAYS * 24 * 60 * 60 * 1000;
   const stale = files.filter((f) => Number(f.split("-")[0]) < cutoff);
   const over = files.slice(0, Math.max(0, files.length - HOLD_MAX));
-  for (const f of new Set([...stale, ...over]))
+  for (const f of new Set([...stale, ...over])) {
     fs5.rmSync(path6.join(held, f), { force: true });
+    removed();
+  }
 }
-function spool(record) {
-  const dir = spoolDir();
+function put(dir, record) {
   fs5.mkdirSync(dir, { recursive: true, mode: 448 });
   const name = `${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2, 10)}.json`;
   const tmp = path6.join(dir, `.${name}`);
   fs5.writeFileSync(tmp, JSON.stringify(record), { mode: 384 });
   fs5.renameSync(tmp, path6.join(dir, name));
+  return path6.join(dir, name);
+}
+function spool(record) {
+  put(spoolDir(), record);
 }
 var branchOf = (root) => {
   try {
@@ -12101,9 +12108,11 @@ function captureNotice(file = dbFile()) {
   if (!fs5.existsSync(file))
     return panel("sphica: no database, so conversations are not recorded", [file], "Create it with sphica init");
   const s = readState();
+  if (s.unreadable)
+    return panel("sphica: cannot read the recording queue", [spoolDir(), s.unreadable], "Check with sphica doctor");
   if (s.stuck)
     return panel("sphica: cannot send recordings", [`${s.pending} pending / failed: ${plain2(s.stuck.slice(0, 120))}`], "Check with sphica doctor");
-  if (s.rejected > 0)
+  if (s.rejected)
     return panel(`sphica: the database rejected ${plural(s.rejected, "record")}`, [rejectedDir(), `Move them back to ${spoolDir()} to resend`], "Fix them first, then check with sphica doctor");
   return null;
 }
@@ -12294,38 +12303,49 @@ function onHook(host, input) {
   }
   return { flush: false };
 }
-function writeState(s) {
-  try {
-    fs5.writeFileSync(stateFile(), JSON.stringify(s));
-  } catch {}
-}
-function readState() {
-  const count = (dir) => {
-    try {
-      return fs5.readdirSync(dir).filter((f) => f.endsWith(".json") && !f.startsWith(".")).length;
-    } catch {
-      return 0;
-    }
-  };
-  const counts = {
-    pending: count(spoolDir()),
-    rejected: count(rejectedDir()),
-    unregistered: count(unregisteredDir())
-  };
+function readStateFile() {
   let raw = {};
   try {
     const parsed = JSON.parse(fs5.readFileSync(stateFile(), "utf8"));
     if (parsed && typeof parsed === "object")
       raw = parsed;
   } catch {}
-  const error = typeof raw.error === "string" ? raw.error || "unknown failure" : null;
+  const p = raw.pruned;
   return {
     flushedAt: typeof raw.flushedAt === "string" ? raw.flushedAt : undefined,
-    error,
+    error: typeof raw.error === "string" ? raw.error || "unknown failure" : null,
     deferred: typeof raw.deferred === "number" ? raw.deferred : undefined,
-    ...counts,
-    stuck: error && counts.pending > 0 ? error : null
+    pruned: p && typeof p.at === "string" && typeof p.count === "number" ? { at: p.at, count: p.count } : undefined
+  };
+}
+function writeState(s) {
+  try {
+    fs5.writeFileSync(stateFile(), JSON.stringify({ ...s, pruned: s.pruned ?? readStateFile().pruned }));
+  } catch {}
+}
+function readState() {
+  const queue = listQueue(spoolDir());
+  const counts = {
+    pending: queue.files?.length ?? null,
+    rejected: listQueue(rejectedDir()).files?.length ?? null,
+    unregistered: listQueue(unregisteredDir()).files?.length ?? null
   };
+  const state = readStateFile();
+  const stuck = state.error && (counts.pending ?? 0) > 0 ? state.error : null;
+  return { ...state, ...counts, unreadable: queue.code, stuck };
+}
+function listQueue(dir) {
+  try {
+    const names = fs5.readdirSync(dir);
+    return {
+      files: names.filter((f) => f.endsWith(".json") && !f.startsWith(".")),
+      temp: names.filter((f) => f.endsWith(".json") && f.startsWith(".")),
+      code: null
+    };
+  } catch (e) {
+    const code = e.code ?? "unknown";
+    return code === "ENOENT" ? { files: [], temp: [], code: null } : { files: null, temp: [], code };
+  }
 }
 function lock() {
   const file = path6.join(spoolDir(), ".lock");
@@ -12463,16 +12483,36 @@ var queued = (from) => {
     return [];
   }
 };
+var CODE_NAMES = { 18: "TOOBIG", 19: "CONSTRAINT", 20: "MISMATCH", 25: "RANGE" };
+function setAside(from, name, dir, reason2) {
+  fs5.mkdirSync(dir, { recursive: true, mode: 448 });
+  try {
+    fs5.renameSync(path6.join(from, name), path6.join(dir, name));
+  } catch (e) {
+    if (e.code === "ENOENT")
+      return;
+    throw e;
+  }
+  try {
+    fs5.writeFileSync(path6.join(dir, `${name}.reason`), reason2, { mode: 384 });
+  } catch {}
+}
 async function sendBatch(db, names) {
   const held = unregisteredDir();
   const records = [];
   for (const { name, from } of names) {
+    let raw;
     let r;
     try {
-      r = current(JSON.parse(fs5.readFileSync(path6.join(from, name), "utf8")));
+      raw = JSON.parse(fs5.readFileSync(path6.join(from, name), "utf8"));
+    } catch {
+      setAside(from, name, rejectedDir(), "unreadable");
+      continue;
+    }
+    try {
+      r = current(raw);
     } catch {
-      fs5.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
-      fs5.renameSync(path6.join(from, name), path6.join(rejectedDir(), name));
+      setAside(from, name, rejectedDir(), typeof raw?.project === "string" ? "version" : "no-project");
       continue;
     }
     if (r)
@@ -12499,21 +12539,12 @@ async function sendBatch(db, names) {
       } catch (e2) {
         if (!rejected(e2))
           throw e2;
-        bad.push(x);
-      }
-    }
-  }
-  if (bad.length) {
-    fs5.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
-    for (const x of bad) {
-      try {
-        fs5.renameSync(path6.join(x.from, x.name), path6.join(rejectedDir(), x.name));
-      } catch (e) {
-        if (e.code !== "ENOENT")
-          throw e;
+        bad.push({ ...x, why: `sqlite:${CODE_NAMES[sqliteCode(e2) ?? -1] ?? "unknown"}` });
       }
     }
   }
+  for (const x of bad)
+    setAside(x.from, x.name, rejectedDir(), x.why);
   if (strayed.length) {
     fs5.mkdirSync(held, { recursive: true, mode: 448 });
     for (const x of strayed) {
@@ -12533,6 +12564,60 @@ async function sendBatch(db, names) {
       fs5.rmSync(path6.join(x.from, x.name), { force: true });
   return { sent, rejected: bad.length };
 }
+async function sendObservations(db, names) {
+  const aside = (name, why) => {
+    try {
+      setAside(callsDir(), name, callsRejectedDir(), why);
+    } catch {}
+  };
+  const rows = [];
+  for (const { name, from } of names) {
+    let text;
+    try {
+      text = fs5.readFileSync(path6.join(from, name), "utf8");
+    } catch {
+      continue;
+    }
+    let o = null;
+    let why = "unreadable";
+    try {
+      o = observation(JSON.parse(text));
+      why = "shape";
+    } catch {}
+    if (o)
+      rows.push({ name, o });
+    else
+      aside(name, why);
+  }
+  if (!rows.length)
+    return true;
+  const insert = (part) => db.insertInto("capture_tool_call").values(part.map((x) => observationRow(x.o))).execute();
+  const bad = new Map;
+  try {
+    await insert(rows);
+  } catch (e) {
+    if (!rejected(e))
+      return false;
+    for (const x of rows)
+      try {
+        await insert([x]);
+      } catch (e2) {
+        if (!rejected(e2))
+          return false;
+        bad.set(x.name, `sqlite:${CODE_NAMES[sqliteCode(e2) ?? -1] ?? "unknown"}`);
+      }
+  }
+  for (const x of rows) {
+    const why = bad.get(x.name);
+    if (why)
+      aside(x.name, why);
+    else
+      try {
+        fs5.rmSync(path6.join(callsDir(), x.name), { force: true });
+      } catch {}
+  }
+  return true;
+}
 async function flush(file = dbFile(), budgetMs = FLUSH_BUDGET_MS) {
   const deadline = Date.now() + budgetMs;
   const dir = spoolDir();
@@ -12540,6 +12625,8 @@ async function flush(file = dbFile(), budgetMs = FLUSH_BUDGET_MS) {
   const total = { sent: 0, deferred: 0, rejected: 0 };
   let batches = 0;
   let queueBatches = 0;
+  let removed = 0;
+  const pruned = () => removed ? { at: new Date().toISOString(), count: removed } : undefined;
   for (let first = true;; first = false) {
     let unlock = lock();
     while (!unlock && Date.now() < deadline) {
@@ -12561,12 +12648,20 @@ async function flush(file = dbFile(), budgetMs = FLUSH_BUDGET_MS) {
     const sentBefore = batches;
     try {
       if (first) {
-        prune(held);
+        prune(held, () => removed++);
         for (const part of chunks(queued(held), BATCH)) {
           if (late())
             break;
           await send(part);
         }
+        for (const part of chunks(queued(callsDir()), BATCH)) {
+          if (late())
+            break;
+          client ??= openWriter("capture", file);
+          if (!await sendObservations(client, part))
+            break;
+          batches++;
+        }
       }
       while (queueBatches === 0 || !late()) {
         const part = queued(dir).slice(0, BATCH);
@@ -12575,12 +12670,19 @@ async function flush(file = dbFile(), budgetMs = FLUSH_BUDGET_MS) {
         await send(part);
         queueBatches++;
       }
-      prune(held);
+      prune(held, () => removed++);
       total.deferred = queued(held).length;
       if (batches > sentBefore)
-        writeState({ flushedAt: new Date().toISOString(), error: null, deferred: total.deferred });
+        writeState({
+          flushedAt: new Date().toISOString(),
+          error: null,
+          deferred: total.deferred,
+          pruned: pruned()
+        });
+      else if (removed)
+        writeState({ ...readStateFile(), pruned: pruned() });
     } catch (e) {
-      writeState({ flushedAt: new Date().toISOString(), error: reason(e).slice(0, 300) });
+      writeState({ flushedAt: new Date().toISOString(), error: reason(e).slice(0, 300), pruned: pruned() });
       throw e;
     } finally {
       await client?.destroy().catch(() => {});
@@ -12599,27 +12701,63 @@ async function readInput(stream) {
 }
 var RECORD_TOOLS = "mcp__plugin_sphica_record__";
 var OBSERVE_WAIT_MS = 5000;
-var hostId = (v) => v && /^[\x21-\x7e]{1,200}$/.test(v) ? v : null;
-async function observeRecordCall(file, input, owner) {
-  const session = hostId(input.session_id);
-  const toolUse = hostId(input.tool_use_id);
-  const tool = hostId(input.tool_name);
-  if (!session || !toolUse || !tool?.startsWith(RECORD_TOOLS))
+var hostId = (v) => typeof v === "string" && /^[\x21-\x7e]{1,200}$/.test(v) ? v : null;
+var OBSERVATION_KEYS = ["at", "host", "owner", "session", "tool", "toolUse", "turn", "v"].join();
+function observation(raw) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw))
+    return null;
+  const o = raw;
+  if (Object.keys(o).sort().join() !== OBSERVATION_KEYS)
+    return null;
+  if (o.v !== 1 || o.host !== "claude-code" || !hostId(o.session) || !hostId(o.toolUse))
+    return null;
+  if (!hostId(o.tool)?.startsWith(RECORD_TOOLS))
+    return null;
+  if (o.turn !== null && !hostId(o.turn))
+    return null;
+  if (o.owner !== 0 && o.owner !== 1)
+    return null;
+  const at = typeof o.at === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(o.at) ? o.at : null;
+  if (!at || Number.isNaN(Date.parse(at)) || new Date(at).toISOString() !== at)
+    return null;
+  return o;
+}
+var observationRow = (o) => ({
+  host: o.host,
+  session_external: o.session,
+  turn_id: o.turn,
+  tool_use_id: o.toolUse,
+  tool_name: o.tool,
+  owner_turn: o.owner,
+  observed_at: o.at
+});
+async function observeRecordCall(file, input, owner, waitMs = OBSERVE_WAIT_MS) {
+  const o = observation({
+    v: 1,
+    host: "claude-code",
+    session: hostId(input.session_id),
+    turn: hostId(input.prompt_id),
+    toolUse: hostId(input.tool_use_id),
+    tool: hostId(input.tool_name),
+    owner: owner ? 1 : 0,
+    at: iso(Date.now())
+  });
+  if (!o)
     return;
-  const cap = openWriter("capture", file, OBSERVE_WAIT_MS);
+  let kept = null;
+  try {
+    kept = put(callsDir(), o);
+  } catch {}
+  const cap = openWriter("capture", file, waitMs);
   try {
-    await cap.insertInto("capture_tool_call").values({
-      host: "claude-code",
-      session_external: session,
-      turn_id: hostId(input.prompt_id),
-      tool_use_id: toolUse,
-      tool_name: tool,
-      owner_turn: owner ? 1 : 0,
-      observed_at: iso(Date.now())
-    }).execute();
+    await cap.insertInto("capture_tool_call").values(observationRow(o)).execute();
   } finally {
     await cap.destroy().catch(() => {});
   }
+  if (kept)
+    try {
+      fs5.rmSync(kept, { force: true });
+    } catch {}
 }
 async function main() {
   if (process.argv[2] === "--flush") {