persnally 2.3.1 → 2.4.0

package/README.md

@@ -39,7 +39,7 @@ open http://127.0.0.1:4983            # see it, with evidence for every claim
 ## How it works
 
 ```
-  Your AI clients (Claude, Cursor, agents…)        Importers (claude · chatgpt · git)
+  Your AI clients (Claude, Cursor, agents…)   Importers (claude · claude-code · chatgpt · git)
         │  MCP: context out, signals in                  │  your history → events
         ▼                                                ▼
   ┌──────────────────────── persnallyd (local daemon) ────────────────────────┐

package/build/src/cli.js

@@ -13,11 +13,12 @@ import { runConsolidation } from "./consolidate.js";
 import { chooseExtractor } from "./llm.js";
 import { CATEGORIES, clearScope, loadScopes, setScope } from "./permissions.js";
 import { alreadyImported, DENSITY_QUESTIONS, detectExports, eventsFromAnswers, isThin, markImported } from "./setup.js";
-import { DEFAULT_PORT, startDaemon, VERSION } from "./daemon.js";
+import { autoImportNewSessions, DEFAULT_PORT, startDaemon, VERSION } from "./daemon.js";
 import { extractChatGPTEvents, parseChatGPTExport } from "./importers/chatgpt.js";
 import { extractClaudeEvents, parseClaudeExport } from "./importers/claude.js";
 import { DEFAULT_TRANSCRIPTS_DIR, extractClaudeCodeEvents, parseClaudeCodeTranscripts, } from "./importers/claude-code.js";
 import { gitEvents, scanRepos } from "./importers/git.js";
+import { freshConversations } from "./importers/extract.js";
 import { autostartInstalled, installAutostart, LOG_FILE, removeAutostart, removePidFile, runningPid, startDetached, stopDaemon, writePidFile, } from "./lifecycle.js";
 import { newEvent } from "./events.js";
 import { proseLines } from "./prose.js";
@@ -249,44 +250,77 @@ async function main() {
             const usage = "usage: persnallyd import claude|claude-code|chatgpt|git <path>";
             if (!kind)
                 return die(usage);
-            let events, batch;
-            if (kind === "claude-code") {
-                const engine = await chooseExtractor("extract");
-                const root = path ?? DEFAULT_TRANSCRIPTS_DIR;
-                const { parsed, sessionsFound, sessionsDropped } = parseClaudeCodeTranscripts(root);
-                if (!parsed.conversations.length)
-                    return die(`No usable sessions found at ${root}`);
-                console.error(`Found ${sessionsFound} session(s)${sessionsDropped ? ` — importing the ${parsed.conversations.length} most recent` : ""}. ` +
-                    `Extracting with ${engine.label}...`);
-                ({ events, batch } = await extractClaudeCodeEvents(parsed, engine.extract, engine.model, root));
-            }
-            else if (!path) {
-                return die(usage);
-            }
-            else if (kind === "git") {
+            // Git: offline, deterministic. Dedup by repo so a re-run never doubles the graph.
+            if (kind === "git") {
+                if (!path)
+                    return die(usage);
                 const authorIdx = args.indexOf("--author");
                 const summaries = scanRepos(path, authorIdx > -1 ? args[authorIdx + 1] : undefined);
                 if (!summaries.length)
                     return die(`No git repos with your commits found at ${path}`);
-                console.error(`Found ${summaries.length} repo(s): ${summaries.map((s) => `${s.repo} (${s.commits} commits)`).join(", ")}`);
-                ({ events, batch } = gitEvents(summaries));
+                const store = new EventStore();
+                const seen = store.importedGitRepos();
+                const fresh = summaries.filter((s) => !seen.has(s.repo));
+                const skipped = summaries.length - fresh.length;
+                if (!fresh.length) {
+                    store.close();
+                    console.log(`All ${summaries.length} repo(s) already imported — nothing new.`);
+                    return;
+                }
+                console.error(`Found ${summaries.length} repo(s)${skipped ? ` (${skipped} already imported)` : ""} — importing ${fresh.map((s) => `${s.repo} (${s.commits} commits)`).join(", ")}`);
+                const { events, batch } = gitEvents(fresh);
+                store.append(events);
+                store.rebuild();
+                store.close();
+                console.log(`Imported ${events.length} events from ${fresh.length} repo(s) (batch ${batch}).`);
+                console.log(`Undo with: persnallyd forget --batch ${batch}`);
+                return;
+            }
+            // Conversation imports: dedup by conversation_uuid so re-import only adds new chats.
+            let parsed;
+            let file = "conversations.json";
+            let parseNote = "";
+            if (kind === "claude-code") {
+                const root = path ?? DEFAULT_TRANSCRIPTS_DIR;
+                file = root;
+                const r = parseClaudeCodeTranscripts(root);
+                if (!r.parsed.conversations.length)
+                    return die(`No usable sessions found at ${root}`);
+                parsed = r.parsed;
+                if (r.sessionsDropped)
+                    parseNote = ` (most recent ${r.parsed.conversations.length} of ${r.sessionsFound})`;
             }
-            else if (kind === "claude" || kind === "chatgpt") {
-                const engine = await chooseExtractor("extract");
-                const parsed = kind === "claude" ? parseClaudeExport(path) : parseChatGPTExport(path);
-                console.error(`Parsed ${parsed.conversations.length} conversations. Extracting with ${engine.label}...`);
-                ({ events, batch } = kind === "claude"
-                    ? await extractClaudeEvents(parsed, engine.extract, engine.model)
-                    : await extractChatGPTEvents(parsed, engine.extract, engine.model));
+            else if (kind === "claude") {
+                if (!path)
+                    return die(usage);
+                parsed = parseClaudeExport(path);
+            }
+            else if (kind === "chatgpt") {
+                if (!path)
+                    return die(usage);
+                parsed = parseChatGPTExport(path);
             }
             else {
                 return die(`unknown import source "${kind}" — use claude, claude-code, chatgpt, or git`);
             }
             const store = new EventStore();
+            const seen = store.importedConversationUuids(`import:${kind}`);
+            const { parsed: toExtract, skipped, firstImport } = freshConversations(parsed, seen);
+            if (!toExtract.conversations.length && !firstImport) {
+                store.close();
+                console.log(`Already up to date — all ${parsed.conversations.length} conversation(s) imported. Nothing new.`);
+                return;
+            }
+            const engine = await chooseExtractor("extract");
+            console.error(`Parsed ${parsed.conversations.length} conversation(s)${parseNote}${skipped ? ` — ${skipped} already imported` : ""}. ` +
+                `Extracting ${toExtract.conversations.length} with ${engine.label}...`);
+            const { events, batch } = await (kind === "claude-code" ? extractClaudeCodeEvents(toExtract, engine.extract, engine.model, file)
+                : kind === "claude" ? extractClaudeEvents(toExtract, engine.extract, engine.model)
+                    : extractChatGPTEvents(toExtract, engine.extract, engine.model));
             store.append(events);
             store.rebuild();
             store.close();
-            console.log(`Imported ${events.length} events (batch ${batch}).`);
+            console.log(`Imported ${events.length} events from ${toExtract.conversations.length} conversation(s) (batch ${batch}).`);
             console.log(`Undo with: persnallyd forget --batch ${batch}`);
             return;
         }
@@ -484,6 +518,8 @@ async function main() {
             process.on("uncaughtException", (e) => { console.error("uncaughtException:", e); process.exit(1); });
             console.error(`persnallyd v${VERSION} listening on 127.0.0.1:${port}`);
             console.error(`Dashboard: http://127.0.0.1:${port}`);
+            // Catch up on chats since the daemon last ran; the timer takes it from here.
+            void autoImportNewSessions(store);
             return;
         }
         default:

package/build/src/daemon.d.ts

@@ -7,3 +7,10 @@ import type { EventStore } from "./store.js";
 export declare const DEFAULT_PORT = 4983;
 export declare const VERSION: string;
 export declare function startDaemon(store: EventStore, port?: number): http.Server;
+/**
+ * Ingest Claude Code sessions created since the last pass — the daemon's
+ * automatic capture of new chats (no user action, no per-session hook). A
+ * key-less, Ollama-less machine has no extractor: skip rather than block.
+ * Never throws — capture must not take the daemon down.
+ */
+export declare function autoImportNewSessions(store: EventStore): Promise<void>;

package/build/src/daemon.js

@@ -8,6 +8,7 @@ import { loadConfig } from "./config.js";
 import { runConsolidation, shouldRunNow } from "./consolidate.js";
 import { allowedCategories, loadScopes } from "./permissions.js";
 import { newEvent, validateEvent } from "./events.js";
+import { importNewClaudeCodeSessions } from "./importers/claude-code.js";
 import { chooseExtractor } from "./llm.js";
 import { synthesizeProfile } from "./profile.js";
 export const DEFAULT_PORT = 4983;
@@ -131,8 +132,9 @@ export function startDaemon(store, port = DEFAULT_PORT) {
         }
     });
     server.listen(port, "127.0.0.1");
-    // Nightly reflection: check every 30 min, run once per day at/after the consolidation hour.
+    // Every 30 min: pick up new Claude Code chats, then run the once-a-day reflection.
     const timer = setInterval(async () => {
+        await autoImportNewSessions(store);
         const lastRun = loadConfig().last_consolidation;
         if (!shouldRunNow(typeof lastRun === "string" ? lastRun : undefined, new Date()))
             return;
@@ -149,6 +151,27 @@ export function startDaemon(store, port = DEFAULT_PORT) {
     server.on("close", () => clearInterval(timer));
     return server;
 }
+/**
+ * Ingest Claude Code sessions created since the last pass — the daemon's
+ * automatic capture of new chats (no user action, no per-session hook). A
+ * key-less, Ollama-less machine has no extractor: skip rather than block.
+ * Never throws — capture must not take the daemon down.
+ */
+export async function autoImportNewSessions(store) {
+    try {
+        const engine = await chooseExtractor("extract").catch(() => null);
+        if (!engine)
+            return;
+        const r = await importNewClaudeCodeSessions(store, engine.extract, engine.model);
+        if (r.events) {
+            store.rebuild();
+            console.error(`auto-import: ${r.newSessions} new Claude Code session(s) → ${r.events} events`);
+        }
+    }
+    catch (e) {
+        console.error("auto-import failed:", e instanceof Error ? e.message : e);
+    }
+}
 let cachedHtml;
 function dashboardHtml() {
     cachedHtml ??= readFileSync(new URL("./dashboard.html", import.meta.url), "utf-8");

package/build/src/dashboard.html

@@ -109,7 +109,7 @@
   /* ── constellation ── */
   .constellation-wrap { position: relative; }
   .constellation { background: var(--panel); border: 1px solid var(--line); border-radius: var(--r); overflow: hidden; }
-  #graph { display: block; width: 100%; height: 520px; cursor: grab; }
+  #graph { display: block; width: 100%; height: 520px; cursor: grab; touch-action: none; }
   #graph:active { cursor: grabbing; }
   .ghint { position: absolute; top: 12px; right: 14px; font-size: 11px; color: var(--faint); pointer-events: none; }
   .legend { position: absolute; bottom: 12px; left: 14px; display: flex; gap: 14px; flex-wrap: wrap; font-size: 11px; color: var(--dim); pointer-events: none; }
@@ -173,6 +173,17 @@
   .preview-ribbon code { color: var(--text); }
   a { color: var(--text); }
   :focus-visible { outline: 2px solid #FFFFFF; outline-offset: 3px; border-radius: 4px; }
+  @media (max-width: 640px) {
+    .wrap { padding: 18px 16px 96px; }
+    #graph { height: 380px; }
+    .archetype { font-size: clamp(22px, 7vw, 30px); }
+    header { margin-bottom: 24px; }
+    .value-num { font-size: 32px; }
+    #topicList { overflow-x: auto; }   /* wide table scrolls inside its card — body never does */
+    .bar-track { width: 64px; }
+    td { padding: 8px 6px; font-size: 13px; }
+    .del { padding: 3px 7px; }
+  }
   @media (prefers-reduced-motion: reduce) { .reveal { animation: none; opacity: 1; transform: none; } .read.fresh { animation: none; } }
 </style>
 </head>
@@ -215,7 +226,7 @@
       <div class="view-toggle"><button id="vGraph" class="on">map</button><button id="vList">list</button></div>
     </div>
     <div class="constellation-wrap">
-      <div class="constellation" id="graphHost"><canvas id="graph"></canvas><div class="legend" id="legend"></div><div class="ghint">drag · scroll to zoom · dbl-click resets</div></div>
+      <div class="constellation" id="graphHost"><canvas id="graph"></canvas><div class="legend" id="legend"></div><div class="ghint">drag · pinch or scroll to zoom</div></div>
       <div class="card" id="topicList" style="display:none;padding:6px 14px"><table id="topics"></table></div>
       <div class="node-pop" id="nodePop"></div>
     </div>
@@ -276,6 +287,20 @@ function clientOf(ev) {
 }
 const prettyClient = (c) => ({ "claude-code":"Claude Code","claude-desktop":"Claude Desktop","cursor":"Cursor","cli":"CLI" }[c] || c);
 const clientInitials = (c) => prettyClient(c).split(/[\s-]/).map(w=>w[0]).join("").slice(0,2).toUpperCase();
+const IMPORT_TOOL = { "import:claude":"Claude export", "import:chatgpt":"ChatGPT export", "import:claude-code":"Claude Code" };
+// Human-readable provenance for the "why?" panel — where a fact actually came from.
+function provLabel(e) {
+  const p = e.provenance || {};
+  if (p.kind === "import") {
+    const base = IMPORT_TOOL[e.source] || p.file || "import";
+    return p.conversation_uuid ? `${base} · conversation ${String(p.conversation_uuid).slice(0,8)}` : base;
+  }
+  if (p.kind === "mcp") return `${prettyClient(p.client)}${p.session ? " · session " + String(p.session).slice(0,8) : ""} · live`;
+  if (p.kind === "git") return `git: ${p.repo}${p.ref ? " @ " + p.ref : ""}`;
+  if (p.kind === "derived") return `inferred from ${(p.from||[]).length} signal${(p.from||[]).length===1?"":"s"}`;
+  if (p.kind === "local") return `you · ${p.surface || "local"}`;
+  return p.kind || "local";
+}
 
 const SNAP_KEY = "persnally.snapshot.v1";
 let DEMO = false;
@@ -344,8 +369,7 @@ async function toggleEvidence(btn) {
   box.innerHTML = events.map(e => {
     const p = e.payload || {};
     const summary = e.type === "signal.topic" ? `${p.topic} (${p.intent})` : (p.claim ?? p.topic ?? JSON.stringify(p).slice(0,90));
-    const origin = e.provenance && e.provenance.kind === "import" ? `${(e.provenance.file||"import")}` : (e.provenance ? e.provenance.kind : "local");
-    return `<div class="ev-item"><code>${esc(e.type)}</code> ${esc(summary)} — <i>${esc(origin)}, ${esc((e.ts||"").slice(0,10))}</i></div>`;
+    return `<div class="ev-item"><code>${esc(e.type)}</code> ${esc(summary)} — <i>${esc(provLabel(e))}, ${esc((e.ts||"").slice(0,10))}</i></div>`;
   }).join("") || `<div class="ev-item">evidence not found (deleted?)</div>`;
   box.dataset.loaded = "1";
 }
@@ -436,7 +460,7 @@ function renderMap(topics) {
   const ctx = canvas.getContext("2d");
   const dpr = Math.min(devicePixelRatio || 1, 2);
   let w, h;
-  const resize = () => { w = host.clientWidth; h = 520; canvas.width = w*dpr; canvas.height = h*dpr; canvas.style.height = h+"px"; };
+  const resize = () => { w = host.clientWidth; h = window.innerWidth < 640 ? 380 : 520; canvas.width = w*dpr; canvas.height = h*dpr; canvas.style.height = h+"px"; };
   resize();
 
   const maxW = Math.max(...data.map(t=>t.weight), 0.01);
@@ -510,24 +534,55 @@ function renderMap(topics) {
   $("legend").innerHTML = cats.map(c => `<span><i style="color:${catColor(c)}"></i><span style="color:var(--dim)">${esc(c)}</span></span>`).join("");
 
   const pop=$("nodePop");
-  const nodeAt=(mx,my)=>{ const wx=S2Wx(mx),wy=S2Wy(my); for (let i=nodes.length-1;i>=0;i--) if (Math.hypot(wx-nodes[i].x,wy-nodes[i].y)<=nodes[i].r+5/cam.k) return i; return -1; };
+  const nodeAt=(mx,my)=>{ const wx=S2Wx(mx),wy=S2Wy(my); for (let i=nodes.length-1;i>=0;i--) if (Math.hypot(wx-nodes[i].x,wy-nodes[i].y)<=nodes[i].r+8/cam.k) return i; return -1; };
   const rel=(e)=>{ const r=canvas.getBoundingClientRect(); return [e.clientX-r.left, e.clientY-r.top]; };
-  canvas.onmousedown=(e)=>{ const [mx,my]=rel(e); const n=nodeAt(mx,my); if (n>=0) dragNode=n; else panning=true; lastX=mx; lastY=my; };
-  const onMove=(e)=>{ const [mx,my]=rel(e);
+  function showPop(n) { const p=nodes[n].t;
+    pop.innerHTML=`<div class="npt">${esc(p.topic)}</div><div class="npm">${esc(p.category)} · ${esc(p.dominant_intent)} · weight ${p.weight.toFixed(2)} · ${p.signals}×</div>${(p.entities||[]).length?`<div class="npe">${esc(p.entities.slice(0,4).join(", "))}</div>`:""}`;
+    pop.style.left=Math.min(W2Sx(nodes[n].x)+14,w-250)+"px"; pop.style.top=(W2Sy(nodes[n].y)+14)+"px"; pop.classList.add("show"); }
+
+  // Pointer events unify mouse + touch + pen; a second pointer drives pinch-zoom, a tap inspects.
+  const pts=new Map();
+  let downX=0, downY=0, moved=false, pinchD=0, pinchX=0, pinchY=0;
+  canvas.onpointerdown=(e)=>{
+    const [mx,my]=rel(e); pts.set(e.pointerId,{x:mx,y:my});
+    try { canvas.setPointerCapture(e.pointerId); } catch { /* capture unsupported — degrade to no-capture */ }
+    if (pts.size>=2) { dragNode=-1; panning=false; pop.classList.remove("show"); hover=-1;
+      const [a,b]=[...pts.values()]; pinchD=Math.hypot(a.x-b.x,a.y-b.y)||1; pinchX=(a.x+b.x)/2; pinchY=(a.y+b.y)/2; return; }
+    downX=mx; downY=my; moved=false; lastX=mx; lastY=my;
+    const n=nodeAt(mx,my); if (n>=0) dragNode=n; else panning=true;
+  };
+  canvas.onpointermove=(e)=>{
+    const [mx,my]=rel(e); if (pts.has(e.pointerId)) pts.set(e.pointerId,{x:mx,y:my});
+    if (pts.size>=2) { const [a,b]=[...pts.values()];
+      const d=Math.hypot(a.x-b.x,a.y-b.y)||1, cx=(a.x+b.x)/2, cy=(a.y+b.y)/2, wx=S2Wx(cx), wy=S2Wy(cy);
+      cam.k=Math.max(0.45,Math.min(3,cam.k*(d/pinchD)));
+      cam.x=cx-wx*cam.k+(cx-pinchX); cam.y=cy-wy*cam.k+(cy-pinchY);
+      pinchD=d; pinchX=cx; pinchY=cy; if(!alive)draw(); return; }
+    if ((dragNode>=0||panning) && (Math.abs(mx-downX)>4||Math.abs(my-downY)>4)) moved=true;
     if (dragNode>=0) { nodes[dragNode].x=S2Wx(mx); nodes[dragNode].y=S2Wy(my); nodes[dragNode].vx=0; nodes[dragNode].vy=0; pop.classList.remove("show"); wake(); if(!alive)draw(); return; }
     if (panning) { cam.x+=mx-lastX; cam.y+=my-lastY; lastX=mx; lastY=my; if(!alive)draw(); return; }
+    if (e.pointerType==="touch") return;  // no hover on touch — a tap inspects instead
     if (mx<0||my<0||mx>w||my>h) { if (hover!==-1){hover=-1;pop.classList.remove("show");if(!alive)draw();} return; }
     const n=nodeAt(mx,my);
-    if (n!==hover) { hover=n; if(!alive)draw();
-      if (n>=0) { const p=nodes[n].t; pop.innerHTML=`<div class="npt">${esc(p.topic)}</div><div class="npm">${esc(p.category)} · ${esc(p.dominant_intent)} · weight ${p.weight.toFixed(2)} · ${p.signals}×</div>${(p.entities||[]).length?`<div class="npe">${esc(p.entities.slice(0,4).join(", "))}</div>`:""}`;
-        pop.style.left=Math.min(W2Sx(nodes[n].x)+14,w-250)+"px"; pop.style.top=(W2Sy(nodes[n].y)+14)+"px"; pop.classList.add("show"); }
-      else pop.classList.remove("show"); } };
-  const onUp=()=>{ if (dragNode>=0){ dragNode=-1; wake(); } panning=false; };
+    if (n!==hover) { hover=n; if(!alive)draw(); if (n>=0) showPop(n); else pop.classList.remove("show"); }
+  };
+  const endPointer=(e)=>{
+    const tapped = (!moved && pts.size<=1 && e.pointerType==="touch") ? nodeAt(...rel(e)) : -1;
+    pts.delete(e.pointerId);
+    if (dragNode>=0) { dragNode=-1; wake(); }
+    panning=false;
+    if (tapped>=0) { if (hover===tapped && pop.classList.contains("show")) { hover=-1; pop.classList.remove("show"); } else { hover=tapped; showPop(tapped); } if(!alive)draw(); }
+    else if (!moved && e.pointerType==="touch") { hover=-1; pop.classList.remove("show"); if(!alive)draw(); }
+    if (pts.size===1) { const [p]=[...pts.values()]; lastX=p.x; lastY=p.y; panning=true; moved=true; }  // pinch → one finger: resume pan
+  };
+  canvas.onpointerup=endPointer; canvas.onpointercancel=endPointer;
   canvas.onwheel=(e)=>{ e.preventDefault(); const [mx,my]=rel(e); const wx=S2Wx(mx),wy=S2Wy(my); const f=e.deltaY<0?1.12:1/1.12; cam.k=Math.max(0.45,Math.min(3,cam.k*f)); cam.x=mx-wx*cam.k; cam.y=my-wy*cam.k; if(!alive)draw(); };
   canvas.ondblclick=()=>{ cam.k=1; cam.x=0; cam.y=0; if(!alive)draw(); };
   let rt; const onWinResize=()=>{ clearTimeout(rt); rt=setTimeout(()=>{ resize(); if(!alive)draw(); },150); };
-  window.addEventListener("mousemove", onMove); window.addEventListener("mouseup", onUp); window.addEventListener("resize", onWinResize, { passive:true });
-  mapCleanup=()=>{ window.removeEventListener("mousemove",onMove); window.removeEventListener("mouseup",onUp); window.removeEventListener("resize",onWinResize); };
+  window.addEventListener("resize", onWinResize, { passive:true });
+  mapCleanup=()=>{ window.removeEventListener("resize",onWinResize); };
+
+  if (window.innerWidth < 640) switchView("list");  // phones land on the list; the map is one tap away
 }
 function switchView(v) {
   const g = v==="graph" || v==="map";
@@ -554,6 +609,7 @@ async function loadAll() {
   const st = $("status");
   st.querySelector(".dot").classList.toggle("off", DEMO);
   st.querySelector(".lbl").textContent = DEMO ? "preview" : "active";
+  if (!DEMO) get("/health").then(h => { if (h && h.version) $("ver").textContent = `· v${h.version}`; });
   const total = (stats.byType && stats.byType["context.read"]) || reads.length;
   renderDelta(topics, reads, assertions);
   renderScaleBeat(stats);

package/build/src/importers/chatgpt.js

@@ -3,16 +3,16 @@
  * conversation as a node tree ("mapping"); user text lives in author.role
  * === "user" nodes with content.parts. Multimodal parts are skipped.
  */
-import { readFileSync, existsSync, statSync } from "node:fs";
+import { existsSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { safeIso } from "../events.js";
 import { anthropicExtract, DEFAULT_EXTRACT_MODEL } from "../llm.js";
-import { extractEvents } from "./extract.js";
+import { extractEvents, readImportFile } from "./extract.js";
 export function parseChatGPTExport(path) {
     const file = statSync(path).isDirectory() ? join(path, "conversations.json") : path;
     if (!existsSync(file))
         throw new Error(`No conversations.json at ${path}`);
-    const raw = JSON.parse(readFileSync(file, "utf-8"));
+    const raw = JSON.parse(readImportFile(file));
     const conversations = raw.map((c) => {
         const nodes = Object.values(c.mapping ?? {})
             .filter((n) => n.message?.author?.role === "user")

package/build/src/importers/claude-code.d.ts

@@ -4,6 +4,7 @@
  * (Phase 0 finding), and available immediately with no export wait.
  */
 import { type LlmExtract } from "../llm.js";
+import type { EventStore } from "../store.js";
 import { type ImportResult, type ParsedExport } from "./extract.js";
 export declare const DEFAULT_TRANSCRIPTS_DIR: string;
 export declare const DEFAULT_MAX_SESSIONS = 200;
@@ -14,3 +15,16 @@ export interface ClaudeCodeParse {
 }
 export declare function parseClaudeCodeTranscripts(root?: string, maxSessions?: number): ClaudeCodeParse;
 export declare function extractClaudeCodeEvents(parsed: ParsedExport, extract?: LlmExtract, model?: string, file?: string): Promise<ImportResult>;
+export interface IncrementalImport {
+    newSessions: number;
+    events: number;
+    skipped: number;
+}
+/**
+ * Import only the Claude Code sessions not already in the store — the path the
+ * daemon runs on its loop so new chats accrue without re-extracting old ones.
+ * Sessions are matched by the conversation_uuid recorded in each topic's
+ * provenance; a session that yields zero topics leaves no marker and may be
+ * retried, which is cheap and rare (a real session produces topics).
+ */
+export declare function importNewClaudeCodeSessions(store: EventStore, extract: LlmExtract, model?: string, root?: string): Promise<IncrementalImport>;

package/build/src/importers/claude-code.js

@@ -97,3 +97,23 @@ function humanText(content) {
 export async function extractClaudeCodeEvents(parsed, extract = anthropicExtract, model = DEFAULT_EXTRACT_MODEL, file = DEFAULT_TRANSCRIPTS_DIR) {
     return extractEvents(parsed, { source: "import:claude-code", importer: "claude-code", file }, extract, model);
 }
+/**
+ * Import only the Claude Code sessions not already in the store — the path the
+ * daemon runs on its loop so new chats accrue without re-extracting old ones.
+ * Sessions are matched by the conversation_uuid recorded in each topic's
+ * provenance; a session that yields zero topics leaves no marker and may be
+ * retried, which is cheap and rare (a real session produces topics).
+ */
+export async function importNewClaudeCodeSessions(store, extract, model = DEFAULT_EXTRACT_MODEL, root = DEFAULT_TRANSCRIPTS_DIR) {
+    if (!existsSync(root))
+        return { newSessions: 0, events: 0, skipped: 0 };
+    const { parsed } = parseClaudeCodeTranscripts(root);
+    const seen = store.importedConversationUuids("import:claude-code");
+    const fresh = parsed.conversations.filter((c) => !seen.has(c.uuid));
+    const skipped = parsed.conversations.length - fresh.length;
+    if (!fresh.length)
+        return { newSessions: 0, events: 0, skipped };
+    const { events } = await extractClaudeCodeEvents({ ...parsed, conversations: fresh }, extract, model, root);
+    store.append(events);
+    return { newSessions: fresh.length, events: events.length, skipped };
+}

package/build/src/importers/claude.js

@@ -5,12 +5,12 @@
 import { readFileSync, existsSync, readdirSync } from "node:fs";
 import { join } from "node:path";
 import { anthropicExtract, DEFAULT_EXTRACT_MODEL } from "../llm.js";
-import { extractEvents } from "./extract.js";
+import { extractEvents, readImportFile } from "./extract.js";
 export function parseClaudeExport(dir) {
     const convPath = join(dir, "conversations.json");
     if (!existsSync(convPath))
         throw new Error(`No conversations.json in ${dir}`);
-    const raw = JSON.parse(readFileSync(convPath, "utf-8"));
+    const raw = JSON.parse(readImportFile(convPath));
     const conversations = raw.map((c) => ({
         uuid: String(c.uuid ?? ""),
         name: String(c.name ?? ""),

package/build/src/importers/extract.d.ts

@@ -4,6 +4,8 @@
  */
 import { type PersnallyEvent } from "../events.js";
 import { type LlmExtract } from "../llm.js";
+/** Reads an export file, refusing oversized ones with a clear message instead of an opaque OOM/crash. */
+export declare function readImportFile(path: string, maxBytes?: number): string;
 export interface ParsedConversation {
     uuid: string;
     name: string;
@@ -24,6 +26,17 @@ export interface ImportResult {
     batch: string;
     conversationsProcessed: number;
 }
+/**
+ * Filters a parsed export to the conversations not already imported (matched by
+ * uuid), so a re-import only adds new chats instead of doubling the graph. The
+ * one-time memory/projects snapshot carries no per-conversation id, so it's kept
+ * only on the first import of a source.
+ */
+export declare function freshConversations(parsed: ParsedExport, seen: Set<string>): {
+    parsed: ParsedExport;
+    skipped: number;
+    firstImport: boolean;
+};
 export declare function extractEvents(parsed: ParsedExport, opts: {
     source: string;
     importer: string;

package/build/src/importers/extract.js

@@ -2,12 +2,38 @@
  * Shared extraction pipeline for conversation-export importers.
  * Parsers produce a ParsedExport; this turns it into provenance-linked events.
  */
+import { readFileSync, statSync } from "node:fs";
 import { z } from "zod";
 import { newEvent, safeIso, uuidv7, PAYLOAD_SCHEMAS } from "../events.js";
 import { anthropicExtract, DEFAULT_EXTRACT_MODEL } from "../llm.js";
 import { proseLines, stripNoise } from "../prose.js";
 import { analyzeVoice } from "../stylometry.js";
 const MAX_CONVO_CHARS = 30_000;
+const MAX_IMPORT_FILE_BYTES = 400 * 1024 * 1024; // ~400 MB — under Node's ~512 MB string cap; larger needs streaming
+/** Reads an export file, refusing oversized ones with a clear message instead of an opaque OOM/crash. */
+export function readImportFile(path, maxBytes = MAX_IMPORT_FILE_BYTES) {
+    const { size } = statSync(path);
+    if (size > maxBytes) {
+        throw new Error(`${path} is ${Math.round(size / 1e6)} MB, over the ${Math.round(maxBytes / 1e6)} MB import limit. ` +
+            `Very large exports aren't supported yet — import a smaller export or split conversations.json.`);
+    }
+    return readFileSync(path, "utf-8");
+}
+/**
+ * Filters a parsed export to the conversations not already imported (matched by
+ * uuid), so a re-import only adds new chats instead of doubling the graph. The
+ * one-time memory/projects snapshot carries no per-conversation id, so it's kept
+ * only on the first import of a source.
+ */
+export function freshConversations(parsed, seen) {
+    const firstImport = seen.size === 0;
+    const conversations = parsed.conversations.filter((c) => !c.uuid || !seen.has(c.uuid));
+    const skipped = parsed.conversations.length - conversations.length;
+    const next = firstImport
+        ? { ...parsed, conversations }
+        : { ...parsed, conversations, memoryText: "", projects: [] };
+    return { parsed: next, skipped, firstImport };
+}
 const topicsExtraction = z.object({ topics: z.array(PAYLOAD_SCHEMAS["signal.topic"]) });
 const assertionsExtraction = z.object({ assertions: z.array(PAYLOAD_SCHEMAS["signal.assertion"]) });
 export async function extractEvents(parsed, opts, extract = anthropicExtract, model = DEFAULT_EXTRACT_MODEL) {
@@ -18,19 +44,27 @@ export async function extractEvents(parsed, opts, extract = anthropicExtract, mo
         if (!convo.userMessages.length)
             continue;
         const joined = convo.userMessages.join("\n");
-        voiceCorpus.push(...proseLines(joined));
+        voiceCorpus.push(...proseLines(joined)); // prose feeds the deterministic voice fingerprint even if topic extraction fails
         const text = stripNoise(joined).slice(0, MAX_CONVO_CHARS); // strip pasted paths/URLs/logs before the LLM sees it
         if (!text)
             continue;
-        const result = await extract({
-            model,
-            instruction: "Extract 1-5 topic signals from this conversation's user messages. Weight = centrality, depth = engagement level, sentiment = user's attitude toward the topic. Capture decisions and rejected options as their own signals.",
-            schema: topicsExtraction,
-            content: `Conversation title: ${convo.name}\n\nUser messages:\n${text}`,
-        });
-        const { topics } = topicsExtraction.parse(result);
-        for (const t of topics) {
-            events.push(newEvent("signal.topic", opts.source, t, { kind: "import", batch, file: opts.file, conversation_uuid: convo.uuid }, safeIso(convo.created_at)));
+        try {
+            const result = await extract({
+                model,
+                instruction: "Extract 1-5 topic signals from this conversation's user messages. Weight = centrality, depth = engagement level, sentiment = user's attitude toward the topic. Capture decisions and rejected options as their own signals.",
+                schema: topicsExtraction,
+                content: `Conversation title: ${convo.name}\n\nUser messages:\n${text}`,
+            });
+            const { topics } = topicsExtraction.parse(result);
+            for (const t of topics) {
+                events.push(newEvent("signal.topic", opts.source, t, { kind: "import", batch, file: opts.file, conversation_uuid: convo.uuid }, safeIso(convo.created_at)));
+            }
+        }
+        catch (e) {
+            // One malformed extraction (e.g. the model returns an out-of-enum value)
+            // must not abort a whole multi-conversation import. Skip it — leaving no
+            // conversation_uuid marker, so the next pass retries it — and keep the rest.
+            console.error(`extract: skipped "${convo.name}" — ${(e instanceof Error ? e.message : String(e)).split("\n")[0]}`);
         }
     }
     if (parsed.memoryText.trim() || parsed.projects.length) {
@@ -38,15 +72,21 @@ export async function extractEvents(parsed, opts, extract = anthropicExtract, mo
             parsed.memoryText.trim() && `Assistant's accumulated memory of the user:\n${parsed.memoryText}`,
             parsed.projects.length && `User-created projects:\n${parsed.projects.map((p) => `- ${p.name}: ${p.description}`).join("\n")}`,
         ].filter(Boolean).join("\n\n");
-        const result = await extract({
-            model,
-            instruction: "Extract structured assertions about this person: facts, preferences, behaviors, skills, and context. Confidence reflects how directly the source supports the claim.",
-            schema: assertionsExtraction,
-            content: context,
-        });
-        const { assertions } = assertionsExtraction.parse(result);
-        for (const a of assertions) {
-            events.push(newEvent("signal.assertion", opts.source, a, { kind: "import", batch, file: opts.file }));
+        try {
+            const result = await extract({
+                model,
+                instruction: "Extract structured assertions about this person: facts, preferences, behaviors, skills, and context. Confidence reflects how directly the source supports the claim.",
+                schema: assertionsExtraction,
+                content: context,
+            });
+            const { assertions } = assertionsExtraction.parse(result);
+            for (const a of assertions) {
+                events.push(newEvent("signal.assertion", opts.source, a, { kind: "import", batch, file: opts.file }));
+            }
+        }
+        catch (e) {
+            // A malformed assertions response shouldn't discard the topics already gathered.
+            console.error(`extract: skipped memory/projects assertions — ${(e instanceof Error ? e.message : String(e)).split("\n")[0]}`);
         }
     }
     // Deterministic voice fingerprint over the user's own prose — no LLM, no tokens.

package/build/src/store.d.ts

@@ -42,6 +42,10 @@ export declare class EventStore {
     append(events: PersnallyEvent[]): number;
     query(opts?: QueryOpts): PersnallyEvent[];
     getEvents(ids: string[]): PersnallyEvent[];
+    /** conversation_uuids already imported from a source — lets a re-import top up only new chats, never double. */
+    importedConversationUuids(source: string): Set<string>;
+    /** repo names already imported via `import git` — lets a re-import skip repos already on file. */
+    importedGitRepos(): Set<string>;
     stats(): {
         total: number;
         byType: Record<string, number>;

package/build/src/store.js

@@ -123,6 +123,26 @@ export class EventStore {
             .all(...ids)
             .map(rowToEvent);
     }
+    /** conversation_uuids already imported from a source — lets a re-import top up only new chats, never double. */
+    importedConversationUuids(source) {
+        const uuids = new Set();
+        for (const e of this.query({ source, limit: 1_000_000 })) {
+            const uuid = e.provenance.conversation_uuid;
+            if (uuid)
+                uuids.add(uuid);
+        }
+        return uuids;
+    }
+    /** repo names already imported via `import git` — lets a re-import skip repos already on file. */
+    importedGitRepos() {
+        const repos = new Set();
+        for (const e of this.query({ source: "import:git", limit: 1_000_000 })) {
+            const repo = e.provenance.repo;
+            if (repo)
+                repos.add(repo);
+        }
+        return repos;
+    }
     stats() {
         const total = this.db.prepare("SELECT COUNT(*) n FROM events").get().n;
         const group = (col) => Object.fromEntries(this.db.prepare(`SELECT ${col} k, COUNT(*) n FROM events GROUP BY ${col}`).all()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "persnally",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "license": "FSL-1.1-MIT",
   "description": "Your own context engine — local-first, across every AI. So every AI finally knows you.",
   "type": "module",