pi-session-memory 0.1.4 → 0.2.0

package/spec.md

@@ -1,172 +1,28 @@
-# pi-session-memory — Spec
+# Native Codex-to-Pi Project Session Migration — Spec
 
 ## Goal
 
-A pi extension that persists every conversation turn to SQLite and exposes a
-`recall_memory` tool so the LLM can retrieve relevant past turns when the user
-references previous discussions.
+Allow a user to convert each historical Codex session for the active project into a separate, native Pi session that can be selected through Pi `/resume` and continued normally.
 
-## Architecture
+This is distinct from SQLite historical import and `recall_memory`:
 
-```
-pi turn_end event ──────────────────────────┐
-                                             ▼
-                                 Writer (src/writer.ts)
-                                 - writes current pi turns
-                                             │
-session_start ───────────┐                 │
-                          ▼                 ▼
-                 Incremental source sync → SQLite ~/.pi/agent/memory.db
-                 - per-file size + mtime quick check
-                 - SHA-256 only for changed candidates
-                 - Pi / Claude / Codex JSONL adapters
+- Native migration writes Pi session JSONL files for direct continuation in Pi.
+- `recall_memory` searches local SQLite excerpts and does not restore a client session.
 
-/memory-backfill forces a full rescan.
-                                             │
-                                             ▼
-                                 recall_memory tool
-                                 - LLM supplies entities[]
-                                 - LIKE substring match + hit-score ranking
-                                 - returns top-5 turns
-```
+## Design
 
-## Tables
+- Export `migrateCodexProjectSessions(cwd)` from `src/session-migration.ts`.
+- Scan Codex JSONL sessions and select only sessions with `session.cwd === cwd`.
+- Create one Pi v3 session JSONL per Codex session under Pi's default session directory for that cwd.
+- Write a `Migrated from Codex: <session-id>` session name so it is recognizable in `/resume`.
+- Convert user and assistant textual messages only. Do not represent Codex system/developer prompts, tool calls, or tool results as Pi conversation messages.
+- Use deterministic output file names and skip an already migrated Codex session, making reruns idempotent.
+- Register `/project-session-migration` for users and `migrate_codex_project_sessions` for Pi agents. Both descriptions must state that this is native Pi continuation, not ordinary recall.
 
-### sessions
-| column     | type    | note                          |
-|------------|---------|-------------------------------|
-| session_id | TEXT PK | native source session ID (UUID) |
-| source     | TEXT    | `pi`, `claude`, or `codex`     |
-| cwd        | TEXT    | working directory             |
-| started_at | INTEGER | unix ms                       |
-| model_id   | TEXT    | first model_change value      |
-| jsonl_path | TEXT    | absolute path to source file  |
+## Acceptance criteria
 
-### turns
-| column      | type    | note                                      |
-|-------------|---------|-------------------------------------------|
-| turn_id     | TEXT PK | `{session_id}:{user_message_id}` — stable across live write and backfill |
-| session_id  | TEXT FK |                                           |
-| turn_index  | INTEGER | display order within session; never used as identity |
-| ts          | INTEGER | user message timestamp (unix ms)          |
-| user_text   | TEXT    | user message content                      |
-| reply_text  | TEXT    | assistant final text (all text blocks)    |
-| tool_names  | TEXT    | JSON array e.g. `["bash","read"]`         |
-
-There is intentionally no full-text virtual table. Retrieval uses escaped
-SQLite `LIKE` against the complete `user_text` and `reply_text`, because literal
-substring coverage and retrieval quality are prioritized over index performance.
-
-## Write Path
-
-Trigger: `agent_settled` event, after the agent run and any automatic
-continuations have completed.
-
-Steps:
-1. Locate the latest user `SessionEntry` in `ctx.sessionManager.getBranch()` and
-   use its stable entry ID as `user_message_id`; do not use pi's transient `turnIndex`.
-2. Extract `user_text` from that user entry's text content blocks.
-3. Walk branch entries after that user entry to collect assistant text blocks →
-   `reply_text`, and toolCall names → `tool_names`.
-4. Upsert source=`pi` session row (INSERT OR IGNORE).
-5. Insert turn by stable ID (INSERT OR IGNORE — live writing and backfill target
-   the same row).
-
-## Retrieval Path (recall_memory tool)
-
-### source_files
-| column | type | note |
-|---|---|---|
-| jsonl_path | TEXT PK | absolute source file path |
-| source | TEXT | `pi`, `claude`, or `codex` |
-| size | INTEGER | file size at last sync |
-| mtime_ms | REAL | modification time at last sync |
-| sha256 | TEXT | content hash for changed candidates |
-
-### Tool Invocation Policy
-
-- **Direct recall:** Call `recall_memory` immediately when the user explicitly
-  asks to review, remember, summarize, continue, or compare a prior discussion
-  about a topic.
-- **Knowledge-gap recall:** When the user asks about a topic you cannot answer
-  confidently from the current conversation and your general knowledge, but it
-  may have been discussed in the user's past sessions, ask the user whether they
-  want you to search their conversation history. Call `recall_memory` only after
-  the user agrees.
-- Do not search history merely because a question is difficult when the user has
-  not indicated that their own prior work or discussions are relevant.
-
-Input: `{ entities: string[] }` — 2-5 key terms extracted by LLM from user query.
-
-Steps:
-1. Build per-entity LIKE hit score:
-   - `user_text` match = 2 points
-   - `reply_text` match = 1 point
-2. `SELECT ... WHERE (LOWER(user_text) LIKE ? OR LOWER(reply_text) LIKE ?) OR ...`
-3. Escape `%`, `_`, and `\\` in every entity, then use `LIKE ? ESCAPE '\\'` so
-   technical names containing LIKE wildcards remain literal substring matches.
-4. `ORDER BY hits DESC, ts DESC LIMIT 5`
-
-No FTS5 or write-time preprocessing. LIKE is a literal substring match after
-escaping, so it does not lose substring matches through tokenization.
-
-## Backfill
-
-At `session_start`, `syncChangedHistory()` recursively enumerates the three
-source roots. It records one row per source JSONL file in `source_files`:
-`jsonl_path`, source, size, `mtime_ms`, and SHA-256. Unchanged size/mtime files
-are skipped without reading; changed candidates are SHA-256 checked, and only
-new content is parsed and imported.
-
-`/memory-backfill` forces a full reparse of all historical records. All writes
-use `INSERT OR IGNORE`, so both automatic sync and forced backfill are safe and
-idempotent to run repeatedly.
-
-| source | scan root | accepted user/assistant records | excluded records |
-|---|---|---|---|
-| pi | `~/.pi/agent/sessions/**/*.jsonl` | `message.role=user|assistant` | thinking, tool results, non-text content |
-| claude | `~/.claude/projects/**/*.jsonl` | `type=user|assistant`, `message.role=user|assistant` | `isMeta`, sidechains, slash commands, local-command tags, continuation summaries, system/snapshot/attachments |
-| codex | `~/.codex/sessions/**/*.jsonl` | `type=response_item`, `payload.type=message`, `role=user|assistant` | developer/system context, AGENTS.md and environment-context injection, IDE/image context, reasoning and tool events |
-
-Each adapter outputs the common `ImportedSession` / `ImportedTurn` model.
-Turns pair one accepted user message with all following accepted assistant text
-until the next accepted user message. Each source adapter preserves the native
-user-message ID (`pi entry.id`, Claude `uuid`, Codex `payload.id`) so rerunning
-backfill never creates a duplicate of an already live-written pi turn.
-
-## Acceptance Criteria
-
-- A pi session with multiple user turns produces one distinct `turns` row per
-  user message; no row is dropped because `turnIndex` restarts.
-- Running backfill after live pi writing does not duplicate those pi turns.
-- Claude and Codex injected context records listed above are absent from
-  `turns.user_text`.
-- Entity text containing `%`, `_`, or `\\` only matches its literal occurrence.
-- Automated tests cover stable turn identity, LIKE escaping/ranking, and
-  cross-source backfill parsing.
-
-## File Structure
-
-```
-pi-session-memory/
-├── spec.md
-├── package.json
-├── tsconfig.json
-├── extensions/
-│   └── index.ts       ← extension entry: tool + /memory-backfill
-├── tests/
-│   └── core.test.ts   ← persistence identity and literal-LIKE tests
-└── src/
-    ├── db.ts          ← DatabaseSync schema + upsert helpers
-    ├── writer.ts      ← live pi turn_end writer
-    ├── retriever.ts   ← LIKE query + hit-score ranking
-    └── backfill.ts    ← Pi / Claude / Codex adapters and import runner
-```
-
-## Constraints
-
-- Zero external dependencies (use `node:sqlite`, `node:fs`, `node:path`, `node:os`)
-- Idempotent writes (INSERT OR IGNORE on turn_id)
-- Cross-platform paths: storage is `join(homedir(), ".pi", "agent", "memory.db")`; source roots are derived with `join(homedir(), ...)`, never hard-coded POSIX paths.
-- Requires a Pi-supported Node.js runtime that exposes `node:sqlite` (`DatabaseSync`).
-- No fallback / silent failure — let errors surface
+1. Each current-project Codex fixture produces one independently resumable Pi session with the expected user/assistant message sequence.
+2. Another project's Codex session is not migrated.
+3. A second migration skips existing output session files.
+4. A malformed source file becomes an isolated issue and does not block other sessions.
+5. `npm test` and `git diff --check` pass.

package/src/backfill.ts

@@ -1,4 +1,4 @@
-import { readdirSync, readFileSync, statSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { join } from "node:path";
@@ -25,6 +25,12 @@ interface ImportedSession {
   messages: ImportedMessage[];
 }
 
+export interface BackfillIssue {
+  source: Source;
+  jsonlPath: string | null;
+  error: string;
+}
+
 export interface BackfillStats {
   pi: number;
   claude: number;
@@ -32,8 +38,16 @@ export interface BackfillStats {
   turns: number;
   scannedFiles: number;
   skippedFiles: number;
+  issues: BackfillIssue[];
 }
 
+/** Versions whose Claude Code and Codex JSONL schemas this importer was verified against. */
+export const HISTORY_SCHEMA_REFERENCE_VERSIONS = {
+  pi: "0.85.1",
+  claude: "2.1.234",
+  codex: "0.154.0",
+} as const;
+
 const SOURCES: Array<{ source: Source; root: string; parse: (path: string) => ImportedSession | undefined }> = [
   { source: "pi", root: join(homedir(), ".pi", "agent", "sessions"), parse: _parsePi },
   { source: "claude", root: join(homedir(), ".claude", "projects"), parse: _parseClaude },
@@ -50,35 +64,61 @@ export function syncChangedHistory(): BackfillStats {
   return _syncHistory(false);
 }
 
+/** Synchronize all configured JSONL sources. */
 function _syncHistory(force: boolean): BackfillStats {
-  const stats: BackfillStats = { pi: 0, claude: 0, codex: 0, turns: 0, scannedFiles: 0, skippedFiles: 0 };
+  const stats: BackfillStats = { pi: 0, claude: 0, codex: 0, turns: 0, scannedFiles: 0, skippedFiles: 0, issues: [] };
   for (const definition of SOURCES) {
-    for (const jsonlPath of _jsonlFiles(definition.root)) {
-      const metadata = statSync(jsonlPath);
-      const known = getSourceFile(jsonlPath);
-      if (!force && known?.size === metadata.size && known.mtime_ms === metadata.mtimeMs) {
-        stats.skippedFiles++;
-        continue;
-      }
-
-      const sha256 = _sha256(jsonlPath);
-      if (!force && known?.sha256 === sha256) {
-        upsertSourceFile({ jsonl_path: jsonlPath, source: definition.source, size: metadata.size, mtime_ms: metadata.mtimeMs, sha256 });
-        stats.skippedFiles++;
-        continue;
-      }
-
-      const session = definition.parse(jsonlPath);
+    if (!existsSync(definition.root)) continue;
+    try {
+      for (const jsonlPath of _jsonlFiles(definition.root)) _syncSourceFile(definition, jsonlPath, force, stats);
+    } catch (error) {
+      stats.issues.push(_backfillIssue(definition.source, null, error));
+    }
+  }
+  return stats;
+}
+
+/** Synchronize one source file without allowing its failure to block other files or sources. */
+function _syncSourceFile(definition: typeof SOURCES[number], jsonlPath: string, force: boolean, stats: BackfillStats): void {
+  try {
+    const metadata = statSync(jsonlPath);
+    const known = getSourceFile(jsonlPath);
+    if (!force && known?.size === metadata.size && known.mtime_ms === metadata.mtimeMs) {
+      stats.skippedFiles++;
+      return;
+    }
+
+    const sha256 = _sha256(jsonlPath);
+    if (!force && known?.sha256 === sha256) {
       upsertSourceFile({ jsonl_path: jsonlPath, source: definition.source, size: metadata.size, mtime_ms: metadata.mtimeMs, sha256 });
-      stats.scannedFiles++;
-      if (!session) continue;
+      stats.skippedFiles++;
+      return;
+    }
+
+    const session = definition.parse(jsonlPath);
+    if (session) {
       stats[definition.source]++;
       stats.turns += _persist(session);
+      upsertSourceFile({ jsonl_path: jsonlPath, source: definition.source, size: metadata.size, mtime_ms: metadata.mtimeMs, sha256 });
     }
+    stats.scannedFiles++;
+  } catch (error) {
+    stats.issues.push(_backfillIssue(definition.source, jsonlPath, error));
   }
-  return stats;
 }
 
+/** Format an isolated source failure with the reference version for schema comparison. */
+function _backfillIssue(source: Source, jsonlPath: string | null, error: unknown): BackfillIssue {
+  const location = jsonlPath ? ` (${jsonlPath})` : "";
+  const message = error instanceof Error ? error.message : String(error);
+  return {
+    source,
+    jsonlPath,
+    error: `${source} history import failed${location}: ${message}. Compare the local ${source} version with the supported reference ${HISTORY_SCHEMA_REFERENCE_VERSIONS[source]}; this may be a JSONL schema compatibility issue.`,
+  };
+}
+
+/** Convert one normalized source session into paired, idempotently stored memory turns. */
 function _persist(session: ImportedSession): number {
   const sessionId = `${session.source}:${session.nativeSessionId}`;
   upsertSession({
@@ -96,6 +136,7 @@ function _persist(session: ImportedSession): number {
   const toolNames: string[] = [];
   let persisted = 0;
 
+  /** Persist the current user-plus-assistant accumulation when a turn boundary is reached. */
   const flush = () => {
     if (!user || !replyText.trim()) return;
     const inserted = insertTurn({
@@ -127,6 +168,7 @@ function _persist(session: ImportedSession): number {
   return persisted;
 }
 
+/** Parse a Pi JSONL session into the source-neutral import representation. */
 function _parsePi(jsonlPath: string): ImportedSession | undefined {
   const entries = _readJsonl(jsonlPath);
   const header = entries.find((entry) => entry.type === "session");
@@ -143,7 +185,13 @@ function _parsePi(jsonlPath: string): ImportedSession | undefined {
     const toolNames = message.role === "assistant"
       ? message.content.filter((block: any) => block.type === "toolCall").map((block: any) => block.name)
       : [];
-    messages.push({ id: entry.id, role: message.role, text, ts: message.timestamp, toolNames });
+    messages.push({
+      id: _messageId("Pi entry.id", entry.id),
+      role: message.role,
+      text,
+      ts: message.timestamp,
+      toolNames,
+    });
   }
 
   return {
@@ -157,6 +205,7 @@ function _parsePi(jsonlPath: string): ImportedSession | undefined {
   };
 }
 
+/** Parse non-meta Claude Code conversation records into normalized messages. */
 function _parseClaude(jsonlPath: string): ImportedSession | undefined {
   const entries = _readJsonl(jsonlPath);
   const firstConversation = entries.find((entry) =>
@@ -174,7 +223,14 @@ function _parseClaude(jsonlPath: string): ImportedSession | undefined {
     const toolNames = role === "assistant"
       ? entry.message.content.filter((block: any) => block.type === "tool_use").map((block: any) => block.name)
       : [];
-    messages.push({ id: entry.uuid, role, text, ts: Date.parse(entry.timestamp), toolNames });
+    // Workaround: Claude Code JSONL schema differs by version; message IDs may be in uuid or id.
+    messages.push({
+      id: _messageId("Claude entry.uuid or entry.id", entry.uuid, entry.id),
+      role,
+      text,
+      ts: Date.parse(entry.timestamp),
+      toolNames,
+    });
   }
 
   return {
@@ -188,6 +244,7 @@ function _parseClaude(jsonlPath: string): ImportedSession | undefined {
   };
 }
 
+/** Parse Codex session metadata and response-message envelopes into normalized messages. */
 function _parseCodex(jsonlPath: string): ImportedSession | undefined {
   const entries = _readJsonl(jsonlPath);
   const meta = entries.find((entry) => entry.type === "session_meta")?.payload;
@@ -200,8 +257,14 @@ function _parseCodex(jsonlPath: string): ImportedSession | undefined {
     if (payload?.type !== "message" || (payload.role !== "user" && payload.role !== "assistant")) continue;
     const text = _codexText(payload);
     if (!text || (payload.role === "user" && _isCodexInjectedContext(text))) continue;
+    // Workaround: Codex JSONL schema differs by version; legacy sessions store the ID as metadata.turn_id.
     messages.push({
-      id: payload.id,
+      id: _messageId(
+        "Codex payload.id, entry.id, or metadata.turn_id",
+        payload.id,
+        entry.id,
+        payload.internal_chat_message_metadata_passthrough?.turn_id,
+      ),
       role: payload.role,
       text,
       ts: Date.parse(entry.timestamp),
@@ -220,6 +283,14 @@ function _parseCodex(jsonlPath: string): ImportedSession | undefined {
   };
 }
 
+/** Read a source message ID from a known schema field without inventing one for malformed records. */
+function _messageId(field: string, ...values: unknown[]): string {
+  const id = values.find((value): value is string => typeof value === "string" && value.length > 0);
+  if (!id) throw new Error(`Invalid ${field}: expected a non-empty string`);
+  return id;
+}
+
+/** Extract Pi text content while excluding thinking and non-text blocks. */
 function _piText(message: any): string {
   if (typeof message.content === "string") return message.content.trim();
   return message.content
@@ -229,6 +300,7 @@ function _piText(message: any): string {
     .trim();
 }
 
+/** Extract Claude Code text content from either legacy strings or content blocks. */
 function _claudeText(message: any): string {
   if (typeof message.content === "string") return message.content.trim();
   if (!Array.isArray(message.content)) return "";
@@ -239,6 +311,7 @@ function _claudeText(message: any): string {
     .trim();
 }
 
+/** Extract user input and assistant output text from a Codex message payload. */
 function _codexText(message: any): string {
   return message.content
     .filter((block: any) => block.type === "input_text" || block.type === "output_text")
@@ -247,6 +320,7 @@ function _codexText(message: any): string {
     .trim();
 }
 
+/** Identify Claude Code client-injected text that must not become user memory. */
 function _isClaudeInjectedContext(text: string): boolean {
   return text.startsWith("<command-name>")
     || text.startsWith("<command-message>")
@@ -255,6 +329,7 @@ function _isClaudeInjectedContext(text: string): boolean {
     || text.startsWith("This session is being continued from a previous conversation");
 }
 
+/** Identify Codex environment or IDE context that must not become user memory. */
 function _isCodexInjectedContext(text: string): boolean {
   return text.startsWith("# AGENTS.md instructions")
     || text.startsWith("<environment_context>")
@@ -262,10 +337,12 @@ function _isCodexInjectedContext(text: string): boolean {
     || text.startsWith("<image name=");
 }
 
+/** Hash a source JSONL file so unchanged content can skip reparsing. */
 function _sha256(jsonlPath: string): string {
   return createHash("sha256").update(readFileSync(jsonlPath)).digest("hex");
 }
 
+/** Read every non-empty JSONL line into its ordered JSON record. */
 function _readJsonl(jsonlPath: string): any[] {
   return readFileSync(jsonlPath, "utf8")
     .split("\n")
@@ -273,6 +350,7 @@ function _readJsonl(jsonlPath: string): any[] {
     .map((line) => JSON.parse(line));
 }
 
+/** Recursively discover JSONL session files under a source root. */
 function _jsonlFiles(root: string): string[] {
   const files: string[] = [];
   for (const entry of readdirSync(root, { withFileTypes: true })) {