psychmem 1.0.0

package/dist/transcript/parser.js

@@ -0,0 +1,373 @@
+/**
+ * Transcript Parser for Claude Code
+ *
+ * Parses Claude Code's JSONL transcript files incrementally using watermark tracking.
+ * The watermark is a byte offset to avoid re-processing already-seen content.
+ *
+ * Claude Code transcript format (JSONL):
+ * Each line is a JSON object with various message types. Schema is not fully documented,
+ * so we use flexible parsing with fallback handling.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import * as readline from 'readline';
+// =============================================================================
+// Parser Implementation
+// =============================================================================
+export class TranscriptParser {
+    /**
+     * Parse transcript file from a given byte offset
+     * Returns entries from that point forward and the new watermark
+     */
+    async parseFromWatermark(transcriptPath, watermark = 0) {
+        // Resolve path if needed
+        const resolvedPath = this.resolvePath(transcriptPath);
+        // Check if file exists
+        if (!fs.existsSync(resolvedPath)) {
+            return {
+                entries: [],
+                newWatermark: watermark,
+                linesProcessed: 0,
+            };
+        }
+        const entries = [];
+        let newWatermark = watermark;
+        let linesProcessed = 0;
+        // Open file and seek to watermark position
+        const fileHandle = await fs.promises.open(resolvedPath, 'r');
+        try {
+            const stats = await fileHandle.stat();
+            // If watermark is beyond file size, no new content
+            if (watermark >= stats.size) {
+                return {
+                    entries: [],
+                    newWatermark: watermark,
+                    linesProcessed: 0,
+                };
+            }
+            // Create read stream from watermark position
+            const stream = fs.createReadStream(resolvedPath, {
+                start: watermark,
+                encoding: 'utf-8',
+            });
+            // Read raw bytes separately to detect CRLF vs LF line endings
+            const rawStream = fs.createReadStream(resolvedPath, {
+                start: watermark,
+            });
+            // Detect line endings by scanning a small buffer
+            let crlfDetected = false;
+            let crlfSampleDone = false;
+            const sampleChunks = [];
+            rawStream.on('data', (chunk) => {
+                if (!crlfSampleDone) {
+                    sampleChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+                    const combined = Buffer.concat(sampleChunks);
+                    if (combined.includes(0x0d /* \r */)) {
+                        crlfDetected = true;
+                        crlfSampleDone = true;
+                    }
+                    else if (combined.length >= 4096) {
+                        crlfSampleDone = true;
+                    }
+                }
+            });
+            rawStream.on('error', () => { crlfSampleDone = true; });
+            await new Promise((resolve) => rawStream.on('close', resolve));
+            const rl = readline.createInterface({
+                input: stream,
+                crlfDelay: Infinity,
+            });
+            let bytesRead = watermark;
+            // readline with crlfDelay:Infinity strips \r, so each line's byte length
+            // needs +1 for \n (LF files) or +2 for \r\n (CRLF files)
+            const newlineBytes = crlfDetected ? 2 : 1;
+            for await (const line of rl) {
+                // Track byte position (line content + newline byte(s))
+                bytesRead += Buffer.byteLength(line, 'utf-8') + newlineBytes;
+                if (line.trim().length === 0)
+                    continue;
+                try {
+                    const entry = this.parseLine(line);
+                    if (entry) {
+                        entries.push(entry);
+                    }
+                    linesProcessed++;
+                }
+                catch (e) {
+                    // Skip malformed lines but continue processing
+                    console.error(`[TranscriptParser] Skipping malformed line: ${e}`);
+                }
+            }
+            newWatermark = bytesRead;
+        }
+        finally {
+            await fileHandle.close();
+        }
+        return {
+            entries,
+            newWatermark,
+            linesProcessed,
+        };
+    }
+    /**
+     * Parse a single JSONL line into a transcript entry
+     * Handles various Claude Code message formats flexibly
+     */
+    parseLine(line) {
+        const json = JSON.parse(line);
+        // Detect message type and extract content
+        const type = this.detectEntryType(json);
+        const content = this.extractContent(json);
+        if (!content || content.trim().length === 0) {
+            return null;
+        }
+        // Build entry with only defined optional properties
+        const timestamp = this.extractTimestamp(json);
+        const role = this.extractRole(json);
+        const toolName = this.extractToolName(json);
+        const toolInput = this.extractToolInput(json);
+        const toolOutput = this.extractToolOutput(json);
+        const entry = {
+            type,
+            content,
+            rawJson: json,
+        };
+        // Only add optional properties if defined
+        if (timestamp !== undefined)
+            entry.timestamp = timestamp;
+        if (role !== undefined)
+            entry.role = role;
+        if (toolName !== undefined)
+            entry.toolName = toolName;
+        if (toolInput !== undefined)
+            entry.toolInput = toolInput;
+        if (toolOutput !== undefined)
+            entry.toolOutput = toolOutput;
+        return entry;
+    }
+    /**
+     * Detect the type of transcript entry
+     */
+    detectEntryType(json) {
+        // Check for explicit type field
+        const type = json.type;
+        if (type === 'user' || json.role === 'user') {
+            return 'user_message';
+        }
+        if (type === 'assistant' || json.role === 'assistant') {
+            // Check if it's a tool use
+            if (json.tool_use || json.toolName || this.hasToolUseContent(json)) {
+                return 'tool_use';
+            }
+            return 'assistant_message';
+        }
+        if (type === 'tool_result' || json.tool_result || json.toolOutput !== undefined) {
+            return 'tool_result';
+        }
+        if (type === 'system' || json.role === 'system') {
+            return 'system';
+        }
+        // Check content structure for tool patterns
+        if (json.tool_use || json.name || json.input) {
+            return 'tool_use';
+        }
+        // Default based on role
+        const role = json.role;
+        if (role === 'user')
+            return 'user_message';
+        if (role === 'assistant')
+            return 'assistant_message';
+        return 'unknown';
+    }
+    /**
+     * Check if json contains tool use content
+     */
+    hasToolUseContent(json) {
+        // Check various possible locations for tool use
+        if (json.content && Array.isArray(json.content)) {
+            return json.content.some((c) => typeof c === 'object' && c !== null && 'type' in c && c.type === 'tool_use');
+        }
+        return false;
+    }
+    /**
+     * Extract text content from various formats
+     */
+    extractContent(json) {
+        // Direct content string
+        if (typeof json.content === 'string') {
+            return json.content;
+        }
+        // Content array (Claude API format)
+        if (Array.isArray(json.content)) {
+            const textParts = [];
+            for (const part of json.content) {
+                if (typeof part === 'string') {
+                    textParts.push(part);
+                }
+                else if (typeof part === 'object' && part !== null) {
+                    const obj = part;
+                    // Text block
+                    if (obj.type === 'text' && typeof obj.text === 'string') {
+                        textParts.push(obj.text);
+                    }
+                    // Tool use block
+                    if (obj.type === 'tool_use') {
+                        const toolName = obj.name || 'unknown';
+                        const input = obj.input ? JSON.stringify(obj.input) : '';
+                        textParts.push(`[Tool: ${toolName}] ${input}`);
+                    }
+                    // Tool result block
+                    if (obj.type === 'tool_result') {
+                        const content = typeof obj.content === 'string'
+                            ? obj.content
+                            : JSON.stringify(obj.content);
+                        textParts.push(`[Tool Result] ${content}`);
+                    }
+                }
+            }
+            return textParts.join('\n');
+        }
+        // Message field
+        if (typeof json.message === 'string') {
+            return json.message;
+        }
+        // Text field
+        if (typeof json.text === 'string') {
+            return json.text;
+        }
+        // Tool output
+        if (json.toolOutput !== undefined) {
+            return typeof json.toolOutput === 'string'
+                ? json.toolOutput
+                : JSON.stringify(json.toolOutput);
+        }
+        // Fallback: stringify the whole object
+        return '';
+    }
+    /**
+     * Extract timestamp from entry
+     */
+    extractTimestamp(json) {
+        if (typeof json.timestamp === 'string')
+            return json.timestamp;
+        if (typeof json.created_at === 'string')
+            return json.created_at;
+        if (typeof json.time === 'string')
+            return json.time;
+        return undefined;
+    }
+    /**
+     * Extract role from entry
+     */
+    extractRole(json) {
+        const role = json.role;
+        if (role === 'user' || role === 'assistant' || role === 'system' || role === 'tool') {
+            return role;
+        }
+        return undefined;
+    }
+    /**
+     * Extract tool name
+     */
+    extractToolName(json) {
+        if (typeof json.name === 'string')
+            return json.name;
+        if (typeof json.toolName === 'string')
+            return json.toolName;
+        if (typeof json.tool_name === 'string')
+            return json.tool_name;
+        // Check in content array
+        if (Array.isArray(json.content)) {
+            for (const part of json.content) {
+                if (typeof part === 'object' && part !== null) {
+                    const obj = part;
+                    if (obj.type === 'tool_use' && typeof obj.name === 'string') {
+                        return obj.name;
+                    }
+                }
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Extract tool input
+     */
+    extractToolInput(json) {
+        if (json.input !== undefined)
+            return json.input;
+        if (json.toolInput !== undefined)
+            return json.toolInput;
+        if (json.tool_input !== undefined)
+            return json.tool_input;
+        // Check in content array
+        if (Array.isArray(json.content)) {
+            for (const part of json.content) {
+                if (typeof part === 'object' && part !== null) {
+                    const obj = part;
+                    if (obj.type === 'tool_use' && obj.input !== undefined) {
+                        return obj.input;
+                    }
+                }
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Extract tool output
+     */
+    extractToolOutput(json) {
+        if (json.output !== undefined)
+            return json.output;
+        if (json.toolOutput !== undefined)
+            return json.toolOutput;
+        if (json.tool_output !== undefined)
+            return json.tool_output;
+        // Check in content array for tool_result
+        if (Array.isArray(json.content)) {
+            for (const part of json.content) {
+                if (typeof part === 'object' && part !== null) {
+                    const obj = part;
+                    if (obj.type === 'tool_result') {
+                        return obj.content;
+                    }
+                }
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Resolve path, expanding ~ to home directory
+     */
+    resolvePath(filePath) {
+        if (filePath.startsWith('~')) {
+            const home = process.env.HOME || process.env.USERPROFILE || '';
+            return path.join(home, filePath.slice(1));
+        }
+        return path.resolve(filePath);
+    }
+    /**
+     * Get conversation text from entries (for context sweep)
+     */
+    static entriesToConversationText(entries) {
+        return entries
+            .map(entry => {
+            const rolePrefix = entry.role ? `[${entry.role}] ` : '';
+            return `${rolePrefix}${entry.content}`;
+        })
+            .join('\n\n');
+    }
+    /**
+     * Filter entries to only user and assistant messages (no tool details)
+     */
+    static filterConversation(entries) {
+        return entries.filter(e => e.type === 'user_message' || e.type === 'assistant_message');
+    }
+    /**
+     * Get only new entries since last check (stateless helper)
+     */
+    static async getNewEntries(transcriptPath, lastWatermark) {
+        const parser = new TranscriptParser();
+        return parser.parseFromWatermark(transcriptPath, lastWatermark);
+    }
+}
+//# sourceMappingURL=parser.js.map

package/dist/transcript/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/transcript/parser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAqCrC,gFAAgF;AAChF,wBAAwB;AACxB,gFAAgF;AAEhF,MAAM,OAAO,gBAAgB;IAC3B;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CACtB,cAAsB,EACtB,YAAoB,CAAC;QAErB,yBAAyB;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;QAEtD,uBAAuB;QACvB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,SAAS;gBACvB,cAAc,EAAE,CAAC;aAClB,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,IAAI,YAAY,GAAG,SAAS,CAAC;QAC7B,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,2CAA2C;QAC3C,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QAE7D,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;YAEtC,mDAAmD;YACnD,IAAI,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC5B,OAAO;oBACL,OAAO,EAAE,EAAE;oBACX,YAAY,EAAE,SAAS;oBACvB,cAAc,EAAE,CAAC;iBAClB,CAAC;YACJ,CAAC;YAED,6CAA6C;YAC7C,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,YAAY,EAAE;gBAC/C,KAAK,EAAE,SAAS;gBAChB,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,MAAM,SAAS,GAAG,EAAE,CAAC,gBAAgB,CAAC,YAAY,EAAE;gBAClD,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,iDAAiD;YACjD,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAsB,EAAE,EAAE;gBAC9C,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAe,CAAC,CAAC,CAAC;oBACjF,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;oBAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,cAAc,GAAG,IAAI,CAAC;oBACxB,CAAC;yBAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;wBACnC,cAAc,GAAG,IAAI,CAAC;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAErE,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;gBAClC,KAAK,EAAE,MAAM;gBACb,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YAEH,IAAI,SAAS,GAAG,SAAS,CAAC;YAC1B,yEAAyE;YACzE,yDAAyD;YACzD,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAE1C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;gBAC5B,uDAAuD;gBACvD,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,YAAY,CAAC;gBAE7D,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEvC,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;oBACnC,IAAI,KAAK,EAAE,CAAC;wBACV,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACtB,CAAC;oBACD,cAAc,EAAE,CAAC;gBACnB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,+CAA+C;oBAC/C,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,EAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YAED,YAAY,GAAG,SAAS,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;QAED,OAAO;YACL,OAAO;YACP,YAAY;YACZ,cAAc;SACf,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,SAAS,CAAC,IAAY;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;QAEzD,0CAA0C;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAE1C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,oDAAoD;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAEhD,MAAM,KAAK,GAAoB;YAC7B,IAAI;YACJ,OAAO;YACP,OAAO,EAAE,IAAI;SACd,CAAC;QAEF,0CAA0C;QAC1C,IAAI,SAAS,KAAK,SAAS;YAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QACzD,IAAI,IAAI,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS;YAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACtD,IAAI,SAAS,KAAK,SAAS;YAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QACzD,IAAI,UAAU,KAAK,SAAS;YAAE,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;QAE5D,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,IAA6B;QACnD,gCAAgC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,IAA0B,CAAC;QAE7C,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5C,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACtD,2BAA2B;YAC3B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnE,OAAO,UAAU,CAAC;YACpB,CAAC;YACD,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QAED,IAAI,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChF,OAAO,aAAa,CAAC;QACvB,CAAC;QAED,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAChD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,4CAA4C;QAC5C,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7C,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,wBAAwB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAA0B,CAAC;QAC7C,IAAI,IAAI,KAAK,MAAM;YAAE,OAAO,cAAc,CAAC;QAC3C,IAAI,IAAI,KAAK,WAAW;YAAE,OAAO,mBAAmB,CAAC;QAErD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAA6B;QACrD,gDAAgD;QAChD,IAAI,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,OAAQ,IAAI,CAAC,OAAqB,CAAC,IAAI,CACrC,CAAC,CAAU,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,MAAM,IAAI,CAAC,IAAK,CAA6B,CAAC,IAAI,KAAK,UAAU,CACzH,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,IAA6B;QAClD,wBAAwB;QACxB,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,oCAAoC;QACpC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,MAAM,SAAS,GAAa,EAAE,CAAC;YAE/B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAoB,EAAE,CAAC;gBAC7C,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;qBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrD,MAAM,GAAG,GAAG,IAA+B,CAAC;oBAE5C,aAAa;oBACb,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACxD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC3B,CAAC;oBAED,iBAAiB;oBACjB,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;wBAC5B,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC;wBACvC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBACzD,SAAS,CAAC,IAAI,CAAC,UAAU,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;oBACjD,CAAC;oBAED,oBAAoB;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;wBAC/B,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;4BAC7C,CAAC,CAAC,GAAG,CAAC,OAAO;4BACb,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBAChC,SAAS,CAAC,IAAI,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;oBAC7C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,aAAa;QACb,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QAED,cAAc;QACd,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;gBACxC,CAAC,CAAC,IAAI,CAAC,UAAU;gBACjB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;QAED,uCAAuC;QACvC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,IAA6B;QACpD,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QAC9D,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,IAA6B;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAA0B,CAAC;QAC7C,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,IAA6B;QACnD,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC;QACpD,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC;QAC5D,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QAE9D,yBAAyB;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAoB,EAAE,CAAC;gBAC7C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,IAA+B,CAAC;oBAC5C,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC5D,OAAO,GAAG,CAAC,IAAI,CAAC;oBAClB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,IAA6B;QACpD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QAChD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QACxD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;QAE1D,yBAAyB;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAoB,EAAE,CAAC;gBAC7C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,IAA+B,CAAC;oBAC5C,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;wBACvD,OAAO,GAAG,CAAC,KAAK,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAA6B;QACrD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QAClD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;QAC1D,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAE5D,yCAAyC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAoB,EAAE,CAAC;gBAC7C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,IAA+B,CAAC;oBAC5C,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;wBAC/B,OAAO,GAAG,CAAC,OAAO,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,QAAgB;QAClC,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,yBAAyB,CAAC,OAA0B;QACzD,OAAO,OAAO;aACX,GAAG,CAAC,KAAK,CAAC,EAAE;YACX,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,OAAO,GAAG,UAAU,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QACzC,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB,CAAC,OAA0B;QAClD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,IAAI,CAAC,CAAC,IAAI,KAAK,mBAAmB,CACjE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,cAAsB,EACtB,aAAqB;QAErB,MAAM,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC,kBAAkB,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;IAClE,CAAC;CACF"}

package/dist/transcript/sweep.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Transcript Sweep - Memory extraction from transcript with deduplication
+ *
+ * This module coordinates:
+ * 1. Parsing new transcript entries (using watermark)
+ * 2. Running context sweep on new content
+ * 3. Deduplicating against existing session memories (70% keyword overlap)
+ * 4. Limiting to maxMemoriesPerStop (based on Cowan's working memory: 4±1)
+ */
+import type { MemoryCandidate, MemoryUnit, PsychMemConfig } from '../types/index.js';
+export interface TranscriptSweepResult {
+    /** Memory candidates after deduplication */
+    candidates: MemoryCandidate[];
+    /** New watermark to save */
+    newWatermark: number;
+    /** Number of candidates before deduplication */
+    rawCandidateCount: number;
+    /** Number filtered by deduplication */
+    deduplicatedCount: number;
+    /** Number filtered by limit */
+    limitedCount: number;
+}
+export interface TranscriptSweepOptions {
+    /** Session ID for deduplication lookup */
+    sessionId: string;
+    /** Existing memories for this session */
+    existingMemories: MemoryUnit[];
+    /** Configuration */
+    config?: Partial<PsychMemConfig>;
+}
+export declare class TranscriptSweep {
+    private parser;
+    private contextSweep;
+    private config;
+    constructor(config?: Partial<PsychMemConfig>);
+    /**
+     * Main entry point: extract memories from transcript since last watermark
+     * with deduplication and limits
+     */
+    sweepTranscript(transcriptPath: string, watermark: number, options: TranscriptSweepOptions): Promise<TranscriptSweepResult>;
+    /**
+     * Extract memory candidates from conversation text
+     * Creates synthetic events for context sweep compatibility
+     */
+    private extractFromText;
+    /**
+     * Map transcript entry type to hook type for context sweep
+     */
+    private entryTypeToHookType;
+    /**
+     * Deduplicate candidates against existing memories using keyword overlap
+     */
+    private deduplicateCandidates;
+    /**
+     * Apply the max memories per stop limit
+     * Sort by importance and take top N
+     */
+    private applyLimit;
+    /**
+     * Extract keywords from text for overlap calculation
+     * Returns a Set of normalized keywords
+     */
+    private extractKeywords;
+    /**
+     * Calculate Jaccard similarity (overlap) between two keyword sets
+     * Returns a value between 0 and 1
+     */
+    private calculateKeywordOverlap;
+    /**
+     * Static helper: quick check if transcript has new content
+     */
+    static hasNewContent(transcriptPath: string, watermark: number): Promise<boolean>;
+}
+export { TranscriptParser } from './parser.js';
+//# sourceMappingURL=sweep.d.ts.map

package/dist/transcript/sweep.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../../src/transcript/sweep.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,UAAU,EACV,cAAc,EACf,MAAM,mBAAmB,CAAC;AAS3B,MAAM,WAAW,qBAAqB;IACpC,4CAA4C;IAC5C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,gDAAgD;IAChD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACrC,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,gBAAgB,EAAE,UAAU,EAAE,CAAC;IAC/B,oBAAoB;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;CAClC;AAMD,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAiB;gBAEnB,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IAMhD;;;OAGG;IACG,eAAe,CACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,qBAAqB,CAAC;IAoCjC;;;OAGG;IACH,OAAO,CAAC,eAAe;IAgCvB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAY3B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA0C7B;;;OAGG;IACH,OAAO,CAAC,UAAU;IAkBlB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAsBvB;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAmB/B;;OAEG;WACU,aAAa,CACxB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,OAAO,CAAC;CAIpB;AAMD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC"}

package/dist/transcript/sweep.js

@@ -0,0 +1,202 @@
+/**
+ * Transcript Sweep - Memory extraction from transcript with deduplication
+ *
+ * This module coordinates:
+ * 1. Parsing new transcript entries (using watermark)
+ * 2. Running context sweep on new content
+ * 3. Deduplicating against existing session memories (70% keyword overlap)
+ * 4. Limiting to maxMemoriesPerStop (based on Cowan's working memory: 4±1)
+ */
+import { TranscriptParser } from './parser.js';
+import { ContextSweep } from '../memory/context-sweep.js';
+import { DEFAULT_CONFIG } from '../types/index.js';
+// =============================================================================
+// Transcript Sweep Implementation
+// =============================================================================
+export class TranscriptSweep {
+    parser;
+    contextSweep;
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.parser = new TranscriptParser();
+        this.contextSweep = new ContextSweep(this.config.sweep);
+    }
+    /**
+     * Main entry point: extract memories from transcript since last watermark
+     * with deduplication and limits
+     */
+    async sweepTranscript(transcriptPath, watermark, options) {
+        // 1. Parse new transcript entries
+        const parseResult = await this.parser.parseFromWatermark(transcriptPath, watermark);
+        if (parseResult.entries.length === 0) {
+            return {
+                candidates: [],
+                newWatermark: parseResult.newWatermark,
+                rawCandidateCount: 0,
+                deduplicatedCount: 0,
+                limitedCount: 0,
+            };
+        }
+        // 2. Convert entries to conversation text and run context sweep
+        const conversationText = TranscriptParser.entriesToConversationText(parseResult.entries);
+        const rawCandidates = this.extractFromText(conversationText, parseResult.entries);
+        // 3. Deduplicate against existing session memories
+        const dedupResult = this.deduplicateCandidates(rawCandidates, options.existingMemories);
+        // 4. Apply limit (maxMemoriesPerStop)
+        const limited = this.applyLimit(dedupResult.candidates);
+        return {
+            candidates: limited.candidates,
+            newWatermark: parseResult.newWatermark,
+            rawCandidateCount: rawCandidates.length,
+            deduplicatedCount: dedupResult.removedCount,
+            limitedCount: limited.removedCount,
+        };
+    }
+    /**
+     * Extract memory candidates from conversation text
+     * Creates synthetic events for context sweep compatibility
+     */
+    extractFromText(text, entries) {
+        // Create synthetic events for the context sweep
+        const syntheticEvents = entries.map((entry, index) => ({
+            id: `transcript-${index}`,
+            sessionId: 'transcript',
+            hookType: this.entryTypeToHookType(entry.type),
+            timestamp: entry.timestamp ? new Date(entry.timestamp) : new Date(),
+            content: entry.content,
+            toolName: entry.toolName,
+            toolInput: entry.toolInput ? JSON.stringify(entry.toolInput) : undefined,
+            toolOutput: entry.toolOutput ? JSON.stringify(entry.toolOutput) : undefined,
+        }));
+        // Also add a Stop event with full conversation for comprehensive analysis
+        syntheticEvents.push({
+            id: 'transcript-stop',
+            sessionId: 'transcript',
+            hookType: 'Stop',
+            timestamp: new Date(),
+            content: text,
+            toolName: undefined,
+            toolInput: undefined,
+            toolOutput: undefined,
+        });
+        // Run context sweep
+        return this.contextSweep.extractCandidates(syntheticEvents);
+    }
+    /**
+     * Map transcript entry type to hook type for context sweep
+     */
+    entryTypeToHookType(type) {
+        switch (type) {
+            case 'user_message':
+                return 'UserPromptSubmit';
+            case 'tool_use':
+            case 'tool_result':
+                return 'PostToolUse';
+            default:
+                return 'Stop';
+        }
+    }
+    /**
+     * Deduplicate candidates against existing memories using keyword overlap
+     */
+    deduplicateCandidates(candidates, existingMemories) {
+        if (existingMemories.length === 0) {
+            return { candidates, removedCount: 0 };
+        }
+        // Pre-compute keywords for existing memories
+        const existingKeywordSets = existingMemories.map(mem => ({
+            memory: mem,
+            keywords: this.extractKeywords(mem.summary),
+        }));
+        const filtered = [];
+        let removedCount = 0;
+        for (const candidate of candidates) {
+            const candidateKeywords = this.extractKeywords(candidate.summary);
+            // Check overlap with each existing memory
+            let isDuplicate = false;
+            for (const { keywords: existingKeywords } of existingKeywordSets) {
+                const overlap = this.calculateKeywordOverlap(candidateKeywords, existingKeywords);
+                if (overlap >= this.config.deduplicationThreshold) {
+                    isDuplicate = true;
+                    break;
+                }
+            }
+            if (isDuplicate) {
+                removedCount++;
+            }
+            else {
+                filtered.push(candidate);
+            }
+        }
+        return { candidates: filtered, removedCount };
+    }
+    /**
+     * Apply the max memories per stop limit
+     * Sort by importance and take top N
+     */
+    applyLimit(candidates) {
+        if (candidates.length <= this.config.maxMemoriesPerStop) {
+            return { candidates, removedCount: 0 };
+        }
+        // Sort by preliminary importance (descending)
+        const sorted = [...candidates].sort((a, b) => b.preliminaryImportance - a.preliminaryImportance);
+        const limited = sorted.slice(0, this.config.maxMemoriesPerStop);
+        const removedCount = candidates.length - limited.length;
+        return { candidates: limited, removedCount };
+    }
+    /**
+     * Extract keywords from text for overlap calculation
+     * Returns a Set of normalized keywords
+     */
+    extractKeywords(text) {
+        // Stopwords to filter out
+        const stopwords = new Set([
+            'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+            'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+            'should', 'may', 'might', 'can', 'to', 'of', 'in', 'for', 'on', 'with',
+            'at', 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after',
+            'this', 'that', 'these', 'those', 'it', 'its', 'they', 'them', 'their',
+            'and', 'or', 'but', 'if', 'so', 'because', 'while', 'when', 'where',
+            'what', 'which', 'who', 'how', 'why', 'all', 'each', 'some', 'any',
+            'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she', 'him', 'her',
+        ]);
+        const words = text
+            .toLowerCase()
+            .replace(/[^a-z0-9\s]/g, ' ')
+            .split(/\s+/)
+            .filter(word => word.length > 2 && !stopwords.has(word));
+        return new Set(words);
+    }
+    /**
+     * Calculate Jaccard similarity (overlap) between two keyword sets
+     * Returns a value between 0 and 1
+     */
+    calculateKeywordOverlap(setA, setB) {
+        if (setA.size === 0 || setB.size === 0) {
+            return 0;
+        }
+        let intersection = 0;
+        for (const word of setA) {
+            if (setB.has(word)) {
+                intersection++;
+            }
+        }
+        const union = setA.size + setB.size - intersection;
+        if (union === 0)
+            return 0;
+        return intersection / union;
+    }
+    /**
+     * Static helper: quick check if transcript has new content
+     */
+    static async hasNewContent(transcriptPath, watermark) {
+        const result = await TranscriptParser.getNewEntries(transcriptPath, watermark);
+        return result.entries.length > 0;
+    }
+}
+// =============================================================================
+// Export index for module
+// =============================================================================
+export { TranscriptParser } from './parser.js';
+//# sourceMappingURL=sweep.js.map