spiracha 2.0.0 → 2.2.0

package/src/lib/codex-thread-recovery.ts

@@ -1,8 +1,8 @@
 import { Database } from 'bun:sqlite';
-import { copyFile, utimes } from 'node:fs/promises';
+import { copyFile, readdir, rm, utimes } from 'node:fs/promises';
 import path from 'node:path';
 import type { RecoverProjectThreadsResult } from './codex-browser-types';
-import { getPortablePathBasename } from './shared';
+import { getPortablePathBasename } from './portable-path';
 import { runWithSqliteRetry } from './sqlite-retry';
 
 type RecoveryThreadRow = {
@@ -18,6 +18,8 @@ type GlobalState = {
     'project-order'?: string[];
 };
 
+const RECOVERY_BACKUP_RETENTION_COUNT = 5;
+
 const backupFile = async (filePath: string, label: string) => {
     const stamp = new Date()
         .toISOString()
@@ -26,6 +28,14 @@ const backupFile = async (filePath: string, label: string) => {
         .replace('T', '-');
     const backupPath = `${filePath}.bak-${label}-${stamp}`;
     await copyFile(filePath, backupPath);
+    const directory = path.dirname(filePath);
+    const prefix = `${path.basename(filePath)}.bak-${label}-`;
+    const backups = (await readdir(directory))
+        .filter((entry) => entry.startsWith(prefix))
+        .sort((left, right) => right.localeCompare(left));
+    await Promise.all(
+        backups.slice(RECOVERY_BACKUP_RETENTION_COUNT).map((entry) => rm(path.join(directory, entry), { force: true })),
+    );
     return backupPath;
 };
 
@@ -100,9 +110,9 @@ const refreshThreadRows = (db: Database, threadIds: string[]) => {
     return Number(result.changes);
 };
 
-const refreshSessionIndex = async (sessionIndexPath: string, threadIds: string[]) => {
+const prepareSessionIndexRefresh = async (sessionIndexPath: string, threadIds: string[]) => {
     if (threadIds.length === 0) {
-        return 0;
+        return { content: null, updated: 0 };
     }
 
     const now = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
@@ -116,7 +126,19 @@ const refreshSessionIndex = async (sessionIndexPath: string, threadIds: string[]
             continue;
         }
 
-        const parsed = JSON.parse(line) as { id?: string; updated_at?: string };
+        let parsed: { id?: string; updated_at?: string };
+        try {
+            const value: unknown = JSON.parse(line);
+            if (!value || typeof value !== 'object' || Array.isArray(value)) {
+                rewrittenLines.push(line);
+                continue;
+            }
+            parsed = value as { id?: string; updated_at?: string };
+        } catch {
+            rewrittenLines.push(line);
+            continue;
+        }
+
         if (parsed.id && threadIdSet.has(parsed.id)) {
             parsed.updated_at = now;
             updated += 1;
@@ -125,8 +147,10 @@ const refreshSessionIndex = async (sessionIndexPath: string, threadIds: string[]
         rewrittenLines.push(JSON.stringify(parsed));
     }
 
-    await Bun.write(sessionIndexPath, `${rewrittenLines.join('\n')}\n`);
-    return updated;
+    return {
+        content: `${rewrittenLines.join('\n')}\n`,
+        updated,
+    };
 };
 
 const touchRolloutFiles = async (codexDir: string, rolloutPaths: string[]) => {
@@ -177,12 +201,15 @@ export const recoverCodexProjectThreads = async (
         const topLevelThreads = getProjectTopLevelThreads(db, projectName);
         const projectCwds = [...new Set(topLevelThreads.map((thread) => thread.cwd))];
         const rootUpdateResult = updateGlobalRoots(globalState, projectCwds);
-        await writeGlobalState(globalStatePath, rootUpdateResult.state);
-
         const threadIds = topLevelThreads.map((thread) => thread.id);
         const rolloutPaths = topLevelThreads.map((thread) => thread.rollout_path);
+        const sessionIndexRefresh = await prepareSessionIndexRefresh(sessionIndexPath, threadIds);
+
+        await writeGlobalState(globalStatePath, rootUpdateResult.state);
         const threadDbRowsUpdated = refreshThreadRows(db, threadIds);
-        const sessionIndexRowsUpdated = await refreshSessionIndex(sessionIndexPath, threadIds);
+        if (sessionIndexRefresh.content !== null) {
+            await Bun.write(sessionIndexPath, sessionIndexRefresh.content);
+        }
         const rolloutFilesTouched = await touchRolloutFiles(codexDir, rolloutPaths);
 
         return {
@@ -192,7 +219,7 @@ export const recoverCodexProjectThreads = async (
             resolvedCwds: projectCwds,
             rolloutFilesTouched,
             savedRootsAdded: rootUpdateResult.savedRootsAdded,
-            sessionIndexRowsUpdated,
+            sessionIndexRowsUpdated: sessionIndexRefresh.updated,
             threadDbRowsUpdated,
             topLevelThreadsFound: threadIds.length,
         };

package/src/lib/codex-transcript-filter.ts

@@ -0,0 +1,31 @@
+import type { ThreadEvent } from './codex-browser-types';
+
+const isCommentaryMessage = (event: ThreadEvent) =>
+    event.kind === 'message' && event.role === 'assistant' && event.phase === 'commentary';
+
+export type CodexTranscriptEventFilters = {
+    showCommentary: boolean;
+    showExtraEvents: boolean;
+    showToolCalls: boolean;
+    showUserMessages: boolean;
+};
+
+export const shouldShowCodexTranscriptEvent = (event: ThreadEvent, filters: CodexTranscriptEventFilters) => {
+    if (isCommentaryMessage(event) && !filters.showCommentary) {
+        return false;
+    }
+
+    if (event.kind === 'message') {
+        if (event.role === 'user' && !filters.showUserMessages) {
+            return false;
+        }
+
+        return !event.isHiddenByDefault || filters.showExtraEvents;
+    }
+
+    if (event.kind === 'tool_call' || event.kind === 'tool_output') {
+        return filters.showToolCalls;
+    }
+
+    return filters.showExtraEvents;
+};

package/src/lib/codex-transcript-renderer.ts

@@ -1,7 +1,13 @@
 import { createReadStream } from 'node:fs';
 import { rm } from 'node:fs/promises';
 import path from 'node:path';
-import { finished } from 'node:stream/promises';
+import type { Readable, Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import {
+    parseCodexTranscriptRecord,
+    shouldHideCodexTranscriptText,
+    stripCodexMemoryCitationBlocks,
+} from './codex-thread-parser';
 import {
     type CodexTranscriptExportTarget,
     type CodexTranscriptRenderOptions,
@@ -23,10 +29,17 @@ import {
     type JsonValue,
     type MetadataEntry,
     readJsonlObjects,
+    renderCodeBlock,
     renderDocumentTitle,
     renderMetadataBlock,
     renderSection,
+    stripCodexAppDirectiveLines,
 } from './shared';
+import { runWithTranscriptLoadLimit } from './transcript-load-limiter';
+
+export const pipeCodexExportStream = async (source: Readable, destination: Writable) => {
+    await pipeline(source, destination, { end: false });
+};
 
 export const renderCodexSessionFile = async (
     target: CodexTranscriptExportTarget,
@@ -35,7 +48,14 @@ export const renderCodexSessionFile = async (
     let transcriptState: CodexTranscriptState;
 
     try {
-        transcriptState = await collectCodexTranscript(target.sessionFile, options, target.thread?.model ?? null);
+        transcriptState = await runWithTranscriptLoadLimit(
+            () => collectCodexTranscript(target.sessionFile, options, target.thread?.model ?? null),
+            {
+                id: target.thread?.id,
+                path: target.sessionFile,
+                source: 'codex-export-inline',
+            },
+        );
     } catch (error) {
         const message = error instanceof Error ? error.message : String(error);
         throw new Error(`Failed to read Codex transcript ${target.sessionFile}: ${message}`);
@@ -78,19 +98,28 @@ export const writeCodexSessionFileExport = async (
     let wroteSection = false;
 
     try {
-        transcriptStream = await createExportWriteStream(transcriptOutputPath);
-        for await (const parsed of readJsonlObjects(target.sessionFile)) {
-            captureSessionMeta(parsed, state.sessionMeta);
-            const block = renderCodexTranscriptRecord(parsed, options, state);
-            if (!block) {
-                continue;
-            }
-
-            transcriptStream.write(transform(wroteSection ? `${getSectionSeparator()}${block}` : block));
-            wroteSection = true;
-        }
-        await finalizeExportWriteStream(transcriptStream);
-        transcriptStream = null;
+        await runWithTranscriptLoadLimit(
+            async () => {
+                transcriptStream = await createExportWriteStream(transcriptOutputPath);
+                for await (const parsed of readJsonlObjects(target.sessionFile)) {
+                    captureSessionMeta(parsed, state.sessionMeta);
+                    const block = renderCodexTranscriptRecord(parsed, options, state);
+                    if (!block) {
+                        continue;
+                    }
+
+                    transcriptStream.write(transform(wroteSection ? `${getSectionSeparator()}${block}` : block));
+                    wroteSection = true;
+                }
+                await finalizeExportWriteStream(transcriptStream);
+                transcriptStream = null;
+            },
+            {
+                id: target.thread?.id,
+                path: target.sessionFile,
+                source: 'codex-export-stream',
+            },
+        );
 
         if (!wroteSection) {
             return false;
@@ -104,8 +133,7 @@ export const writeCodexSessionFileExport = async (
             }
 
             const transcriptReadStream = createReadStream(transcriptOutputPath, { encoding: 'utf8' });
-            transcriptReadStream.pipe(outputStream, { end: false });
-            await finished(transcriptReadStream);
+            await pipeCodexExportStream(transcriptReadStream, outputStream);
             outputStream.write('\n');
             await finalizeExportWriteStream(outputStream);
         } catch (error) {
@@ -189,7 +217,7 @@ const processCodexMessageRecord = (
     options: CodexTranscriptRenderOptions,
     state: CodexTranscriptState,
 ) => {
-    return renderMessageBlock(message, options.outputFormat, state.assistantModel, options.includeCommentary);
+    return renderMessageBlock(message, options.outputFormat, state, options.includeCommentary);
 };
 
 const buildStreamExportPrefix = (
@@ -494,45 +522,22 @@ const normalizeMessage = (value: Record<string, JsonValue>): MessageRecord | nul
 };
 
 const extractToolRecord = (parsed: Record<string, JsonValue>): ToolRecord | null => {
-    if (parsed.type !== 'response_item') {
-        return null;
-    }
-
-    const payload = asObject(parsed.payload);
-    if (!payload) {
-        return null;
-    }
-
-    if (payload.type === 'function_call') {
-        const name = asString(payload.name);
-        const argumentsText = asString(payload.arguments);
-        const callId = asString(payload.call_id);
-
-        if (name !== 'exec_command') {
-            return null;
-        }
-
+    const event = parseCodexTranscriptRecord(parsed);
+    if (event?.kind === 'tool_call') {
         return {
-            argumentsText: argumentsText ?? undefined,
-            callId,
+            argumentsText: event.argumentsText ?? undefined,
+            callId: event.callId,
             kind: 'call',
-            name,
+            name: event.name,
         };
     }
 
-    if (payload.type === 'function_call_output') {
-        const callId = asString(payload.call_id);
-        const outputText = asString(payload.output);
-
-        if (!outputText?.includes('Command: ')) {
-            return null;
-        }
-
+    if (event?.kind === 'tool_output') {
         return {
-            callId,
+            callId: event.callId,
             kind: 'output',
-            name: 'function_call_output',
-            outputText: outputText ?? undefined,
+            name: 'tool_output',
+            outputText: event.outputText,
         };
     }
 
@@ -542,7 +547,7 @@ const extractToolRecord = (parsed: Record<string, JsonValue>): ToolRecord | null
 const renderMessageBlock = (
     message: MessageRecord,
     outputFormat: ExportFormat,
-    assistantModel: string | null,
+    state: CodexTranscriptState,
     includeCommentary: boolean,
 ): string => {
     if (message.role !== 'user' && message.role !== 'assistant') {
@@ -553,12 +558,15 @@ const renderMessageBlock = (
         return '';
     }
 
-    const text = cleanExtractedText(stripPreviewBlock(extractText(message.content))).trim();
-    if (!text || shouldSkipMessage(message.role, text)) {
+    const extractedText = extractText(message.content);
+    const text = stripCodexMemoryCitationBlocks(
+        stripCodexAppDirectiveLines(cleanExtractedText(stripPreviewBlock(extractedText))),
+    );
+    if (!text || shouldHideCodexTranscriptText(message.role, text)) {
         return '';
     }
 
-    const title = message.role === 'user' ? 'User' : formatModelLabel(message.model ?? assistantModel);
+    const title = message.role === 'user' ? 'User' : formatModelLabel(message.model ?? state.assistantModel);
     const body = message.phase ? `Phase: ${message.phase}\n\n${text}` : text;
 
     return renderSection(title, body, outputFormat);
@@ -567,11 +575,18 @@ const renderMessageBlock = (
 const renderToolBlock = (tool: ToolRecord, outputFormat: ExportFormat): string => {
     if (tool.kind === 'call') {
         const details = formatToolCallDetails(tool, outputFormat);
-        return details ? renderSection('Tool', details, outputFormat) : '';
+        return renderSection('Tool Call', details, outputFormat);
+    }
+
+    const outputText = tool.outputText ?? '';
+    const summary = formatToolOutputSummary(outputText, outputFormat);
+    if (!summary && !outputText.trim()) {
+        return '';
     }
 
-    const summary = formatToolOutputSummary(tool.outputText ?? '', outputFormat);
-    return summary ? renderSection('Tool Output', summary, outputFormat) : '';
+    const lines = tool.callId ? [`Call ID: ${tool.callId}`, ''] : [];
+    lines.push(summary || renderCodeBlock(outputText.trim(), outputFormat));
+    return renderSection('Tool Output', lines.join('\n'), outputFormat);
 };
 
 const stripPreviewBlock = (text: string): string => {
@@ -586,7 +601,7 @@ const stripPreviewBlock = (text: string): string => {
 
     const first = parts[0];
     const second = parts[1];
-    const isTranscriptHeading = (value: string) => /^##\s+.+$/i.test(value);
+    const isTranscriptHeading = (value: string) => /^##\s+(?:assistant|user)$/i.test(value);
     const looksLikePreview =
         !/^([UA]):/i.test(first) &&
         !isTranscriptHeading(first) &&
@@ -600,33 +615,26 @@ const stripPreviewBlock = (text: string): string => {
     return parts.slice(1).join('\n\n');
 };
 
-const shouldSkipMessage = (role: string, text: string): boolean => {
-    if (text.startsWith('<environment_context>')) {
-        return true;
-    }
-
-    if (text.startsWith('AGENTS.md instructions for ')) {
-        return true;
-    }
-
-    if (text.startsWith('# AGENTS.md instructions for ')) {
-        return true;
+const formatToolCallDetails = (tool: ToolRecord, outputFormat: ExportFormat): string => {
+    const lines = [`Tool: ${formatInlineLiteral(tool.name, outputFormat)}`];
+    if (tool.callId) {
+        lines.push(`Call ID: ${tool.callId}`);
     }
 
-    if (role === 'user' && text.includes('<environment_context>')) {
-        return true;
+    if (tool.name === 'exec_command') {
+        const details = parseExecCommandArguments(tool.argumentsText);
+        if (details.cmd) {
+            lines.push(`Command: ${formatInlineLiteral(details.cmd, outputFormat)}`);
+            return lines.join('\n');
+        }
     }
 
-    return false;
-};
-
-const formatToolCallDetails = (tool: ToolRecord, outputFormat: ExportFormat): string => {
-    if (tool.name !== 'exec_command') {
-        return '';
+    const argumentsText = tool.argumentsText?.trim();
+    if (argumentsText) {
+        lines.push('', 'Input:', '', renderCodeBlock(argumentsText, outputFormat));
     }
 
-    const details = parseExecCommandArguments(tool.argumentsText);
-    return details.cmd ? `Command: ${formatInlineLiteral(details.cmd, outputFormat)}` : '';
+    return lines.join('\n');
 };
 
 const extractText = (content: JsonValue): string => {

package/src/lib/concurrency.ts

@@ -11,15 +11,68 @@ export const mapWithConcurrency = async <T, TResult>(
     const workerLimit = Math.max(1, requestedLimit);
     const results = new Array<TResult>(values.length);
     let nextIndex = 0;
+    let failed = false;
 
     const worker = async () => {
-        while (nextIndex < values.length) {
+        while (!failed && nextIndex < values.length) {
             const currentIndex = nextIndex;
             nextIndex += 1;
-            results[currentIndex] = await mapper(values[currentIndex]!, currentIndex);
+            try {
+                results[currentIndex] = await mapper(values[currentIndex]!, currentIndex);
+            } catch (error) {
+                failed = true;
+                throw error;
+            }
         }
     };
 
-    await Promise.all(Array.from({ length: Math.min(workerLimit, values.length) }, () => worker()));
+    const settledWorkers = await Promise.allSettled(
+        Array.from({ length: Math.min(workerLimit, values.length) }, () => worker()),
+    );
+    const failure = settledWorkers.find((result): result is PromiseRejectedResult => result.status === 'rejected');
+    if (failure) {
+        throw failure.reason;
+    }
     return results;
 };
+
+export const createConcurrencyLimiter = (limit: number) => {
+    const requestedLimit = Number.isFinite(limit) ? Math.floor(limit) : 1;
+    const workerLimit = Math.max(1, requestedLimit);
+    const queue: Array<() => void> = [];
+    let activeCount = 0;
+
+    const drain = () => {
+        if (activeCount >= workerLimit) {
+            return;
+        }
+
+        const next = queue.shift();
+        if (next) {
+            next();
+        }
+    };
+
+    return async <T>(task: () => Promise<T>): Promise<T> => {
+        await new Promise<void>((resolve) => {
+            const start = () => {
+                activeCount += 1;
+                resolve();
+            };
+
+            if (activeCount < workerLimit) {
+                start();
+                return;
+            }
+
+            queue.push(start);
+        });
+
+        try {
+            return await task();
+        } finally {
+            activeCount -= 1;
+            drain();
+        }
+    };
+};