spiracha 1.5.0 → 2.1.0

package/src/lib/codex-browser-export.ts

@@ -2,11 +2,17 @@ import { randomUUID } from 'node:crypto';
 import { mkdir, mkdtemp, rm, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { getThreadBrowseData } from './codex-browser-db';
-import { convertSessionFile, writeSessionFileExport } from './codex-exporter-transcript';
-import type { CodexCliOptions } from './codex-exporter-types';
+import type { CodexTranscriptRenderOptions } from './codex-thread-types';
+import { renderCodexSessionFile, writeCodexSessionFileExport } from './codex-transcript-renderer';
 import { applyPathTransforms, type PathDisplaySettings } from './path-transforms';
-import { type ExportFormat, getPortablePathBasename } from './shared';
-import { getExportMimeType, sanitizeExportFileName, zipExportDirectory, zipExportFile } from './ui-export-archive';
+import type { ExportFormat } from './shared';
+import {
+    buildBatchExportBaseName,
+    buildConversationExportBaseName,
+    getExportMimeType,
+    zipExportDirectory,
+    zipExportFile,
+} from './ui-export-archive';
 import { buildUiExportDownloadUrl, ensureUiExportDir } from './ui-export-files';
 
 type RenderCodexThreadDownloadInput = {
@@ -42,31 +48,15 @@ export type CodexThreadDownload =
 
 const LARGE_BROWSER_EXPORT_THRESHOLD_BYTES = 128 * 1024 * 1024;
 
-const formatReadableExportDate = (value: number) => {
-    const date = new Date(value);
-    const year = date.getUTCFullYear();
-    const month = String(date.getUTCMonth() + 1).padStart(2, '0');
-    const day = String(date.getUTCDate()).padStart(2, '0');
-    const hours = String(date.getUTCHours()).padStart(2, '0');
-    const minutes = String(date.getUTCMinutes()).padStart(2, '0');
-    return `${year}-${month}-${day}-${hours}${minutes}`;
-};
-
 const buildExportBaseName = (thread: ReturnType<typeof getThreadBrowseData>['thread']) => {
-    const projectName = sanitizeExportFileName(getPortablePathBasename(thread.cwd) || 'thread') || 'thread';
-    const timestamp = thread.updated_at_ms ?? thread.updated_at * 1000;
-    return `${projectName}-${formatReadableExportDate(timestamp)}-${thread.id.slice(0, 8)}`;
-};
-
-const buildBatchExportBaseName = (threads: Array<ReturnType<typeof getThreadBrowseData>['thread']>) => {
-    const firstThread = threads[0];
-    if (!firstThread) {
-        throw new Error('No threads selected for export');
-    }
-
-    const projectName = sanitizeExportFileName(getPortablePathBasename(firstThread.cwd) || 'threads') || 'threads';
-    const latestTimestamp = Math.max(...threads.map((thread) => thread.updated_at_ms ?? thread.updated_at * 1000));
-    return `${projectName}-${formatReadableExportDate(latestTimestamp)}-threads-${threads.length}`;
+    return buildConversationExportBaseName(
+        {
+            cwd: thread.cwd,
+            id: thread.id,
+            updatedAtMs: thread.updated_at_ms ?? thread.updated_at * 1000,
+        },
+        'thread',
+    );
 };
 
 const buildUniqueArchivePath = (exportDir: string, exportBaseName: string) => {
@@ -89,19 +79,12 @@ type RolloutSnapshot = {
     sizeBytes: number;
 };
 
-const toDownloadOptions = (input: RenderCodexThreadDownloadInput): CodexCliOptions => {
+const toDownloadOptions = (input: RenderCodexThreadDownloadInput): CodexTranscriptRenderOptions => {
     return {
-        cwdFilter: null,
-        dbPath: input.dbPath,
-        flat: false,
         includeCommentary: input.includeCommentary,
         includeMetadata: input.includeMetadata,
         includeTools: input.includeTools,
-        inputDir: '',
-        outputDir: '',
         outputFormat: input.outputFormat,
-        projectFilter: null,
-        threadIds: [input.threadId],
     };
 };
 
@@ -214,7 +197,7 @@ export const renderCodexThreadDownload = async (
             const savedPath = path.join(workspaceDir, `${exportBaseName}.${extension}`);
             const zipPath = buildUniqueArchivePath(exportDir, exportBaseName);
             try {
-                const saved = await writeSessionFileExport(
+                const saved = await writeCodexSessionFileExport(
                     {
                         fallbackReason: null,
                         outputRelativePath: fileName,
@@ -257,7 +240,7 @@ export const renderCodexThreadDownload = async (
             };
         }
 
-        const content = await convertSessionFile(
+        const content = await renderCodexSessionFile(
             {
                 fallbackReason: null,
                 outputRelativePath: fileName,
@@ -309,7 +292,13 @@ export const renderCodexThreadsDownload = async (
     const browseEntries = threadIds.map((threadId) => getThreadBrowseData(input.dbPath, threadId));
     const threads = browseEntries.map((entry) => entry.thread);
     const exportDir = await resolvePublicExportDir(input.publicExportDir);
-    const exportBaseName = buildBatchExportBaseName(threads);
+    const exportBaseName = buildBatchExportBaseName(
+        threads.map((thread) => ({
+            cwd: thread.cwd,
+            updatedAtMs: thread.updated_at_ms ?? thread.updated_at * 1000,
+        })),
+        'threads',
+    );
     const bundleDirectory = await createExportWorkspace(exportDir, exportBaseName);
     const zipPath = buildUniqueArchivePath(exportDir, exportBaseName);
     const usedBatchEntryBaseNames = new Set<string>();
@@ -349,7 +338,7 @@ export const renderCodexThreadsDownload = async (
                 });
             }
 
-            const saved = await writeSessionFileExport(
+            const saved = await writeCodexSessionFileExport(
                 {
                     fallbackReason: null,
                     outputRelativePath: relativeFileName,
@@ -362,7 +351,6 @@ export const renderCodexThreadsDownload = async (
                         ...input,
                         threadId: entry.thread.id,
                     }),
-                    threadIds: [entry.thread.id],
                 },
                 savedPath,
                 transform,

package/src/lib/codex-browser-types.ts

@@ -1,4 +1,4 @@
-import type { SessionMeta, ThreadRelations, ThreadRow } from './codex-exporter-types';
+import type { SessionMeta, ThreadRelations, ThreadRow } from './codex-thread-types';
 import type { JsonValue } from './shared';
 
 export type DynamicToolDefinition = {

package/src/lib/codex-thread-cache.ts

@@ -2,10 +2,15 @@ import { stat } from 'node:fs/promises';
 import path from 'node:path';
 import type { ParsedCodexTranscript } from './codex-browser-types';
 import { parseCodexTranscriptFile } from './codex-thread-parser';
+import type { CodexTranscriptEventFilters } from './codex-transcript-filter';
+import { shouldShowCodexTranscriptEvent } from './codex-transcript-filter';
+import { runWithTranscriptLoadLimit } from './transcript-load-limiter';
 import { getFileFingerprint, hashCacheKeyParts, withCachedJson } from './ui-cache';
 
-export const LARGE_THREAD_SIZE_BYTES = 100 * 1024 * 1024;
+// Keep initial thread payloads below sizes that make TanStack Start SSR responses unreliable.
+export const LARGE_THREAD_SIZE_BYTES = 8 * 1024 * 1024;
 export const LARGE_THREAD_PREVIEW_EVENT_LIMIT = 200;
+const CODEX_TRANSCRIPT_CACHE_VERSION = 'v2';
 
 const isMissingFileError = (error: unknown) => {
     return error instanceof Error && 'code' in error && error.code === 'ENOENT';
@@ -13,12 +18,18 @@ const isMissingFileError = (error: unknown) => {
 
 export const getCachedParsedCodexTranscript = async (sessionFile: string): Promise<ParsedCodexTranscript> => {
     const fingerprint = await getFileFingerprint(sessionFile);
-    const key = `thread-${hashCacheKeyParts(path.basename(sessionFile), fingerprint)}`;
+    const key = `thread-${hashCacheKeyParts(CODEX_TRANSCRIPT_CACHE_VERSION, path.basename(sessionFile), fingerprint)}`;
 
-    return withCachedJson(key, async () => parseCodexTranscriptFile(sessionFile));
+    return withCachedJson(key, async () =>
+        runWithTranscriptLoadLimit(() => parseCodexTranscriptFile(sessionFile), {
+            path: sessionFile,
+            source: 'codex-full',
+        }),
+    );
 };
 
 type CachedThreadTranscriptPreviewOptions = {
+    filters?: CodexTranscriptEventFilters;
     largeTranscriptThresholdBytes?: number;
     previewEventLimit?: number;
 };
@@ -53,22 +64,39 @@ export const getCachedThreadTranscriptPreview = async (
 ): Promise<ParsedCodexTranscript> => {
     const threshold = options.largeTranscriptThresholdBytes ?? LARGE_THREAD_SIZE_BYTES;
     const previewEventLimit = options.previewEventLimit ?? LARGE_THREAD_PREVIEW_EVENT_LIMIT;
+    const filters = options.filters;
     const fingerprint = await getFileFingerprint(sessionFile);
     const { fileSizeBytes, shouldDeferTranscriptLoad } = await getThreadRolloutLoadState(sessionFile, threshold);
-    const key = `thread-preview-${hashCacheKeyParts(path.basename(sessionFile), fingerprint, String(threshold), String(previewEventLimit))}`;
+    const filterKey = filters ? JSON.stringify(filters) : 'all';
+    const key = `thread-preview-${hashCacheKeyParts(CODEX_TRANSCRIPT_CACHE_VERSION, path.basename(sessionFile), fingerprint, String(threshold), String(previewEventLimit), filterKey)}`;
 
     return withCachedJson(key, async () => {
         if (!shouldDeferTranscriptLoad) {
-            return parseCodexTranscriptFile(sessionFile, {
-                sourceFileSizeBytes: fileSizeBytes,
-            });
+            return runWithTranscriptLoadLimit(
+                () =>
+                    parseCodexTranscriptFile(sessionFile, {
+                        sourceFileSizeBytes: fileSizeBytes,
+                    }),
+                {
+                    path: sessionFile,
+                    source: 'codex-preview-full',
+                },
+            );
         }
 
-        return parseCodexTranscriptFile(sessionFile, {
-            includeRaw: false,
-            maxEvents: previewEventLimit,
-            maxTurnContexts: 0,
-            sourceFileSizeBytes: fileSizeBytes,
-        });
+        return runWithTranscriptLoadLimit(
+            () =>
+                parseCodexTranscriptFile(sessionFile, {
+                    eventFilter: filters ? (event) => shouldShowCodexTranscriptEvent(event, filters) : undefined,
+                    includeRaw: false,
+                    maxTurnContexts: 0,
+                    sourceFileSizeBytes: fileSizeBytes,
+                    tailEventLimit: previewEventLimit,
+                }),
+            {
+                path: sessionFile,
+                source: 'codex-preview',
+            },
+        );
     });
 };

package/src/lib/codex-thread-parser.ts

@@ -14,13 +14,27 @@ import type {
     TurnContextRecord,
     WebSearchEvent,
 } from './codex-browser-types';
-import { asNumber, asObject, asString, type JsonValue, readJsonlObjects } from './shared';
+import { asNumber, asObject, asString, type JsonValue, readJsonlObjects, stripCodexAppDirectiveLines } from './shared';
 
 type ParseCodexTranscriptOptions = {
+    eventFilter?: (event: ThreadEvent) => boolean;
     includeRaw?: boolean;
     maxEvents?: number;
     maxTurnContexts?: number;
     sourceFileSizeBytes?: number | null;
+    tailEventLimit?: number;
+};
+
+type ParseCodexTranscriptState = {
+    events: ThreadEvent[];
+    eventFilter: (event: ThreadEvent) => boolean;
+    includeRaw: boolean;
+    maxEvents: number;
+    maxTurnContexts: number;
+    sequence: number;
+    shouldStop: boolean;
+    tailEventLimit: number;
+    turnContexts: TurnContextRecord[];
 };
 
 const createEmptyStats = (): ThreadTranscriptStats => {
@@ -64,45 +78,74 @@ export const parseCodexTranscriptFile = async (
     const includeRaw = options.includeRaw ?? true;
     const maxEvents = options.maxEvents ?? Number.POSITIVE_INFINITY;
     const maxTurnContexts = options.maxTurnContexts ?? Number.POSITIVE_INFINITY;
-    let sequence = 0;
+    const tailEventLimit = options.tailEventLimit ?? Number.POSITIVE_INFINITY;
+    const eventFilter = options.eventFilter ?? (() => true);
+    const state: ParseCodexTranscriptState = {
+        eventFilter,
+        events,
+        includeRaw,
+        maxEvents,
+        maxTurnContexts,
+        sequence: 0,
+        shouldStop: false,
+        tailEventLimit,
+        turnContexts,
+    };
 
     for await (const parsed of readJsonlObjects(sessionFile)) {
         captureSessionMeta(parsed, sessionMeta);
-        const topLevelType = asString(parsed.type);
-
-        if (topLevelType === 'turn_context') {
-            if (turnContexts.length < maxTurnContexts) {
-                captureTurnContext(parsed, turnContexts);
-            }
-            continue;
-        }
-
-        const event = toThreadEvent(parsed, sequence, includeRaw);
-        if (!event) {
-            continue;
+        captureTranscriptRecord(parsed, state);
+        if (state.shouldStop) {
+            break;
         }
+    }
 
-        events.push(event);
+    for (const event of events) {
         updateTranscriptStats(stats, event);
-        sequence += 1;
-
-        if (events.length >= maxEvents) {
-            break;
-        }
     }
 
     return {
         events,
-        isPartial: Number.isFinite(maxEvents) || Number.isFinite(maxTurnContexts),
+        isPartial:
+            Number.isFinite(maxEvents) ||
+            Number.isFinite(maxTurnContexts) ||
+            Number.isFinite(tailEventLimit) ||
+            options.eventFilter !== undefined,
         rawIncluded: includeRaw,
         sessionMeta,
         sourceFileSizeBytes: options.sourceFileSizeBytes ?? null,
         stats,
-        statsArePartial: Number.isFinite(maxEvents),
+        statsArePartial:
+            Number.isFinite(maxEvents) || Number.isFinite(tailEventLimit) || options.eventFilter !== undefined,
         turnContexts,
     };
 };
 
+const captureTranscriptRecord = (parsed: Record<string, JsonValue>, state: ParseCodexTranscriptState) => {
+    if (asString(parsed.type) === 'turn_context') {
+        if (state.turnContexts.length < state.maxTurnContexts) {
+            captureTurnContext(parsed, state.turnContexts);
+        }
+        return;
+    }
+
+    const event = toThreadEvent(parsed, state.sequence, state.includeRaw);
+    if (!event) {
+        return;
+    }
+
+    state.sequence += 1;
+    if (!state.eventFilter(event)) {
+        return;
+    }
+
+    state.events.push(event);
+    if (state.events.length > state.tailEventLimit) {
+        state.events.shift();
+    }
+    state.shouldStop = state.events.length >= state.maxEvents;
+};
+
 const captureSessionMeta = (parsed: Record<string, JsonValue>, sessionMeta: SessionMetaExtended) => {
     if (parsed.type !== 'session_meta') {
         return;
@@ -261,12 +304,13 @@ const createMessageEvent = (
 ): MessageEvent | null => {
     const role = asString(payload.role);
     const content = payload.content;
+    const text = extractText(content);
     if (!role || content === undefined) {
         return null;
     }
 
     return {
-        isHiddenByDefault: shouldHideTranscriptText(role, extractText(content)),
+        isHiddenByDefault: shouldHideTranscriptText(role, text),
         kind: 'message',
         memoryCitation: null,
         model: asString(payload.model),
@@ -274,7 +318,7 @@ const createMessageEvent = (
         raw,
         role,
         sequence,
-        text: extractText(content),
+        text,
         timestamp,
         variant: 'message',
     };
@@ -286,8 +330,9 @@ const createUserMessageEvent = (
     sequence: number,
     timestamp: string | null,
 ): MessageEvent => {
+    const text = stripMemoryCitationBlocks(asString(payload.message) ?? '');
     return {
-        isHiddenByDefault: shouldHideTranscriptText('user', asString(payload.message)?.trim() ?? ''),
+        isHiddenByDefault: shouldHideTranscriptText('user', text),
         kind: 'message',
         memoryCitation: null,
         model: null,
@@ -295,7 +340,7 @@ const createUserMessageEvent = (
         raw,
         role: 'user',
         sequence,
-        text: asString(payload.message)?.trim() ?? '',
+        text,
         timestamp,
         variant: 'user_message',
     };
@@ -307,8 +352,9 @@ const createAgentMessageEvent = (
     sequence: number,
     timestamp: string | null,
 ): MessageEvent => {
+    const text = stripMemoryCitationBlocks(asString(payload.message) ?? '');
     return {
-        isHiddenByDefault: false,
+        isHiddenByDefault: shouldHideTranscriptText('assistant', text),
         kind: 'message',
         memoryCitation: payload.memory_citation ?? null,
         model: asString(payload.model),
@@ -316,7 +362,7 @@ const createAgentMessageEvent = (
         raw,
         role: 'assistant',
         sequence,
-        text: asString(payload.message)?.trim() ?? '',
+        text,
         timestamp,
         variant: 'agent_message',
     };
@@ -548,24 +594,29 @@ const parseExecCommandArguments = (argumentsText: string | null) => {
 
 const extractText = (content: JsonValue): string => {
     if (typeof content === 'string') {
-        return content.trim();
+        return stripMemoryCitationBlocks(content);
     }
 
     if (Array.isArray(content)) {
-        return content
-            .map((entry) => extractTextPart(entry))
-            .filter(Boolean)
-            .join('\n\n')
-            .trim();
+        return stripMemoryCitationBlocks(
+            content
+                .map((entry) => extractTextPart(entry))
+                .filter(Boolean)
+                .join('\n\n'),
+        );
     }
 
     if (content && typeof content === 'object') {
-        return asString((content as Record<string, JsonValue>).text)?.trim() ?? '';
+        return stripMemoryCitationBlocks(asString((content as Record<string, JsonValue>).text) ?? '');
     }
 
     return '';
 };
 
+const stripMemoryCitationBlocks = (text: string): string => {
+    return stripCodexAppDirectiveLines(text.replace(/<oai-mem-citation>[\s\S]*?<\/oai-mem-citation>/gu, ''));
+};
+
 const extractTextPart = (entry: JsonValue): string => {
     const objectValue = asObject(entry);
     if (!objectValue) {

package/src/lib/{codex-exporter-types.ts → codex-thread-types.ts}

@@ -2,31 +2,11 @@ import os from 'node:os';
 import path from 'node:path';
 import type { ExportFormat, JsonValue } from './shared';
 
-export type CodexCliOptions = {
-    dbPath: string;
-    inputDir: string;
-    outputDir: string;
-    cwdFilter: string | null;
-    projectFilter: string | null;
-    threadIds: string[];
+export type CodexTranscriptRenderOptions = {
     includeMetadata: boolean;
     includeCommentary: boolean;
     includeTools: boolean;
     outputFormat: ExportFormat;
-    flat: boolean;
-};
-
-export type CodexExportedFile = {
-    sourcePath: string;
-    outputPath: string;
-    threadId: string | null;
-};
-
-export type CodexExportRunResult = {
-    outputDir: string;
-    exportedCount: number;
-    files: CodexExportedFile[];
-    missingThreadIds: string[];
 };
 
 export type SessionMeta = {
@@ -98,7 +78,7 @@ export type ThreadRelations = {
     childEdges: SpawnEdgeRow[];
 };
 
-export type ExportTarget = {
+export type CodexTranscriptExportTarget = {
     sessionFile: string;
     outputRelativePath: string;
     thread: ThreadRow | null;
@@ -106,13 +86,5 @@ export type ExportTarget = {
     fallbackReason: string | null;
 };
 
-export type ThreadData = {
-    threadsById: Map<string, ThreadRow>;
-    parentByChildId: Map<string, SpawnEdgeRow>;
-    childEdgesByParentId: Map<string, SpawnEdgeRow[]>;
-};
-
 export const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
 export const DEFAULT_DB_PATH = path.join(DEFAULT_CODEX_DIR, 'state_5.sqlite');
-export const DEFAULT_INPUT_DIR = path.join(DEFAULT_CODEX_DIR, 'sessions');
-export const DEFAULT_OUTPUT_DIR = path.join(process.cwd(), 'exports');

package/src/lib/codex-transcript-filter.ts

@@ -0,0 +1,31 @@
+import type { ThreadEvent } from './codex-browser-types';
+
+const isCommentaryMessage = (event: ThreadEvent) =>
+    event.kind === 'message' && event.role === 'assistant' && event.phase === 'commentary';
+
+export type CodexTranscriptEventFilters = {
+    showCommentary: boolean;
+    showExtraEvents: boolean;
+    showToolCalls: boolean;
+    showUserMessages: boolean;
+};
+
+export const shouldShowCodexTranscriptEvent = (event: ThreadEvent, filters: CodexTranscriptEventFilters) => {
+    if (isCommentaryMessage(event) && !filters.showCommentary) {
+        return false;
+    }
+
+    if (event.kind === 'message') {
+        if (event.role === 'user' && !filters.showUserMessages) {
+            return false;
+        }
+
+        return !event.isHiddenByDefault || filters.showExtraEvents;
+    }
+
+    if (event.kind === 'tool_call' || event.kind === 'tool_output') {
+        return filters.showToolCalls;
+    }
+
+    return filters.showExtraEvents;
+};