spiracha 1.0.0 → 1.1.0

package/src/lib/codex-exporter-transcript.ts

@@ -1,4 +1,7 @@
+import { createReadStream } from 'node:fs';
+import { rm } from 'node:fs/promises';
 import path from 'node:path';
+import { finished } from 'node:stream/promises';
 import { matchesFilters, toCodexRelativePath } from './codex-exporter-db';
 import type { CodexCliOptions, ExportTarget, MessageRecord, SessionMeta, ToolRecord } from './codex-exporter-types';
 import {
@@ -6,8 +9,11 @@ import {
     asString,
     cleanExtractedText,
     cleanInlineTitle,
+    createExportWriteStream,
     type ExportFormat,
+    finalizeExportWriteStream,
     formatInlineLiteral,
+    formatModelLabel,
     type JsonValue,
     type MetadataEntry,
     readJsonlObjects,
@@ -49,7 +55,64 @@ export const convertSessionFile = async (target: ExportTarget, options: CodexCli
     return parts.join('\n').trimEnd() + '\n';
 };
 
+type TranscriptTextTransform = (text: string) => string;
+
+export const writeSessionFileExport = async (
+    target: ExportTarget,
+    options: CodexCliOptions,
+    outputPath: string,
+    transform: TranscriptTextTransform = (text) => text,
+): Promise<boolean> => {
+    const transcriptOutputPath = `${outputPath}.transcript.tmp`;
+    const transcriptStream = await createExportWriteStream(transcriptOutputPath);
+    const state: CodexTranscriptState = {
+        assistantModel: target.thread?.model ?? null,
+        sections: [],
+        sessionMeta: {},
+        startedTranscript: false,
+    };
+    let wroteSection = false;
+
+    try {
+        for await (const parsed of readJsonlObjects(target.sessionFile)) {
+            captureSessionMeta(parsed, state.sessionMeta);
+            const block = renderCodexTranscriptRecord(parsed, options, state);
+            if (!block) {
+                continue;
+            }
+
+            transcriptStream.write(transform(wroteSection ? `${getSectionSeparator(options)}${block}` : block));
+            wroteSection = true;
+        }
+    } catch (error) {
+        transcriptStream.destroy();
+        throw error;
+    }
+
+    await finalizeExportWriteStream(transcriptStream);
+
+    if (!matchesFilters(target.thread?.cwd ?? state.sessionMeta.cwd ?? null, options) || !wroteSection) {
+        await rm(transcriptOutputPath, { force: true });
+        return false;
+    }
+
+    const outputStream = await createExportWriteStream(outputPath);
+    const prefix = buildStreamExportPrefix(target, state.sessionMeta, options);
+    if (prefix) {
+        outputStream.write(transform(prefix));
+    }
+
+    const transcriptReadStream = createReadStream(transcriptOutputPath, { encoding: 'utf8' });
+    transcriptReadStream.pipe(outputStream, { end: false });
+    await finished(transcriptReadStream);
+    outputStream.write('\n');
+    await finalizeExportWriteStream(outputStream);
+    await rm(transcriptOutputPath, { force: true });
+    return true;
+};
+
 type CodexTranscriptState = {
+    assistantModel: string | null;
     sessionMeta: SessionMeta;
     sections: string[];
     startedTranscript: boolean;
@@ -57,6 +120,7 @@ type CodexTranscriptState = {
 
 const collectCodexTranscript = async (sessionFile: string, options: CodexCliOptions): Promise<CodexTranscriptState> => {
     const state: CodexTranscriptState = {
+        assistantModel: null,
         sections: [],
         sessionMeta: {},
         startedTranscript: false,
@@ -69,46 +133,52 @@ const collectCodexTranscript = async (sessionFile: string, options: CodexCliOpti
     return state;
 };
 
+const getSectionSeparator = (options: CodexCliOptions) => {
+    return options.optimized ? '\n\n' : '\n';
+};
+
 const processCodexTranscriptRecord = (
     parsed: Record<string, JsonValue>,
     options: CodexCliOptions,
     state: CodexTranscriptState,
 ) => {
     captureSessionMeta(parsed, state.sessionMeta);
+    const block = renderCodexTranscriptRecord(parsed, options, state);
+    if (block) {
+        state.sections.push(block);
+    }
+};
 
+const renderCodexTranscriptRecord = (
+    parsed: Record<string, JsonValue>,
+    options: CodexCliOptions,
+    state: CodexTranscriptState,
+) => {
     const message = extractMessageRecord(parsed);
     if (message) {
-        processCodexMessageRecord(message, options, state);
-        return;
+        return processCodexMessageRecord(message, options, state);
     }
 
     if (!options.includeTools) {
-        return;
+        return '';
     }
 
     const tool = extractToolRecord(parsed);
     if (!tool) {
-        return;
+        return '';
     }
 
-    const block = options.optimized
+    return options.optimized
         ? renderCompactToolBlock(tool, options.outputFormat)
         : renderToolBlock(tool, options.outputFormat);
-    if (block) {
-        state.sections.push(block);
-    }
 };
 
 const processCodexMessageRecord = (message: MessageRecord, options: CodexCliOptions, state: CodexTranscriptState) => {
     if (options.optimized) {
-        processOptimizedCodexMessageRecord(message, options, state);
-        return;
+        return processOptimizedCodexMessageRecord(message, options, state);
     }
 
-    const block = renderMessageBlock(message, options.outputFormat);
-    if (block) {
-        state.sections.push(block);
-    }
+    return renderMessageBlock(message, options.outputFormat, state.assistantModel, options.includeCommentary);
 };
 
 const processOptimizedCodexMessageRecord = (
@@ -117,25 +187,44 @@ const processOptimizedCodexMessageRecord = (
     state: CodexTranscriptState,
 ) => {
     if (message.role !== 'user' && message.role !== 'assistant') {
-        return;
+        return '';
+    }
+
+    if (message.role === 'assistant' && message.phase === 'commentary' && !options.includeCommentary) {
+        return '';
     }
 
     const compact = compactMessageText(message, true);
     if (!compact) {
-        return;
+        return '';
     }
 
     if (!state.startedTranscript) {
         if (shouldSkipOptimizedPrelude(message.role, compact)) {
-            return;
+            return '';
         }
         state.startedTranscript = true;
     }
 
-    const rendered = renderCompactBlock(message, compact, options.outputFormat);
-    if (rendered) {
-        state.sections.push(rendered);
+    return renderCompactBlock(message, compact, options.outputFormat, state.assistantModel);
+};
+
+const buildStreamExportPrefix = (target: ExportTarget, sessionMeta: SessionMeta, options: CodexCliOptions) => {
+    if (options.optimized) {
+        return '';
     }
+
+    const title = getTitle(target, sessionMeta);
+    const metadata = buildMetadataEntries(target, sessionMeta, options);
+    const parts = [
+        renderDocumentTitle(title, options.outputFormat),
+        '',
+        renderMetadataBlock(metadata, options.outputFormat),
+    ]
+        .filter(Boolean)
+        .join('\n');
+
+    return `${parts}\n`;
 };
 
 export const compactMessageText = (message: MessageRecord, optimized: boolean): string => {
@@ -183,17 +272,18 @@ export const formatToolOutputSummary = (outputText: string, outputFormat: Export
 
 export const parseExecCommandArguments = (argumentsText?: string) => {
     if (!argumentsText) {
-        return { cmd: null as string | null, workdir: null as string | null };
+        return { argumentsParseFailed: false, cmd: null as string | null, workdir: null as string | null };
     }
 
     try {
         const parsed = JSON.parse(argumentsText) as Record<string, unknown>;
         return {
+            argumentsParseFailed: false,
             cmd: typeof parsed.cmd === 'string' ? parsed.cmd : null,
             workdir: typeof parsed.workdir === 'string' ? parsed.workdir : null,
         };
     } catch {
-        return { cmd: null as string | null, workdir: null as string | null };
+        return { argumentsParseFailed: true, cmd: null as string | null, workdir: null as string | null };
     }
 };
 
@@ -397,7 +487,11 @@ const extractMessageRecord = (parsed: Record<string, JsonValue>): MessageRecord
     }
 
     const payload = asObject(parsed.payload);
-    if (!payload || payload.type !== 'message') {
+    if (!payload) {
+        return null;
+    }
+
+    if (payload.type !== 'message' && payload.type !== 'agent_message' && payload.type !== 'user_message') {
         return null;
     }
 
@@ -405,15 +499,17 @@ const extractMessageRecord = (parsed: Record<string, JsonValue>): MessageRecord
 };
 
 const normalizeMessage = (value: Record<string, JsonValue>): MessageRecord | null => {
-    const role = asString(value.role);
-    const content = value.content;
+    const type = asString(value.type);
+    const role =
+        asString(value.role) ?? (type === 'agent_message' ? 'assistant' : type === 'user_message' ? 'user' : null);
+    const content = value.content ?? asString(value.message);
     const phase = asString(value.phase);
 
     if (!role || content === undefined) {
         return null;
     }
 
-    return { content, phase: phase ?? undefined, role };
+    return { content, model: asString(value.model), phase: phase ?? undefined, role };
 };
 
 const extractToolRecord = (parsed: Record<string, JsonValue>): ToolRecord | null => {
@@ -462,17 +558,26 @@ const extractToolRecord = (parsed: Record<string, JsonValue>): ToolRecord | null
     return null;
 };
 
-const renderMessageBlock = (message: MessageRecord, outputFormat: ExportFormat): string => {
+const renderMessageBlock = (
+    message: MessageRecord,
+    outputFormat: ExportFormat,
+    assistantModel: string | null,
+    includeCommentary: boolean,
+): string => {
     if (message.role !== 'user' && message.role !== 'assistant') {
         return '';
     }
 
+    if (message.role === 'assistant' && message.phase === 'commentary' && !includeCommentary) {
+        return '';
+    }
+
     const text = cleanExtractedText(extractText(message.content)).trim();
     if (!text || shouldSkipMessage(message.role, text)) {
         return '';
     }
 
-    const title = message.role === 'user' ? 'User' : 'Assistant';
+    const title = message.role === 'user' ? 'User' : formatModelLabel(message.model ?? assistantModel);
     const body = message.phase ? `Phase: ${message.phase}\n\n${text}` : text;
 
     return renderSection(title, body, outputFormat);
@@ -488,8 +593,13 @@ const renderToolBlock = (tool: ToolRecord, outputFormat: ExportFormat): string =
     return summary ? renderSection('Tool Output', summary, outputFormat) : '';
 };
 
-const renderCompactBlock = (message: MessageRecord, text: string, outputFormat: ExportFormat): string => {
-    const prefix = message.role === 'user' ? 'U:' : 'A:';
+const renderCompactBlock = (
+    message: MessageRecord,
+    text: string,
+    outputFormat: ExportFormat,
+    assistantModel: string | null,
+): string => {
+    const prefix = message.role === 'user' ? 'U:' : `${formatModelLabel(message.model ?? assistantModel)}:`;
     const lines = text.split('\n');
     const [firstLine, ...rest] = lines;
 
@@ -525,11 +635,12 @@ const stripPreviewBlock = (text: string): string => {
 
     const first = parts[0];
     const second = parts[1];
+    const isTranscriptHeading = (value: string) => /^##\s+.+$/i.test(value);
     const looksLikePreview =
         !/^([UA]):/i.test(first) &&
-        !/^##\s+(User|Assistant)\s*$/i.test(first) &&
+        !isTranscriptHeading(first) &&
         /^([UA]):/i.test(second) === false &&
-        /^##\s+(User|Assistant)\s*$/i.test(second);
+        isTranscriptHeading(second);
 
     if (!looksLikePreview) {
         return text.trim();

package/src/lib/codex-exporter-types.ts

@@ -10,6 +10,7 @@ export type CodexCliOptions = {
     projectFilter: string | null;
     threadIds: string[];
     optimized: boolean;
+    includeCommentary: boolean;
     includeTools: boolean;
     outputFormat: ExportFormat;
     flat: boolean;
@@ -35,11 +36,14 @@ export type SessionMeta = {
     source?: string;
     originator?: string;
     cli_version?: string;
+    thread_source?: string;
+    model_provider?: string;
 };
 
 export type MessageRecord = {
     role: string;
     content: JsonValue;
+    model?: string | null;
     phase?: string;
 };
 
@@ -77,6 +81,10 @@ export type ThreadRow = {
     model: string | null;
     reasoning_effort: string | null;
     agent_path: string | null;
+    created_at_ms: number | null;
+    updated_at_ms: number | null;
+    thread_source: string | null;
+    preview: string;
 };
 
 export type SpawnEdgeRow = {

package/src/lib/codex-thread-cache.ts

@@ -0,0 +1,58 @@
+import { stat } from 'node:fs/promises';
+import path from 'node:path';
+import type { ParsedCodexTranscript } from './codex-browser-types';
+import { parseCodexTranscriptFile } from './codex-thread-parser';
+import { getFileFingerprint, hashCacheKeyParts, withCachedJson } from './ui-cache';
+
+export const LARGE_THREAD_SIZE_BYTES = 100 * 1024 * 1024;
+export const LARGE_THREAD_PREVIEW_EVENT_LIMIT = 200;
+
+export const getCachedParsedCodexTranscript = async (sessionFile: string): Promise<ParsedCodexTranscript> => {
+    const fingerprint = await getFileFingerprint(sessionFile);
+    const key = `thread-${hashCacheKeyParts(path.basename(sessionFile), fingerprint)}`;
+
+    return withCachedJson(key, async () => parseCodexTranscriptFile(sessionFile));
+};
+
+type CachedThreadTranscriptPreviewOptions = {
+    largeTranscriptThresholdBytes?: number;
+    previewEventLimit?: number;
+};
+
+export const getThreadRolloutLoadState = async (
+    sessionFile: string,
+    largeTranscriptThresholdBytes = LARGE_THREAD_SIZE_BYTES,
+) => {
+    const metadata = await stat(sessionFile);
+
+    return {
+        fileSizeBytes: metadata.size,
+        shouldDeferTranscriptLoad: metadata.size > largeTranscriptThresholdBytes,
+    };
+};
+
+export const getCachedThreadTranscriptPreview = async (
+    sessionFile: string,
+    options: CachedThreadTranscriptPreviewOptions = {},
+): Promise<ParsedCodexTranscript> => {
+    const threshold = options.largeTranscriptThresholdBytes ?? LARGE_THREAD_SIZE_BYTES;
+    const previewEventLimit = options.previewEventLimit ?? LARGE_THREAD_PREVIEW_EVENT_LIMIT;
+    const fingerprint = await getFileFingerprint(sessionFile);
+    const { fileSizeBytes, shouldDeferTranscriptLoad } = await getThreadRolloutLoadState(sessionFile, threshold);
+    const key = `thread-preview-${hashCacheKeyParts(path.basename(sessionFile), fingerprint, String(threshold), String(previewEventLimit))}`;
+
+    return withCachedJson(key, async () => {
+        if (!shouldDeferTranscriptLoad) {
+            return parseCodexTranscriptFile(sessionFile, {
+                sourceFileSizeBytes: fileSizeBytes,
+            });
+        }
+
+        return parseCodexTranscriptFile(sessionFile, {
+            includeRaw: false,
+            maxEvents: previewEventLimit,
+            maxTurnContexts: 0,
+            sourceFileSizeBytes: fileSizeBytes,
+        });
+    });
+};