spiracha 1.0.0 → 1.1.1

package/src/lib/codex-thread-parser.ts

@@ -0,0 +1,604 @@
+import type {
+    DynamicToolDefinition,
+    MessageEvent,
+    ParsedCodexTranscript,
+    ReasoningEvent,
+    SessionMetaExtended,
+    TaskCompleteEvent,
+    TaskStartedEvent,
+    ThreadEvent,
+    ThreadTranscriptStats,
+    TokenCountEvent,
+    ToolCallEvent,
+    ToolOutputEvent,
+    TurnContextRecord,
+    WebSearchEvent,
+} from './codex-browser-types';
+import { asNumber, asObject, asString, type JsonValue, readJsonlObjects } from './shared';
+
+type ParseCodexTranscriptOptions = {
+    includeRaw?: boolean;
+    maxEvents?: number;
+    maxTurnContexts?: number;
+    sourceFileSizeBytes?: number | null;
+};
+
+const createEmptyStats = (): ThreadTranscriptStats => {
+    return {
+        assistantMessageCount: 0,
+        commentaryCount: 0,
+        execCommandCount: 0,
+        finalAnswerCount: 0,
+        messageCount: 0,
+        toolCallCount: 0,
+        toolOutputCount: 0,
+        userMessageCount: 0,
+        webSearchEventCount: 0,
+    };
+};
+
+const createEmptySessionMeta = (): SessionMetaExtended => {
+    return {
+        baseInstructions: null,
+        cli_version: undefined,
+        cwd: undefined,
+        dynamicTools: [],
+        git: null,
+        id: undefined,
+        modelProvider: null,
+        originator: undefined,
+        source: undefined,
+        threadSource: null,
+        timestamp: undefined,
+    };
+};
+
+export const parseCodexTranscriptFile = async (
+    sessionFile: string,
+    options: ParseCodexTranscriptOptions = {},
+): Promise<ParsedCodexTranscript> => {
+    const sessionMeta = createEmptySessionMeta();
+    const turnContexts: TurnContextRecord[] = [];
+    const events: ThreadEvent[] = [];
+    const stats = createEmptyStats();
+    const includeRaw = options.includeRaw ?? true;
+    const maxEvents = options.maxEvents ?? Number.POSITIVE_INFINITY;
+    const maxTurnContexts = options.maxTurnContexts ?? Number.POSITIVE_INFINITY;
+    let sequence = 0;
+
+    for await (const parsed of readJsonlObjects(sessionFile)) {
+        captureSessionMeta(parsed, sessionMeta);
+        const topLevelType = asString(parsed.type);
+
+        if (topLevelType === 'turn_context') {
+            if (turnContexts.length < maxTurnContexts) {
+                captureTurnContext(parsed, turnContexts);
+            }
+            continue;
+        }
+
+        const event = toThreadEvent(parsed, sequence, includeRaw);
+        if (!event) {
+            continue;
+        }
+
+        events.push(event);
+        updateTranscriptStats(stats, event);
+        sequence += 1;
+
+        if (events.length >= maxEvents) {
+            break;
+        }
+    }
+
+    return {
+        events,
+        isPartial: Number.isFinite(maxEvents) || Number.isFinite(maxTurnContexts),
+        rawIncluded: includeRaw,
+        sessionMeta,
+        sourceFileSizeBytes: options.sourceFileSizeBytes ?? null,
+        stats,
+        statsArePartial: Number.isFinite(maxEvents),
+        turnContexts,
+    };
+};
+
+const captureSessionMeta = (parsed: Record<string, JsonValue>, sessionMeta: SessionMetaExtended) => {
+    if (parsed.type !== 'session_meta') {
+        return;
+    }
+
+    const payload = asObject(parsed.payload);
+    if (!payload) {
+        return;
+    }
+
+    sessionMeta.baseInstructions = payload.base_instructions ?? sessionMeta.baseInstructions;
+    sessionMeta.cli_version = asString(payload.cli_version) ?? sessionMeta.cli_version;
+    sessionMeta.cwd = asString(payload.cwd) ?? sessionMeta.cwd;
+    sessionMeta.dynamicTools = parseDynamicTools(payload.dynamic_tools) ?? sessionMeta.dynamicTools;
+    sessionMeta.git = asObject(payload.git) ?? sessionMeta.git;
+    sessionMeta.id = asString(payload.id) ?? sessionMeta.id;
+    sessionMeta.modelProvider = asString(payload.model_provider) ?? sessionMeta.modelProvider;
+    sessionMeta.originator = asString(payload.originator) ?? sessionMeta.originator;
+    sessionMeta.source = asString(payload.source) ?? sessionMeta.source;
+    sessionMeta.threadSource = asString(payload.thread_source) ?? sessionMeta.threadSource;
+    sessionMeta.timestamp = asString(payload.timestamp) ?? sessionMeta.timestamp;
+};
+
+const parseDynamicTools = (value: JsonValue | undefined): DynamicToolDefinition[] | null => {
+    if (!Array.isArray(value)) {
+        return null;
+    }
+
+    return value.flatMap((entry) => {
+        const tool = asObject(entry);
+        if (!tool) {
+            return [];
+        }
+
+        return [
+            {
+                deferLoading: tool.deferLoading === true || tool.defer_loading === true,
+                description: asString(tool.description) ?? '',
+                inputSchema: asObject(tool.inputSchema) ?? asObject(tool.input_schema) ?? null,
+                name: asString(tool.name) ?? 'unknown',
+                namespace: asString(tool.namespace),
+            },
+        ];
+    });
+};
+
+const captureTurnContext = (parsed: Record<string, JsonValue>, turnContexts: TurnContextRecord[]) => {
+    const payload = asObject(parsed.payload);
+    if (!payload) {
+        return;
+    }
+
+    turnContexts.push({
+        payload,
+        timestamp: asString(parsed.timestamp),
+    });
+};
+
+const toThreadEvent = (
+    parsed: Record<string, JsonValue>,
+    sequence: number,
+    includeRaw: boolean,
+): ThreadEvent | null => {
+    const payload = asObject(parsed.payload);
+    if (!payload) {
+        return null;
+    }
+
+    const payloadType = asString(payload.type);
+    const timestamp = asString(parsed.timestamp);
+
+    if (parsed.type === 'event_msg') {
+        return buildEventMessage(payload, payloadType, includeRaw ? parsed : {}, sequence, timestamp);
+    }
+
+    if (parsed.type !== 'response_item') {
+        return null;
+    }
+
+    return buildResponseItemEvent(payload, payloadType, includeRaw ? parsed : {}, sequence, timestamp);
+};
+
+const buildEventMessage = (
+    payload: Record<string, JsonValue>,
+    payloadType: string | null,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+) => {
+    if (payloadType === 'task_started') {
+        return createTaskStartedEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'task_complete') {
+        return createTaskCompleteEvent(payload, raw, sequence, timestamp);
+    }
+
+    return null;
+};
+
+const buildResponseItemEvent = (
+    payload: Record<string, JsonValue>,
+    payloadType: string | null,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+) => {
+    if (payloadType === 'message') {
+        return createMessageEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'user_message') {
+        return createUserMessageEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'agent_message') {
+        return createAgentMessageEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'function_call') {
+        return createToolCallEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'function_call_output') {
+        return createToolOutputEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'reasoning') {
+        return createReasoningEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'token_count') {
+        return createTokenCountEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'web_search_call' || payloadType === 'web_search_end') {
+        return createWebSearchEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'task_started') {
+        return createTaskStartedEvent(payload, raw, sequence, timestamp);
+    }
+
+    if (payloadType === 'task_complete') {
+        return createTaskCompleteEvent(payload, raw, sequence, timestamp);
+    }
+
+    return null;
+};
+
+const createMessageEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): MessageEvent | null => {
+    const role = asString(payload.role);
+    const content = payload.content;
+    if (!role || content === undefined) {
+        return null;
+    }
+
+    return {
+        isHiddenByDefault: shouldHideTranscriptText(role, extractText(content)),
+        kind: 'message',
+        memoryCitation: null,
+        model: asString(payload.model),
+        phase: asString(payload.phase),
+        raw,
+        role,
+        sequence,
+        text: extractText(content),
+        timestamp,
+        variant: 'message',
+    };
+};
+
+const createUserMessageEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): MessageEvent => {
+    return {
+        isHiddenByDefault: shouldHideTranscriptText('user', asString(payload.message)?.trim() ?? ''),
+        kind: 'message',
+        memoryCitation: null,
+        model: null,
+        phase: null,
+        raw,
+        role: 'user',
+        sequence,
+        text: asString(payload.message)?.trim() ?? '',
+        timestamp,
+        variant: 'user_message',
+    };
+};
+
+const createAgentMessageEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): MessageEvent => {
+    return {
+        isHiddenByDefault: false,
+        kind: 'message',
+        memoryCitation: payload.memory_citation ?? null,
+        model: asString(payload.model),
+        phase: asString(payload.phase),
+        raw,
+        role: 'assistant',
+        sequence,
+        text: asString(payload.message)?.trim() ?? '',
+        timestamp,
+        variant: 'agent_message',
+    };
+};
+
+const createToolCallEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): ToolCallEvent => {
+    const name = asString(payload.name) ?? 'unknown';
+    const argumentsText = asString(payload.arguments);
+    const parsedArguments = parseExecCommandArguments(argumentsText);
+
+    return {
+        argumentsParseFailed: parsedArguments.argumentsParseFailed,
+        argumentsText,
+        callId: asString(payload.call_id),
+        command: parsedArguments.cmd,
+        kind: 'tool_call',
+        name,
+        raw,
+        sequence,
+        timestamp,
+        workdir: parsedArguments.workdir,
+    };
+};
+
+const createToolOutputEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): ToolOutputEvent => {
+    const outputText = asString(payload.output) ?? '';
+
+    return {
+        callId: asString(payload.call_id),
+        exitCode: parseExitCode(outputText),
+        kind: 'tool_output',
+        outputText,
+        raw,
+        sequence,
+        summary: formatToolOutputSummary(outputText),
+        timestamp,
+        wallTime: parseWallTime(outputText),
+    };
+};
+
+const createReasoningEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): ReasoningEvent => {
+    return {
+        content: payload.content ?? null,
+        hasEncryptedContent: Boolean(asString(payload.encrypted_content)),
+        kind: 'reasoning',
+        raw,
+        sequence,
+        summary: toStringArray(payload.summary),
+        timestamp,
+    };
+};
+
+const createTokenCountEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): TokenCountEvent => {
+    return {
+        info: payload.info ?? null,
+        kind: 'token_count',
+        rateLimits: payload.rate_limits ?? null,
+        raw,
+        sequence,
+        timestamp,
+    };
+};
+
+const createTaskStartedEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): TaskStartedEvent => {
+    return {
+        collaborationModeKind: asString(payload.collaboration_mode_kind),
+        kind: 'task_started',
+        modelContextWindow: asNumber(payload.model_context_window),
+        raw,
+        sequence,
+        startedAt: asNumber(payload.started_at),
+        timestamp,
+        turnId: asString(payload.turn_id),
+    };
+};
+
+const createTaskCompleteEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): TaskCompleteEvent => {
+    return {
+        completedAt: asNumber(payload.completed_at),
+        durationMs: asNumber(payload.duration_ms),
+        kind: 'task_complete',
+        lastAgentMessage: asString(payload.last_agent_message),
+        raw,
+        sequence,
+        timestamp,
+        timeToFirstTokenMs: asNumber(payload.time_to_first_token_ms),
+        turnId: asString(payload.turn_id),
+    };
+};
+
+const createWebSearchEvent = (
+    payload: Record<string, JsonValue>,
+    raw: Record<string, JsonValue>,
+    sequence: number,
+    timestamp: string | null,
+): WebSearchEvent => {
+    const payloadType = asString(payload.type);
+
+    return {
+        action: payload.action ?? null,
+        callId: asString(payload.call_id),
+        kind: 'web_search',
+        phase: payloadType === 'web_search_end' ? 'end' : 'call',
+        query: asString(payload.query),
+        raw,
+        sequence,
+        status: asString(payload.status),
+        timestamp,
+    };
+};
+
+const updateTranscriptStats = (stats: ThreadTranscriptStats, event: ThreadEvent) => {
+    if (event.kind === 'message') {
+        stats.messageCount += 1;
+        if (event.role === 'assistant') {
+            stats.assistantMessageCount += 1;
+        }
+        if (event.role === 'user') {
+            stats.userMessageCount += 1;
+        }
+        if (event.phase === 'commentary') {
+            stats.commentaryCount += 1;
+        }
+        if (event.phase === 'final_answer') {
+            stats.finalAnswerCount += 1;
+        }
+        return;
+    }
+
+    if (event.kind === 'tool_call') {
+        stats.toolCallCount += 1;
+        if (event.name === 'exec_command') {
+            stats.execCommandCount += 1;
+        }
+        return;
+    }
+
+    if (event.kind === 'tool_output') {
+        stats.toolOutputCount += 1;
+        return;
+    }
+
+    if (event.kind === 'web_search') {
+        stats.webSearchEventCount += 1;
+    }
+};
+
+const toStringArray = (value: JsonValue | undefined): string[] => {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+
+    return value.map((entry) => asString(entry)).filter((entry): entry is string => Boolean(entry));
+};
+
+const parseExitCode = (outputText: string): number | null => {
+    const match = /Process exited with code (\d+)/u.exec(outputText);
+    return match ? Number(match[1]) : null;
+};
+
+const parseWallTime = (outputText: string): string | null => {
+    const match = /Wall time: ([^\n]+)/u.exec(outputText);
+    return match?.[1] ?? null;
+};
+
+const formatToolOutputSummary = (outputText: string): string => {
+    const lines = outputText
+        .split('\n')
+        .map((line) => line.trim())
+        .filter(Boolean);
+
+    return lines
+        .filter((line) => {
+            return (
+                line.startsWith('Command: ') ||
+                line.startsWith('Process exited with code ') ||
+                line.startsWith('Wall time: ')
+            );
+        })
+        .join('\n');
+};
+
+const parseExecCommandArguments = (argumentsText: string | null) => {
+    if (!argumentsText) {
+        return { argumentsParseFailed: false, cmd: null as string | null, workdir: null as string | null };
+    }
+
+    try {
+        const parsed = JSON.parse(argumentsText) as Record<string, unknown>;
+        return {
+            argumentsParseFailed: false,
+            cmd: typeof parsed.cmd === 'string' ? parsed.cmd : null,
+            workdir: typeof parsed.workdir === 'string' ? parsed.workdir : null,
+        };
+    } catch {
+        return { argumentsParseFailed: true, cmd: null as string | null, workdir: null as string | null };
+    }
+};
+
+const extractText = (content: JsonValue): string => {
+    if (typeof content === 'string') {
+        return content.trim();
+    }
+
+    if (Array.isArray(content)) {
+        return content
+            .map((entry) => extractTextPart(entry))
+            .filter(Boolean)
+            .join('\n\n')
+            .trim();
+    }
+
+    if (content && typeof content === 'object') {
+        return asString((content as Record<string, JsonValue>).text)?.trim() ?? '';
+    }
+
+    return '';
+};
+
+const extractTextPart = (entry: JsonValue): string => {
+    const objectValue = asObject(entry);
+    if (!objectValue) {
+        return '';
+    }
+
+    const type = asString(objectValue.type);
+    const text = asString(objectValue.text);
+
+    if (type === 'input_image') {
+        return '[Image attached]';
+    }
+
+    return text ?? '';
+};
+
+const shouldHideTranscriptText = (role: string, text: string) => {
+    if (!text) {
+        return true;
+    }
+
+    if (role === 'developer') {
+        return true;
+    }
+
+    return (
+        text.startsWith('# AGENTS.md instructions for ') ||
+        text.startsWith('<permissions instructions>') ||
+        text.startsWith('<app-context>') ||
+        text.startsWith('<environment_context>') ||
+        text.startsWith('<collaboration_mode>') ||
+        text.startsWith('<skills_instructions>') ||
+        text.startsWith('<plugins_instructions>') ||
+        text.includes('Filesystem sandboxing defines which files can be read or written.')
+    );
+};

package/src/lib/interactive-cli.ts

@@ -1,13 +1,12 @@
-import { Database } from 'bun:sqlite';
 import { access, lstat } from 'node:fs/promises';
-import os from 'node:os';
 import path from 'node:path';
 import { stdin as input, stdout as output } from 'node:process';
 import { createInterface, type Interface } from 'node:readline/promises';
 import { checkbox } from '@inquirer/prompts';
 import { type ClaudeCliOptions, runClaudeExport } from './claude-exporter';
+import { resolveCodexThreadDbPath, withReadonlyDb } from './codex-browser-db';
 import { type CodexCliOptions, runCodexExport } from './codex-exporter';
-import { DEFAULT_DB_PATH, DEFAULT_INPUT_DIR } from './codex-exporter-types';
+import { DEFAULT_INPUT_DIR } from './codex-exporter-types';
 import { type ExportFormat, expandHome, getPortablePathBasename } from './shared';
 
 type InteractiveTargetKind =
@@ -23,7 +22,7 @@ type InteractiveInference = {
     value: string | null;
 };
 
-type InteractiveExportResult =
+export type InteractiveExportResult =
     | {
           mode: 'codex';
           outputDir: string;
@@ -309,6 +308,7 @@ const promptForCommonCodexOptions = async (
 ): Promise<CodexCliOptions> => {
     const outputFormat = await promptForOutputFormat(rl);
     const optimized = await promptYesNo(rl, 'Use optimized output? [y/N]: ', false);
+    const includeCommentary = await promptYesNo(rl, 'Include commentary messages? [y/N]: ', false);
     const includeTools = await promptYesNo(rl, 'Include tool logs? [y/N]: ', false);
     const flat = await promptYesNo(rl, 'Write to a flat output folder? [y/N]: ', false);
     const outputDir = await promptOptionalPath(rl, 'Optional output directory (leave blank for default):\n> ');
@@ -317,6 +317,7 @@ const promptForCommonCodexOptions = async (
         cwdFilter: target.cwdFilter,
         dbPath,
         flat,
+        includeCommentary,
         includeTools,
         inputDir: DEFAULT_INPUT_DIR,
         optimized,
@@ -397,29 +398,16 @@ const normalizeInteractiveThreadSelections = (value: string): string[] => {
 };
 
 const listCodexProjects = (dbPath: string): string[] => {
-    const db = new Database(dbPath, { readonly: true });
-    try {
+    return withReadonlyDb(dbPath, (db) => {
         const rows = db.query("SELECT DISTINCT cwd FROM threads WHERE cwd IS NOT NULL AND cwd != ''").all() as Array<{
             cwd: string;
         }>;
         return [...new Set(rows.map((row) => getPortablePathBasename(row.cwd)).filter(Boolean))].sort();
-    } finally {
-        db.close();
-    }
+    });
 };
 
 const resolveInteractiveDbPath = (): string => {
-    const candidates = [DEFAULT_DB_PATH, path.join(os.homedir(), '.codex', 'sqlite', 'state_5.sqlite')];
-
-    for (const candidate of candidates) {
-        try {
-            const db = new Database(candidate, { readonly: true });
-            db.close();
-            return candidate;
-        } catch {}
-    }
-
-    throw new Error(`Unable to open Codex thread database. Tried: ${candidates.join(', ')}`);
+    return resolveCodexThreadDbPath();
 };
 
 const listThreadIdsForProjects = (dbPath: string, projectNames: string[]): string[] => {
@@ -427,16 +415,13 @@ const listThreadIdsForProjects = (dbPath: string, projectNames: string[]): strin
         return [];
     }
 
-    const db = new Database(dbPath, { readonly: true });
-    try {
+    return withReadonlyDb(dbPath, (db) => {
         const projectNameSet = new Set(projectNames);
         const rows = db
             .query("SELECT id, cwd FROM threads WHERE cwd IS NOT NULL AND cwd != '' ORDER BY updated_at DESC")
             .all() as Array<{ id: string; cwd: string }>;
         return rows.filter((row) => projectNameSet.has(getPortablePathBasename(row.cwd))).map((row) => row.id);
-    } finally {
-        db.close();
-    }
+    });
 };
 
 const createPromptInterface = (): Interface => {