spiracha 1.0.0 → 1.1.1

package/src/lib/codex-browser-export.ts

@@ -0,0 +1,429 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, mkdtemp, rm, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { getThreadBrowseData } from './codex-browser-db';
+import { convertSessionFile, writeSessionFileExport } from './codex-exporter-transcript';
+import type { CodexCliOptions } from './codex-exporter-types';
+import { applyPathTransforms, type PathDisplaySettings } from './path-transforms';
+import { type ExportFormat, getPortablePathBasename } from './shared';
+import { buildUiExportDownloadUrl, ensureUiExportDir } from './ui-export-files';
+
+type RenderCodexThreadDownloadInput = {
+    dbPath: string;
+    includeCommentary: boolean;
+    includeTools: boolean;
+    largeExportThresholdBytes?: number;
+    optimized: boolean;
+    outputFormat: ExportFormat;
+    pathDisplaySettings?: Pick<PathDisplaySettings, 'convertToProjectRoot' | 'redactUsername'>;
+    publicExportDir?: string;
+    threadId: string;
+};
+
+type RenderCodexThreadsDownloadInput = Omit<RenderCodexThreadDownloadInput, 'threadId'> & {
+    threadIds: string[];
+};
+
+export type CodexThreadDownload =
+    | {
+          content: string;
+          fileName: string;
+          mimeType: string;
+          mode: 'download';
+      }
+    | {
+          downloadUrl: string;
+          fileName: string;
+          mimeType: string;
+          mode: 'download_url';
+      };
+
+const LARGE_BROWSER_EXPORT_THRESHOLD_BYTES = 128 * 1024 * 1024;
+
+const toSanitizedFileName = (value: string) => {
+    return value
+        .replace(/[<>:"/\\|?*\u0000-\u001f]/gu, ' ')
+        .replace(/\.\.+/gu, ' ')
+        .replace(/\s+/gu, ' ')
+        .trim();
+};
+
+const formatReadableExportDate = (value: number) => {
+    const date = new Date(value);
+    const year = date.getUTCFullYear();
+    const month = String(date.getUTCMonth() + 1).padStart(2, '0');
+    const day = String(date.getUTCDate()).padStart(2, '0');
+    const hours = String(date.getUTCHours()).padStart(2, '0');
+    const minutes = String(date.getUTCMinutes()).padStart(2, '0');
+    return `${year}-${month}-${day}-${hours}${minutes}`;
+};
+
+const buildExportBaseName = (thread: ReturnType<typeof getThreadBrowseData>['thread']) => {
+    const projectName = toSanitizedFileName(getPortablePathBasename(thread.cwd) || 'thread') || 'thread';
+    const timestamp = thread.updated_at_ms ?? thread.updated_at * 1000;
+    return `${projectName}-${formatReadableExportDate(timestamp)}-${thread.id.slice(0, 8)}`;
+};
+
+const buildBatchExportBaseName = (threads: Array<ReturnType<typeof getThreadBrowseData>['thread']>) => {
+    const firstThread = threads[0];
+    if (!firstThread) {
+        throw new Error('No threads selected for export');
+    }
+
+    const projectName = toSanitizedFileName(getPortablePathBasename(firstThread.cwd) || 'threads') || 'threads';
+    const latestTimestamp = Math.max(...threads.map((thread) => thread.updated_at_ms ?? thread.updated_at * 1000));
+    return `${projectName}-${formatReadableExportDate(latestTimestamp)}-threads-${threads.length}`;
+};
+
+const buildUniqueArchivePath = (exportDir: string, exportBaseName: string) => {
+    return path.join(exportDir, `${exportBaseName}-${randomUUID()}.zip`);
+};
+
+type RolloutSnapshot = {
+    mtimeMs: number;
+    sizeBytes: number;
+};
+
+const toDownloadOptions = (input: RenderCodexThreadDownloadInput): CodexCliOptions => {
+    return {
+        cwdFilter: null,
+        dbPath: input.dbPath,
+        flat: false,
+        includeCommentary: input.includeCommentary,
+        includeTools: input.includeTools,
+        inputDir: '',
+        optimized: input.optimized,
+        outputDir: '',
+        outputFormat: input.outputFormat,
+        projectFilter: null,
+        threadIds: [input.threadId],
+    };
+};
+
+const getMimeType = (outputFormat: ExportFormat) => {
+    return outputFormat === 'md' ? 'text/markdown; charset=utf-8' : 'text/plain; charset=utf-8';
+};
+
+const resolvePublicExportDir = async (publicExportDir?: string) => {
+    if (publicExportDir) {
+        await ensureDirectory(publicExportDir);
+        return publicExportDir;
+    }
+
+    return ensureUiExportDir();
+};
+
+const ensureDirectory = async (directoryPath: string) => {
+    await mkdir(directoryPath, { recursive: true });
+};
+
+const createExportWorkspace = async (exportDir: string, exportBaseName: string) => {
+    return mkdtemp(path.join(exportDir, `${exportBaseName}-`));
+};
+
+const getRolloutSnapshot = async (rolloutPath: string): Promise<RolloutSnapshot> => {
+    const metadata = await stat(rolloutPath);
+    return {
+        mtimeMs: metadata.mtimeMs,
+        sizeBytes: metadata.size,
+    };
+};
+
+const logExportEvent = (level: 'error' | 'info' | 'warn', event: string, details: Record<string, unknown>) => {
+    console[level](`[spiracha:export] ${event}`, details);
+};
+
+const logRolloutChangeIfDetected = (
+    threadId: string,
+    rolloutPath: string,
+    beforeSnapshot: RolloutSnapshot,
+    afterSnapshot: RolloutSnapshot,
+) => {
+    if (beforeSnapshot.mtimeMs === afterSnapshot.mtimeMs && beforeSnapshot.sizeBytes === afterSnapshot.sizeBytes) {
+        return;
+    }
+
+    logExportEvent('warn', 'rollout_changed_during_export', {
+        afterMtimeMs: afterSnapshot.mtimeMs,
+        afterSizeBytes: afterSnapshot.sizeBytes,
+        beforeMtimeMs: beforeSnapshot.mtimeMs,
+        beforeSizeBytes: beforeSnapshot.sizeBytes,
+        rolloutPath,
+        threadId,
+    });
+};
+
+const cleanupExportWorkspace = async (workspacePath: string) => {
+    try {
+        await rm(workspacePath, { force: true, recursive: true });
+    } catch (error) {
+        logExportEvent('warn', 'workspace_cleanup_failed', {
+            error: error instanceof Error ? error.message : String(error),
+            workspacePath,
+        });
+    }
+};
+
+const zipExportFile = async (sourcePath: string, zipPath: string) => {
+    const proc = Bun.spawn(['zip', '-9', '-j', zipPath, sourcePath], {
+        stderr: 'pipe',
+        stdout: 'pipe',
+    });
+    const [stdoutText, stderrText, exitCode] = await Promise.all([
+        new Response(proc.stdout).text(),
+        new Response(proc.stderr).text(),
+        proc.exited,
+    ]);
+
+    if (exitCode !== 0) {
+        throw new Error(`zip failed (${exitCode}): ${(stderrText || stdoutText).trim()}`);
+    }
+};
+
+const zipExportDirectory = async (sourceDirectory: string, zipPath: string) => {
+    const proc = Bun.spawn(['zip', '-9', '-r', zipPath, '.'], {
+        cwd: sourceDirectory,
+        stderr: 'pipe',
+        stdout: 'pipe',
+    });
+    const [stdoutText, stderrText, exitCode] = await Promise.all([
+        new Response(proc.stdout).text(),
+        new Response(proc.stderr).text(),
+        proc.exited,
+    ]);
+
+    if (exitCode !== 0) {
+        throw new Error(`zip failed (${exitCode}): ${(stderrText || stdoutText).trim()}`);
+    }
+};
+
+export const renderCodexThreadDownload = async (
+    input: RenderCodexThreadDownloadInput,
+): Promise<CodexThreadDownload> => {
+    const startedAt = Date.now();
+    const browseData = getThreadBrowseData(input.dbPath, input.threadId);
+    const extension = input.outputFormat === 'md' ? 'md' : 'txt';
+    const fileBaseName = buildExportBaseName(browseData.thread);
+    const fileName = `${fileBaseName}.${extension}`;
+    const mimeType = getMimeType(input.outputFormat);
+    const transform = (text: string) =>
+        input.pathDisplaySettings
+            ? applyPathTransforms(text, {
+                  ...input.pathDisplaySettings,
+                  projectPath: browseData.thread.cwd,
+              })
+            : text;
+    const rolloutSnapshotBefore = await getRolloutSnapshot(browseData.thread.rollout_path);
+
+    logExportEvent('info', 'single_start', {
+        fileName,
+        rolloutPath: browseData.thread.rollout_path,
+        sizeBytes: rolloutSnapshotBefore.sizeBytes,
+        threadId: input.threadId,
+    });
+
+    try {
+        if (
+            rolloutSnapshotBefore.sizeBytes > (input.largeExportThresholdBytes ?? LARGE_BROWSER_EXPORT_THRESHOLD_BYTES)
+        ) {
+            const exportBaseName = fileBaseName;
+            const exportDir = await resolvePublicExportDir(input.publicExportDir);
+            const workspaceDir = await createExportWorkspace(exportDir, exportBaseName);
+            const savedPath = path.join(workspaceDir, `${exportBaseName}.${extension}`);
+            const zipPath = buildUniqueArchivePath(exportDir, exportBaseName);
+            try {
+                const saved = await writeSessionFileExport(
+                    {
+                        fallbackReason: null,
+                        outputRelativePath: fileName,
+                        relations: browseData.relations,
+                        sessionFile: browseData.thread.rollout_path,
+                        thread: browseData.thread,
+                    },
+                    toDownloadOptions(input),
+                    savedPath,
+                    transform,
+                );
+
+                if (!saved) {
+                    throw new Error(`Thread ${input.threadId} produced no exportable content`);
+                }
+
+                await zipExportFile(savedPath, zipPath);
+            } finally {
+                await cleanupExportWorkspace(workspaceDir);
+            }
+
+            const rolloutSnapshotAfter = await getRolloutSnapshot(browseData.thread.rollout_path);
+            logRolloutChangeIfDetected(
+                input.threadId,
+                browseData.thread.rollout_path,
+                rolloutSnapshotBefore,
+                rolloutSnapshotAfter,
+            );
+
+            const zipStat = await Bun.file(zipPath).stat();
+            logExportEvent('info', 'single_zip_ready', {
+                downloadUrl: buildUiExportDownloadUrl(zipPath),
+                durationMs: Date.now() - startedAt,
+                fileName: `${exportBaseName}.zip`,
+                sizeBytes: zipStat.size,
+                threadId: input.threadId,
+                zipPath,
+            });
+
+            return {
+                downloadUrl: buildUiExportDownloadUrl(zipPath),
+                fileName: `${exportBaseName}.zip`,
+                mimeType: 'application/zip',
+                mode: 'download_url',
+            };
+        }
+
+        const content = await convertSessionFile(
+            {
+                fallbackReason: null,
+                outputRelativePath: fileName,
+                relations: browseData.relations,
+                sessionFile: browseData.thread.rollout_path,
+                thread: browseData.thread,
+            },
+            toDownloadOptions(input),
+        );
+
+        if (!content) {
+            throw new Error(`Thread ${input.threadId} produced no exportable content`);
+        }
+
+        const rolloutSnapshotAfter = await getRolloutSnapshot(browseData.thread.rollout_path);
+        logRolloutChangeIfDetected(
+            input.threadId,
+            browseData.thread.rollout_path,
+            rolloutSnapshotBefore,
+            rolloutSnapshotAfter,
+        );
+        logExportEvent('info', 'single_inline_ready', {
+            durationMs: Date.now() - startedAt,
+            fileName,
+            sizeBytes: content.length,
+            threadId: input.threadId,
+        });
+
+        return {
+            content: transform(content),
+            fileName,
+            mimeType,
+            mode: 'download',
+        };
+    } catch (error) {
+        logExportEvent('error', 'single_error', {
+            error: error instanceof Error ? error.message : String(error),
+            fileName,
+            threadId: input.threadId,
+        });
+        throw error;
+    }
+};
+
+export const renderCodexThreadsDownload = async (
+    input: RenderCodexThreadsDownloadInput,
+): Promise<CodexThreadDownload> => {
+    const startedAt = Date.now();
+    const threadIds = [...new Set(input.threadIds)];
+    if (threadIds.length === 0) {
+        throw new Error('No threads selected for export');
+    }
+
+    const browseEntries = threadIds.map((threadId) => getThreadBrowseData(input.dbPath, threadId));
+    const threads = browseEntries.map((entry) => entry.thread);
+    const exportDir = await resolvePublicExportDir(input.publicExportDir);
+    const exportBaseName = buildBatchExportBaseName(threads);
+    const bundleDirectory = await createExportWorkspace(exportDir, exportBaseName);
+    const zipPath = buildUniqueArchivePath(exportDir, exportBaseName);
+
+    logExportEvent('info', 'batch_start', {
+        exportBaseName,
+        selectedThreadCount: threadIds.length,
+        selectedThreadIds: threadIds,
+        zipPath,
+    });
+
+    try {
+        for (const entry of browseEntries) {
+            const rolloutSnapshotBefore = await getRolloutSnapshot(entry.thread.rollout_path);
+            const singleBaseName = buildExportBaseName(entry.thread);
+            const extension = input.outputFormat === 'md' ? 'md' : 'txt';
+            const relativeFileName = `${singleBaseName}.${extension}`;
+            const savedPath = path.join(bundleDirectory, relativeFileName);
+            const transform = (text: string) =>
+                input.pathDisplaySettings
+                    ? applyPathTransforms(text, {
+                          ...input.pathDisplaySettings,
+                          projectPath: entry.thread.cwd,
+                      })
+                    : text;
+
+            const saved = await writeSessionFileExport(
+                {
+                    fallbackReason: null,
+                    outputRelativePath: relativeFileName,
+                    relations: entry.relations,
+                    sessionFile: entry.thread.rollout_path,
+                    thread: entry.thread,
+                },
+                {
+                    ...toDownloadOptions({
+                        ...input,
+                        threadId: entry.thread.id,
+                    }),
+                    threadIds: [entry.thread.id],
+                },
+                savedPath,
+                transform,
+            );
+
+            if (!saved) {
+                throw new Error(`Thread ${entry.thread.id} produced no exportable content`);
+            }
+
+            const rolloutSnapshotAfter = await getRolloutSnapshot(entry.thread.rollout_path);
+            logRolloutChangeIfDetected(
+                entry.thread.id,
+                entry.thread.rollout_path,
+                rolloutSnapshotBefore,
+                rolloutSnapshotAfter,
+            );
+        }
+
+        await zipExportDirectory(bundleDirectory, zipPath);
+    } catch (error) {
+        logExportEvent('error', 'batch_error', {
+            error: error instanceof Error ? error.message : String(error),
+            exportBaseName,
+            selectedThreadCount: threadIds.length,
+            selectedThreadIds: threadIds,
+            zipPath,
+        });
+        throw error;
+    } finally {
+        await cleanupExportWorkspace(bundleDirectory);
+    }
+
+    const zipStat = await Bun.file(zipPath).stat();
+    logExportEvent('info', 'batch_ready', {
+        downloadUrl: buildUiExportDownloadUrl(zipPath),
+        durationMs: Date.now() - startedAt,
+        fileName: `${exportBaseName}.zip`,
+        selectedThreadCount: threadIds.length,
+        selectedThreadIds: threadIds,
+        sizeBytes: zipStat.size,
+        zipPath,
+    });
+
+    return {
+        downloadUrl: buildUiExportDownloadUrl(zipPath),
+        fileName: `${exportBaseName}.zip`,
+        mimeType: 'application/zip',
+        mode: 'download_url',
+    };
+};

package/src/lib/codex-browser-types.ts

@@ -0,0 +1,224 @@
+import type { SessionMeta, ThreadRelations, ThreadRow } from './codex-exporter-types';
+import type { JsonValue } from './shared';
+
+export type DynamicToolDefinition = {
+    deferLoading: boolean;
+    description: string;
+    inputSchema: JsonValue | null;
+    name: string;
+    namespace: string | null;
+};
+
+export type DynamicToolRow = DynamicToolDefinition & {
+    position: number;
+    threadId: string;
+};
+
+export type SessionMetaExtended = SessionMeta & {
+    baseInstructions: JsonValue | null;
+    dynamicTools: DynamicToolDefinition[];
+    git: Record<string, JsonValue> | null;
+    modelProvider: string | null;
+    threadSource: string | null;
+};
+
+export type TurnContextRecord = {
+    payload: Record<string, JsonValue>;
+    timestamp: string | null;
+};
+
+type BaseThreadEvent = {
+    kind:
+        | 'message'
+        | 'reasoning'
+        | 'task_complete'
+        | 'task_started'
+        | 'token_count'
+        | 'tool_call'
+        | 'tool_output'
+        | 'web_search';
+    raw: Record<string, JsonValue>;
+    sequence: number;
+    timestamp: string | null;
+};
+
+export type MessageEvent = BaseThreadEvent & {
+    kind: 'message';
+    isHiddenByDefault: boolean;
+    memoryCitation: JsonValue | null;
+    model: string | null;
+    phase: string | null;
+    role: string;
+    text: string;
+    variant: 'agent_message' | 'message' | 'user_message';
+};
+
+export type ToolCallEvent = BaseThreadEvent & {
+    argumentsText: string | null;
+    argumentsParseFailed: boolean;
+    callId: string | null;
+    command: string | null;
+    kind: 'tool_call';
+    name: string;
+    workdir: string | null;
+};
+
+export type ToolOutputEvent = BaseThreadEvent & {
+    callId: string | null;
+    exitCode: number | null;
+    kind: 'tool_output';
+    outputText: string;
+    summary: string;
+    wallTime: string | null;
+};
+
+export type ReasoningEvent = BaseThreadEvent & {
+    content: JsonValue | null;
+    hasEncryptedContent: boolean;
+    kind: 'reasoning';
+    summary: string[];
+};
+
+export type TokenCountEvent = BaseThreadEvent & {
+    info: JsonValue | null;
+    kind: 'token_count';
+    rateLimits: JsonValue | null;
+};
+
+export type TaskStartedEvent = BaseThreadEvent & {
+    collaborationModeKind: string | null;
+    kind: 'task_started';
+    modelContextWindow: number | null;
+    startedAt: number | null;
+    turnId: string | null;
+};
+
+export type TaskCompleteEvent = BaseThreadEvent & {
+    completedAt: number | null;
+    durationMs: number | null;
+    kind: 'task_complete';
+    lastAgentMessage: string | null;
+    timeToFirstTokenMs: number | null;
+    turnId: string | null;
+};
+
+export type WebSearchEvent = BaseThreadEvent & {
+    action: JsonValue | null;
+    callId: string | null;
+    kind: 'web_search';
+    phase: 'call' | 'end';
+    query: string | null;
+    status: string | null;
+};
+
+export type ThreadEvent =
+    | MessageEvent
+    | ReasoningEvent
+    | TaskCompleteEvent
+    | TaskStartedEvent
+    | TokenCountEvent
+    | ToolCallEvent
+    | ToolOutputEvent
+    | WebSearchEvent;
+
+export type ThreadTranscriptStats = {
+    assistantMessageCount: number;
+    commentaryCount: number;
+    execCommandCount: number;
+    finalAnswerCount: number;
+    messageCount: number;
+    toolCallCount: number;
+    toolOutputCount: number;
+    userMessageCount: number;
+    webSearchEventCount: number;
+};
+
+export type ParsedCodexTranscript = {
+    events: ThreadEvent[];
+    isPartial: boolean;
+    rawIncluded: boolean;
+    sessionMeta: SessionMetaExtended;
+    sourceFileSizeBytes: number | null;
+    stats: ThreadTranscriptStats;
+    statsArePartial: boolean;
+    turnContexts: TurnContextRecord[];
+};
+
+export type ProjectSummary = {
+    archivedThreadCount: number;
+    cwdPaths: string[];
+    lastUpdatedAtMs: number | null;
+    modelNames: string[];
+    name: string;
+    threadCount: number;
+    totalTokens: number;
+};
+
+export type ThreadListEntry = {
+    project: string;
+    rolloutSizeBytes: number | null;
+    stats: Pick<ThreadTranscriptStats, 'execCommandCount' | 'toolCallCount' | 'webSearchEventCount'> & {
+        deferred: boolean;
+    };
+    thread: ThreadRow;
+};
+
+export type ThreadBrowseData = {
+    dynamicTools: DynamicToolRow[];
+    project: string;
+    relations: ThreadRelations;
+    thread: ThreadRow;
+};
+
+export type DashboardSummary = {
+    activeThreads: number;
+    archivedThreads: number;
+    recentThreads: ThreadRow[];
+    threadsWithRelations: number;
+    topProjectsByThreadCount: ProjectSummary[];
+    topProjectsByTokens: ProjectSummary[];
+    totalProjects: number;
+    totalThreads: number;
+    totalTokens: number;
+};
+
+export type DeleteThreadsResult = {
+    deletedSessionFiles: string[];
+    deletedThreadIds: string[];
+};
+
+export type DeleteProjectResult = DeleteThreadsResult & {
+    projectName: string;
+};
+
+export type ToolUsageSummary = {
+    count: number;
+    name: string;
+};
+
+export type ModelTokenSummary = {
+    model: string;
+    threadCount: number;
+    totalTokens: number;
+};
+
+export type DistributionItem = {
+    count: number;
+    label: string;
+};
+
+export type CodexAnalyticsSummary = {
+    archivedThreads: number;
+    averageTokensPerThread: number;
+    distinctToolNames: number;
+    threadsWithWebSearch: number;
+    totalProjects: number;
+    totalThreads: number;
+    totalTokens: number;
+};
+
+export type CodexAnalytics = {
+    modelsByTokens: ModelTokenSummary[];
+    summary: CodexAnalyticsSummary;
+    toolUsage: ToolUsageSummary[];
+};

package/src/lib/codex-exporter-cli.ts

@@ -11,6 +11,7 @@ export const parseCodexCliArgs = (argv: string[]): CodexCliOptions => {
     let threadIds: string[] = [];
     let outputProvided = false;
     let optimized = false;
+    let includeCommentary = true;
     let includeTools = false;
     let outputFormat: ExportFormat = 'md';
     let flat = false;
@@ -20,6 +21,7 @@ export const parseCodexCliArgs = (argv: string[]): CodexCliOptions => {
             cwdFilter,
             dbPath,
             flat,
+            includeCommentary,
             includeTools,
             inputDir,
             optimized,
@@ -34,6 +36,7 @@ export const parseCodexCliArgs = (argv: string[]): CodexCliOptions => {
             cwdFilter,
             dbPath,
             flat,
+            includeCommentary,
             includeTools,
             inputDir,
             optimized,
@@ -54,6 +57,7 @@ export const parseCodexCliArgs = (argv: string[]): CodexCliOptions => {
         cwdFilter,
         dbPath,
         flat,
+        includeCommentary,
         includeTools,
         inputDir,
         optimized,
@@ -68,6 +72,7 @@ type CodexCliState = {
     cwdFilter: string | null;
     dbPath: string;
     flat: boolean;
+    includeCommentary: boolean;
     includeTools: boolean;
     inputDir: string;
     optimized: boolean;
@@ -256,7 +261,7 @@ export const resolveDefaultOutputDir = (cwdFilter: string | null): string => {
 };
 
 const requireValue = (value: string | undefined, flag: string): string => {
-    if (!value || value.startsWith('--')) {
+    if (!value || (value.startsWith('-') && value !== '-')) {
         throw new CliUsageError(`Missing value for ${flag}`);
     }