spiracha 1.4.0 → 2.0.0

package/src/lib/codex-browser-export.ts

@@ -2,8 +2,8 @@ import { randomUUID } from 'node:crypto';
 import { mkdir, mkdtemp, rm, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { getThreadBrowseData } from './codex-browser-db';
-import { convertSessionFile, writeSessionFileExport } from './codex-exporter-transcript';
-import type { CodexCliOptions } from './codex-exporter-types';
+import type { CodexTranscriptRenderOptions } from './codex-thread-types';
+import { renderCodexSessionFile, writeCodexSessionFileExport } from './codex-transcript-renderer';
 import { applyPathTransforms, type PathDisplaySettings } from './path-transforms';
 import { type ExportFormat, getPortablePathBasename } from './shared';
 import { getExportMimeType, sanitizeExportFileName, zipExportDirectory, zipExportFile } from './ui-export-archive';
@@ -89,19 +89,12 @@ type RolloutSnapshot = {
     sizeBytes: number;
 };
 
-const toDownloadOptions = (input: RenderCodexThreadDownloadInput): CodexCliOptions => {
+const toDownloadOptions = (input: RenderCodexThreadDownloadInput): CodexTranscriptRenderOptions => {
     return {
-        cwdFilter: null,
-        dbPath: input.dbPath,
-        flat: false,
         includeCommentary: input.includeCommentary,
         includeMetadata: input.includeMetadata,
         includeTools: input.includeTools,
-        inputDir: '',
-        outputDir: '',
         outputFormat: input.outputFormat,
-        projectFilter: null,
-        threadIds: [input.threadId],
     };
 };
 
@@ -130,6 +123,22 @@ const getRolloutSnapshot = async (rolloutPath: string): Promise<RolloutSnapshot>
     };
 };
 
+const isMissingFileError = (error: unknown) => {
+    return error instanceof Error && 'code' in error && error.code === 'ENOENT';
+};
+
+const getExistingRolloutSnapshot = async (threadId: string, rolloutPath: string): Promise<RolloutSnapshot> => {
+    try {
+        return await getRolloutSnapshot(rolloutPath);
+    } catch (error) {
+        if (isMissingFileError(error)) {
+            throw new Error(`Thread ${threadId} rollout file is missing: ${rolloutPath}`, { cause: error });
+        }
+
+        throw error;
+    }
+};
+
 const logExportEvent = (level: 'error' | 'info' | 'warn', event: string, details: Record<string, unknown>) => {
     console[level](`[spiracha:export] ${event}`, details);
 };
@@ -179,7 +188,7 @@ export const renderCodexThreadDownload = async (
                   projectPath: browseData.thread.cwd,
               })
             : text;
-    const rolloutSnapshotBefore = await getRolloutSnapshot(browseData.thread.rollout_path);
+    const rolloutSnapshotBefore = await getExistingRolloutSnapshot(input.threadId, browseData.thread.rollout_path);
 
     logExportEvent('info', 'single_start', {
         fileName,
@@ -198,7 +207,7 @@ export const renderCodexThreadDownload = async (
             const savedPath = path.join(workspaceDir, `${exportBaseName}.${extension}`);
             const zipPath = buildUniqueArchivePath(exportDir, exportBaseName);
             try {
-                const saved = await writeSessionFileExport(
+                const saved = await writeCodexSessionFileExport(
                     {
                         fallbackReason: null,
                         outputRelativePath: fileName,
@@ -241,7 +250,7 @@ export const renderCodexThreadDownload = async (
             };
         }
 
-        const content = await convertSessionFile(
+        const content = await renderCodexSessionFile(
             {
                 fallbackReason: null,
                 outputRelativePath: fileName,
@@ -307,7 +316,7 @@ export const renderCodexThreadsDownload = async (
 
     try {
         for (const entry of browseEntries) {
-            const rolloutSnapshotBefore = await getRolloutSnapshot(entry.thread.rollout_path);
+            const rolloutSnapshotBefore = await getExistingRolloutSnapshot(entry.thread.id, entry.thread.rollout_path);
             const singleBaseName = buildExportBaseName(entry.thread);
             const uniqueBaseName = buildUniqueBatchEntryBaseName(
                 singleBaseName,
@@ -333,7 +342,7 @@ export const renderCodexThreadsDownload = async (
                 });
             }
 
-            const saved = await writeSessionFileExport(
+            const saved = await writeCodexSessionFileExport(
                 {
                     fallbackReason: null,
                     outputRelativePath: relativeFileName,
@@ -346,7 +355,6 @@ export const renderCodexThreadsDownload = async (
                         ...input,
                         threadId: entry.thread.id,
                     }),
-                    threadIds: [entry.thread.id],
                 },
                 savedPath,
                 transform,

package/src/lib/codex-browser-types.ts

@@ -1,4 +1,4 @@
-import type { SessionMeta, ThreadRelations, ThreadRow } from './codex-exporter-types';
+import type { SessionMeta, ThreadRelations, ThreadRow } from './codex-thread-types';
 import type { JsonValue } from './shared';
 
 export type DynamicToolDefinition = {

package/src/lib/codex-thread-cache.ts

@@ -7,6 +7,10 @@ import { getFileFingerprint, hashCacheKeyParts, withCachedJson } from './ui-cach
 export const LARGE_THREAD_SIZE_BYTES = 100 * 1024 * 1024;
 export const LARGE_THREAD_PREVIEW_EVENT_LIMIT = 200;
 
+const isMissingFileError = (error: unknown) => {
+    return error instanceof Error && 'code' in error && error.code === 'ENOENT';
+};
+
 export const getCachedParsedCodexTranscript = async (sessionFile: string): Promise<ParsedCodexTranscript> => {
     const fingerprint = await getFileFingerprint(sessionFile);
     const key = `thread-${hashCacheKeyParts(path.basename(sessionFile), fingerprint)}`;
@@ -23,7 +27,19 @@ export const getThreadRolloutLoadState = async (
     sessionFile: string,
     largeTranscriptThresholdBytes = LARGE_THREAD_SIZE_BYTES,
 ) => {
-    const metadata = await stat(sessionFile);
+    let metadata: Awaited<ReturnType<typeof stat>>;
+    try {
+        metadata = await stat(sessionFile);
+    } catch (error) {
+        if (isMissingFileError(error)) {
+            return {
+                fileSizeBytes: null,
+                shouldDeferTranscriptLoad: false,
+            };
+        }
+
+        throw error;
+    }
 
     return {
         fileSizeBytes: metadata.size,

package/src/lib/{codex-exporter-types.ts → codex-thread-types.ts}

@@ -2,31 +2,11 @@ import os from 'node:os';
 import path from 'node:path';
 import type { ExportFormat, JsonValue } from './shared';
 
-export type CodexCliOptions = {
-    dbPath: string;
-    inputDir: string;
-    outputDir: string;
-    cwdFilter: string | null;
-    projectFilter: string | null;
-    threadIds: string[];
+export type CodexTranscriptRenderOptions = {
     includeMetadata: boolean;
     includeCommentary: boolean;
     includeTools: boolean;
     outputFormat: ExportFormat;
-    flat: boolean;
-};
-
-export type CodexExportedFile = {
-    sourcePath: string;
-    outputPath: string;
-    threadId: string | null;
-};
-
-export type CodexExportRunResult = {
-    outputDir: string;
-    exportedCount: number;
-    files: CodexExportedFile[];
-    missingThreadIds: string[];
 };
 
 export type SessionMeta = {
@@ -98,7 +78,7 @@ export type ThreadRelations = {
     childEdges: SpawnEdgeRow[];
 };
 
-export type ExportTarget = {
+export type CodexTranscriptExportTarget = {
     sessionFile: string;
     outputRelativePath: string;
     thread: ThreadRow | null;
@@ -106,13 +86,5 @@ export type ExportTarget = {
     fallbackReason: string | null;
 };
 
-export type ThreadData = {
-    threadsById: Map<string, ThreadRow>;
-    parentByChildId: Map<string, SpawnEdgeRow>;
-    childEdgesByParentId: Map<string, SpawnEdgeRow[]>;
-};
-
 export const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
 export const DEFAULT_DB_PATH = path.join(DEFAULT_CODEX_DIR, 'state_5.sqlite');
-export const DEFAULT_INPUT_DIR = path.join(DEFAULT_CODEX_DIR, 'sessions');
-export const DEFAULT_OUTPUT_DIR = path.join(process.cwd(), 'exports');

package/src/lib/{codex-exporter-transcript.ts → codex-transcript-renderer.ts}

@@ -2,8 +2,14 @@ import { createReadStream } from 'node:fs';
 import { rm } from 'node:fs/promises';
 import path from 'node:path';
 import { finished } from 'node:stream/promises';
-import { matchesFilters, toCodexRelativePath } from './codex-exporter-db';
-import type { CodexCliOptions, ExportTarget, MessageRecord, SessionMeta, ToolRecord } from './codex-exporter-types';
+import {
+    type CodexTranscriptExportTarget,
+    type CodexTranscriptRenderOptions,
+    DEFAULT_CODEX_DIR,
+    type MessageRecord,
+    type SessionMeta,
+    type ToolRecord,
+} from './codex-thread-types';
 import {
     asObject,
     asString,
@@ -22,7 +28,10 @@ import {
     renderSection,
 } from './shared';
 
-export const convertSessionFile = async (target: ExportTarget, options: CodexCliOptions): Promise<string | null> => {
+export const renderCodexSessionFile = async (
+    target: CodexTranscriptExportTarget,
+    options: CodexTranscriptRenderOptions,
+): Promise<string | null> => {
     let transcriptState: CodexTranscriptState;
 
     try {
@@ -32,10 +41,6 @@ export const convertSessionFile = async (target: ExportTarget, options: CodexCli
         throw new Error(`Failed to read Codex transcript ${target.sessionFile}: ${message}`);
     }
 
-    if (!matchesFilters(target.thread?.cwd ?? transcriptState.sessionMeta.cwd ?? null, options)) {
-        return null;
-    }
-
     if (transcriptState.sections.length === 0) {
         return null;
     }
@@ -57,9 +62,9 @@ export const convertSessionFile = async (target: ExportTarget, options: CodexCli
 
 type TranscriptTextTransform = (text: string) => string;
 
-export const writeSessionFileExport = async (
-    target: ExportTarget,
-    options: CodexCliOptions,
+export const writeCodexSessionFileExport = async (
+    target: CodexTranscriptExportTarget,
+    options: CodexTranscriptRenderOptions,
     outputPath: string,
     transform: TranscriptTextTransform = (text) => text,
 ): Promise<boolean> => {
@@ -87,7 +92,7 @@ export const writeSessionFileExport = async (
         await finalizeExportWriteStream(transcriptStream);
         transcriptStream = null;
 
-        if (!matchesFilters(target.thread?.cwd ?? state.sessionMeta.cwd ?? null, options) || !wroteSection) {
+        if (!wroteSection) {
             return false;
         }
 
@@ -127,7 +132,7 @@ type CodexTranscriptState = {
 
 const collectCodexTranscript = async (
     sessionFile: string,
-    options: CodexCliOptions,
+    options: CodexTranscriptRenderOptions,
     assistantModel: string | null = null,
 ): Promise<CodexTranscriptState> => {
     const state: CodexTranscriptState = {
@@ -147,7 +152,7 @@ const getSectionSeparator = () => '\n';
 
 const processCodexTranscriptRecord = (
     parsed: Record<string, JsonValue>,
-    options: CodexCliOptions,
+    options: CodexTranscriptRenderOptions,
     state: CodexTranscriptState,
 ) => {
     captureSessionMeta(parsed, state.sessionMeta);
@@ -159,7 +164,7 @@ const processCodexTranscriptRecord = (
 
 const renderCodexTranscriptRecord = (
     parsed: Record<string, JsonValue>,
-    options: CodexCliOptions,
+    options: CodexTranscriptRenderOptions,
     state: CodexTranscriptState,
 ) => {
     const message = extractMessageRecord(parsed);
@@ -179,16 +184,24 @@ const renderCodexTranscriptRecord = (
     return renderToolBlock(tool, options.outputFormat);
 };
 
-const processCodexMessageRecord = (message: MessageRecord, options: CodexCliOptions, state: CodexTranscriptState) => {
+const processCodexMessageRecord = (
+    message: MessageRecord,
+    options: CodexTranscriptRenderOptions,
+    state: CodexTranscriptState,
+) => {
     return renderMessageBlock(message, options.outputFormat, state.assistantModel, options.includeCommentary);
 };
 
-const buildStreamExportPrefix = (target: ExportTarget, sessionMeta: SessionMeta, options: CodexCliOptions) => {
+const buildStreamExportPrefix = (
+    target: CodexTranscriptExportTarget,
+    sessionMeta: SessionMeta,
+    options: CodexTranscriptRenderOptions,
+) => {
+    const title = getTitle(target, sessionMeta);
     if (!options.includeMetadata) {
-        return '';
+        return `${renderDocumentTitle(title, options.outputFormat)}\n`;
     }
 
-    const title = getTitle(target, sessionMeta);
     const parts = [
         renderDocumentTitle(title, options.outputFormat),
         '',
@@ -253,7 +266,7 @@ export const parseExecCommandArguments = (argumentsText?: string) => {
     }
 };
 
-const getTitle = (target: ExportTarget, sessionMeta: SessionMeta): string => {
+const getTitle = (target: CodexTranscriptExportTarget, sessionMeta: SessionMeta): string => {
     if (target.thread?.title) {
         return cleanInlineTitle(target.thread.title);
     }
@@ -261,10 +274,21 @@ const getTitle = (target: ExportTarget, sessionMeta: SessionMeta): string => {
     return sessionMeta.id ?? path.basename(target.sessionFile, '.jsonl');
 };
 
+const toCodexRelativePath = (targetPath: string): string => {
+    const codexRoot = path.resolve(DEFAULT_CODEX_DIR);
+    const normalized = path.resolve(targetPath);
+
+    if (normalized.startsWith(`${codexRoot}${path.sep}`)) {
+        return path.relative(codexRoot, normalized);
+    }
+
+    return normalized;
+};
+
 const buildMetadataEntries = (
-    target: ExportTarget,
+    target: CodexTranscriptExportTarget,
     sessionMeta: SessionMeta,
-    options: CodexCliOptions,
+    options: CodexTranscriptRenderOptions,
 ): MetadataEntry[] => {
     return [
         ...buildCodexExportIdentityMetadata(target, sessionMeta),
@@ -275,7 +299,10 @@ const buildMetadataEntries = (
     ];
 };
 
-const buildCodexExportIdentityMetadata = (target: ExportTarget, sessionMeta: SessionMeta): MetadataEntry[] => {
+const buildCodexExportIdentityMetadata = (
+    target: CodexTranscriptExportTarget,
+    sessionMeta: SessionMeta,
+): MetadataEntry[] => {
     const thread = target.thread;
 
     return [
@@ -289,7 +316,10 @@ const buildCodexExportIdentityMetadata = (target: ExportTarget, sessionMeta: Ses
     ];
 };
 
-const buildCodexExportPathMetadata = (target: ExportTarget, options: CodexCliOptions): MetadataEntry[] => {
+const buildCodexExportPathMetadata = (
+    target: CodexTranscriptExportTarget,
+    options: CodexTranscriptRenderOptions,
+): MetadataEntry[] => {
     const relativeOutputPath = target.outputRelativePath;
 
     return [
@@ -306,7 +336,7 @@ const buildCodexExportPathMetadata = (target: ExportTarget, options: CodexCliOpt
     ];
 };
 
-const buildCodexRelationMetadata = (target: ExportTarget): MetadataEntry[] => {
+const buildCodexRelationMetadata = (target: CodexTranscriptExportTarget): MetadataEntry[] => {
     const childThreadIds = target.relations.childEdges.map((edge) => edge.child_thread_id);
     const childEdges = target.relations.childEdges.map((edge) => ({
         child_thread_id: edge.child_thread_id,
@@ -320,14 +350,17 @@ const buildCodexRelationMetadata = (target: ExportTarget): MetadataEntry[] => {
     ];
 };
 
-const buildCodexThreadMetadata = (target: ExportTarget, sessionMeta: SessionMeta): MetadataEntry[] => {
+const buildCodexThreadMetadata = (target: CodexTranscriptExportTarget, sessionMeta: SessionMeta): MetadataEntry[] => {
     return [
         ...buildCodexThreadTimingMetadata(target, sessionMeta),
         ...buildCodexThreadIdentityMetadata(target, sessionMeta),
     ];
 };
 
-const buildCodexThreadTimingMetadata = (target: ExportTarget, sessionMeta: SessionMeta): MetadataEntry[] => {
+const buildCodexThreadTimingMetadata = (
+    target: CodexTranscriptExportTarget,
+    sessionMeta: SessionMeta,
+): MetadataEntry[] => {
     const thread = target.thread;
 
     return [
@@ -341,7 +374,10 @@ const buildCodexThreadTimingMetadata = (target: ExportTarget, sessionMeta: Sessi
     ];
 };
 
-const buildCodexThreadIdentityMetadata = (target: ExportTarget, sessionMeta: SessionMeta): MetadataEntry[] => {
+const buildCodexThreadIdentityMetadata = (
+    target: CodexTranscriptExportTarget,
+    sessionMeta: SessionMeta,
+): MetadataEntry[] => {
     const thread = target.thread;
 
     return [
@@ -367,7 +403,7 @@ const buildCodexThreadIdentityMetadata = (target: ExportTarget, sessionMeta: Ses
     ];
 };
 
-const buildCodexAgentMetadata = (target: ExportTarget): MetadataEntry[] => {
+const buildCodexAgentMetadata = (target: CodexTranscriptExportTarget): MetadataEntry[] => {
     const thread = target.thread;
 
     return [

package/src/lib/concurrency.ts

@@ -0,0 +1,25 @@
+export const mapWithConcurrency = async <T, TResult>(
+    values: T[],
+    limit: number,
+    mapper: (value: T, index: number) => Promise<TResult>,
+) => {
+    if (values.length === 0) {
+        return [];
+    }
+
+    const requestedLimit = Number.isFinite(limit) ? Math.floor(limit) : 1;
+    const workerLimit = Math.max(1, requestedLimit);
+    const results = new Array<TResult>(values.length);
+    let nextIndex = 0;
+
+    const worker = async () => {
+        while (nextIndex < values.length) {
+            const currentIndex = nextIndex;
+            nextIndex += 1;
+            results[currentIndex] = await mapper(values[currentIndex]!, currentIndex);
+        }
+    };
+
+    await Promise.all(Array.from({ length: Math.min(workerLimit, values.length) }, () => worker()));
+    return results;
+};