spiracha 1.3.3 → 1.5.0

package/src/lib/codex-browser-db.ts

@@ -1,7 +1,10 @@
-import { Database } from 'bun:sqlite';
+import { constants, Database } from 'bun:sqlite';
+import { closeSync, openSync, readdirSync, readSync, statSync } from 'node:fs';
 import { rm } from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
+import { pathToFileURL } from 'node:url';
 import type {
     DashboardSummary,
     DeleteProjectResult,
@@ -14,6 +17,7 @@ import type {
 import type { ThreadRelations, ThreadRow } from './codex-exporter-types';
 import { DEFAULT_CODEX_DIR, DEFAULT_DB_PATH } from './codex-exporter-types';
 import { getCachedParsedCodexTranscript, getThreadRolloutLoadState } from './codex-thread-cache';
+import { mapWithConcurrency } from './concurrency';
 import { cleanInlineTitle, getPortablePathBasename } from './shared';
 import { runWithSqliteRetry } from './sqlite-retry';
 import { invalidateCacheByPrefix } from './ui-cache';
@@ -29,6 +33,44 @@ type DeleteProjectOptions = {
 const SQLITE_DELETE_BATCH_SIZE = 400;
 const SESSION_FILE_DELETE_CONCURRENCY = 16;
 const THREAD_LIST_IO_CONCURRENCY = 8;
+const CODEX_READONLY_DB_OPEN_FLAGS = constants.SQLITE_OPEN_READONLY | constants.SQLITE_OPEN_URI;
+const JSONL_READ_CHUNK_BYTES = 64 * 1024;
+const SESSION_META_READ_LIMIT_BYTES = 4 * 1024 * 1024;
+const THREAD_ID_PATTERN = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/iu;
+
+type SessionIndexEntry = {
+    id: string;
+    thread_name?: string;
+    updated_at?: string;
+};
+
+type FallbackSessionMeta = {
+    agent_nickname?: string;
+    agent_path?: string;
+    agent_role?: string;
+    cli_version?: string;
+    cwd?: string;
+    forked_from_id?: string;
+    id?: string;
+    model_provider?: string;
+    parent_thread_id?: string;
+    source?: unknown;
+    thread_source?: string;
+    timestamp?: string;
+};
+
+type ReadFallbackThreadRowsOptions = {
+    includeSubagents?: boolean;
+};
+
+type FallbackRolloutStats = {
+    model: string | null;
+    tokensUsed: number;
+};
+
+const isSqliteCantOpenError = (error: unknown) => {
+    return (error as { code?: unknown }).code === 'SQLITE_CANTOPEN';
+};
 
 const chunkValues = <T>(values: T[], chunkSize: number) => {
     const chunks: T[][] = [];
@@ -48,40 +90,20 @@ const isPromiseLike = (value: unknown): value is PromiseLike<unknown> => {
     return 'then' in value && typeof value.then === 'function';
 };
 
-const mapWithConcurrency = async <T, TResult>(
-    values: T[],
-    limit: number,
-    mapper: (value: T, index: number) => Promise<TResult>,
-) => {
-    const results = new Array<TResult>(values.length);
-    let nextIndex = 0;
-
-    const worker = async () => {
-        while (true) {
-            const currentIndex = nextIndex;
-            nextIndex += 1;
-
-            if (currentIndex >= values.length) {
-                return;
-            }
-
-            results[currentIndex] = await mapper(values[currentIndex]!, currentIndex);
-        }
-    };
-
-    await Promise.all(Array.from({ length: Math.min(limit, values.length) }, () => worker()));
-    return results;
-};
-
-const openReadonlyDb = (dbPath: string, busyTimeoutMs: number) => {
+const openReadonlyDb = (dbPath: string) => {
     const db = new Database(dbPath, { readonly: true });
     try {
-        db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`);
+        db.query('SELECT name FROM sqlite_master LIMIT 1').get();
         return db;
     } catch (error) {
         db.close();
-        throw error;
+        if (!isSqliteCantOpenError(error)) {
+            throw error;
+        }
     }
+
+    // Codex uses WAL mode; immutable URI reads keep Bun usable after clean shutdown removes sidecar files.
+    return new Database(`${pathToFileURL(dbPath).href}?immutable=1`, CODEX_READONLY_DB_OPEN_FLAGS);
 };
 
 const openWritableDb = (dbPath: string, busyTimeoutMs: number) => {
@@ -126,7 +148,7 @@ const parseJsonSafely = (value: string | null) => {
 export const withReadonlyDb = <T>(dbPath: string, callback: (db: Database) => T): T => {
     return runWithSqliteRetry({
         action: () => {
-            const db = openReadonlyDb(dbPath, 5000);
+            const db = openReadonlyDb(dbPath);
             try {
                 const result = callback(db);
                 if (isPromiseLike(result)) {
@@ -173,12 +195,10 @@ export const resolveCodexThreadDbPath = () => {
 
     for (const candidate of candidates) {
         try {
-            const db = runWithSqliteRetry({
-                action: () => {
-                    return openReadonlyDb(candidate, 1500);
-                },
-            });
-            db.close();
+            // Avoid opening candidates as a probe: Bun can make later read-only opens fail on Codex WAL databases.
+            if (!statSync(candidate).isFile()) {
+                continue;
+            }
             return candidate;
         } catch {}
     }
@@ -194,6 +214,399 @@ const readAllThreads = (dbPath: string): ThreadRow[] => {
     });
 };
 
+const resolveCodexDirFromDbPath = (dbPath: string) => {
+    const dbDir = path.dirname(dbPath);
+    return path.basename(dbDir) === 'sqlite' ? path.dirname(dbDir) : dbDir;
+};
+
+const parseJsonlObject = <T>(line: string): T | null => {
+    try {
+        return JSON.parse(line) as T;
+    } catch {
+        return null;
+    }
+};
+
+const emitJsonlLine = <T>(line: string, onRecord: (record: T) => void) => {
+    const trimmed = line.trim();
+    const parsed = trimmed ? parseJsonlObject<T>(trimmed) : null;
+    if (parsed) {
+        onRecord(parsed);
+    }
+};
+
+const emitCompleteJsonlLines = <T>(text: string, onRecord: (record: T) => void): string => {
+    const lines = text.split(/\r?\n/u);
+    const pending = lines.pop() ?? '';
+    for (const line of lines) {
+        emitJsonlLine(line, onRecord);
+    }
+    return pending;
+};
+
+const readJsonlObjects = <T>(filePath: string, onRecord: (record: T) => void) => {
+    let descriptor: number | null = null;
+    try {
+        const stats = statSync(filePath);
+        if (!stats.isFile()) {
+            return;
+        }
+
+        descriptor = openSync(filePath, 'r');
+        const buffer = Buffer.alloc(JSONL_READ_CHUNK_BYTES);
+        const decoder = new StringDecoder('utf8');
+        let position = 0;
+        let pending = '';
+
+        while (true) {
+            const bytesRead = readSync(descriptor, buffer, 0, buffer.length, position);
+            if (bytesRead === 0) {
+                break;
+            }
+
+            position += bytesRead;
+            pending += decoder.write(buffer.subarray(0, bytesRead));
+            pending = emitCompleteJsonlLines(pending, onRecord);
+        }
+
+        emitJsonlLine(pending + decoder.end(), onRecord);
+    } catch {
+        return;
+    } finally {
+        if (descriptor !== null) {
+            closeSync(descriptor);
+        }
+    }
+};
+
+const collectJsonlObjects = <T>(filePath: string): T[] => {
+    const records: T[] = [];
+    readJsonlObjects<T>(filePath, (record) => {
+        records.push(record);
+    });
+    return records;
+};
+
+const readSessionIndexEntries = (codexDir: string): SessionIndexEntry[] => {
+    return collectJsonlObjects<SessionIndexEntry>(path.join(codexDir, 'session_index.jsonl')).filter(
+        (entry) => typeof entry.id === 'string' && entry.id.length > 0,
+    );
+};
+
+const collectSessionFilesByThreadId = (sessionsDir: string): Map<string, string> => {
+    const sessionFiles = new Map<string, string>();
+    const visit = (directory: string) => {
+        const entries = (() => {
+            try {
+                return readdirSync(directory, { withFileTypes: true });
+            } catch {
+                return null;
+            }
+        })();
+        if (!entries) {
+            return;
+        }
+
+        for (const entry of entries) {
+            const entryPath = path.join(directory, entry.name);
+            if (entry.isDirectory()) {
+                visit(entryPath);
+                continue;
+            }
+
+            if (!entry.isFile()) {
+                continue;
+            }
+
+            const threadId = THREAD_ID_PATTERN.exec(entry.name)?.[1];
+            if (threadId && !sessionFiles.has(threadId)) {
+                sessionFiles.set(threadId, entryPath);
+            }
+        }
+    };
+
+    visit(sessionsDir);
+    return sessionFiles;
+};
+
+const readSessionMetaLine = (sessionFile: string): string | null => {
+    let descriptor: number | null = null;
+    try {
+        descriptor = openSync(sessionFile, 'r');
+        const chunks: Buffer[] = [];
+        let totalBytes = 0;
+
+        while (totalBytes < SESSION_META_READ_LIMIT_BYTES) {
+            const buffer = Buffer.alloc(Math.min(64 * 1024, SESSION_META_READ_LIMIT_BYTES - totalBytes));
+            const bytesRead = readSync(descriptor, buffer, 0, buffer.length, totalBytes);
+            if (bytesRead === 0) {
+                break;
+            }
+
+            chunks.push(buffer.subarray(0, bytesRead));
+            totalBytes += bytesRead;
+            if (buffer.subarray(0, bytesRead).includes(10)) {
+                break;
+            }
+        }
+
+        const firstLine = Buffer.concat(chunks).toString('utf8').split(/\r?\n/u)[0]?.trim();
+        return firstLine || null;
+    } catch {
+        return null;
+    } finally {
+        if (descriptor !== null) {
+            closeSync(descriptor);
+        }
+    }
+};
+
+const readFallbackSessionMeta = (sessionFile: string): FallbackSessionMeta | null => {
+    const line = readSessionMetaLine(sessionFile);
+    if (!line) {
+        return null;
+    }
+
+    try {
+        const record = JSON.parse(line) as { payload?: FallbackSessionMeta; type?: string };
+        return record.type === 'session_meta' && record.payload ? record.payload : null;
+    } catch {
+        return null;
+    }
+};
+
+const parseIsoMs = (value: string | undefined, fallback: number) => {
+    if (!value) {
+        return fallback;
+    }
+
+    const parsed = Date.parse(value);
+    return Number.isFinite(parsed) ? parsed : fallback;
+};
+
+const stringOrNull = (value: unknown) => (typeof value === 'string' && value.length > 0 ? value : null);
+
+const numberOrNull = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) ? value : null);
+
+const objectOrNull = (value: unknown) => {
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? (value as Record<string, unknown>)
+        : null;
+};
+
+const isFallbackSubagent = (sessionMeta: FallbackSessionMeta) => {
+    return Boolean(
+        sessionMeta.thread_source === 'subagent' ||
+            stringOrNull(sessionMeta.parent_thread_id) ||
+            stringOrNull(sessionMeta.forked_from_id),
+    );
+};
+
+const readFallbackRolloutStats = (sessionFile: string): FallbackRolloutStats => {
+    let model: string | null = null;
+    let tokensUsed = 0;
+
+    readJsonlObjects<Record<string, unknown>>(sessionFile, (record) => {
+        const payload = objectOrNull(record.payload);
+        if (!payload) {
+            return;
+        }
+
+        if (record.type === 'turn_context') {
+            model = stringOrNull(payload.model) ?? model;
+            return;
+        }
+
+        const payloadType = stringOrNull(payload.type);
+        if (payloadType === 'message' || payloadType === 'agent_message') {
+            model = stringOrNull(payload.model) ?? model;
+            return;
+        }
+
+        if (payloadType !== 'token_count') {
+            return;
+        }
+
+        const info = objectOrNull(payload.info);
+        const totalTokenUsage = objectOrNull(info?.total_token_usage);
+        tokensUsed = numberOrNull(totalTokenUsage?.total_tokens) ?? tokensUsed;
+    });
+
+    return {
+        model,
+        tokensUsed,
+    };
+};
+
+const buildFallbackThreadRow = (
+    entry: SessionIndexEntry,
+    sessionFile: string,
+    sessionMeta: FallbackSessionMeta,
+    rolloutStats: FallbackRolloutStats,
+): ThreadRow | null => {
+    const cwd = stringOrNull(sessionMeta.cwd);
+    if (!cwd) {
+        return null;
+    }
+
+    let mtimeMs = Date.now();
+    try {
+        mtimeMs = statSync(sessionFile).mtimeMs;
+    } catch {}
+
+    const updatedAtMs = parseIsoMs(entry.updated_at, mtimeMs);
+    const createdAtMs = parseIsoMs(sessionMeta.timestamp, updatedAtMs);
+    const title = entry.thread_name?.trim() || path.basename(sessionFile, '.jsonl');
+    const source = stringOrNull(sessionMeta.source) ?? 'session_file';
+
+    return {
+        agent_nickname: sessionMeta.agent_nickname ?? null,
+        agent_path: sessionMeta.agent_path ?? null,
+        agent_role: sessionMeta.agent_role ?? null,
+        approval_mode: 'unknown',
+        archived: 0,
+        archived_at: null,
+        cli_version: sessionMeta.cli_version ?? '',
+        created_at: Math.floor(createdAtMs / 1000),
+        created_at_ms: Math.floor(createdAtMs),
+        cwd,
+        first_user_message: title,
+        git_branch: null,
+        git_origin_url: null,
+        git_sha: null,
+        has_user_event: sessionMeta.thread_source === 'user' ? 1 : 0,
+        id: entry.id,
+        memory_mode: 'enabled',
+        model: rolloutStats.model,
+        model_provider: sessionMeta.model_provider ?? 'unknown',
+        preview: title,
+        reasoning_effort: null,
+        rollout_path: sessionFile,
+        sandbox_policy: '{}',
+        source,
+        thread_source: sessionMeta.thread_source ?? null,
+        title,
+        tokens_used: rolloutStats.tokensUsed,
+        updated_at: Math.floor(updatedAtMs / 1000),
+        updated_at_ms: Math.floor(updatedAtMs),
+    };
+};
+
+const readFallbackThreadRows = (
+    dbPath: string,
+    existingThreadIds: Set<string>,
+    projectName: string | null = null,
+    options: ReadFallbackThreadRowsOptions = {},
+): ThreadRow[] => {
+    const codexDir = resolveCodexDirFromDbPath(dbPath);
+    const sessionFilesByThreadId = collectSessionFilesByThreadId(path.join(codexDir, 'sessions'));
+    const fallbackThreads: ThreadRow[] = [];
+
+    for (const entry of readSessionIndexEntries(codexDir)) {
+        if (existingThreadIds.has(entry.id)) {
+            continue;
+        }
+
+        const sessionFile = sessionFilesByThreadId.get(entry.id);
+        if (!sessionFile) {
+            continue;
+        }
+
+        const sessionMeta = readFallbackSessionMeta(sessionFile);
+        if (!sessionMeta) {
+            continue;
+        }
+
+        if (!options.includeSubagents && isFallbackSubagent(sessionMeta)) {
+            continue;
+        }
+
+        const fallbackThread = buildFallbackThreadRow(
+            entry,
+            sessionFile,
+            sessionMeta,
+            readFallbackRolloutStats(sessionFile),
+        );
+        if (!fallbackThread || (projectName && getPortablePathBasename(fallbackThread.cwd) !== projectName)) {
+            continue;
+        }
+
+        fallbackThreads.push(fallbackThread);
+    }
+
+    return fallbackThreads;
+};
+
+const mergeFallbackThreadRows = (dbPath: string, threads: ThreadRow[], projectName: string | null = null) => {
+    const threadIds = new Set(threads.map((thread) => thread.id));
+    return [...threads, ...readFallbackThreadRows(dbPath, threadIds, projectName)].sort((left, right) => {
+        const updatedDifference = toTimestampMs(right) - toTimestampMs(left);
+        if (updatedDifference !== 0) {
+            return updatedDifference;
+        }
+
+        return right.id.localeCompare(left.id);
+    });
+};
+
+const applyRolloutActivityTimestamps = (threads: ThreadRow[]) => {
+    return threads
+        .map((thread) => {
+            let rolloutUpdatedAtMs = toTimestampMs(thread);
+            try {
+                rolloutUpdatedAtMs = Math.max(rolloutUpdatedAtMs, statSync(thread.rollout_path).mtimeMs);
+            } catch {}
+
+            if (rolloutUpdatedAtMs <= toTimestampMs(thread)) {
+                return thread;
+            }
+
+            return {
+                ...thread,
+                updated_at: Math.floor(rolloutUpdatedAtMs / 1000),
+                updated_at_ms: Math.floor(rolloutUpdatedAtMs),
+            };
+        })
+        .sort((left, right) => {
+            const updatedDifference = toTimestampMs(right) - toTimestampMs(left);
+            if (updatedDifference !== 0) {
+                return updatedDifference;
+            }
+
+            return right.id.localeCompare(left.id);
+        });
+};
+
+const buildDashboardRecentThreads = (threads: ThreadRow[]) => {
+    const bestThreadByProject = new Map<string, ThreadRow>();
+    for (const thread of threads) {
+        const project = getPortablePathBasename(thread.cwd);
+        if (!project) {
+            continue;
+        }
+
+        const current = bestThreadByProject.get(project);
+        if (!current || toTimestampMs(thread) > toTimestampMs(current)) {
+            bestThreadByProject.set(project, thread);
+        }
+    }
+
+    return [...bestThreadByProject.values()]
+        .sort((left, right) => {
+            const updatedDifference = toTimestampMs(right) - toTimestampMs(left);
+            if (updatedDifference !== 0) {
+                return updatedDifference;
+            }
+
+            return right.id.localeCompare(left.id);
+        })
+        .slice(0, 5)
+        .map((thread) => ({
+            project: getPortablePathBasename(thread.cwd),
+            thread: compactThreadListRow(thread),
+        }));
+};
+
 const filterThreadsByProject = (threads: ThreadRow[], projectName: string | null) => {
     if (!projectName) {
         return threads;
@@ -391,7 +804,9 @@ const deleteThreadSessionFiles = async (sessionFiles: string[]) => {
 };
 
 export const listCodexProjects = (dbPath: string): ProjectSummary[] => {
-    return mapProjectSummaries(buildProjectSummaryMap(readAllThreads(dbPath)));
+    return mapProjectSummaries(
+        buildProjectSummaryMap(applyRolloutActivityTimestamps(mergeFallbackThreadRows(dbPath, readAllThreads(dbPath)))),
+    );
 };
 
 type ListProjectThreadsOptions = {
@@ -411,10 +826,29 @@ export const listProjectThreads = async (
     projectName: string,
     options: ListProjectThreadsOptions = {},
 ): Promise<ThreadListEntry[]> => {
-    const threads = filterThreadsByProject(readAllThreads(dbPath), projectName);
-    const entries = await mapWithConcurrency(threads, THREAD_LIST_IO_CONCURRENCY, async (thread) => {
+    const threads = mergeFallbackThreadRows(
+        dbPath,
+        filterThreadsByProject(readAllThreads(dbPath), projectName),
+        projectName,
+    );
+    const activeThreads = applyRolloutActivityTimestamps(threads);
+    const entries = await mapWithConcurrency(activeThreads, THREAD_LIST_IO_CONCURRENCY, async (thread) => {
         const rollout = await getThreadRolloutLoadState(thread.rollout_path, options.largeTranscriptThresholdBytes);
 
+        if (rollout.fileSizeBytes === null) {
+            return {
+                project: projectName,
+                rolloutSizeBytes: null,
+                stats: {
+                    deferred: false,
+                    execCommandCount: 0,
+                    toolCallCount: 0,
+                    webSearchEventCount: 0,
+                },
+                thread: compactThreadListRow(thread),
+            };
+        }
+
         if (rollout.shouldDeferTranscriptLoad) {
             return {
                 project: projectName,
@@ -450,18 +884,25 @@ export const listProjectThreads = async (
 export const getThreadBrowseData = (dbPath: string, threadId: string): ThreadBrowseData => {
     return withReadonlyDb(dbPath, (db) => {
         const existingTableNames = getExistingTableNames(db);
-        const thread = db.query('SELECT * FROM threads WHERE id = ? LIMIT 1').get(threadId) as ThreadRow | null;
+        const dbThread = db.query('SELECT * FROM threads WHERE id = ? LIMIT 1').get(threadId) as ThreadRow | null;
+        const thread =
+            dbThread ??
+            readFallbackThreadRows(dbPath, new Set(), null, { includeSubagents: true }).find(
+                (fallbackThread) => fallbackThread.id === threadId,
+            ) ??
+            null;
         if (!thread) {
             throw new Error(`Thread not found: ${threadId}`);
         }
 
-        const dynamicTools = existingTableNames.has('thread_dynamic_tools')
-            ? (db
-                  .query(
-                      'SELECT thread_id, position, name, description, input_schema, defer_loading, namespace FROM thread_dynamic_tools WHERE thread_id = ? ORDER BY position ASC',
-                  )
-                  .all(threadId) as Array<Record<string, number | string | null>>)
-            : [];
+        const dynamicTools =
+            dbThread && existingTableNames.has('thread_dynamic_tools')
+                ? (db
+                      .query(
+                          'SELECT thread_id, position, name, description, input_schema, defer_loading, namespace FROM thread_dynamic_tools WHERE thread_id = ? ORDER BY position ASC',
+                      )
+                      .all(threadId) as Array<Record<string, number | string | null>>)
+                : [];
 
         return {
             dynamicTools: dynamicTools.map((row) => parseDynamicToolRow(row)),
@@ -473,7 +914,7 @@ export const getThreadBrowseData = (dbPath: string, threadId: string): ThreadBro
 };
 
 export const getCodexDashboardSummary = (dbPath: string): DashboardSummary => {
-    const threads = readAllThreads(dbPath);
+    const threads = applyRolloutActivityTimestamps(mergeFallbackThreadRows(dbPath, readAllThreads(dbPath)));
     const projects = mapProjectSummaries(buildProjectSummaryMap(threads));
     const threadsWithRelations = withReadonlyDb(dbPath, (db) => {
         if (!getExistingTableNames(db).has('thread_spawn_edges')) {
@@ -491,13 +932,7 @@ export const getCodexDashboardSummary = (dbPath: string): DashboardSummary => {
     return {
         activeThreads: threads.filter((thread) => !thread.archived).length,
         archivedThreads: threads.filter((thread) => Boolean(thread.archived)).length,
-        recentThreads: threads
-            .slice(0, 5)
-            .filter((thread) => Boolean(getPortablePathBasename(thread.cwd)))
-            .map((thread) => ({
-                project: getPortablePathBasename(thread.cwd),
-                thread: compactThreadListRow(thread),
-            })),
+        recentThreads: buildDashboardRecentThreads(threads),
         threadsWithRelations,
         topProjectsByThreadCount: [...projects]
             .sort((left, right) => {
@@ -603,7 +1038,7 @@ export const deleteCodexProject = async (
 };
 
 export const listScopedThreads = (dbPath: string, projectName: string | null): ThreadRow[] => {
-    return filterThreadsByProject(readAllThreads(dbPath), projectName);
+    return mergeFallbackThreadRows(dbPath, filterThreadsByProject(readAllThreads(dbPath), projectName), projectName);
 };
 
 export const invalidateCodexUiCaches = async () => {