spiracha 1.3.3 → 1.5.0

package/src/lib/claude-code-exporter-types.ts

@@ -0,0 +1,110 @@
+import os from 'node:os';
+import path from 'node:path';
+import type { ExportFormat, JsonValue } from './shared';
+
+export type { ClaudeCodeAssistantMessagePhase } from './claude-code-transcript-phase';
+export { getClaudeCodeAssistantMessagePhase } from './claude-code-transcript-phase';
+
+export const getDefaultClaudeCodeDataDir = (_env: NodeJS.ProcessEnv = process.env, homeDir = os.homedir()): string => {
+    return path.join(homeDir, '.claude');
+};
+
+export const DEFAULT_CLAUDE_CODE_DATA_DIR = getDefaultClaudeCodeDataDir();
+
+export const resolveClaudeCodeDataDir = (): string => {
+    const configured =
+        process.env.SPIRACHA_CLAUDE_CODE_DATA_DIR?.trim() ||
+        process.env.SPIRACHA_CLAUDE_CODE_DIR?.trim() ||
+        process.env.SPIRACHA_CLAUDE_HOME?.trim();
+    return configured ? configured : DEFAULT_CLAUDE_CODE_DATA_DIR;
+};
+
+export const resolveClaudeCodeProjectsDir = (): string => {
+    const configured = process.env.SPIRACHA_CLAUDE_CODE_PROJECTS_DIR?.trim();
+    return configured ? configured : path.join(resolveClaudeCodeDataDir(), 'projects');
+};
+
+export type ClaudeCodeWorkspaceGroup = {
+    assistantMessageCount: number;
+    directoryName: string;
+    key: string;
+    label: string;
+    lastActiveAtIso: string | null;
+    lastActiveAtMs: number | null;
+    messageCount: number;
+    sessionCount: number;
+    toolCallCount: number;
+    toolResultCount: number;
+    uri: string;
+    userMessageCount: number;
+    worktree: string;
+};
+
+export type ClaudeCodeSessionSummary = {
+    assistantMessageCount: number;
+    attachmentCount: number;
+    cacheCreationInputTokens: number;
+    cacheReadInputTokens: number;
+    createdAtIso: string | null;
+    createdAtMs: number | null;
+    cwd: string | null;
+    filePath: string;
+    gitBranch: string | null;
+    inputTokens: number;
+    lastActiveAtIso: string | null;
+    lastActiveAtMs: number | null;
+    messageCount: number;
+    model: string | null;
+    outputTokens: number;
+    renderablePartCount: number;
+    sessionId: string;
+    title: string;
+    toolCallCount: number;
+    toolResultCount: number;
+    totalTokens: number;
+    userMessageCount: number;
+    version: string | null;
+    workspaceKey: string;
+    workspaceLabel: string;
+    worktree: string;
+};
+
+export type ClaudeCodePartType = 'attachment' | 'text' | 'thinking' | 'tool_result' | 'tool_use' | 'unknown';
+
+export type ClaudeCodeTranscriptPart = {
+    argumentsText?: string | null;
+    attachmentType?: string | null;
+    isError?: boolean | null;
+    outputText?: string | null;
+    raw: Record<string, JsonValue>;
+    text?: string;
+    toolName?: string | null;
+    toolUseId?: string | null;
+    type: ClaudeCodePartType;
+};
+
+export type ClaudeCodeTranscriptEntry = {
+    cwd: string | null;
+    entryId: string;
+    model?: string | null;
+    parentEntryId?: string | null;
+    parts: ClaudeCodeTranscriptPart[];
+    raw: Record<string, JsonValue>;
+    role: string;
+    timestamp: string | null;
+    type: string;
+};
+
+export type ClaudeCodeSessionTranscript = {
+    entries: ClaudeCodeTranscriptEntry[];
+    rawEvents: Record<string, JsonValue>[];
+    renderablePartCount: number;
+    session: ClaudeCodeSessionSummary;
+};
+
+export type ClaudeCodeExportOptions = {
+    includeCommentary: boolean;
+    includeMetadata: boolean;
+    includeTools: boolean;
+    outputFormat: ExportFormat;
+};

package/src/lib/claude-code-transcript-phase.ts

@@ -0,0 +1,25 @@
+import type { JsonValue } from './shared';
+
+export type ClaudeCodeAssistantMessagePhase = 'commentary' | 'final_answer';
+
+type ClaudeCodeAssistantPhaseEntry = {
+    raw: Record<string, JsonValue>;
+    role: string;
+};
+
+const asMessageObject = (value: JsonValue | undefined): Record<string, JsonValue> | null => {
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? (value as Record<string, JsonValue>)
+        : null;
+};
+
+export const getClaudeCodeAssistantMessagePhase = (
+    entry: ClaudeCodeAssistantPhaseEntry,
+): ClaudeCodeAssistantMessagePhase | null => {
+    if (entry.role !== 'assistant') {
+        return null;
+    }
+
+    const stopReason = asMessageObject(entry.raw.message)?.stop_reason;
+    return stopReason === 'tool_use' ? 'commentary' : 'final_answer';
+};

package/src/lib/claude-code-transcript.ts

@@ -0,0 +1,180 @@
+import type {
+    ClaudeCodeExportOptions,
+    ClaudeCodeSessionSummary,
+    ClaudeCodeSessionTranscript,
+    ClaudeCodeTranscriptEntry,
+    ClaudeCodeTranscriptPart,
+} from './claude-code-exporter-types';
+import { getClaudeCodeAssistantMessagePhase } from './claude-code-transcript-phase';
+import {
+    cleanExtractedText,
+    cleanInlineTitle,
+    formatInlineLiteral,
+    type MetadataEntry,
+    renderCodeBlock,
+    renderDocumentTitle,
+    renderMetadataBlock,
+    renderSection,
+} from './shared';
+
+const TOOL_OUTPUT_PREVIEW_LIMIT = 4000;
+
+const getSessionTitle = (session: ClaudeCodeSessionSummary): string => {
+    return cleanInlineTitle(session.title || session.sessionId);
+};
+
+const buildMetadataEntries = (session: ClaudeCodeSessionSummary): MetadataEntry[] => [
+    { key: 'exported_from', value: 'claude_code_local_jsonl' },
+    { key: 'session_id', value: session.sessionId },
+    { key: 'title', value: session.title },
+    { key: 'source_transcript_path', value: session.filePath },
+    { key: 'workspace_key', value: session.workspaceKey },
+    { key: 'worktree', value: session.worktree },
+    { key: 'cwd', value: session.cwd },
+    { key: 'model', value: session.model },
+    { key: 'version', value: session.version },
+    { key: 'git_branch', value: session.gitBranch },
+    { key: 'created_at_iso', value: session.createdAtIso },
+    { key: 'last_active_at_iso', value: session.lastActiveAtIso },
+    { key: 'message_count', value: session.messageCount },
+    { key: 'tool_call_count', value: session.toolCallCount },
+    { key: 'tool_result_count', value: session.toolResultCount },
+    { key: 'total_tokens', value: session.totalTokens },
+];
+
+const roleTitle = (role: string): string => {
+    if (role === 'assistant') {
+        return 'Assistant';
+    }
+
+    if (role === 'user') {
+        return 'User';
+    }
+
+    if (role === 'system') {
+        return 'System';
+    }
+
+    return role ? cleanInlineTitle(role) : 'Message';
+};
+
+const truncateOutput = (text: string): string => {
+    if (text.length <= TOOL_OUTPUT_PREVIEW_LIMIT) {
+        return text;
+    }
+
+    return `${text.slice(0, TOOL_OUTPUT_PREVIEW_LIMIT)}\n... (truncated)`;
+};
+
+const renderTextPart = (part: ClaudeCodeTranscriptPart, role: string, options: ClaudeCodeExportOptions): string => {
+    const text = cleanExtractedText(part.text ?? '').trim();
+    return text ? renderSection(roleTitle(role), text, options.outputFormat) : '';
+};
+
+const renderThinkingPart = (part: ClaudeCodeTranscriptPart, options: ClaudeCodeExportOptions): string => {
+    if (!options.includeCommentary) {
+        return '';
+    }
+
+    const text = cleanExtractedText(part.text ?? '').trim();
+    return text ? renderSection('Reasoning', text, options.outputFormat) : '';
+};
+
+const renderToolUsePart = (part: ClaudeCodeTranscriptPart, options: ClaudeCodeExportOptions): string => {
+    if (!options.includeTools) {
+        return '';
+    }
+
+    const toolName = part.toolName ?? 'unknown';
+    const lines = [`Tool: ${formatInlineLiteral(toolName, options.outputFormat)}`];
+    if (part.toolUseId) {
+        lines.push(`Call ID: ${part.toolUseId}`);
+    }
+    if (part.argumentsText?.trim()) {
+        lines.push('', 'Input:', '', renderCodeBlock(part.argumentsText.trim(), options.outputFormat));
+    }
+
+    return renderSection('Tool Call', lines.join('\n'), options.outputFormat);
+};
+
+const renderToolResultPart = (part: ClaudeCodeTranscriptPart, options: ClaudeCodeExportOptions): string => {
+    if (!options.includeTools) {
+        return '';
+    }
+
+    const outputText = part.outputText?.trim();
+    if (!outputText) {
+        return '';
+    }
+
+    const lines: string[] = [];
+    if (part.isError) {
+        lines.push('Error: true', '');
+    }
+    if (part.toolUseId) {
+        lines.push(`Call ID: ${part.toolUseId}`, '');
+    }
+    lines.push(renderCodeBlock(truncateOutput(outputText), options.outputFormat));
+    return renderSection('Tool Output', lines.join('\n'), options.outputFormat);
+};
+
+const renderAttachmentPart = (part: ClaudeCodeTranscriptPart, options: ClaudeCodeExportOptions): string => {
+    if (!options.includeCommentary) {
+        return '';
+    }
+
+    const text = cleanExtractedText(part.text ?? '').trim();
+    if (!text) {
+        return '';
+    }
+
+    const title = part.attachmentType ? `Attachment: ${part.attachmentType}` : 'Attachment';
+    return renderSection(title, text, options.outputFormat);
+};
+
+const renderPart = (
+    entry: ClaudeCodeTranscriptEntry,
+    part: ClaudeCodeTranscriptPart,
+    options: ClaudeCodeExportOptions,
+): string => {
+    switch (part.type) {
+        case 'text':
+            if (getClaudeCodeAssistantMessagePhase(entry) === 'commentary' && !options.includeCommentary) {
+                return '';
+            }
+            return renderTextPart(part, entry.role, options);
+        case 'thinking':
+            return renderThinkingPart(part, options);
+        case 'tool_use':
+            return renderToolUsePart(part, options);
+        case 'tool_result':
+            return renderToolResultPart(part, options);
+        case 'attachment':
+            return renderAttachmentPart(part, options);
+        case 'unknown':
+            return '';
+    }
+};
+
+export const renderClaudeCodeTranscript = (
+    transcript: ClaudeCodeSessionTranscript,
+    options: ClaudeCodeExportOptions,
+): string | null => {
+    const sections = transcript.entries.flatMap((entry) =>
+        entry.parts.map((part) => renderPart(entry, part, options)).filter(Boolean),
+    );
+    if (sections.length === 0) {
+        return null;
+    }
+
+    const parts = [
+        renderDocumentTitle(getSessionTitle(transcript.session), options.outputFormat),
+        '',
+        options.includeMetadata
+            ? renderMetadataBlock(buildMetadataEntries(transcript.session), options.outputFormat)
+            : '',
+        ...sections,
+    ].filter(Boolean);
+
+    return `${parts.join('\n').trimEnd()}\n`;
+};

package/src/lib/codex-analytics.ts

@@ -1,13 +1,37 @@
 import { listScopedThreads } from './codex-browser-db';
 import type { CodexAnalytics, DistributionItem, ModelTokenSummary } from './codex-browser-types';
 import type { ThreadRow } from './codex-exporter-types';
-import { getCachedParsedCodexTranscript } from './codex-thread-cache';
-import { getPortablePathBasename } from './shared';
-import { getFileFingerprint, hashCacheKeyParts, withCachedJson } from './ui-cache';
+import { mapWithConcurrency } from './concurrency';
+import { asObject, asString, getPortablePathBasename, readJsonlObjects } from './shared';
+import { hashCacheKeyParts, hashCacheKeyPartsIterable, withCachedJson } from './ui-cache';
 
 export type CodexAnalyticsInput = {
     dbPath: string;
     project: string | null;
+    transcriptConcurrency?: number;
+};
+
+export type ThreadAnalyticsSummary = {
+    hasWebSearch: boolean;
+    toolNames: string[];
+};
+
+export type ComputeCodexAnalyticsOptions = {
+    loadThreadAnalytics?: (thread: ThreadRow) => Promise<ThreadAnalyticsSummary>;
+    transcriptConcurrency?: number;
+};
+
+export const DEFAULT_ANALYTICS_TRANSCRIPT_CONCURRENCY = 8;
+
+export const resolveAnalyticsTranscriptConcurrency = (
+    configuredValue = process.env.SPIRACHA_ANALYTICS_TRANSCRIPT_CONCURRENCY,
+) => {
+    const parsed = Number(configuredValue);
+    if (!Number.isInteger(parsed) || parsed < 1) {
+        return DEFAULT_ANALYTICS_TRANSCRIPT_CONCURRENCY;
+    }
+
+    return parsed;
 };
 
 const roundToTwoDecimals = (value: number) => {
@@ -52,28 +76,100 @@ const buildModelsByTokens = (threads: ThreadRow[]): ModelTokenSummary[] => {
         });
 };
 
-const buildAnalyticsCacheKey = async (dbPath: string, threads: ThreadRow[], project: string | null) => {
-    const dbFingerprint = await getFileFingerprint(dbPath);
-    const rolloutFingerprints = await Promise.all(threads.map((thread) => getFileFingerprint(thread.rollout_path)));
-    return `analytics-${hashCacheKeyParts(dbFingerprint, project ?? 'all', ...rolloutFingerprints)}`;
+const timestampSignature = (thread: ThreadRow) => {
+    return String(thread.updated_at_ms ?? thread.updated_at * 1000);
 };
 
-const computeCodexAnalytics = async (threads: ThreadRow[]): Promise<CodexAnalytics> => {
+const threadMetadataCacheKeyParts = (thread: ThreadRow) => [
+    thread.id,
+    thread.rollout_path,
+    timestampSignature(thread),
+    String(thread.created_at_ms ?? thread.created_at * 1000),
+    String(thread.tokens_used),
+    String(thread.archived),
+    String(thread.archived_at ?? ''),
+    thread.cwd,
+    thread.model ?? '',
+    thread.model_provider,
+    thread.cli_version,
+    thread.title,
+    thread.preview,
+];
+
+export const buildCodexAnalyticsCacheKey = (dbPath: string, threads: ThreadRow[], project: string | null) => {
+    const parts = (function* () {
+        yield 'v2';
+        yield dbPath;
+        yield project ?? 'all';
+        yield String(threads.length);
+        for (const thread of threads) {
+            yield* threadMetadataCacheKeyParts(thread);
+        }
+    })();
+
+    return `analytics-${hashCacheKeyPartsIterable(parts)}`;
+};
+
+const buildThreadAnalyticsCacheKey = (thread: ThreadRow) => {
+    return `thread-analytics-${hashCacheKeyParts('v1', ...threadMetadataCacheKeyParts(thread))}`;
+};
+
+const parseThreadAnalyticsFile = async (sessionFile: string): Promise<ThreadAnalyticsSummary> => {
+    const toolNames: string[] = [];
+    let hasWebSearch = false;
+
+    for await (const parsed of readJsonlObjects(sessionFile)) {
+        if (parsed.type !== 'response_item') {
+            continue;
+        }
+
+        const payload = asObject(parsed.payload);
+        if (!payload) {
+            continue;
+        }
+
+        const payloadType = asString(payload.type);
+        if (payloadType === 'function_call') {
+            toolNames.push(asString(payload.name) ?? 'unknown');
+            continue;
+        }
+
+        if (payloadType === 'web_search_call' || payloadType === 'web_search_end') {
+            hasWebSearch = true;
+        }
+    }
+
+    return {
+        hasWebSearch,
+        toolNames,
+    };
+};
+
+const getCachedThreadAnalytics = async (thread: ThreadRow): Promise<ThreadAnalyticsSummary> => {
+    return withCachedJson(buildThreadAnalyticsCacheKey(thread), () => parseThreadAnalyticsFile(thread.rollout_path));
+};
+
+export const computeCodexAnalyticsFromThreads = async (
+    threads: ThreadRow[],
+    options: ComputeCodexAnalyticsOptions = {},
+): Promise<CodexAnalytics> => {
     const totalTokens = threads.reduce((sum, thread) => sum + thread.tokens_used, 0);
     const projectNames = new Set(threads.map((thread) => getPortablePathBasename(thread.cwd)).filter(Boolean));
     const toolUsage = new Map<string, number>();
-    const transcripts = await Promise.all(threads.map((thread) => getCachedParsedCodexTranscript(thread.rollout_path)));
     let threadsWithWebSearch = 0;
-
-    for (const transcript of transcripts) {
-        if (transcript.stats.webSearchEventCount > 0) {
+    const loadThreadAnalytics = options.loadThreadAnalytics ?? getCachedThreadAnalytics;
+    const transcriptConcurrency = options.transcriptConcurrency ?? resolveAnalyticsTranscriptConcurrency();
+    const threadAnalytics = await mapWithConcurrency(threads, transcriptConcurrency, (thread) =>
+        loadThreadAnalytics(thread),
+    );
+
+    for (const analytics of threadAnalytics) {
+        if (analytics.hasWebSearch) {
             threadsWithWebSearch += 1;
         }
 
-        for (const event of transcript.events) {
-            if (event.kind === 'tool_call') {
-                incrementCount(toolUsage, event.name);
-            }
+        for (const toolName of analytics.toolNames) {
+            incrementCount(toolUsage, toolName);
         }
     }
 
@@ -94,7 +190,11 @@ const computeCodexAnalytics = async (threads: ThreadRow[]): Promise<CodexAnalyti
 
 export const getCodexAnalytics = async (input: CodexAnalyticsInput): Promise<CodexAnalytics> => {
     const threads = listScopedThreads(input.dbPath, input.project);
-    const key = await buildAnalyticsCacheKey(input.dbPath, threads, input.project);
+    const key = buildCodexAnalyticsCacheKey(input.dbPath, threads, input.project);
 
-    return withCachedJson(key, async () => computeCodexAnalytics(threads));
+    return withCachedJson(key, async () =>
+        computeCodexAnalyticsFromThreads(threads, {
+            transcriptConcurrency: input.transcriptConcurrency,
+        }),
+    );
 };