tokentrace 0.1.0

package/src/db/seed.ts

@@ -0,0 +1,110 @@
+import { db } from "./client";
+import { models, providers, settings, tools } from "./schema";
+
+const now = new Date("2026-01-01T00:00:00.000Z");
+
+const seedProviders = [
+  { id: "openai", name: "OpenAI", type: "llm-provider" },
+  { id: "anthropic", name: "Anthropic", type: "llm-provider" },
+  { id: "generic", name: "Generic", type: "local-log" }
+];
+
+const seedTools = [
+  { id: "codex-cli", providerId: "openai", name: "Codex CLI" },
+  { id: "claude-code", providerId: "anthropic", name: "Claude Code" },
+  { id: "generic-jsonl", providerId: "generic", name: "Generic JSONL" },
+  { id: "generic-json", providerId: "generic", name: "Generic JSON" },
+  { id: "generic-log", providerId: "generic", name: "Generic Log" }
+];
+
+const seedModels = [
+  {
+    id: "openai-gpt-4-1",
+    providerId: "openai",
+    name: "gpt-4.1",
+    inputTokenPrice: 2,
+    outputTokenPrice: 8,
+    cachedInputTokenPrice: 0.5,
+    currency: "USD"
+  },
+  {
+    id: "openai-gpt-4-1-mini",
+    providerId: "openai",
+    name: "gpt-4.1-mini",
+    inputTokenPrice: 0.4,
+    outputTokenPrice: 1.6,
+    cachedInputTokenPrice: 0.1,
+    currency: "USD"
+  },
+  {
+    id: "openai-gpt-4o",
+    providerId: "openai",
+    name: "gpt-4o",
+    inputTokenPrice: 2.5,
+    outputTokenPrice: 10,
+    cachedInputTokenPrice: 1.25,
+    currency: "USD"
+  },
+  {
+    id: "anthropic-claude-sonnet-4",
+    providerId: "anthropic",
+    name: "claude-sonnet-4",
+    inputTokenPrice: 3,
+    outputTokenPrice: 15,
+    cachedInputTokenPrice: 0.3,
+    currency: "USD"
+  },
+  {
+    id: "anthropic-claude-haiku",
+    providerId: "anthropic",
+    name: "claude-haiku",
+    inputTokenPrice: 0.8,
+    outputTokenPrice: 4,
+    cachedInputTokenPrice: 0.08,
+    currency: "USD"
+  },
+  {
+    id: "generic-unknown",
+    providerId: "generic",
+    name: "unknown",
+    inputTokenPrice: null,
+    outputTokenPrice: null,
+    cachedInputTokenPrice: null,
+    currency: "USD"
+  }
+];
+
+export function seedDatabase() {
+  for (const provider of seedProviders) {
+    db.insert(providers).values(provider).onConflictDoNothing().run();
+  }
+
+  for (const tool of seedTools) {
+    db.insert(tools).values(tool).onConflictDoNothing().run();
+  }
+
+  for (const model of seedModels) {
+    db.insert(models)
+      .values({
+        ...model,
+        effectiveFrom: now,
+        rawMetadata: {
+          note: "Editable placeholder price. Verify current provider pricing before financial use."
+        }
+      })
+      .onConflictDoNothing()
+      .run();
+  }
+
+  db.insert(settings)
+    .values({
+      key: "app",
+      value: {
+        customFolders: [],
+        storeRawMessageContent: false
+      },
+      updatedAt: new Date()
+    })
+    .onConflictDoNothing()
+    .run();
+}

package/src/db/settings.ts

@@ -0,0 +1,47 @@
+import { eq } from "drizzle-orm";
+import { db } from "./client";
+import { settings } from "./schema";
+
+export type AppSettings = {
+  customFolders: string[];
+  storeRawMessageContent: boolean;
+};
+
+const defaultSettings: AppSettings = {
+  customFolders: [],
+  storeRawMessageContent: false
+};
+
+function normalizeSettings(value: unknown): AppSettings {
+  if (!value || typeof value !== "object") return defaultSettings;
+  const candidate = value as Partial<AppSettings>;
+  return {
+    customFolders: Array.isArray(candidate.customFolders)
+      ? candidate.customFolders.filter((item): item is string => typeof item === "string")
+      : [],
+    storeRawMessageContent: Boolean(candidate.storeRawMessageContent)
+  };
+}
+
+export function getAppSettings(): AppSettings {
+  const row = db.select().from(settings).where(eq(settings.key, "app")).get();
+  return normalizeSettings(row?.value);
+}
+
+export function saveAppSettings(nextSettings: AppSettings) {
+  db.insert(settings)
+    .values({
+      key: "app",
+      value: nextSettings,
+      updatedAt: new Date()
+    })
+    .onConflictDoUpdate({
+      target: settings.key,
+      set: {
+        value: nextSettings,
+        updatedAt: new Date()
+      }
+    })
+    .run();
+  return nextSettings;
+}

package/src/ingestion/adapters/claude-code.ts

@@ -0,0 +1,78 @@
+import path from "node:path";
+import { IngestionAdapter } from "../types";
+import { buildSessionsFromRecords } from "./generic-records";
+import { asObject, fileLooksLikeJsonl, firstString, readFileText, readTextSample, safeJsonParse } from "./helpers";
+
+function projectPathFromClaudeProjectFile(filePath: string) {
+  const parts = filePath.split(path.sep);
+  const projectsIndex = parts.lastIndexOf("projects");
+  if (projectsIndex === -1 || !parts[projectsIndex + 1]) return null;
+  const encoded = parts[projectsIndex + 1];
+  if (!encoded.startsWith("-")) return null;
+  return encoded.replace(/-/g, path.sep);
+}
+
+export const claudeCodeAdapter: IngestionAdapter = {
+  id: "claude-code",
+  displayName: "Claude Code",
+
+  async detect(file) {
+    const normalized = file.path.toLowerCase();
+    const extension = path.extname(file.path).toLowerCase();
+    if (normalized.includes(`${path.sep}.claude${path.sep}`) && [".jsonl", ".json"].includes(extension)) {
+      return { detected: true, confidence: 0.95, reason: "Path is inside a .claude directory" };
+    }
+
+    if (![".jsonl", ".json", ".log"].includes(extension)) {
+      return { detected: false, confidence: 0 };
+    }
+
+    const sample = await readTextSample(file.path);
+    if (/claude|anthropic|cache_creation_input_tokens|cache_read_input_tokens/i.test(sample)) {
+      return { detected: true, confidence: 0.72, reason: "Claude/Anthropic fields found" };
+    }
+    return { detected: false, confidence: 0 };
+  },
+
+  async parse(file, context) {
+    const warnings: string[] = [];
+    const records: Record<string, unknown>[] = [];
+    const text = await readFileText(file.path);
+
+    if (fileLooksLikeJsonl(text)) {
+      text.split(/\r?\n/).forEach((line, index) => {
+        const trimmed = line.trim();
+        if (!trimmed) return;
+        const object = asObject(safeJsonParse(trimmed));
+        if (!object) {
+          warnings.push(`Line ${index + 1} is not a JSON object.`);
+          return;
+        }
+        const message = asObject(object.message);
+        records.push({
+          ...object,
+          model: firstString(object.model, message?.model),
+          role: object.type ?? message?.role ?? object.role
+        });
+      });
+    } else {
+      const object = asObject(safeJsonParse(text));
+      if (object) records.push(object);
+    }
+
+    const projectPath = projectPathFromClaudeProjectFile(file.path);
+
+    return {
+      sessions: buildSessionsFromRecords({
+        file,
+        records,
+        provider: { id: "anthropic", name: "Anthropic", type: "llm-provider" },
+        tool: { id: "claude-code", name: "Claude Code" },
+        storeRawMessageContent: context.storeRawMessageContent,
+        defaultProjectPath: projectPath
+      }),
+      warnings,
+      errors: records.length ? [] : ["No Claude Code JSON records were parsed."]
+    };
+  }
+};

package/src/ingestion/adapters/codex-cli.ts

@@ -0,0 +1,82 @@
+import path from "node:path";
+import { IngestionAdapter } from "../types";
+import { buildSessionsFromRecords } from "./generic-records";
+import { asObject, fileLooksLikeJsonl, firstString, readFileText, readTextSample, safeJsonParse } from "./helpers";
+
+function flattenCodexRecord(record: Record<string, unknown>) {
+  const payload = asObject(record.payload);
+  const response = asObject(payload?.response) ?? asObject(record.response);
+  const usage = asObject(response?.usage) ?? asObject(payload?.usage) ?? asObject(record.usage);
+  const message = asObject(payload?.message) ?? asObject(record.message);
+  const type = firstString(record.type, payload?.type, message?.type);
+
+  return {
+    ...record,
+    ...payload,
+    response,
+    usage,
+    model: firstString(record.model, payload?.model, response?.model),
+    role: firstString(record.role, payload?.role, message?.role) ?? (type?.includes("response") ? "assistant" : undefined),
+    content: record.content ?? payload?.content ?? message?.content,
+    cwd: record.cwd ?? payload?.cwd,
+    id: firstString(record.id, payload?.id, response?.id, message?.id),
+    timestamp: record.timestamp ?? payload?.timestamp
+  };
+}
+
+export const codexCliAdapter: IngestionAdapter = {
+  id: "codex-cli",
+  displayName: "Codex CLI",
+
+  async detect(file) {
+    const normalized = file.path.toLowerCase();
+    const extension = path.extname(file.path).toLowerCase();
+    if (normalized.includes(`${path.sep}.codex${path.sep}`) && [".jsonl", ".json", ".log"].includes(extension)) {
+      return { detected: true, confidence: 0.95, reason: "Path is inside a .codex directory" };
+    }
+
+    if (![".jsonl", ".json", ".log"].includes(extension)) {
+      return { detected: false, confidence: 0 };
+    }
+
+    const sample = await readTextSample(file.path);
+    if (/codex|openai|response\.completed|response_completed|turn_context/i.test(sample)) {
+      return { detected: true, confidence: 0.72, reason: "Codex/OpenAI event fields found" };
+    }
+    return { detected: false, confidence: 0 };
+  },
+
+  async parse(file, context) {
+    const warnings: string[] = [];
+    const records: Record<string, unknown>[] = [];
+    const text = await readFileText(file.path);
+
+    if (fileLooksLikeJsonl(text)) {
+      text.split(/\r?\n/).forEach((line, index) => {
+        const trimmed = line.trim();
+        if (!trimmed) return;
+        const object = asObject(safeJsonParse(trimmed));
+        if (!object) {
+          warnings.push(`Line ${index + 1} is not a JSON object.`);
+          return;
+        }
+        records.push(flattenCodexRecord(object));
+      });
+    } else {
+      const object = asObject(safeJsonParse(text));
+      if (object) records.push(flattenCodexRecord(object));
+    }
+
+    return {
+      sessions: buildSessionsFromRecords({
+        file,
+        records,
+        provider: { id: "openai", name: "OpenAI", type: "llm-provider" },
+        tool: { id: "codex-cli", name: "Codex CLI" },
+        storeRawMessageContent: context.storeRawMessageContent
+      }),
+      warnings,
+      errors: records.length ? [] : ["No Codex CLI JSON records were parsed."]
+    };
+  }
+};

package/src/ingestion/adapters/generic-json.ts

@@ -0,0 +1,93 @@
+import path from "node:path";
+import { IngestionAdapter } from "../types";
+import { buildSessionsFromRecords } from "./generic-records";
+import { asArray, asObject, readFileText, readTextSample, safeJsonParse } from "./helpers";
+
+function collectRecords(value: unknown): Record<string, unknown>[] {
+  const object = asObject(value);
+  if (Array.isArray(value)) {
+    return value.map(asObject).filter((item): item is Record<string, unknown> => Boolean(item));
+  }
+  if (!object) return [];
+
+  const sessions = asArray(object.sessions);
+  if (sessions.length) {
+    return sessions.flatMap((session, sessionIndex) => {
+      const sessionObject = asObject(session);
+      if (!sessionObject) return [];
+      const messages = [
+        ...asArray(sessionObject.messages),
+        ...asArray(sessionObject.interactions),
+        ...asArray(sessionObject.events)
+      ];
+      if (!messages.length) return [sessionObject];
+      return messages
+        .map(asObject)
+        .filter((item): item is Record<string, unknown> => Boolean(item))
+        .map((message, messageIndex) => ({
+          ...message,
+          session_id:
+            sessionObject.session_id ??
+            sessionObject.sessionId ??
+            sessionObject.id ??
+            `session-${sessionIndex}`,
+          cwd: message.cwd ?? sessionObject.cwd ?? sessionObject.project_path,
+          title: message.title ?? sessionObject.title,
+          id: message.id ?? `${sessionIndex}-${messageIndex}`
+        }));
+    });
+  }
+
+  const records = [
+    ...asArray(object.messages),
+    ...asArray(object.interactions),
+    ...asArray(object.events),
+    ...asArray(object.records)
+  ];
+
+  if (records.length) {
+    return records.map(asObject).filter((item): item is Record<string, unknown> => Boolean(item));
+  }
+
+  return [object];
+}
+
+export const genericJsonAdapter: IngestionAdapter = {
+  id: "generic-json",
+  displayName: "Generic JSON",
+
+  async detect(file) {
+    if (path.extname(file.path).toLowerCase() !== ".json") {
+      return { detected: false, confidence: 0 };
+    }
+
+    const sample = await readTextSample(file.path);
+    const parsed = safeJsonParse(sample);
+    return parsed
+      ? { detected: true, confidence: 0.65, reason: "JSON extension and valid JSON sample" }
+      : { detected: true, confidence: 0.35, reason: "JSON extension" };
+  },
+
+  async parse(file, context) {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+    const parsed = safeJsonParse(await readFileText(file.path));
+    const records = collectRecords(parsed);
+
+    if (!records.length) {
+      errors.push("No usable JSON records were found.");
+    }
+
+    return {
+      sessions: buildSessionsFromRecords({
+        file,
+        records,
+        provider: { id: "generic", name: "Generic", type: "local-log" },
+        tool: { id: "generic-json", name: "Generic JSON" },
+        storeRawMessageContent: context.storeRawMessageContent
+      }),
+      warnings,
+      errors
+    };
+  }
+};

package/src/ingestion/adapters/generic-jsonl.ts

@@ -0,0 +1,62 @@
+import path from "node:path";
+import { IngestionAdapter } from "../types";
+import { buildSessionsFromRecords } from "./generic-records";
+import { asObject, fileLooksLikeJsonl, readFileText, readTextSample, safeJsonParse } from "./helpers";
+
+export const genericJsonlAdapter: IngestionAdapter = {
+  id: "generic-jsonl",
+  displayName: "Generic JSONL",
+
+  async detect(file) {
+    const extension = path.extname(file.path).toLowerCase();
+    if (extension === ".jsonl" || file.path.endsWith(".jsonl.gz")) {
+      return { detected: true, confidence: 0.75, reason: "JSONL extension" };
+    }
+
+    if (![".log", ".txt", ""].includes(extension)) {
+      return { detected: false, confidence: 0 };
+    }
+
+    const sample = await readTextSample(file.path);
+    if (fileLooksLikeJsonl(sample)) {
+      return { detected: true, confidence: 0.55, reason: "First lines are JSON objects" };
+    }
+
+    return { detected: false, confidence: 0 };
+  },
+
+  async parse(file, context) {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+    const text = await readFileText(file.path);
+    const records: Record<string, unknown>[] = [];
+
+    text.split(/\r?\n/).forEach((line, index) => {
+      const trimmed = line.trim();
+      if (!trimmed) return;
+      const parsed = safeJsonParse(trimmed);
+      const object = asObject(parsed);
+      if (!object) {
+        warnings.push(`Line ${index + 1} is not a JSON object.`);
+        return;
+      }
+      records.push(object);
+    });
+
+    if (!records.length) {
+      errors.push("No JSON objects were parsed.");
+    }
+
+    return {
+      sessions: buildSessionsFromRecords({
+        file,
+        records,
+        provider: { id: "generic", name: "Generic", type: "local-log" },
+        tool: { id: "generic-jsonl", name: "Generic JSONL" },
+        storeRawMessageContent: context.storeRawMessageContent
+      }),
+      warnings,
+      errors
+    };
+  }
+};

package/src/ingestion/adapters/generic-log.ts

@@ -0,0 +1,144 @@
+import path from "node:path";
+import { estimateTokensFromText, previewText } from "@/src/lib/token-estimator";
+import { IngestionAdapter, NormalizedInteraction } from "../types";
+import { firstNumber, parseTimestamp, readFileText, readTextSample, sessionNameFromFile } from "./helpers";
+
+function numberAfter(line: string, patterns: RegExp[]) {
+  for (const pattern of patterns) {
+    const match = line.match(pattern);
+    if (match?.[1]) return firstNumber(match[1]);
+  }
+  return null;
+}
+
+function textAfter(line: string, patterns: RegExp[]) {
+  for (const pattern of patterns) {
+    const match = line.match(pattern);
+    if (match?.[1]) return match[1].trim();
+  }
+  return null;
+}
+
+export const genericLogAdapter: IngestionAdapter = {
+  id: "generic-log",
+  displayName: "Generic Text Log",
+
+  async detect(file) {
+    const extension = path.extname(file.path).toLowerCase();
+    if (![".log", ".txt", ".md", ""].includes(extension)) {
+      return { detected: false, confidence: 0 };
+    }
+
+    const sample = await readTextSample(file.path);
+    if (/(tokens?|prompt_tokens|completion_tokens|model|session|cost|\$[0-9.]+)/i.test(sample)) {
+      return { detected: true, confidence: 0.45, reason: "Token/model/cost-like text found" };
+    }
+
+    return { detected: false, confidence: 0 };
+  },
+
+  async parse(file, context) {
+    const text = await readFileText(file.path);
+    const lines = text.split(/\r?\n/);
+    const interactions: NormalizedInteraction[] = [];
+    const warnings: string[] = [];
+    let currentSession = sessionNameFromFile(file.path);
+    let projectPath: string | null = null;
+
+    lines.forEach((line, index) => {
+      const session = textAfter(line, [/session(?:_id)?\s*[:=]\s*([^\s,]+)/i]);
+      if (session) currentSession = session;
+      projectPath =
+        projectPath ?? textAfter(line, [/(?:cwd|project|path)\s*[:=]\s*(.+)$/i]);
+
+      const model = textAfter(line, [/model\s*[:=]\s*([A-Za-z0-9_.:/-]+)/i]);
+      const inputTokens = numberAfter(line, [
+        /(?:input_tokens|prompt_tokens|input tokens|prompt tokens)\s*[:=]\s*([0-9,]+)/i
+      ]);
+      const outputTokens = numberAfter(line, [
+        /(?:output_tokens|completion_tokens|output tokens|completion tokens)\s*[:=]\s*([0-9,]+)/i
+      ]);
+      const totalTokens = numberAfter(line, [/(?:total_tokens|total tokens|tokens)\s*[:=]\s*([0-9,]+)/i]);
+      const reasoningTokens = numberAfter(line, [/(?:reasoning_tokens|reasoning tokens)\s*[:=]\s*([0-9,]+)/i]);
+      const hasStructuredTokens =
+        inputTokens != null || outputTokens != null || totalTokens != null || reasoningTokens != null;
+
+      if (!model && !hasStructuredTokens) return;
+
+      const timestamp =
+        parseTimestamp(textAfter(line, [/^(\d{4}-\d{2}-\d{2}T[^\s]+)/, /^(\d{4}-\d{2}-\d{2} [^\]]+)/])) ??
+        file.modifiedTime;
+
+      const estimated = !hasStructuredTokens;
+      const estimate = estimated ? estimateTokensFromText(line) : { tokens: 0 };
+      const structuredTotal =
+        totalTokens ?? (inputTokens ?? 0) + (outputTokens ?? 0) + (reasoningTokens ?? 0);
+      interactions.push({
+        externalId: `${currentSession}-${index}`,
+        timestamp,
+        role: outputTokens || reasoningTokens ? "assistant" : "unknown",
+        modelName: model,
+        inputTokens: inputTokens ?? (estimated ? estimate.tokens : 0),
+        outputTokens: outputTokens ?? 0,
+        reasoningTokens: reasoningTokens ?? 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        totalTokens: structuredTotal || estimate.tokens,
+        estimatedTokens: estimated,
+        tokenConfidence: estimated ? "low-confidence estimate" : "high-confidence estimate",
+        rawText: context.storeRawMessageContent ? line : null,
+        rawTextPreview: previewText(line),
+        rawMetadata: {
+          source: "generic-log-line",
+          line: index + 1
+        },
+        toolCalls: []
+      });
+    });
+
+    if (!interactions.length && text.trim()) {
+      const estimate = estimateTokensFromText(text);
+      warnings.push("No structured token lines found; created one estimated file-level interaction.");
+      interactions.push({
+        externalId: `${currentSession}-estimated-file`,
+        timestamp: file.modifiedTime,
+        role: "unknown",
+        modelName: "unknown",
+        inputTokens: estimate.tokens,
+        outputTokens: 0,
+        reasoningTokens: 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        totalTokens: estimate.tokens,
+        estimatedTokens: true,
+        tokenConfidence: "low-confidence estimate",
+        rawText: context.storeRawMessageContent ? text : null,
+        rawTextPreview: previewText(text),
+        rawMetadata: { source: "generic-log-file-estimate" },
+        toolCalls: []
+      });
+    }
+
+    return {
+      sessions: interactions.length
+        ? [
+            {
+              externalId: currentSession,
+              provider: { id: "generic", name: "Generic", type: "local-log" },
+              tool: { id: "generic-log", name: "Generic Log" },
+              projectPath,
+              projectName: projectPath ? undefined : "Unknown project",
+              startedAt: interactions[0]?.timestamp ?? file.modifiedTime,
+              endedAt: interactions[interactions.length - 1]?.timestamp ?? file.modifiedTime,
+              title: sessionNameFromFile(file.path),
+              sourceFile: file.path,
+              rawMetadata: { parser: "generic-log" },
+              interactions
+            }
+          ]
+        : [],
+      warnings,
+      errors: interactions.length ? [] : ["No text log records were inferred."]
+    };
+  }
+};