tokentrace 0.1.0

package/src/ingestion/persist.ts

@@ -0,0 +1,283 @@
+import fs from "node:fs";
+import path from "node:path";
+import { sqlite } from "@/src/db/client";
+import { calculateInteractionCost } from "@/src/lib/cost";
+import { stableId, slugify } from "@/src/lib/ids";
+import { estimateTokensFromText, previewText } from "@/src/lib/token-estimator";
+import { NormalizedInteraction, NormalizedSession } from "./types";
+
+export type ImportSessionResult = {
+  sessionsImported: number;
+  interactionsImported: number;
+  toolCallsImported: number;
+  warnings: string[];
+};
+
+function json(value: unknown) {
+  return JSON.stringify(value ?? null);
+}
+
+function insertIgnore(sql: string, values: unknown[]) {
+  return sqlite.prepare(sql).run(...values).changes;
+}
+
+function upsertProvider(session: NormalizedSession) {
+  insertIgnore(
+    "INSERT OR IGNORE INTO providers (id, name, type) VALUES (?, ?, ?)",
+    [session.provider.id, session.provider.name, session.provider.type]
+  );
+}
+
+function upsertTool(session: NormalizedSession) {
+  insertIgnore(
+    "INSERT OR IGNORE INTO tools (id, provider_id, name) VALUES (?, ?, ?)",
+    [session.tool.id, session.provider.id, session.tool.name]
+  );
+}
+
+function providerForTool(toolId: string) {
+  return sqlite.prepare("SELECT provider_id FROM tools WHERE id = ?").get(toolId) as
+    | { provider_id: string }
+    | undefined;
+}
+
+function getModel(providerId: string, modelName: string) {
+  return sqlite
+    .prepare(
+      `SELECT id, input_token_price, output_token_price, cached_input_token_price, currency
+       FROM models WHERE provider_id = ? AND lower(name) = lower(?)`
+    )
+    .get(providerId, modelName) as
+    | {
+        id: string;
+        input_token_price: number | null;
+        output_token_price: number | null;
+        cached_input_token_price: number | null;
+        currency: string;
+      }
+    | undefined;
+}
+
+function ensureModel(providerId: string, modelName: string | null | undefined) {
+  const name = modelName?.trim() || "unknown";
+  const existing = getModel(providerId, name);
+  if (existing) return existing;
+
+  const id = stableId("model", [providerId, name]);
+  insertIgnore(
+    `INSERT OR IGNORE INTO models
+      (id, provider_id, name, input_token_price, output_token_price, cached_input_token_price, currency, raw_metadata)
+     VALUES (?, ?, ?, NULL, NULL, NULL, 'USD', ?)`,
+    [
+      id,
+      providerId,
+      name,
+      json({
+        note: "Observed during import. Add prices on the Pricing page to enable cost calculation."
+      })
+    ]
+  );
+  return getModel(providerId, name) ?? {
+    id,
+    input_token_price: null,
+    output_token_price: null,
+    cached_input_token_price: null,
+    currency: "USD"
+  };
+}
+
+function findProjectRoot(startFile: string) {
+  let current = fs.statSync(startFile).isDirectory() ? startFile : path.dirname(startFile);
+  const home = process.env.HOME ?? path.parse(current).root;
+
+  while (current !== path.dirname(current)) {
+    if (
+      fs.existsSync(path.join(current, ".git")) ||
+      fs.existsSync(path.join(current, "package.json")) ||
+      fs.existsSync(path.join(current, "pyproject.toml")) ||
+      fs.existsSync(path.join(current, "Cargo.toml"))
+    ) {
+      return current;
+    }
+    if (current === home) break;
+    current = path.dirname(current);
+  }
+
+  return path.dirname(startFile);
+}
+
+function ensureProject(session: NormalizedSession) {
+  const projectPath = session.projectPath || findProjectRoot(session.sourceFile);
+  const resolved = path.resolve(projectPath);
+  const name = session.projectName || path.basename(resolved) || "Unknown project";
+  const id = stableId("project", [resolved]);
+  insertIgnore("INSERT OR IGNORE INTO projects (id, name, path) VALUES (?, ?, ?)", [
+    id,
+    name,
+    resolved
+  ]);
+  return id;
+}
+
+function normalizeTokens(interaction: NormalizedInteraction) {
+  const providedAnyToken = [
+    interaction.inputTokens,
+    interaction.outputTokens,
+    interaction.cacheReadTokens,
+    interaction.cacheWriteTokens,
+    interaction.reasoningTokens,
+    interaction.totalTokens
+  ].some((value) => value != null && value > 0);
+  const baseText = interaction.rawText || interaction.rawTextPreview || "";
+  const estimate = estimateTokensFromText(baseText).tokens;
+  const estimatedTokens = Boolean(interaction.estimatedTokens || !providedAnyToken);
+  let tokenConfidence = interaction.tokenConfidence ?? "unknown";
+
+  let inputTokens = interaction.inputTokens ?? 0;
+  let outputTokens = interaction.outputTokens ?? 0;
+  const cacheReadTokens = interaction.cacheReadTokens ?? 0;
+  const cacheWriteTokens = interaction.cacheWriteTokens ?? 0;
+  const reasoningTokens = interaction.reasoningTokens ?? 0;
+
+  if (!providedAnyToken && estimate > 0) {
+    if (interaction.role === "assistant") outputTokens = estimate;
+    else inputTokens = estimate;
+    if (tokenConfidence === "unknown") tokenConfidence = "low-confidence estimate";
+  }
+
+  const summed = inputTokens + outputTokens + cacheReadTokens + cacheWriteTokens + reasoningTokens;
+  const totalTokens = Math.max(interaction.totalTokens ?? 0, summed);
+
+  return {
+    inputTokens,
+    outputTokens,
+    cacheReadTokens,
+    cacheWriteTokens,
+    reasoningTokens,
+    totalTokens,
+    estimatedTokens,
+    tokenConfidence
+  };
+}
+
+export function importSessions(sessions: NormalizedSession[]): ImportSessionResult {
+  const warnings: string[] = [];
+  let sessionsImported = 0;
+  let interactionsImported = 0;
+  let toolCallsImported = 0;
+
+  const transaction = sqlite.transaction((records: NormalizedSession[]) => {
+    for (const session of records) {
+      upsertProvider(session);
+      upsertTool(session);
+      const providerId = providerForTool(session.tool.id)?.provider_id ?? session.provider.id;
+      const projectId = ensureProject(session);
+      const sessionSourceId = stableId("session-source", [
+        session.tool.id,
+        session.sourceFile,
+        session.externalId ?? session.title
+      ]);
+      const sessionId = stableId("session", [sessionSourceId]);
+      const insertedSession = insertIgnore(
+        `INSERT OR IGNORE INTO sessions
+          (id, source_id, tool_id, project_id, started_at, ended_at, title, source_file, raw_metadata)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        [
+          sessionId,
+          sessionSourceId,
+          session.tool.id,
+          projectId,
+          session.startedAt?.getTime() ?? null,
+          session.endedAt?.getTime() ?? null,
+          session.title ?? null,
+          session.sourceFile,
+          json(session.rawMetadata)
+        ]
+      );
+      sessionsImported += insertedSession;
+
+      for (const interaction of session.interactions) {
+        const model = ensureModel(providerId, interaction.modelName);
+        const tokens = normalizeTokens(interaction);
+        const cost = calculateInteractionCost(tokens, {
+          inputTokenPrice: model.input_token_price,
+          outputTokenPrice: model.output_token_price,
+          cachedInputTokenPrice: model.cached_input_token_price,
+          currency: model.currency
+        });
+        const interactionSourceId = stableId("interaction-source", [
+          sessionSourceId,
+          interaction.externalId,
+          interaction.timestamp?.getTime(),
+          interaction.role,
+          interaction.rawTextPreview
+        ]);
+        const interactionId = stableId("interaction", [interactionSourceId]);
+        const insertedInteraction = insertIgnore(
+          `INSERT OR IGNORE INTO interactions
+            (id, source_id, session_id, timestamp, role, model_id, input_tokens, output_tokens,
+             cache_read_tokens, cache_write_tokens, reasoning_tokens, total_tokens, estimated_tokens,
+             token_confidence, cost, cost_estimated, latency_ms, raw_text_preview, raw_text, raw_metadata)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+          [
+            interactionId,
+            interactionSourceId,
+            sessionId,
+            interaction.timestamp?.getTime() ?? session.startedAt?.getTime() ?? null,
+            interaction.role,
+            model.id,
+            tokens.inputTokens,
+            tokens.outputTokens,
+            tokens.cacheReadTokens,
+            tokens.cacheWriteTokens,
+            tokens.reasoningTokens,
+            tokens.totalTokens,
+            tokens.estimatedTokens ? 1 : 0,
+            tokens.tokenConfidence,
+            cost.amount,
+            cost.status === "estimated" ? 1 : 0,
+            interaction.latencyMs ?? null,
+            previewText(interaction.rawTextPreview || interaction.rawText),
+            interaction.rawText ?? null,
+            json({
+              ...(interaction.rawMetadata ?? {}),
+              costStatus: cost.status,
+              costExplanation: cost.explanation
+            })
+          ]
+        );
+        interactionsImported += insertedInteraction;
+
+        for (const [index, toolCall] of (interaction.toolCalls ?? []).entries()) {
+          const toolCallId = stableId("toolcall", [
+            interactionId,
+            toolCall.externalId ?? index,
+            toolCall.name
+          ]);
+          toolCallsImported += insertIgnore(
+            `INSERT OR IGNORE INTO tool_calls
+              (id, interaction_id, name, status, duration_ms, raw_metadata)
+             VALUES (?, ?, ?, ?, ?, ?)`,
+            [
+              toolCallId,
+              interactionId,
+              toolCall.name,
+              toolCall.status ?? null,
+              toolCall.durationMs ?? null,
+              json(toolCall.rawMetadata)
+            ]
+          );
+        }
+      }
+    }
+  });
+
+  transaction(sessions);
+
+  return {
+    sessionsImported,
+    interactionsImported,
+    toolCallsImported,
+    warnings
+  };
+}

package/src/ingestion/scan.ts

@@ -0,0 +1,247 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { sqlite } from "@/src/db/client";
+import { getAppSettings } from "@/src/db/settings";
+import { hashContent, stableId } from "@/src/lib/ids";
+import { adapters } from "./adapters";
+import { discoverFiles, expandHome, getDefaultSearchRoots } from "./discovery";
+import { importSessions } from "./persist";
+import { FileCandidate, IngestionAdapter } from "./types";
+
+export type RunScanOptions = {
+  folders?: string[];
+  force?: boolean;
+  includeDefaults?: boolean;
+  storeRawMessageContent?: boolean;
+};
+
+export type RunScanResult = {
+  scanRunId: string;
+  filesScanned: number;
+  recordsImported: number;
+  warnings: string[];
+  errors: string[];
+};
+
+function json(value: unknown) {
+  return JSON.stringify(value ?? null);
+}
+
+function hasImportedFile(file: FileCandidate) {
+  if (!file.hash) return false;
+  const row = sqlite
+    .prepare(
+      "SELECT id FROM scan_files WHERE path = ? AND file_hash = ? AND status = 'imported' LIMIT 1"
+    )
+    .get(file.path, file.hash);
+  return Boolean(row);
+}
+
+async function hashFile(file: FileCandidate): Promise<FileCandidate> {
+  const content = await fs.readFile(file.path);
+  return {
+    ...file,
+    hash: hashContent(content)
+  };
+}
+
+async function selectAdapter(file: FileCandidate) {
+  const matches: Array<{ adapter: IngestionAdapter; confidence: number; reason?: string }> = [];
+  const warnings: string[] = [];
+
+  for (const adapter of adapters) {
+    try {
+      const result = await adapter.detect(file);
+      if (result.detected) {
+        matches.push({
+          adapter,
+          confidence: result.confidence,
+          reason: result.reason
+        });
+      }
+    } catch (error) {
+      warnings.push(
+        `${adapter.displayName} detection failed: ${
+          error instanceof Error ? error.message : "Unknown error"
+        }`
+      );
+    }
+  }
+
+  matches.sort((a, b) => b.confidence - a.confidence);
+  return { selected: matches[0] ?? null, warnings };
+}
+
+function insertScanFile(args: {
+  scanRunId: string;
+  file: FileCandidate;
+  parser: string | null;
+  status: string;
+  recordsImported: number;
+  warnings: string[];
+  errors: string[];
+  rawMetadata?: Record<string, unknown>;
+}) {
+  const id = stableId("scanfile", [
+    args.scanRunId,
+    args.file.path,
+    args.file.hash,
+    args.status,
+    args.parser
+  ]);
+  sqlite
+    .prepare(
+      `INSERT INTO scan_files
+       (id, scan_run_id, path, modified_time, size_bytes, file_hash, parser, status,
+        records_imported, warnings, errors, raw_metadata)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    )
+    .run(
+      id,
+      args.scanRunId,
+      args.file.path,
+      args.file.modifiedTime?.getTime() ?? null,
+      args.file.sizeBytes,
+      args.file.hash ?? null,
+      args.parser,
+      args.status,
+      args.recordsImported,
+      json(args.warnings),
+      json(args.errors),
+      json(args.rawMetadata ?? {})
+    );
+}
+
+export async function runScan(options: RunScanOptions = {}): Promise<RunScanResult> {
+  const settings = getAppSettings();
+  const explicitFolders = options.folders ?? [];
+  const roots =
+    options.includeDefaults === false
+      ? explicitFolders.map((folder) => path.resolve(expandHome(folder)))
+      : await getDefaultSearchRoots([...settings.customFolders, ...explicitFolders]);
+  const candidates = await discoverFiles(roots);
+  const startedAt = new Date();
+  const scanRunId = stableId("scan", [startedAt.getTime(), roots.join("|")]);
+  const allWarnings: string[] = [];
+  const allErrors: string[] = [];
+  let recordsImported = 0;
+  let filesScanned = 0;
+
+  sqlite
+    .prepare(
+      "INSERT INTO scan_runs (id, started_at, warnings, errors) VALUES (?, ?, '[]', '[]')"
+    )
+    .run(scanRunId, startedAt.getTime());
+
+  for (const candidate of candidates) {
+    filesScanned += 1;
+    let file = candidate;
+    const warnings: string[] = [];
+    const errors: string[] = [];
+
+    try {
+      file = await hashFile(candidate);
+      if (!options.force && hasImportedFile(file)) {
+        insertScanFile({
+          scanRunId,
+          file,
+          parser: null,
+          status: "skipped_duplicate",
+          recordsImported: 0,
+          warnings: ["File hash already imported. Use force rescan to parse again."],
+          errors: []
+        });
+        continue;
+      }
+
+      const adapterChoice = await selectAdapter(file);
+      warnings.push(...adapterChoice.warnings);
+
+      if (!adapterChoice.selected) {
+        insertScanFile({
+          scanRunId,
+          file,
+          parser: null,
+          status: "skipped_unknown",
+          recordsImported: 0,
+          warnings,
+          errors: ["No parser detected a compatible format."]
+        });
+        continue;
+      }
+
+      const parseResult = await adapterChoice.selected.adapter.parse(file, {
+        storeRawMessageContent:
+          options.storeRawMessageContent ?? settings.storeRawMessageContent
+      });
+      warnings.push(...parseResult.warnings);
+      errors.push(...parseResult.errors);
+      const importResult = importSessions(parseResult.sessions);
+      const tokenConfidence = parseResult.sessions
+        .flatMap((session) => session.interactions)
+        .reduce<Record<string, number>>((summary, interaction) => {
+          const key = interaction.tokenConfidence ?? "unknown";
+          summary[key] = (summary[key] ?? 0) + 1;
+          return summary;
+        }, {});
+      warnings.push(...importResult.warnings);
+      recordsImported += importResult.interactionsImported;
+
+      insertScanFile({
+        scanRunId,
+        file,
+        parser: adapterChoice.selected.adapter.id,
+        status: errors.length ? "imported_with_errors" : "imported",
+        recordsImported: importResult.interactionsImported,
+        warnings,
+        errors,
+        rawMetadata: {
+          confidence: adapterChoice.selected.confidence,
+          reason: adapterChoice.selected.reason,
+          tokenConfidence,
+          sessionsParsed: parseResult.sessions.length,
+          sessionsImported: importResult.sessionsImported,
+          toolCallsImported: importResult.toolCallsImported
+        }
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown scan error";
+      errors.push(message);
+      insertScanFile({
+        scanRunId,
+        file,
+        parser: null,
+        status: "failed",
+        recordsImported: 0,
+        warnings,
+        errors
+      });
+    }
+
+    allWarnings.push(...warnings.map((warning) => `${candidate.path}: ${warning}`));
+    allErrors.push(...errors.map((error) => `${candidate.path}: ${error}`));
+  }
+
+  sqlite
+    .prepare(
+      `UPDATE scan_runs
+       SET completed_at = ?, files_scanned = ?, records_imported = ?, warnings = ?, errors = ?
+       WHERE id = ?`
+    )
+    .run(
+      Date.now(),
+      filesScanned,
+      recordsImported,
+      json(allWarnings),
+      json(allErrors),
+      scanRunId
+    );
+
+  return {
+    scanRunId,
+    filesScanned,
+    recordsImported,
+    warnings: allWarnings,
+    errors: allErrors
+  };
+}

package/src/ingestion/types.ts

@@ -0,0 +1,78 @@
+export type FileCandidate = {
+  path: string;
+  modifiedTime: Date | null;
+  sizeBytes: number;
+  hash?: string;
+};
+
+export type DetectionResult = {
+  detected: boolean;
+  confidence: number;
+  reason?: string;
+};
+
+export type NormalizedToolCall = {
+  externalId?: string;
+  name: string;
+  status?: string | null;
+  durationMs?: number | null;
+  rawMetadata?: Record<string, unknown>;
+};
+
+export type NormalizedInteraction = {
+  externalId?: string;
+  timestamp?: Date | null;
+  role: "user" | "assistant" | "system" | "tool" | "unknown";
+  modelName?: string | null;
+  inputTokens?: number | null;
+  outputTokens?: number | null;
+  cacheReadTokens?: number | null;
+  cacheWriteTokens?: number | null;
+  reasoningTokens?: number | null;
+  totalTokens?: number | null;
+  estimatedTokens?: boolean;
+  tokenConfidence?: "exact" | "high-confidence estimate" | "low-confidence estimate" | "unknown";
+  latencyMs?: number | null;
+  rawText?: string | null;
+  rawTextPreview?: string | null;
+  rawMetadata?: Record<string, unknown>;
+  toolCalls?: NormalizedToolCall[];
+};
+
+export type NormalizedSession = {
+  externalId?: string;
+  provider: {
+    id: string;
+    name: string;
+    type: string;
+  };
+  tool: {
+    id: string;
+    name: string;
+  };
+  projectPath?: string | null;
+  projectName?: string | null;
+  startedAt?: Date | null;
+  endedAt?: Date | null;
+  title?: string | null;
+  sourceFile: string;
+  rawMetadata?: Record<string, unknown>;
+  interactions: NormalizedInteraction[];
+};
+
+export type AdapterParseResult = {
+  sessions: NormalizedSession[];
+  warnings: string[];
+  errors: string[];
+};
+
+export type ParseContext = {
+  storeRawMessageContent: boolean;
+};
+
+export interface IngestionAdapter {
+  id: string;
+  displayName: string;
+  detect(file: FileCandidate): Promise<DetectionResult>;
+  parse(file: FileCandidate, context: ParseContext): Promise<AdapterParseResult>;
+}