tokentrace 0.1.0

package/src/ingestion/adapters/generic-records.ts

@@ -0,0 +1,178 @@
+import { FileCandidate, NormalizedInteraction, NormalizedSession } from "../types";
+import {
+  asObject,
+  extractModel,
+  extractRole,
+  extractText,
+  extractToolCalls,
+  extractUsage,
+  firstString,
+  normalizeInteraction,
+  parseTimestamp,
+  sessionNameFromFile
+} from "./helpers";
+
+type BuildOptions = {
+  file: FileCandidate;
+  records: Record<string, unknown>[];
+  provider: NormalizedSession["provider"];
+  tool: NormalizedSession["tool"];
+  storeRawMessageContent: boolean;
+  defaultSessionId?: string;
+  defaultProjectPath?: string | null;
+};
+
+function interactionExternalId(record: Record<string, unknown>, index: number) {
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+  const response = asObject(record.response) ?? asObject(payload?.response);
+  return (
+    firstString(
+      record.uuid,
+      record.id,
+      record.message_id,
+      record.messageId,
+      message?.id,
+      payload?.id,
+      response?.id
+    ) ?? `${index}`
+  );
+}
+
+function sessionExternalId(record: Record<string, unknown>, fallback: string) {
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+  return (
+    firstString(
+      record.session_id,
+      record.sessionId,
+      record.conversation_id,
+      record.conversationId,
+      record.thread_id,
+      record.threadId,
+      message?.session_id,
+      message?.sessionId,
+      payload?.session_id,
+      payload?.sessionId,
+      payload?.conversation_id
+    ) ?? fallback
+  );
+}
+
+function projectPathFromRecord(record: Record<string, unknown>) {
+  const payload = asObject(record.payload);
+  return firstString(
+    record.cwd,
+    record.project_path,
+    record.projectPath,
+    record.repository,
+    payload?.cwd,
+    payload?.project_path,
+    payload?.projectPath
+  );
+}
+
+function titleFromRecord(record: Record<string, unknown>) {
+  const payload = asObject(record.payload);
+  return firstString(record.title, record.summary, payload?.title, payload?.summary);
+}
+
+function hasUsage(usage: ReturnType<typeof extractUsage>) {
+  return Object.values(usage).some((value) => typeof value === "number" && value > 0);
+}
+
+function shouldKeepInteraction(record: Record<string, unknown>) {
+  const usage = extractUsage(record);
+  return Boolean(
+    extractText(record) ||
+      hasUsage(usage) ||
+      extractModel(record) ||
+      extractToolCalls(record).length ||
+      extractRole(record) !== "unknown"
+  );
+}
+
+function tuneRole(interaction: NormalizedInteraction): NormalizedInteraction {
+  if (interaction.role !== "unknown") return interaction;
+  if ((interaction.outputTokens ?? 0) > 0 || (interaction.reasoningTokens ?? 0) > 0) {
+    return { ...interaction, role: "assistant" };
+  }
+  if ((interaction.inputTokens ?? 0) > 0) {
+    return { ...interaction, role: "user" };
+  }
+  return interaction;
+}
+
+export function buildSessionsFromRecords(options: BuildOptions): NormalizedSession[] {
+  const fallbackSessionId = options.defaultSessionId ?? sessionNameFromFile(options.file.path);
+  const grouped = new Map<
+    string,
+    {
+      interactions: NormalizedInteraction[];
+      projectPath: string | null;
+      title: string | null;
+      timestamps: Date[];
+      metadata: Record<string, unknown>[];
+    }
+  >();
+
+  options.records.forEach((record, index) => {
+    const sessionId = sessionExternalId(record, fallbackSessionId);
+    const group =
+      grouped.get(sessionId) ??
+      {
+        interactions: [],
+        projectPath: options.defaultProjectPath ?? null,
+        title: null,
+        timestamps: [],
+        metadata: []
+      };
+
+    group.projectPath = group.projectPath ?? projectPathFromRecord(record);
+    group.title = group.title ?? titleFromRecord(record);
+    group.metadata.push(record);
+
+    const timestamp = parseTimestamp(
+      record.timestamp,
+      record.created_at,
+      record.createdAt,
+      record.time,
+      record.ts
+    );
+    if (timestamp) group.timestamps.push(timestamp);
+
+    if (shouldKeepInteraction(record)) {
+      const interaction = tuneRole(
+        normalizeInteraction(
+          record,
+          interactionExternalId(record, index),
+          options.storeRawMessageContent
+        )
+      );
+      group.interactions.push(interaction);
+    }
+
+    grouped.set(sessionId, group);
+  });
+
+  return Array.from(grouped.entries())
+    .filter(([, group]) => group.interactions.length > 0)
+    .map(([externalId, group]) => {
+      const sorted = [...group.timestamps].sort((a, b) => a.getTime() - b.getTime());
+      return {
+        externalId,
+        provider: options.provider,
+        tool: options.tool,
+        projectPath: group.projectPath,
+        projectName: group.projectPath ? undefined : "Unknown project",
+        startedAt: sorted[0] ?? options.file.modifiedTime,
+        endedAt: sorted[sorted.length - 1] ?? options.file.modifiedTime,
+        title: group.title ?? sessionNameFromFile(options.file.path),
+        sourceFile: options.file.path,
+        rawMetadata: {
+          parserInputRecords: group.metadata.length
+        },
+        interactions: group.interactions
+      };
+    });
+}

package/src/ingestion/adapters/helpers.ts

@@ -0,0 +1,309 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { NormalizedInteraction, NormalizedToolCall } from "../types";
+import { previewText } from "@/src/lib/token-estimator";
+
+export async function readTextSample(filePath: string, bytes = 64_000) {
+  const handle = await fs.open(filePath, "r");
+  try {
+    const buffer = Buffer.alloc(bytes);
+    const { bytesRead } = await handle.read(buffer, 0, bytes, 0);
+    return buffer.subarray(0, bytesRead).toString("utf8");
+  } finally {
+    await handle.close();
+  }
+}
+
+export async function readFileText(filePath: string, maxBytes = 25 * 1024 * 1024) {
+  const stat = await fs.stat(filePath);
+  if (stat.size > maxBytes) {
+    throw new Error(`File is larger than ${Math.round(maxBytes / 1024 / 1024)} MB.`);
+  }
+  return fs.readFile(filePath, "utf8");
+}
+
+export function safeJsonParse(value: string): unknown | null {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+}
+
+export function asObject(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+export function asArray(value: unknown): unknown[] {
+  return Array.isArray(value) ? value : [];
+}
+
+export function firstString(...values: unknown[]) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return null;
+}
+
+export function firstNumber(...values: unknown[]) {
+  for (const value of values) {
+    if (typeof value === "number" && Number.isFinite(value)) return Math.max(0, Math.round(value));
+    if (typeof value === "string" && value.trim() && Number.isFinite(Number(value))) {
+      return Math.max(0, Math.round(Number(value)));
+    }
+  }
+  return null;
+}
+
+export function parseTimestamp(...values: unknown[]): Date | null {
+  for (const value of values) {
+    if (value instanceof Date && !Number.isNaN(value.getTime())) return value;
+    if (typeof value === "number" && Number.isFinite(value)) {
+      const ms = value > 10_000_000_000 ? value : value * 1000;
+      const date = new Date(ms);
+      if (!Number.isNaN(date.getTime())) return date;
+    }
+    if (typeof value === "string" && value.trim()) {
+      const date = new Date(value);
+      if (!Number.isNaN(date.getTime())) return date;
+    }
+  }
+  return null;
+}
+
+function textFromContent(value: unknown): string | null {
+  if (typeof value === "string") return value;
+  if (Array.isArray(value)) {
+    return value
+      .map((item) => {
+        if (typeof item === "string") return item;
+        const object = asObject(item);
+        return firstString(object?.text, object?.content, object?.input, object?.output);
+      })
+      .filter(Boolean)
+      .join("\n");
+  }
+  const object = asObject(value);
+  if (!object) return null;
+  return firstString(object.text, object.content, object.value, object.input, object.output);
+}
+
+export function extractText(record: Record<string, unknown>): string | null {
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+  const response = asObject(record.response);
+  return (
+    textFromContent(record.text) ??
+    textFromContent(record.content) ??
+    textFromContent(record.prompt) ??
+    textFromContent(record.completion) ??
+    textFromContent(record.output) ??
+    textFromContent(message?.content) ??
+    textFromContent(payload?.content) ??
+    textFromContent(payload?.message) ??
+    textFromContent(response?.content) ??
+    null
+  );
+}
+
+export function extractUsage(record: Record<string, unknown>) {
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+  const response = asObject(record.response) ?? asObject(payload?.response);
+  const usage =
+    asObject(record.usage) ??
+    asObject(message?.usage) ??
+    asObject(payload?.usage) ??
+    asObject(response?.usage) ??
+    asObject(record.token_usage) ??
+    asObject(record.tokens);
+  const inputDetails =
+    asObject(usage?.input_tokens_details) ??
+    asObject(usage?.prompt_tokens_details) ??
+    asObject(usage?.cache);
+  const outputDetails = asObject(usage?.output_tokens_details) ?? asObject(usage?.completion_tokens_details);
+
+  return {
+    inputTokens: firstNumber(
+      usage?.input_tokens,
+      usage?.prompt_tokens,
+      usage?.inputTokens,
+      usage?.promptTokens,
+      record.input_tokens,
+      record.prompt_tokens
+    ),
+    outputTokens: firstNumber(
+      usage?.output_tokens,
+      usage?.completion_tokens,
+      usage?.outputTokens,
+      usage?.completionTokens,
+      record.output_tokens,
+      record.completion_tokens
+    ),
+    cacheReadTokens: firstNumber(
+      usage?.cache_read_input_tokens,
+      usage?.cached_input_tokens,
+      inputDetails?.cached_tokens,
+      inputDetails?.cache_read_tokens,
+      record.cache_read_tokens
+    ),
+    cacheWriteTokens: firstNumber(
+      usage?.cache_creation_input_tokens,
+      usage?.cache_write_input_tokens,
+      inputDetails?.cache_creation_tokens,
+      inputDetails?.cache_write_tokens,
+      record.cache_write_tokens
+    ),
+    reasoningTokens: firstNumber(
+      usage?.reasoning_tokens,
+      outputDetails?.reasoning_tokens,
+      record.reasoning_tokens
+    ),
+    totalTokens: firstNumber(
+      usage?.total_tokens,
+      usage?.totalTokens,
+      usage?.tokens,
+      record.total_tokens,
+      record.totalTokens
+    )
+  };
+}
+
+export function extractModel(record: Record<string, unknown>) {
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+  const response = asObject(record.response) ?? asObject(payload?.response);
+  return firstString(
+    record.model,
+    record.model_name,
+    record.modelName,
+    message?.model,
+    payload?.model,
+    response?.model
+  );
+}
+
+export function extractRole(record: Record<string, unknown>): NormalizedInteraction["role"] {
+  const message = asObject(record.message);
+  const raw = firstString(record.role, record.type, message?.role);
+  if (!raw) return "unknown";
+  const normalized = raw.toLowerCase();
+  if (normalized.includes("assistant") || normalized.includes("completion")) return "assistant";
+  if (normalized.includes("user") || normalized.includes("prompt")) return "user";
+  if (normalized.includes("system")) return "system";
+  if (normalized.includes("tool")) return "tool";
+  return "unknown";
+}
+
+export function extractToolCalls(record: Record<string, unknown>): NormalizedToolCall[] {
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+  const possible = [
+    ...asArray(record.tool_calls),
+    ...asArray(record.toolCalls),
+    ...asArray(message?.tool_calls),
+    ...asArray(payload?.tool_calls),
+    ...asArray(payload?.toolCalls)
+  ];
+
+  const calls: NormalizedToolCall[] = [];
+  possible.forEach((item, index) => {
+    const object = asObject(item);
+    if (!object) return;
+    const name = firstString(object.name, object.tool, object.function_name, asObject(object.function)?.name);
+    if (!name) return;
+    calls.push({
+      externalId: firstString(object.id, object.call_id) ?? `${index}`,
+      name,
+      status: firstString(object.status, object.state),
+      durationMs: firstNumber(object.duration_ms, object.durationMs),
+      rawMetadata: object
+    });
+  });
+  return calls;
+}
+
+const sensitiveMetadataKeys = new Set([
+  "content",
+  "text",
+  "prompt",
+  "completion",
+  "input",
+  "output",
+  "arguments",
+  "message"
+]);
+
+export function sanitizeMetadata(value: unknown, depth = 0): unknown {
+  if (value == null) return value;
+  if (typeof value === "string") return previewText(value, 160);
+  if (typeof value !== "object") return value;
+  if (depth > 3) return "[nested metadata]";
+  if (Array.isArray(value)) {
+    return value.slice(0, 5).map((item) => sanitizeMetadata(item, depth + 1));
+  }
+
+  const object = value as Record<string, unknown>;
+  return Object.fromEntries(
+    Object.entries(object).map(([key, item]) => {
+      if (sensitiveMetadataKeys.has(key.toLowerCase())) {
+        return [key, "[redacted: raw storage disabled]"];
+      }
+      return [key, sanitizeMetadata(item, depth + 1)];
+    })
+  );
+}
+
+export function normalizeInteraction(
+  record: Record<string, unknown>,
+  externalId: string,
+  storeRawMessageContent: boolean
+): NormalizedInteraction {
+  const text = extractText(record);
+  const usage = extractUsage(record);
+  const hasUsage = Object.values(usage).some((value) => value != null && value > 0);
+  const message = asObject(record.message);
+  const payload = asObject(record.payload);
+
+  return {
+    externalId,
+    timestamp: parseTimestamp(record.timestamp, record.created_at, record.createdAt, record.time, record.ts),
+    role: extractRole(record),
+    modelName: extractModel(record),
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    cacheReadTokens: usage.cacheReadTokens,
+    cacheWriteTokens: usage.cacheWriteTokens,
+    reasoningTokens: usage.reasoningTokens,
+    totalTokens: usage.totalTokens,
+    estimatedTokens: false,
+    tokenConfidence: hasUsage ? "exact" : text ? "high-confidence estimate" : "unknown",
+    latencyMs: firstNumber(record.latency_ms, record.latencyMs, payload?.latency_ms, message?.latency_ms),
+    rawText: storeRawMessageContent ? text : null,
+    rawTextPreview: previewText(text),
+    rawMetadata: storeRawMessageContent
+      ? record
+      : (sanitizeMetadata(record) as Record<string, unknown>),
+    toolCalls: extractToolCalls(record).map((toolCall) => ({
+      ...toolCall,
+      rawMetadata: storeRawMessageContent
+        ? toolCall.rawMetadata
+        : (sanitizeMetadata(toolCall.rawMetadata) as Record<string, unknown>)
+    }))
+  };
+}
+
+export function sessionNameFromFile(filePath: string) {
+  return path.basename(filePath).replace(/\.(jsonl|json|log|txt|md)$/i, "");
+}
+
+export function fileLooksLikeJsonl(sample: string) {
+  const lines = sample
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .slice(0, 5);
+  return lines.length > 0 && lines.every((line) => safeJsonParse(line) !== null);
+}

package/src/ingestion/adapters/index.ts

@@ -0,0 +1,15 @@
+import { claudeCodeAdapter } from "./claude-code";
+import { codexCliAdapter } from "./codex-cli";
+import { genericJsonAdapter } from "./generic-json";
+import { genericJsonlAdapter } from "./generic-jsonl";
+import { genericLogAdapter } from "./generic-log";
+
+export const adapters = [
+  claudeCodeAdapter,
+  codexCliAdapter,
+  genericJsonlAdapter,
+  genericJsonAdapter,
+  genericLogAdapter
+];
+
+export type AdapterId = (typeof adapters)[number]["id"];

package/src/ingestion/discovery.ts

@@ -0,0 +1,130 @@
+import fs from "node:fs/promises";
+import type { Dirent } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { FileCandidate } from "./types";
+
+const supportedExtensions = new Set([
+  ".jsonl",
+  ".json",
+  ".log",
+  ".txt",
+  ".md",
+  ".db",
+  ".sqlite",
+  ".sqlite3"
+]);
+
+const skippedDirectories = new Set([
+  "node_modules",
+  ".git",
+  ".next",
+  "dist",
+  "out",
+  "build",
+  "coverage",
+  "Library",
+  "Applications"
+]);
+
+export function expandHome(input: string) {
+  if (input === "~") return os.homedir();
+  if (input.startsWith("~/")) return path.join(os.homedir(), input.slice(2));
+  return input;
+}
+
+async function exists(target: string) {
+  try {
+    await fs.access(target);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function getDefaultSearchRoots(customFolders: string[] = []) {
+  const home = os.homedir();
+  const workingDirectory = process.env.TOKENTRACE_WORKDIR ?? process.cwd();
+  const appDataDir = process.env.TOKENTRACE_APP_DATA_DIR;
+  const candidates = [
+    path.join(home, ".claude"),
+    path.join(home, ".config", "claude"),
+    path.join(home, ".codex"),
+    path.join(home, ".config", "codex"),
+    path.join(home, ".openai"),
+    path.join(workingDirectory, ".claude"),
+    path.join(workingDirectory, ".codex"),
+    path.join(workingDirectory, ".openai"),
+    path.join(workingDirectory, ".ai"),
+    ...(appDataDir ? [path.join(appDataDir, "wrapper-runs")] : []),
+    ...customFolders.map(expandHome)
+  ];
+
+  const unique = Array.from(new Set(candidates.map((candidate) => path.resolve(candidate))));
+  const present: string[] = [];
+  for (const candidate of unique) {
+    if (await exists(candidate)) present.push(candidate);
+  }
+  return present;
+}
+
+async function walkDirectory(root: string, depth: number, maxDepth: number, results: FileCandidate[]) {
+  if (depth > maxDepth) return;
+  let entries: Dirent[];
+  try {
+    entries = await fs.readdir(root, { withFileTypes: true });
+  } catch {
+    return;
+  }
+
+  for (const entry of entries) {
+    const fullPath = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      if (skippedDirectories.has(entry.name)) continue;
+      await walkDirectory(fullPath, depth + 1, maxDepth, results);
+      continue;
+    }
+
+    if (!entry.isFile()) continue;
+    const extension = path.extname(entry.name).toLowerCase();
+    if (!supportedExtensions.has(extension)) continue;
+
+    try {
+      const stat = await fs.stat(fullPath);
+      if (stat.size <= 0 || stat.size > 25 * 1024 * 1024) continue;
+      results.push({
+        path: fullPath,
+        modifiedTime: stat.mtime,
+        sizeBytes: stat.size
+      });
+    } catch {
+      continue;
+    }
+  }
+}
+
+export async function discoverFiles(roots: string[]) {
+  const results: FileCandidate[] = [];
+  for (const root of roots) {
+    const stat = await fs.stat(root).catch(() => null);
+    if (!stat) continue;
+    if (stat.isFile()) {
+      const extension = path.extname(root).toLowerCase();
+      if (supportedExtensions.has(extension)) {
+        results.push({
+          path: root,
+          modifiedTime: stat.mtime,
+          sizeBytes: stat.size
+        });
+      }
+      continue;
+    }
+    if (stat.isDirectory()) {
+      await walkDirectory(root, 0, 12, results);
+    }
+  }
+
+  const deduped = new Map<string, FileCandidate>();
+  for (const result of results) deduped.set(result.path, result);
+  return Array.from(deduped.values()).sort((a, b) => a.path.localeCompare(b.path));
+}