tokenmaxing 0.1.0

package/src/scanners/claude.js

@@ -0,0 +1,198 @@
+import fs from "node:fs";
+import path from "node:path";
+import {
+  addModelUsage,
+  addMonthlyTokens,
+  addSpend,
+  addTokens,
+  countJsonlEntries,
+  createEmptyReport,
+  fileExists,
+  findDirs,
+  normalizeClaudeUsage,
+  readJson,
+  registerTimestamp,
+  toDay,
+  toMonth,
+  tokenTotal,
+  walkFiles,
+} from "../utils.js";
+import { finalizeReport } from "../metrics.js";
+
+export async function scanClaude(options = {}) {
+  const claudeHome = options.claudeHome;
+  const report = createEmptyReport("Claude Code", "claude", claudeHome);
+
+  if (!fileExists(claudeHome)) {
+    report.notes.push("Claude home not found.");
+    return finalizeReport(report);
+  }
+
+  report.available = true;
+  report.projectScanRoot = options.scanRoot;
+  report.projectConfigExamples = findProjectClaudeDirs(options.scanRoot, claudeHome, options.maxScanDepth);
+  report.projectConfigDirs = report.projectConfigExamples.length;
+  report.historyEntries = countJsonlEntries(path.join(claudeHome, "history.jsonl"));
+
+  const projectRoot = path.join(claudeHome, "projects");
+  const sessionFiles = walkFiles(projectRoot, (_file, name) => name.endsWith(".jsonl"));
+  report.sessionFiles = sessionFiles.length;
+
+  const transcriptStats = scanClaudeTranscripts(sessionFiles);
+  report.dailyActivity = transcriptStats.dailyActivity;
+  report.hourlyActivity = transcriptStats.hourlyActivity;
+  report.totalMessages = transcriptStats.messageCount;
+  report.totalSessions = transcriptStats.sessionIds.size || sessionFiles.length;
+  for (const timestamp of transcriptStats.seenTimestamps) registerTimestamp(report, timestamp);
+
+  const stats = readJson(path.join(claudeHome, "stats-cache.json"));
+  if (stats) {
+    applyClaudeStats(report, stats);
+    report.notes.push(`Used stats-cache.json last computed ${stats.lastComputedDate || "unknown date"}.`);
+  } else {
+    applyTranscriptUsage(report, transcriptStats);
+    report.notes.push("stats-cache.json not found; token usage inferred from transcripts.");
+  }
+
+  if (!report.totalMessages && transcriptStats.messageCount) report.totalMessages = transcriptStats.messageCount;
+  if (!report.totalSessions && transcriptStats.sessionIds.size) report.totalSessions = transcriptStats.sessionIds.size;
+  if (report.projectConfigDirs > 0) {
+    report.notes.push(
+      `Found ${report.projectConfigDirs} project .claude directories under ${report.projectScanRoot}; these are treated as project config, not usage transcripts.`,
+    );
+  }
+
+  return finalizeReport(report);
+}
+
+function findProjectClaudeDirs(scanRoot, claudeHome, maxScanDepth) {
+  if (!scanRoot || !fileExists(scanRoot)) return [];
+  const globalRoot = path.resolve(claudeHome);
+  const dirs = findDirs(scanRoot, ".claude", { maxDepth: maxScanDepth });
+  return dirs.filter((dir) => path.resolve(dir) !== globalRoot);
+}
+
+function scanClaudeTranscripts(files) {
+  const result = {
+    dailyActivity: {},
+    hourlyActivity: Array.from({ length: 24 }, () => 0),
+    messageCount: 0,
+    sessionIds: new Set(),
+    seenTimestamps: [],
+    usageEvents: [],
+  };
+
+  for (const file of files) {
+    for (const line of readJsonLines(file)) {
+      const timestamp = line.timestamp;
+      if (timestamp) result.seenTimestamps.push(timestamp);
+      if (line.sessionId) result.sessionIds.add(line.sessionId);
+      if (line.type === "user" || line.type === "assistant") {
+        result.messageCount += 1;
+        const date = timestamp ? new Date(timestamp) : null;
+        if (date && Number.isFinite(date.getTime())) {
+          result.hourlyActivity[date.getHours()] += 1;
+          const day = toDay(date);
+          if (day) result.dailyActivity[day] = (result.dailyActivity[day] || 0) + 1;
+        }
+      }
+
+      const usage = line.message?.usage;
+      const model = line.message?.model || line.model;
+      if (usage && model) {
+        result.usageEvents.push({
+          timestamp,
+          model,
+          tokens: normalizeClaudeUsage(usage),
+        });
+      }
+    }
+  }
+
+  return result;
+}
+
+function applyClaudeStats(report, stats) {
+  report.totalSessions = Number(stats.totalSessions || report.totalSessions || 0);
+  report.totalMessages = Number(stats.totalMessages || report.totalMessages || 0);
+  if (stats.firstSessionDate) registerTimestamp(report, stats.firstSessionDate);
+
+  const modelTotals = {};
+  for (const [model, usage] of Object.entries(stats.modelUsage || {})) {
+    const tokens = normalizeClaudeUsage(usage);
+    const costUSD = Number(usage.costUSD || 0);
+    modelTotals[model] = tokenTotal(tokens);
+    addTokens(report.tokens, tokens);
+    addModelUsage(report, model, tokens, { costUSD });
+  }
+
+  for (const day of stats.dailyModelTokens || []) {
+    const month = day.date ? day.date.slice(0, 7) : null;
+    if (!month || !day.tokensByModel) continue;
+    for (const [model, tokensForDay] of Object.entries(day.tokensByModel)) {
+      const totalForModel = modelTotals[model] || 0;
+      const modelUsage = stats.modelUsage?.[model] || {};
+      const cost = totalForModel > 0 ? (Number(modelUsage.costUSD || 0) * Number(tokensForDay || 0)) / totalForModel : 0;
+      addSpend(report, month, cost);
+      addMonthlyTokens(report, month, { total: Number(tokensForDay || 0) });
+    }
+  }
+
+  if (!Object.keys(report.monthlySpendUSD).length) {
+    const totalCost = Object.values(stats.modelUsage || {}).reduce((sum, usage) => sum + Number(usage.costUSD || 0), 0);
+    if (totalCost > 0) {
+      report.knownSpendUSD += totalCost;
+      report.notes.push("Claude spend total found, but daily model tokens were unavailable for monthly allocation.");
+    }
+  }
+
+  mergeStatsDailyMessages(report, stats.dailyActivity || []);
+}
+
+// Transcripts older than the cleanup window are pruned from disk, but
+// stats-cache.json keeps recorded per-day message counts for the full history.
+// Merge those recorded counts in for days the transcripts no longer cover.
+function mergeStatsDailyMessages(report, statDays) {
+  let merged = 0;
+  for (const day of statDays) {
+    const recorded = Number(day.messageCount);
+    if (!day.date || !recorded) continue;
+    if (!report.dailyActivity[day.date]) merged += 1;
+    report.dailyActivity[day.date] = Math.max(report.dailyActivity[day.date] || 0, recorded);
+  }
+  if (merged) {
+    report.notes.push(
+      `Added ${merged} days of message counts recorded in stats-cache (transcripts pruned from disk).`,
+    );
+  }
+}
+
+function applyTranscriptUsage(report, transcriptStats) {
+  for (const event of transcriptStats.usageEvents) {
+    addTokens(report.tokens, event.tokens);
+    addModelUsage(report, event.model, event.tokens);
+    addMonthlyTokens(report, toMonth(event.timestamp), event.tokens);
+  }
+  report.unknownSpend = true;
+}
+
+function readJsonLines(file) {
+  let text;
+  try {
+    text = fs.readFileSync(file, "utf8");
+  } catch {
+    return [];
+  }
+
+  const parsed = [];
+  for (const line of text.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      parsed.push(JSON.parse(line));
+    } catch {
+      // Ignore corrupt or partial lines.
+    }
+  }
+  return parsed;
+}
+

package/src/scanners/codex.js

@@ -0,0 +1,103 @@
+import fs from "node:fs";
+import path from "node:path";
+import {
+  addModelUsage,
+  addMonthlyTokens,
+  addTokens,
+  countJsonlEntries,
+  createEmptyReport,
+  diffTokens,
+  fileExists,
+  normalizeCodexUsage,
+  registerMessageEvent,
+  registerTimestamp,
+  toMonth,
+  tokenTotal,
+  walkFiles,
+} from "../utils.js";
+import { finalizeReport } from "../metrics.js";
+
+export async function scanCodex(options = {}) {
+  const codexHome = options.codexHome;
+  const report = createEmptyReport("Codex", "codex", codexHome);
+
+  if (!fileExists(codexHome)) {
+    report.notes.push("Codex home not found.");
+    return finalizeReport(report);
+  }
+
+  report.available = true;
+  report.historyEntries = countJsonlEntries(path.join(codexHome, "history.jsonl"));
+
+  const sessionRoot = path.join(codexHome, "sessions");
+  const sessionFiles = walkFiles(sessionRoot, (_file, name) => name.endsWith(".jsonl"));
+  report.sessionFiles = sessionFiles.length;
+  report.totalSessions = sessionFiles.length;
+  report.unknownSpend = true;
+
+  for (const file of sessionFiles) {
+    scanCodexSessionFile(file, report);
+  }
+
+  if (!sessionFiles.length) {
+    report.notes.push("No Codex session JSONL files found.");
+  } else {
+    report.notes.push("Spend is not estimated for Codex because local files do not include dollar prices.");
+  }
+
+  return finalizeReport(report);
+}
+
+function scanCodexSessionFile(file, report) {
+  let currentModel = "unknown";
+  let previousUsage = null;
+
+  for (const line of readJsonLines(file)) {
+    if (line.timestamp) {
+      registerTimestamp(report, line.timestamp);
+    }
+
+    const payload = line.payload || {};
+    if (payload.model) currentModel = payload.model;
+    if (payload.collaboration_mode?.settings?.model) {
+      currentModel = payload.collaboration_mode.settings.model;
+    }
+
+    if (payload.type === "user_message" || payload.type === "agent_message" || payload.type === "message") {
+      report.totalMessages += 1;
+      registerMessageEvent(report, line.timestamp);
+    }
+
+    const totalUsage = payload.info?.total_token_usage;
+    if (!totalUsage) continue;
+
+    const currentUsage = normalizeCodexUsage(totalUsage);
+    const increment = previousUsage ? diffTokens(currentUsage, previousUsage) : currentUsage;
+    previousUsage = currentUsage;
+
+    if (tokenTotal(increment) <= 0) continue;
+    addTokens(report.tokens, increment);
+    addModelUsage(report, currentModel, increment);
+    addMonthlyTokens(report, toMonth(line.timestamp), increment);
+  }
+}
+
+function readJsonLines(file) {
+  let text;
+  try {
+    text = fs.readFileSync(file, "utf8");
+  } catch {
+    return [];
+  }
+
+  const parsed = [];
+  for (const line of text.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      parsed.push(JSON.parse(line));
+    } catch {
+      // Ignore corrupt or partial lines.
+    }
+  }
+  return parsed;
+}

package/src/utils.js

@@ -0,0 +1,281 @@
+import fs from "node:fs";
+import path from "node:path";
+
+export function readJson(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+export function fileExists(file) {
+  try {
+    return fs.existsSync(file);
+  } catch {
+    return false;
+  }
+}
+
+export function walkFiles(root, predicate, options = {}) {
+  const files = [];
+  const maxDepth = options.maxDepth ?? Infinity;
+  const skipDirs = options.skipDirs || defaultSkipDirs;
+
+  function walk(dir, depth) {
+    if (depth > maxDepth) return;
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (skipDirs(entry.name, fullPath)) continue;
+        walk(fullPath, depth + 1);
+      } else if (entry.isFile() && predicate(fullPath, entry.name)) {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  walk(root, 0);
+  return files;
+}
+
+export function findDirs(root, targetName, options = {}) {
+  const dirs = [];
+  const maxDepth = options.maxDepth ?? 8;
+  const skipDirs = options.skipDirs || defaultProjectSkipDirs;
+
+  function walk(dir, depth) {
+    if (depth > maxDepth) return;
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const fullPath = path.join(dir, entry.name);
+      if (entry.name === targetName) {
+        dirs.push(fullPath);
+        continue;
+      }
+      if (skipDirs(entry.name, fullPath)) continue;
+      walk(fullPath, depth + 1);
+    }
+  }
+
+  walk(root, 0);
+  return dirs;
+}
+
+export function countJsonlEntries(file) {
+  try {
+    const text = fs.readFileSync(file, "utf8");
+    return text.split("\n").filter(Boolean).length;
+  } catch {
+    return 0;
+  }
+}
+
+export function createEmptyReport(provider, key, root) {
+  return {
+    provider,
+    key,
+    root,
+    available: false,
+    sessionFiles: 0,
+    projectConfigDirs: 0,
+    projectConfigExamples: [],
+    projectScanRoot: null,
+    historyEntries: 0,
+    totalSessions: 0,
+    totalMessages: 0,
+    firstSeen: null,
+    lastSeen: null,
+    dailyActivity: {},
+    monthlySpendUSD: {},
+    monthlyTokens: {},
+    hourlyActivity: Array.from({ length: 24 }, () => 0),
+    tokens: emptyTokens(),
+    modelUsage: {},
+    knownSpendUSD: 0,
+    unknownSpend: false,
+    favoriteModel: null,
+    notes: [],
+  };
+}
+
+export function emptyTokens() {
+  return {
+    input: 0,
+    output: 0,
+    cacheRead: 0,
+    cacheCreation: 0,
+    reasoning: 0,
+    total: 0,
+  };
+}
+
+export function addTokens(target, source) {
+  for (const key of ["input", "output", "cacheRead", "cacheCreation", "reasoning", "total"]) {
+    target[key] = (target[key] || 0) + (Number(source?.[key]) || 0);
+  }
+  return target;
+}
+
+export function diffTokens(current, previous) {
+  const diff = emptyTokens();
+  for (const key of ["input", "output", "cacheRead", "cacheCreation", "reasoning", "total"]) {
+    diff[key] = Math.max(0, (Number(current?.[key]) || 0) - (Number(previous?.[key]) || 0));
+  }
+  return diff;
+}
+
+export function tokenTotal(tokens) {
+  return Number(tokens?.total) || sumTokenParts(tokens);
+}
+
+function sumTokenParts(tokens) {
+  return (
+    (Number(tokens?.input) || 0) +
+    (Number(tokens?.output) || 0) +
+    (Number(tokens?.cacheRead) || 0) +
+    (Number(tokens?.cacheCreation) || 0) +
+    (Number(tokens?.reasoning) || 0)
+  );
+}
+
+export function normalizeClaudeUsage(usage = {}) {
+  const tokens = emptyTokens();
+  tokens.input = Number(usage.input_tokens || usage.inputTokens || 0);
+  tokens.output = Number(usage.output_tokens || usage.outputTokens || 0);
+  tokens.cacheRead = Number(usage.cache_read_input_tokens || usage.cacheReadInputTokens || 0);
+  tokens.cacheCreation = Number(usage.cache_creation_input_tokens || usage.cacheCreationInputTokens || 0);
+  tokens.reasoning = Number(usage.reasoning_output_tokens || usage.reasoningOutputTokens || 0);
+  tokens.total = sumTokenParts(tokens);
+  return tokens;
+}
+
+export function normalizeCodexUsage(usage = {}) {
+  const tokens = emptyTokens();
+  tokens.input = Number(usage.input_tokens || usage.inputTokens || 0);
+  tokens.output = Number(usage.output_tokens || usage.outputTokens || 0);
+  tokens.cacheRead = Number(usage.cached_input_tokens || usage.cacheReadInputTokens || 0);
+  tokens.cacheCreation = Number(usage.cache_creation_input_tokens || usage.cacheCreationInputTokens || 0);
+  tokens.reasoning = Number(usage.reasoning_output_tokens || usage.reasoningOutputTokens || 0);
+  tokens.total = Number(usage.total_tokens || usage.totalTokens || tokens.input + tokens.output);
+  return tokens;
+}
+
+export function addMonthlyTokens(report, month, tokens) {
+  if (!month) return;
+  report.monthlyTokens[month] ||= emptyTokens();
+  addTokens(report.monthlyTokens[month], tokens);
+}
+
+export function addModelUsage(report, model, tokens, extra = {}) {
+  const key = model || "unknown";
+  report.modelUsage[key] ||= { ...emptyTokens(), model: key, provider: report.provider, costUSD: 0 };
+  addTokens(report.modelUsage[key], tokens);
+  if (extra.costUSD) report.modelUsage[key].costUSD += extra.costUSD;
+}
+
+export function addSpend(report, month, spend) {
+  if (!month || !Number.isFinite(spend) || spend <= 0) return;
+  report.monthlySpendUSD[month] = (report.monthlySpendUSD[month] || 0) + spend;
+  report.knownSpendUSD += spend;
+}
+
+export function registerTimestamp(report, timestamp) {
+  if (!timestamp) return;
+  const date = new Date(timestamp);
+  if (!Number.isFinite(date.getTime())) return;
+  const iso = date.toISOString();
+  if (!report.firstSeen || iso < report.firstSeen) report.firstSeen = iso;
+  if (!report.lastSeen || iso > report.lastSeen) report.lastSeen = iso;
+}
+
+export function registerMessageEvent(report, timestamp) {
+  if (!timestamp) return;
+  const date = new Date(timestamp);
+  if (!Number.isFinite(date.getTime())) return;
+  report.hourlyActivity[date.getHours()] += 1;
+  const day = toDay(date);
+  if (day) report.dailyActivity[day] = (report.dailyActivity[day] || 0) + 1;
+}
+
+export function toMonth(value) {
+  const date = value instanceof Date ? value : new Date(value);
+  if (!Number.isFinite(date.getTime())) return null;
+  return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}`;
+}
+
+export function toDay(value) {
+  const date = value instanceof Date ? value : new Date(value);
+  if (!Number.isFinite(date.getTime())) return null;
+  return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`;
+}
+
+export function makeObjectMap(object) {
+  return Object.fromEntries(Object.entries(object).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+export function sortedEntries(object) {
+  return Object.entries(object || {}).sort(([a], [b]) => a.localeCompare(b));
+}
+
+export function compactNumber(value) {
+  const number = Number(value) || 0;
+  const abs = Math.abs(number);
+  if (abs >= 1_000_000_000) return `${(number / 1_000_000_000).toFixed(2)}B`;
+  if (abs >= 1_000_000) return `${(number / 1_000_000).toFixed(2)}M`;
+  if (abs >= 1_000) return `${(number / 1_000).toFixed(1)}K`;
+  return formatInteger(number);
+}
+
+export function formatInteger(value) {
+  return new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 }).format(Number(value) || 0);
+}
+
+export function dollars(value) {
+  return `$${(Number(value) || 0).toFixed(2)}`;
+}
+
+export function formatPercent(value) {
+  return `${((Number(value) || 0) * 100).toFixed(1)}%`;
+}
+
+const defaultSkipDirs = (name) => name === "node_modules" || name === ".git";
+
+const skippedProjectDirs = new Set([
+  ".cache",
+  ".cargo",
+  ".git",
+  ".gradle",
+  ".local",
+  ".npm",
+  ".pnpm-store",
+  ".rustup",
+  ".Trash",
+  "Applications",
+  "Caches",
+  "Library",
+  "Movies",
+  "Music",
+  "node_modules",
+  "Pictures",
+]);
+
+function defaultProjectSkipDirs(name) {
+  if (name === ".claude") return false;
+  if (skippedProjectDirs.has(name)) return true;
+  return name.startsWith(".") && name !== ".claude";
+}