token-studio 4.8.0

package/src/source-health.mjs

@@ -0,0 +1,195 @@
+const SOURCE_MATCHERS = {
+  claude: ['claude', 'anthropic'],
+  codex: ['codex', 'openai'],
+  gemini: ['gemini'],
+  opencode: ['opencode', 'open code'],
+  openclaw: ['openclaw', 'clawdbot', 'moltbot', 'moldbot'],
+  hermes: ['hermes'],
+  cursor: ['cursor'],
+  copilot: ['copilot'],
+  qwen: ['qwen'],
+  kimi: ['kimi', 'moonshot'],
+  goose: ['goose'],
+  ccusage: ['import:ccusage', 'ccusage']
+};
+
+export function buildSourceHealth({
+  collectors = [],
+  dailyRows = [],
+  sessionRows = [],
+  eventRows = [],
+  runs = []
+} = {}) {
+  const daily = groupSourceRows(dailyRows, 'dailyRows');
+  const sessions = groupSourceRows(sessionRows, 'sessions');
+  const events = groupSourceRows(eventRows, 'tokenEvents');
+  const latestRuns = latestRunBySource(runs);
+
+  return collectors.map(collector => {
+    const matchedKeys = matchingSourceKeys(collector, [
+      ...daily.keys(),
+      ...sessions.keys(),
+      ...events.keys(),
+      ...latestRuns.keys()
+    ]);
+    const stats = mergeStats(matchedKeys, { daily, sessions, events });
+    const run = firstRun(matchedKeys, latestRuns);
+    const lastSeenAt = latestDate([
+      stats.latestDailyAt,
+      stats.latestSessionAt,
+      stats.latestEventAt,
+      run?.collectedAt
+    ]);
+
+    return {
+      id: collector.id,
+      label: collector.label,
+      supportStatus: collector.supportStatus,
+      coverageTier: coverageTier(collector.supportStatus),
+      privacyLevel: collector.privacyLevel,
+      defaultEnabled: Boolean(collector.defaultEnabled),
+      detected: Boolean(collector.detected),
+      detectedRootCount: Array.isArray(collector.existingRoots) ? collector.existingRoots.length : 0,
+      configuredRootCount: Array.isArray(collector.configuredRoots) ? collector.configuredRoots.length : 0,
+      readsConversationContent: Boolean(collector.readsConversationContent),
+      tokenReliability: collector.tokenReliability || 'unknown',
+      fixtureBacked: Boolean(collector.fixtureBacked),
+      auditRecommended: Boolean(collector.auditRecommended),
+      dataFields: collector.dataFields || [],
+      workflow: workflowFor(collector),
+      commandHint: commandHintFor(collector),
+      matchedSources: matchedKeys,
+      dailyRows: stats.dailyRows,
+      sessions: stats.sessions,
+      tokenEvents: stats.tokenEvents,
+      totalTokens: stats.totalTokens,
+      lastSeenAt,
+      lastRunStatus: run?.status || null,
+      lastRunAt: run?.collectedAt || null,
+      health: healthStatus({ collector, stats, run, lastSeenAt }),
+      note: collector.note || null
+    };
+  });
+}
+
+function groupSourceRows(rows, countField) {
+  const map = new Map();
+  for (const row of rows) {
+    const key = normalizeKey(row.source);
+    if (!key) continue;
+    const current = map.get(key) || emptyStats();
+    current[countField] += Number(row.count ?? row[countField] ?? 0);
+    current.totalTokens += Number(row.totalTokens || 0);
+    if (row.latestDailyAt) current.latestDailyAt = latestDate([current.latestDailyAt, row.latestDailyAt]);
+    if (row.latestSessionAt) current.latestSessionAt = latestDate([current.latestSessionAt, row.latestSessionAt]);
+    if (row.latestEventAt) current.latestEventAt = latestDate([current.latestEventAt, row.latestEventAt]);
+    map.set(key, current);
+  }
+  return map;
+}
+
+function latestRunBySource(runs) {
+  const map = new Map();
+  for (const run of runs) {
+    const key = normalizeKey(run.source);
+    if (!key) continue;
+    const existing = map.get(key);
+    if (!existing || new Date(run.collectedAt || 0) > new Date(existing.collectedAt || 0)) {
+      map.set(key, {
+        source: run.source,
+        status: run.status || null,
+        collectedAt: run.collectedAt || null
+      });
+    }
+  }
+  return map;
+}
+
+function matchingSourceKeys(collector, keys) {
+  const unique = [...new Set(keys)].filter(Boolean);
+  const patterns = SOURCE_MATCHERS[collector.id] || [collector.id, collector.label];
+  const normalizedPatterns = patterns.map(normalizeKey).filter(Boolean);
+  return unique.filter(key => normalizedPatterns.some(pattern => key.includes(pattern) || pattern.includes(key)));
+}
+
+function mergeStats(keys, groups) {
+  const merged = emptyStats();
+  for (const key of keys) {
+    for (const group of Object.values(groups)) {
+      const row = group.get(key);
+      if (!row) continue;
+      merged.dailyRows += row.dailyRows;
+      merged.sessions += row.sessions;
+      merged.tokenEvents += row.tokenEvents;
+      merged.totalTokens += row.totalTokens;
+      merged.latestDailyAt = latestDate([merged.latestDailyAt, row.latestDailyAt]);
+      merged.latestSessionAt = latestDate([merged.latestSessionAt, row.latestSessionAt]);
+      merged.latestEventAt = latestDate([merged.latestEventAt, row.latestEventAt]);
+    }
+  }
+  return merged;
+}
+
+function firstRun(keys, latestRuns) {
+  return keys
+    .map(key => latestRuns.get(key))
+    .filter(Boolean)
+    .sort((a, b) => new Date(b.collectedAt || 0) - new Date(a.collectedAt || 0))[0] || null;
+}
+
+function emptyStats() {
+  return {
+    dailyRows: 0,
+    sessions: 0,
+    tokenEvents: 0,
+    totalTokens: 0,
+    latestDailyAt: null,
+    latestSessionAt: null,
+    latestEventAt: null
+  };
+}
+
+function coverageTier(status) {
+  if (status === 'stable') return 'native stable';
+  if (status === 'experimental') return 'experimental';
+  if (status === 'import-only') return 'ccusage import-bridge';
+  return 'detected-only';
+}
+
+function workflowFor(collector) {
+  if (collector.supportStatus === 'stable') return 'Native collector can write structured token usage after explicit collect.';
+  if (collector.supportStatus === 'experimental') return 'Run collector audit first; only explicit token fields are eligible.';
+  if (collector.supportStatus === 'import-only') return 'Import saved ccusage JSON or run explicit ccusage CLI bridge from terminal.';
+  return 'Presence detection only; no usage rows are written.';
+}
+
+function commandHintFor(collector) {
+  if (collector.id === 'ccusage') {
+    return 'npx token-studio import-usage --format=ccusage-cli --report=session --dry-run --yes';
+  }
+  if (collector.supportStatus === 'stable') return `npx token-studio collect --sources=${collector.id}`;
+  if (collector.supportStatus === 'experimental') return 'npx token-studio collectors --audit --json';
+  return 'npx token-studio collectors --json';
+}
+
+function healthStatus({ collector, stats, run, lastSeenAt }) {
+  if (stats.sessions || stats.tokenEvents || stats.dailyRows) return 'has-data';
+  if (run?.status === 'error') return 'last-run-error';
+  if (collector.detected) return 'detected-no-data';
+  if (collector.supportStatus === 'import-only') return 'import-ready';
+  if (lastSeenAt) return 'seen';
+  return 'not-detected';
+}
+
+function latestDate(values) {
+  const dates = values
+    .filter(Boolean)
+    .map(value => new Date(value))
+    .filter(date => Number.isFinite(date.getTime()))
+    .sort((a, b) => b.getTime() - a.getTime());
+  return dates[0]?.toISOString() || null;
+}
+
+function normalizeKey(value) {
+  return String(value || '').trim().toLowerCase();
+}

package/src/statusline.mjs

@@ -0,0 +1,156 @@
+import { listAdvisorActions, listBudgetProfiles, listTokenEvents } from './db.mjs';
+import { buildLiveSnapshot } from './live.mjs';
+
+export function buildStatuslineSnapshot(db, {
+  now = new Date(),
+  windowMinutes = 15,
+  source = 'all'
+} = {}) {
+  const sourceFilter = normalizeSourceFilter(source);
+  const tokenEvents = listTokenEvents(db, { limit: 5000 })
+    .filter(event => sourceMatches(event.source, sourceFilter));
+  const budgetProfiles = listBudgetProfiles(db)
+    .filter(profile => profile.enabled)
+    .filter(profile => sourceFilter === 'all' || !profile.source || sourceMatches(profile.source, sourceFilter));
+  const live = buildLiveSnapshot({
+    tokenEvents,
+    budgetProfiles,
+    now,
+    windowMinutes
+  });
+  const openAdvisorActions = listAdvisorActions(db)
+    .filter(action => action.status === 'open')
+    .length;
+  const highestBudgetShare = Math.max(0, ...live.budgetWindows.map(window => Math.max(
+    Number(window.tokenShare || 0),
+    Number(window.costShare || 0)
+  )));
+  const resetInMinutes = live.budgetWindows.length
+    ? Math.min(...live.budgetWindows.map(window => Number(window.resetInMinutes || 0)))
+    : null;
+
+  return {
+    generatedAt: live.generatedAt,
+    source: sourceFilter,
+    windowMinutes: live.windowMinutes,
+    status: live.status,
+    totals: {
+      totalTokens: live.totals.totalTokens,
+      burnRateTokensPerHour: live.totals.burnRateTokensPerHour,
+      cacheHitRate: live.totals.cacheHitRate,
+      costUSD: live.totals.costUSD
+    },
+    budget: {
+      windows: live.budgetWindows,
+      status: budgetStatus(live.budgetWindows),
+      highestShare: highestBudgetShare,
+      resetInMinutes
+    },
+    warnings: live.warnings.map(warning => ({
+      type: warning.type,
+      level: warning.level,
+      message: warning.message,
+      evidence: warning.evidence,
+      action: warning.action
+    })),
+    openAdvisorActions
+  };
+}
+
+export function buildEmptyStatuslineSnapshot({
+  now = new Date(),
+  windowMinutes = 15,
+  source = 'all',
+  warning = null
+} = {}) {
+  const sourceFilter = normalizeSourceFilter(source);
+  return {
+    generatedAt: new Date(now).toISOString(),
+    source: sourceFilter,
+    windowMinutes,
+    status: 'missing-db',
+    totals: {
+      totalTokens: 0,
+      burnRateTokensPerHour: 0,
+      cacheHitRate: 0,
+      costUSD: 0
+    },
+    budget: {
+      windows: [],
+      status: 'none',
+      highestShare: 0,
+      resetInMinutes: null
+    },
+    warnings: warning ? [{
+      type: 'missing-db',
+      level: 'low',
+      message: warning,
+      evidence: 'No local SQLite database was available.',
+      action: 'Run demo, import ccusage JSON, or start real collection explicitly before using statusline data.'
+    }] : [],
+    openAdvisorActions: 0
+  };
+}
+
+export function formatStatuslineText(snapshot, { maxWidth = 100 } = {}) {
+  const totals = snapshot.totals || {};
+  const budget = snapshot.budget || {};
+  const warningTypes = (snapshot.warnings || []).map(warning => warning.type);
+  const warn = warningTypes.includes('missing-db') ? 'no-db'
+    : warningTypes.includes('unpriced-model-active') ? 'unpriced'
+    : warningTypes.length ? warningTypes[0] : 'ok';
+  const reset = budget.resetInMinutes == null ? '-' : `${Math.round(budget.resetInMinutes)}m`;
+  const budgetText = budget.windows?.length
+    ? `${budget.status}:${Math.round(Number(budget.highestShare || 0) * 100)}%`
+    : 'none';
+  const text = [
+    'TS',
+    `tok=${compactInt(totals.totalTokens)}`,
+    `burn=${compactInt(totals.burnRateTokensPerHour)}/h`,
+    `cache=${Number(totals.cacheHitRate || 0).toFixed(0)}%`,
+    `actions=${snapshot.openAdvisorActions || 0}`,
+    `budget=${budgetText}`,
+    `reset=${reset}`,
+    `warn=${warn}`
+  ].join(' ');
+  return truncateStatusline(text, maxWidth);
+}
+
+function budgetStatus(windows) {
+  if (windows.some(window => window.status === 'exceeded')) return 'exceeded';
+  if (windows.some(window => window.status === 'over-pace')) return 'over-pace';
+  if (windows.some(window => window.status === 'near-limit')) return 'near-limit';
+  if (windows.length) return 'ok';
+  return 'none';
+}
+
+function normalizeSourceFilter(source) {
+  const value = String(source || 'all').trim().toLowerCase();
+  if (['all', 'claude', 'codex'].includes(value)) return value;
+  throw new Error('--source must be all, claude, or codex');
+}
+
+function sourceMatches(source, filter) {
+  if (filter === 'all') return true;
+  const value = String(source || '').toLowerCase();
+  if (filter === 'claude') return value.includes('claude') || value.includes('anthropic');
+  if (filter === 'codex') return value.includes('codex') || value.includes('openai');
+  return false;
+}
+
+function compactInt(value) {
+  const number = Math.round(Number(value || 0));
+  if (number >= 1_000_000) return `${trimFixed(number / 1_000_000)}M`;
+  if (number >= 1_000) return `${trimFixed(number / 1_000)}k`;
+  return String(number);
+}
+
+function trimFixed(value) {
+  return value.toFixed(value >= 10 ? 0 : 1).replace(/\.0$/, '');
+}
+
+function truncateStatusline(text, maxWidth) {
+  const width = Math.max(40, Number(maxWidth) || 100);
+  if (text.length <= width) return text;
+  return `${text.slice(0, Math.max(0, width - 3)).trimEnd()}...`;
+}

package/src/terminal-report.mjs

@@ -0,0 +1,245 @@
+import { listAdvisorActions, listBudgetProfiles } from './db.mjs';
+import { buildLiveSnapshot } from './live.mjs';
+
+export function buildTerminalReport(db, { period = 'week', now = new Date() } = {}) {
+  const range = periodRange(period, now);
+  const daily = queryDaily(db, range);
+  const sessions = querySessions(db, range);
+  const tokenEvents = queryTokenEvents(db);
+  const budgetProfiles = listBudgetProfiles(db).filter(profile => profile.enabled);
+  const live = buildLiveSnapshot({ sessions, tokenEvents, budgetProfiles, now });
+  const advisorActions = listAdvisorActions(db, {
+    periodStart: range.start,
+    periodEnd: range.end
+  });
+  const totals = sumDaily(daily);
+  return {
+    period: range,
+    totals,
+    topProjects: topProjects(sessions).slice(0, 8),
+    topModels: topModels(daily).slice(0, 8),
+    budgetWindows: live.budgetWindows || [],
+    budgetWarnings: (live.warnings || []).filter(warning => warning.type?.includes('budget')),
+    advisorActions: advisorActions.slice(0, 8)
+  };
+}
+
+export function formatTerminalReport(report, format = 'table') {
+  if (format === 'json') return JSON.stringify(report, null, 2);
+  if (format === 'markdown') return markdownReport(report);
+  return tableReport(report);
+}
+
+function queryDaily(db, range) {
+  if (range.id === 'all') {
+    return db.prepare(`
+      SELECT device, source, usage_date AS usageDate, model,
+        input_tokens AS inputTokens,
+        output_tokens AS outputTokens,
+        cache_creation_tokens AS cacheCreationTokens,
+        cache_read_tokens AS cacheReadTokens,
+        reasoning_output_tokens AS reasoningOutputTokens,
+        total_tokens AS totalTokens,
+        cost_usd AS costUSD
+      FROM daily_usage
+    `).all();
+  }
+  return db.prepare(`
+    SELECT device, source, usage_date AS usageDate, model,
+      input_tokens AS inputTokens,
+      output_tokens AS outputTokens,
+      cache_creation_tokens AS cacheCreationTokens,
+      cache_read_tokens AS cacheReadTokens,
+      reasoning_output_tokens AS reasoningOutputTokens,
+      total_tokens AS totalTokens,
+      cost_usd AS costUSD
+    FROM daily_usage
+    WHERE usage_date >= ? AND usage_date <= ?
+  `).all(range.start, range.end);
+}
+
+function querySessions(db, range) {
+  const base = `
+    SELECT device, source, session_id AS sessionId,
+      last_activity AS lastActivity,
+      project_path AS projectPath,
+      input_tokens AS inputTokens,
+      output_tokens AS outputTokens,
+      cache_creation_tokens AS cacheCreationTokens,
+      cache_read_tokens AS cacheReadTokens,
+      reasoning_output_tokens AS reasoningOutputTokens,
+      total_tokens AS totalTokens,
+      cost_usd AS costUSD
+    FROM session_usage
+  `;
+  if (range.id === 'all') return db.prepare(base).all();
+  return db.prepare(`${base} WHERE substr(last_activity, 1, 10) >= ? AND substr(last_activity, 1, 10) <= ?`)
+    .all(range.start, range.end);
+}
+
+function queryTokenEvents(db) {
+  return db.prepare(`
+    SELECT event_id AS eventId, device, source, session_id AS sessionId,
+      timestamp, model,
+      input_tokens AS inputTokens,
+      output_tokens AS outputTokens,
+      cache_read_tokens AS cacheReadTokens,
+      cache_creation_tokens AS cacheCreationTokens,
+      reasoning_tokens AS reasoningTokens,
+      tool_category AS toolCategory,
+      file_extension AS fileExtension
+    FROM token_events
+    ORDER BY timestamp DESC
+    LIMIT 5000
+  `).all();
+}
+
+function periodRange(period, now) {
+  const id = String(period || 'week').toLowerCase();
+  const endDate = new Date(now);
+  const startDate = new Date(now);
+  if (id === 'month') startDate.setUTCDate(endDate.getUTCDate() - 29);
+  else if (id === '90d') startDate.setUTCDate(endDate.getUTCDate() - 89);
+  else if (id === 'all') return { id: 'all', label: 'all', start: '', end: '' };
+  else startDate.setUTCDate(endDate.getUTCDate() - 6);
+  return { id, label: id, start: formatDate(startDate), end: formatDate(endDate) };
+}
+
+function sumDaily(rows) {
+  return rows.reduce((acc, row) => {
+    acc.totalTokens += row.totalTokens || 0;
+    acc.inputTokens += row.inputTokens || 0;
+    acc.outputTokens += row.outputTokens || 0;
+    acc.cacheReadTokens += row.cacheReadTokens || 0;
+    acc.cacheCreationTokens += row.cacheCreationTokens || 0;
+    acc.reasoningOutputTokens += row.reasoningOutputTokens || 0;
+    acc.costUSD += row.costUSD || 0;
+    return acc;
+  }, {
+    totalTokens: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheCreationTokens: 0,
+    reasoningOutputTokens: 0,
+    costUSD: 0
+  });
+}
+
+function topProjects(rows) {
+  const map = new Map();
+  for (const row of rows) {
+    const key = row.projectPath || '<unknown>';
+    const acc = map.get(key) || { project: key, sessions: 0, totalTokens: 0, costUSD: 0 };
+    acc.sessions += 1;
+    acc.totalTokens += row.totalTokens || 0;
+    acc.costUSD += row.costUSD || 0;
+    map.set(key, acc);
+  }
+  return [...map.values()].sort((a, b) => b.totalTokens - a.totalTokens);
+}
+
+function topModels(rows) {
+  const map = new Map();
+  for (const row of rows) {
+    const key = `${row.model || '<unknown>'}::${row.source || 'unknown'}`;
+    const acc = map.get(key) || { model: row.model || '<unknown>', source: row.source || 'unknown', totalTokens: 0, costUSD: 0 };
+    acc.totalTokens += row.totalTokens || 0;
+    acc.costUSD += row.costUSD || 0;
+    map.set(key, acc);
+  }
+  return [...map.values()].sort((a, b) => b.totalTokens - a.totalTokens);
+}
+
+function tableReport(report) {
+  return [
+    `Token Studio ROI Report (${report.period.id === 'all' ? 'all' : `${report.period.start}..${report.period.end}`})`,
+    '',
+    `tokens=${formatInt(report.totals.totalTokens)} official_price=${money(report.totals.costUSD)}`,
+    '',
+    'Top projects',
+    ...report.topProjects.slice(0, 5).map(row => `- ${row.project}: ${formatInt(row.totalTokens)} tokens, ${money(row.costUSD)}`),
+    '',
+    'Top models',
+    ...report.topModels.slice(0, 5).map(row => `- ${row.model} (${row.source}): ${formatInt(row.totalTokens)} tokens, ${money(row.costUSD)}`),
+    '',
+    'Budget risks',
+    ...(report.budgetWarnings.length ? report.budgetWarnings.map(w => `- ${w.level}: ${w.message} — ${w.evidence}`) : ['- none']),
+    '',
+    'Advisor actions',
+    ...(report.advisorActions.length ? report.advisorActions.map(a => `- [${a.status}] ${a.title}: ${a.action}`) : ['- none'])
+  ].join('\n');
+}
+
+function markdownReport(report) {
+  return [
+    '# Token Studio ROI Terminal Report',
+    '',
+    `- Period: ${safe(report.period.id === 'all' ? 'all' : `${report.period.start}..${report.period.end}`)}`,
+    `- Total tokens: ${formatInt(report.totals.totalTokens)}`,
+    `- Official price conversion: ${money(report.totals.costUSD)}`,
+    '',
+    '## Top Projects',
+    '',
+    report.topProjects.length ? mdTable(
+      ['Project', 'Sessions', 'Tokens', 'Official Price'],
+      report.topProjects.slice(0, 8).map(row => [row.project, row.sessions, formatInt(row.totalTokens), money(row.costUSD)])
+    ) : 'No project data.',
+    '',
+    '## Top Models',
+    '',
+    report.topModels.length ? mdTable(
+      ['Model', 'Source', 'Tokens', 'Official Price'],
+      report.topModels.slice(0, 8).map(row => [row.model, row.source, formatInt(row.totalTokens), money(row.costUSD)])
+    ) : 'No model data.',
+    '',
+    '## Budget Risks',
+    '',
+    report.budgetWarnings.length ? report.budgetWarnings.map(w => `- **${safe(w.level)}** ${safe(w.message)}: ${safe(w.evidence)}. ${safe(w.action)}`).join('\n') : '- No active budget risk.',
+    '',
+    '## Advisor Actions',
+    '',
+    report.advisorActions.length ? mdTable(
+      ['Status', 'Category', 'Title', 'Action'],
+      report.advisorActions.map(row => [row.status, row.category, row.title, row.action])
+    ) : 'No advisor actions for this period.',
+    '',
+    '## Notes',
+    '',
+    '- Costs are official public token-price conversions, not provider invoices.',
+    '- Completed actions are tracked as review workflow state; they do not prove causal savings.'
+  ].join('\n');
+}
+
+function mdTable(headers, rows) {
+  return [
+    `| ${headers.map(cell).join(' | ')} |`,
+    `| ${headers.map(() => '---').join(' | ')} |`,
+    ...rows.map(row => `| ${row.map(cell).join(' | ')} |`)
+  ].join('\n');
+}
+
+function cell(value) {
+  const text = safe(value);
+  return (/^[=+\-@\t\r]/.test(text) ? `'${text}` : text).replace(/\|/g, '\\|');
+}
+
+function safe(value) {
+  return String(value ?? '').replace(/\r?\n/g, ' ').replace(/\s+/g, ' ').trim();
+}
+
+function formatDate(date) {
+  return [
+    date.getUTCFullYear(),
+    String(date.getUTCMonth() + 1).padStart(2, '0'),
+    String(date.getUTCDate()).padStart(2, '0')
+  ].join('-');
+}
+
+function formatInt(value) {
+  return Math.round(Number(value || 0)).toLocaleString('en-US');
+}
+
+function money(value) {
+  return `$${Number(value || 0).toFixed(4)}`;
+}

package/src/update-pricing.mjs

@@ -0,0 +1,8 @@
+import { resolve } from 'node:path';
+import { loadPricing } from './pricing.mjs';
+
+process.env.PRICING_REFRESH = '1';
+
+const pricingCachePath = resolve(process.cwd(), 'data', 'official-pricing.json');
+const pricing = await loadPricing(pricingCachePath);
+console.log(`[pricing] official-docs verifiedAt=${pricing.verifiedAt} models=${pricing.models.length}`);