token-studio 4.8.0

package/src/collect.mjs

@@ -0,0 +1,247 @@
+import { hostname } from 'node:os';
+import { resolve } from 'node:path';
+import { openDb, recordRun, upsertDaily, upsertSession, upsertTokenEvent } from './db.mjs';
+import { loadPricing } from './pricing.mjs';
+import { collectableCollectors, collectorLabel, enabledCollectorIds } from './collector-registry.mjs';
+
+const args = parseArgs(process.argv.slice(2));
+const device = args.device || hostname();
+const db = openDb(args.db);
+const exportPayload = {
+  device,
+  collectedAt: new Date().toISOString(),
+  daily: [],
+  sessions: [],
+  tokenEvents: [],
+  runs: []
+};
+
+// Load official provider pricing once. Unknown models are intentionally unpriced.
+const pricingCachePath = resolve(process.cwd(), 'data', 'official-pricing.json');
+const pricingData = await loadPricing(pricingCachePath);
+
+await collectLocal();
+
+if (args.push) {
+  await pushPayload(args.push, exportPayload, args.token);
+}
+
+db.close();
+
+async function collectLocal() {
+  let anyError = false;
+  const enabled = enabledCollectors();
+  const includeExperimental = Boolean(args.sources || args.collectors || args.experimental);
+  const collectors = collectableCollectors({ includeExperimental }).filter(({ id }) => enabled.has(id));
+
+  console.log(`[collect] enabled=${Array.from(enabled).join(',') || 'none'}`);
+
+  for (const { module, label } of collectors) {
+    let graphJson;
+    let modelsJson;
+    let tokenEvents;
+
+    try {
+      const { collect } = await import(module);
+      ({ graphJson, modelsJson, tokenEvents } = await collect(pricingData));
+    } catch (error) {
+      const run = {
+        device,
+        source: label,
+        status: 'error',
+        message: error.message,
+        collectedAt: exportPayload.collectedAt,
+        command: `js-collector:${module}`
+      };
+      recordRun(db, run);
+      exportPayload.runs.push(run);
+      console.warn(`[${label}] ${error.message}`);
+      anyError = true;
+      continue;
+    }
+
+    const dailyRows = normalizeDailyRows(graphJson, device);
+    runInTransaction(db, () => dailyRows.forEach((row) => upsertDaily(db, row)));
+    exportPayload.daily.push(...dailyRows);
+
+    const sessionRows = normalizeSessionRows(modelsJson, device);
+    runInTransaction(db, () => sessionRows.forEach((row) => upsertSession(db, row)));
+    exportPayload.sessions.push(...sessionRows);
+
+    const eventRows = normalizeTokenEventRows(tokenEvents, device);
+    runInTransaction(db, () => eventRows.forEach((row) => upsertTokenEvent(db, row)));
+    exportPayload.tokenEvents.push(...eventRows);
+
+    const run = {
+      device,
+      source: label,
+      status: dailyRows.length || sessionRows.length ? 'ok' : 'empty',
+      message: `daily=${dailyRows.length}, workspace_model=${sessionRows.length}, token_events=${eventRows.length}`,
+      collectedAt: exportPayload.collectedAt,
+      command: `js-collector:${module}`
+    };
+    recordRun(db, run);
+    exportPayload.runs.push(run);
+    console.log(`[${label}] daily=${dailyRows.length}, workspace_model=${sessionRows.length}, token_events=${eventRows.length}`);
+  }
+
+  if (anyError) process.exitCode = 1;
+}
+
+function enabledCollectors() {
+  const sourceArg = args.sources || args.collectors;
+  if (sourceArg) {
+    return enabledCollectorIds({ includeExperimental: true, values: sourceArg });
+  }
+  return enabledCollectorIds({ includeExperimental: Boolean(args.experimental) });
+}
+
+function runInTransaction(database, work) {
+  database.exec('BEGIN');
+  try {
+    work();
+    database.exec('COMMIT');
+  } catch (error) {
+    database.exec('ROLLBACK');
+    throw error;
+  }
+}
+
+function normalizeDailyRows(json, deviceName) {
+  const days = Array.isArray(json.contributions) ? json.contributions : [];
+  return days.flatMap((day) => {
+    const clients = Array.isArray(day.clients) ? day.clients : [];
+    return clients.map((entry) => {
+      const tokens = normalizeTokens(entry.tokens);
+      return {
+        device: deviceName,
+        source: sourceLabel(entry.client),
+        usageDate: day.date,
+        model: entry.modelId || entry.model_id || 'unknown',
+        inputTokens: tokens.input,
+        outputTokens: tokens.output,
+        cacheCreationTokens: tokens.cacheWrite,
+        cacheReadTokens: tokens.cacheRead,
+        reasoningOutputTokens: tokens.reasoning,
+        totalTokens: tokenTotal(tokens),
+        costUSD: entry.cost || 0
+      };
+    });
+  });
+}
+
+function normalizeSessionRows(json, deviceName) {
+  const entries = Array.isArray(json.entries) ? json.entries : [];
+  return entries.map((entry) => {
+    const tokens = {
+      input: positiveNumber(entry.input),
+      output: positiveNumber(entry.output),
+      cacheRead: positiveNumber(entry.cacheRead),
+      cacheWrite: positiveNumber(entry.cacheWrite),
+      reasoning: positiveNumber(entry.reasoning)
+    };
+    const source = sourceLabel(entry.client);
+    const workspace = entry.workspaceLabel || entry.workspaceKey || '';
+    const model = entry.model || 'unknown';
+    return {
+      device: deviceName,
+      source,
+      sessionId: ['local', entry.client || 'unknown', workspace || 'no-workspace', model].join(':'),
+      lastActivity: exportPayload.collectedAt,
+      projectPath: workspace || null,
+      inputTokens: tokens.input,
+      outputTokens: tokens.output,
+      cacheCreationTokens: tokens.cacheWrite,
+      cacheReadTokens: tokens.cacheRead,
+      reasoningOutputTokens: tokens.reasoning,
+      totalTokens: tokenTotal(tokens),
+      costUSD: entry.cost || 0
+    };
+  });
+}
+
+function normalizeTokenEventRows(events, deviceName) {
+  if (!Array.isArray(events)) return [];
+  return events.map((event) => ({
+    device: deviceName,
+    source: sourceLabel(event.source || event.client),
+    sessionId: event.sessionId || event.session_id || 'unknown-session',
+    timestamp: event.timestamp || exportPayload.collectedAt,
+    model: event.model || 'unknown',
+    inputTokens: positiveNumber(event.inputTokens ?? event.input_tokens),
+    outputTokens: positiveNumber(event.outputTokens ?? event.output_tokens),
+    cacheReadTokens: positiveNumber(event.cacheReadTokens ?? event.cache_read_tokens),
+    cacheCreationTokens: positiveNumber(event.cacheCreationTokens ?? event.cache_creation_tokens),
+    reasoningTokens: positiveNumber(event.reasoningTokens ?? event.reasoning_tokens),
+    toolCategory: event.toolCategory ?? event.tool_category ?? null,
+    fileExtension: event.fileExtension ?? event.file_extension ?? null,
+    repoPathHash: event.repoPathHash ?? event.repo_path_hash ?? null,
+    privacyLevel: event.privacyLevel ?? event.privacy_level ?? 'safe',
+    eventId: event.eventId ?? event.event_id ?? null
+  }));
+}
+
+function normalizeTokens(tokens = {}) {
+  return {
+    input: positiveNumber(tokens.input),
+    output: positiveNumber(tokens.output),
+    cacheRead: positiveNumber(tokens.cacheRead ?? tokens.cache_read),
+    cacheWrite: positiveNumber(tokens.cacheWrite ?? tokens.cache_write),
+    reasoning: positiveNumber(tokens.reasoning)
+  };
+}
+
+function tokenTotal(tokens) {
+  return tokens.input + tokens.output + tokens.cacheRead + tokens.cacheWrite + tokens.reasoning;
+}
+
+function positiveNumber(value) {
+  const number = Number(value || 0);
+  return Number.isFinite(number) && number > 0 ? number : 0;
+}
+
+function sourceLabel(client) {
+  return collectorLabel(client) || client || 'unknown';
+}
+
+// ---------------------------------------------------------------------------
+// CLI argument parser
+// ---------------------------------------------------------------------------
+
+function parseArgs(argv) {
+  const parsed = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--device') {
+      parsed.device = argv[++i];
+    } else if (arg === '--db') {
+      parsed.db = argv[++i];
+    } else if (arg === '--push') {
+      parsed.push = argv[++i];
+    } else if (arg === '--token') {
+      parsed.token = argv[++i];
+    } else if (arg === '--sources' || arg === '--collectors') {
+      parsed.sources = argv[++i];
+    }
+  }
+  return parsed;
+}
+
+// ---------------------------------------------------------------------------
+// Remote push helper
+// ---------------------------------------------------------------------------
+
+async function pushPayload(url, payload, token) {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      ...(token ? { authorization: `Bearer ${token}` } : {})
+    },
+    body: JSON.stringify(payload)
+  });
+  if (!response.ok) {
+    throw new Error(`上报失败：HTTP ${response.status} ${await response.text()}`);
+  }
+  console.log(`[push] ${url}`);
+}

package/src/collector-config.mjs

@@ -0,0 +1,82 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+
+let cachedConfig;
+
+export function loadCollectorConfig() {
+  if (cachedConfig) return cachedConfig;
+
+  const configPath = process.env.TOKEN_STUDIO_CONFIG ||
+    process.env.AI_TOKEN_DASHBOARD_CONFIG ||
+    resolve(process.cwd(), 'config', 'collectors.json');
+
+  try {
+    cachedConfig = JSON.parse(readFileSync(configPath, 'utf8').replace(/^\uFEFF/, ''));
+  } catch {
+    cachedConfig = { collectors: {} };
+  }
+
+  return cachedConfig;
+}
+
+export function collectorConfig(name) {
+  return loadCollectorConfig().collectors?.[name] || {};
+}
+
+export function configuredPaths(name, key, fallback = []) {
+  const value = collectorConfig(name)[key];
+  const paths = Array.isArray(value) ? value : fallback;
+  return paths
+    .map((item) => expandPath(item))
+    .filter(Boolean);
+}
+
+export function configuredPath(name, key, fallback = null) {
+  const value = collectorConfig(name)[key] ?? fallback;
+  return expandPath(value);
+}
+
+export function configuredBool(name, key, fallback = false) {
+  const value = collectorConfig(name)[key];
+  return typeof value === 'boolean' ? value : fallback;
+}
+
+export function configuredStrings(name, key, fallback = []) {
+  const value = collectorConfig(name)[key];
+  return Array.isArray(value)
+    ? value.map((item) => String(item)).filter(Boolean)
+    : fallback;
+}
+
+export function envPathList(value, fallback = []) {
+  const paths = String(value || '')
+    .split(',')
+    .map((item) => expandPath(item.trim()))
+    .filter(Boolean);
+  return paths.length ? paths : fallback;
+}
+
+export function existingPaths(paths) {
+  return paths.filter((path) => existsSync(path));
+}
+
+export function expandPath(value) {
+  if (typeof value !== 'string' || !value.trim()) return null;
+
+  let expanded = value.trim();
+  if (expanded === '~') {
+    expanded = homedir();
+  } else if (expanded.startsWith('~/')) {
+    expanded = `${homedir()}${expanded.slice(1)}`;
+  }
+
+  expanded = expanded.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, name) => {
+    return process.env[name] || '';
+  });
+  expanded = expanded.replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (_, name) => {
+    return process.env[name] || '';
+  });
+
+  return expanded;
+}

package/src/collector-registry.mjs

@@ -0,0 +1,333 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { configuredPath, configuredPaths } from './collector-config.mjs';
+import { auditStructuredUsage } from './collectors/structured-usage.mjs';
+
+const STABLE_FIELDS = [
+  'date',
+  'source',
+  'session_id',
+  'project',
+  'model',
+  'input_tokens',
+  'output_tokens',
+  'cache_tokens',
+  'reasoning_tokens'
+];
+
+const EXPERIMENTAL_FIELDS = [
+  'timestamp',
+  'source',
+  'session_id',
+  'project_label',
+  'model',
+  'input_tokens',
+  'output_tokens',
+  'cache_tokens',
+  'tool_category',
+  'file_extension'
+];
+
+const IMPORT_FIELDS = [
+  'date',
+  'session',
+  'model',
+  'input_tokens',
+  'output_tokens',
+  'cache_tokens',
+  'cost_usd'
+];
+
+export const COLLECTOR_REGISTRY = [
+  stableCollector('claude', 'Claude Code', './collectors/claude-code.mjs', {
+    privacyLevel: 'metadata-only',
+    roots: () => configuredPaths('claude', 'roots', ['~/.config/claude', '~/.claude'])
+  }),
+  stableCollector('codex', 'Codex CLI', './collectors/codex.mjs', {
+    privacyLevel: 'metadata-only',
+    roots: () => configuredPaths('codex', 'homes', ['~/.codex'])
+  }),
+  stableCollector('gemini', 'Gemini CLI', './collectors/gemini.mjs', {
+    privacyLevel: 'metadata-only',
+    roots: () => [join(homedir(), '.gemini', 'tmp')]
+  }),
+  stableCollector('opencode', 'OpenCode', './collectors/opencode.mjs', {
+    privacyLevel: 'metadata-only',
+    roots: () => [configuredPath('opencode', 'dataDir', '~/.local/share/opencode')]
+  }),
+  stableCollector('openclaw', 'OpenClaw', './collectors/openclaw.mjs', {
+    privacyLevel: 'metadata-only',
+    roots: () => configuredPaths('openclaw', 'agentRoots', [
+      '~/.openclaw/agents',
+      '~/.clawdbot/agents',
+      '~/.moltbot/agents',
+      '~/.moldbot/agents'
+    ])
+  }),
+  stableCollector('hermes', 'Hermes Agent', './collectors/hermes.mjs', {
+    privacyLevel: 'metadata-only',
+    roots: () => [configuredPath('hermes', 'dbPath', '~/.hermes/state.db')]
+  }),
+  experimentalCollector('cursor', 'Cursor', {
+    module: './collectors/cursor.mjs',
+    privacyLevel: 'metadata-only',
+    roots: () => cursorRoots(),
+    note: 'Experimental: only explicit local usage records with token fields are imported; chat content is ignored.'
+  }),
+  experimentalCollector('copilot', 'GitHub Copilot CLI', {
+    module: './collectors/copilot.mjs',
+    privacyLevel: 'metadata-only',
+    roots: () => copilotRoots(),
+    note: 'Experimental: local token rows are imported only when token fields are present.'
+  }),
+  experimentalCollector('qwen', 'Qwen Code', {
+    module: './collectors/qwen.mjs',
+    privacyLevel: 'metadata-only',
+    roots: () => configuredPaths('qwen', 'roots', ['~/.qwen', '~/.qwen-code']),
+    note: 'Experimental: supports fixture-backed structured usage logs without message-body ingestion.'
+  }),
+  experimentalCollector('kimi', 'Kimi / Moonshot Coding CLI', {
+    module: './collectors/kimi.mjs',
+    privacyLevel: 'metadata-only',
+    roots: () => configuredPaths('kimi', 'roots', ['~/.kimi', '~/.moonshot']),
+    note: 'Experimental: supports fixture-backed structured usage logs without message-body ingestion.'
+  }),
+  experimentalCollector('goose', 'Goose', {
+    module: './collectors/goose.mjs',
+    privacyLevel: 'metadata-only',
+    roots: () => configuredPaths('goose', 'roots', ['~/.config/goose', '~/.goose']),
+    note: 'Experimental: supports explicit token metadata only; message bodies are not imported.'
+  }),
+  importOnlyCollector('ccusage', 'ccusage Import Bridge', {
+    roots: () => configuredPaths('ccusage', 'roots', []),
+    note: 'Import-only: use token-studio import-usage --format=ccusage-json for saved JSON or --format=ccusage-cli for an explicit ccusage CLI bridge.'
+  }),
+  detectedOnlyCollector('amp', 'Amp', ['~/.config/amp', '~/.amp']),
+  detectedOnlyCollector('droid', 'Droid', ['~/.droid', '~/.config/droid']),
+  detectedOnlyCollector('codebuff', 'Codebuff', ['~/.codebuff', '~/.config/codebuff']),
+  detectedOnlyCollector('pi-agent', 'pi-agent', ['~/.pi-agent', '~/.config/pi-agent']),
+  detectedOnlyCollector('roo-code', 'Roo Code', ['~/.roo-code', '~/.config/roo-code']),
+  detectedOnlyCollector('zed-agent', 'Zed Agent', ['~/.config/zed', '~/Library/Application Support/Zed']),
+  detectedOnlyCollector('antigravity', 'Antigravity', ['~/.antigravity', '~/.config/antigravity']),
+  detectedOnlyCollector('cline', 'Cline', ['~/.cline', '~/.config/cline']),
+  detectedOnlyCollector('kiro', 'Kiro', ['~/.kiro', '~/.config/kiro']),
+  detectedOnlyCollector('grok-build', 'Grok Build', ['~/.grok', '~/.config/grok']),
+  detectedOnlyCollector('kilo', 'Kilo', ['~/.kilo', '~/.config/kilo'])
+];
+
+export function listCollectors() {
+  return COLLECTOR_REGISTRY.map(({ detect, roots, ...entry }) => ({
+    ...entry,
+    configuredRoots: roots().filter(Boolean)
+  }));
+}
+
+export function stableCollectors() {
+  return COLLECTOR_REGISTRY.filter(item => item.supportStatus === 'stable');
+}
+
+export function collectableCollectors({ includeExperimental = false } = {}) {
+  return COLLECTOR_REGISTRY.filter(item =>
+    item.module && (item.supportStatus === 'stable' || (includeExperimental && item.supportStatus === 'experimental'))
+  );
+}
+
+export function collectorById(id) {
+  return COLLECTOR_REGISTRY.find(item => item.id === id);
+}
+
+export function collectorLabel(id) {
+  return collectorById(id)?.label || id || 'unknown';
+}
+
+export function detectCollectors() {
+  return COLLECTOR_REGISTRY.map(item => {
+    const roots = item.roots().filter(Boolean);
+    const existingRoots = roots.filter(path => existsSync(path));
+    return {
+      id: item.id,
+      label: item.label,
+      supportStatus: item.supportStatus,
+      privacyLevel: item.privacyLevel,
+      defaultEnabled: item.defaultEnabled,
+      detected: existingRoots.length > 0,
+      configuredRoots: roots,
+      existingRoots,
+      module: item.module || null,
+      fixtures: item.fixtures || null,
+      dataFields: item.dataFields || [],
+      readsConversationContent: Boolean(item.readsConversationContent),
+      tokenReliability: item.tokenReliability || 'unknown',
+      fixtureBacked: Boolean(item.fixtures),
+      auditRecommended: item.supportStatus === 'experimental',
+      lastAudit: null,
+      note: item.note || null
+    };
+  });
+}
+
+export async function auditExperimentalCollectors() {
+  const auditedAt = new Date().toISOString();
+  const collectors = [];
+
+  for (const item of COLLECTOR_REGISTRY.filter(row => row.supportStatus === 'experimental')) {
+    const roots = item.roots().filter(Boolean);
+    const existingRoots = roots.filter(path => existsSync(path));
+    const summary = existingRoots.length
+      ? await auditStructuredUsage({ roots: existingRoots })
+      : emptyAuditSummary();
+    collectors.push({
+      id: item.id,
+      label: item.label,
+      supportStatus: item.supportStatus,
+      auditRecommended: true,
+      detected: existingRoots.length > 0,
+      privacyLevel: item.privacyLevel,
+      tokenReliability: item.tokenReliability || 'unknown',
+      readsConversationContent: Boolean(item.readsConversationContent),
+      auditedAt,
+      summary
+    });
+  }
+
+  return {
+    auditedAt,
+    collectors,
+    totals: collectors.reduce((acc, item) => addAuditSummary(acc, item.summary), emptyAuditSummary())
+  };
+}
+
+export function enabledCollectorIds({ includeExperimental = false, values = null } = {}) {
+  const envValue = process.env.TOKEN_STUDIO_COLLECTORS || process.env.AI_TOKEN_DASHBOARD_COLLECTORS;
+  const configRoot = globalCollectorConfig();
+  const rawValues = values != null
+    ? String(values).split(',')
+    : envValue ? envValue.split(',')
+      : Array.isArray(configRoot.enabledCollectors) ? configRoot.enabledCollectors
+      : stableCollectors().filter(item => item.defaultEnabled).map(item => item.id);
+
+  const ids = rawValues.map(item => String(item).trim().toLowerCase()).filter(Boolean);
+  const allowed = new Set(COLLECTOR_REGISTRY
+    .filter(item => includeExperimental || item.supportStatus === 'stable')
+    .map(item => item.id));
+  return new Set(ids.filter(id => allowed.has(id)));
+}
+
+function stableCollector(id, label, module, options) {
+  return {
+    id,
+    label,
+    module,
+    privacyLevel: options.privacyLevel,
+    defaultEnabled: true,
+    supportStatus: 'stable',
+    fixtures: `test/fixtures/collectors/${id}`,
+    dataFields: STABLE_FIELDS,
+    readsConversationContent: false,
+    tokenReliability: 'native-token-fields',
+    roots: options.roots
+  };
+}
+
+function experimentalCollector(id, label, options) {
+  return {
+    id,
+    label,
+    module: options.module || null,
+    privacyLevel: options.privacyLevel,
+    defaultEnabled: false,
+    supportStatus: options.module ? 'experimental' : 'detected-only',
+    fixtures: `test/fixtures/collectors/${id}`,
+    dataFields: EXPERIMENTAL_FIELDS,
+    readsConversationContent: false,
+    tokenReliability: 'explicit-token-fields-only',
+    roots: options.roots,
+    note: options.note
+  };
+}
+
+function importOnlyCollector(id, label, options) {
+  return {
+    id,
+    label,
+    module: null,
+    privacyLevel: 'metadata-only',
+    defaultEnabled: false,
+    supportStatus: 'import-only',
+    fixtures: null,
+    dataFields: IMPORT_FIELDS,
+    readsConversationContent: false,
+    tokenReliability: 'external-json-token-fields',
+    roots: options.roots,
+    note: options.note
+  };
+}
+
+function detectedOnlyCollector(id, label, roots) {
+  return {
+    id,
+    label,
+    module: null,
+    privacyLevel: 'detected-only',
+    defaultEnabled: false,
+    supportStatus: 'detected-only',
+    fixtures: null,
+    dataFields: [],
+    readsConversationContent: false,
+    tokenReliability: 'unknown-no-usage-import',
+    roots: () => configuredPaths(id, 'roots', roots),
+    note: 'Detected-only: Token Studio can show local presence, but it will not write token usage until a reliable token field is audited.'
+  };
+}
+
+function globalCollectorConfig() {
+  try {
+    const path = join(process.cwd(), 'config', 'collectors.json');
+    if (!existsSync(path)) return {};
+    return JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    return {};
+  }
+}
+
+function cursorRoots() {
+  const appData = process.env.APPDATA;
+  const localAppData = process.env.LOCALAPPDATA;
+  return configuredPaths('cursor', 'roots', [
+    appData ? join(appData, 'Cursor') : null,
+    localAppData ? join(localAppData, 'Programs', 'Cursor') : null,
+    '~/.config/Cursor',
+    '~/Library/Application Support/Cursor'
+  ]);
+}
+
+function copilotRoots() {
+  return configuredPaths('copilot', 'roots', [
+    '~/.config/github-copilot',
+    '~/.copilot',
+    '~/Library/Application Support/github-copilot',
+    process.env.APPDATA ? join(process.env.APPDATA, 'GitHub Copilot') : null
+  ]);
+}
+
+function emptyAuditSummary() {
+  return {
+    candidateFiles: 0,
+    usableTokenRecords: 0,
+    skippedNoTokenRecords: 0,
+    skippedConversationLikeRecords: 0,
+    skippedOversizedFiles: 0,
+    parseErrors: 0
+  };
+}
+
+function addAuditSummary(target, source) {
+  target.candidateFiles += source.candidateFiles || 0;
+  target.usableTokenRecords += source.usableTokenRecords || 0;
+  target.skippedNoTokenRecords += source.skippedNoTokenRecords || 0;
+  target.skippedConversationLikeRecords += source.skippedConversationLikeRecords || 0;
+  target.skippedOversizedFiles += source.skippedOversizedFiles || 0;
+  target.parseErrors += source.parseErrors || 0;
+  return target;
+}