vibeusage 0.5.0 → 0.6.1

package/src/lib/ops/audit-source.js

@@ -0,0 +1,399 @@
+"use strict";
+
+/**
+ * audit-source.js — generic ground-truth auditor.
+ *
+ * Every AI CLI source (claude, opencode, codex, gemini, kimi, ...) carries a
+ * different session-log layout and token schema, but the audit shape is the
+ * same: walk local sessions, dedup by upstream id, sum all channels per day,
+ * then compare against DB totals.
+ *
+ * This module captures that shape. Source-specific knowledge lives in
+ * src/lib/ops/sources/<id>.js as a `strategy` object (see CONTRACT below).
+ *
+ * CONTRACT (strategy shape):
+ *   {
+ *     id: "claude" | "opencode" | ...,
+ *     displayName: "Claude Code",
+ *     sessionRoot({ home, env }) -> absolute path,
+ *     walkSessions({ root }) -> string[]  // list of files/dbs to read
+ *     extractUsage(line, context) -> null | {
+ *       timestamp: "<ISO8601>",
+ *       dedupeId: "<stable id>" | null,
+ *       channels: { input, cache_creation, cache_read, output, reasoning }
+ *     }
+ *     // optional: skip the jsonl line-by-line reader if the source uses sqlite
+ *     //            and must iterate rows differently
+ *     iterateRecords(filePath) -> iterable<{ line, context }>
+ *   }
+ *
+ * Consumers:
+ *   - doctor --audit-tokens --source <id>
+ *   - scripts/ops/compare-<source>-ground-truth.cjs  (thin CLI wrappers)
+ */
+
+const fs = require("node:fs");
+const os = require("node:os");
+const { spawnSync } = require("node:child_process");
+
+const DEFAULT_DAYS = 14;
+const DEFAULT_THRESHOLD_PCT = 25;
+
+// `resolveUserIdViaInsforge` and `queryDbTotalsViaInsforge` interpolate these
+// values into a SQL string handed to `insforge db query` (argv, not shell, but
+// the argv reaches a SQL executor with service-role authority). The three
+// values are validated against strict whitelists so a malicious / typo flag
+// like --user-id "foo'; DROP TABLE users; --" cannot reach the DB.
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const SOURCE_ID_RE = /^[a-z][a-z0-9-]*$/;
+const ISO_TIMESTAMP_RE =
+  /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;
+
+function runSourceAudit({
+  strategy,
+  days = DEFAULT_DAYS,
+  threshold = DEFAULT_THRESHOLD_PCT,
+  userId = null,
+  deviceId = null,
+  dbJsonPath = null,
+  dbJson = null,
+  home = os.homedir(),
+  env = process.env,
+  sessionRootOverride = null,
+} = {}) {
+  if (!strategy || typeof strategy !== "object") {
+    throw new Error("runSourceAudit requires a strategy object");
+  }
+  for (const key of ["id", "sessionRoot", "walkSessions", "extractUsage"]) {
+    if (typeof strategy[key] !== "function" && typeof strategy[key] !== "string") {
+      throw new Error(`strategy.${key} is required`);
+    }
+  }
+  if (!Number.isFinite(days) || days <= 0) {
+    throw new Error(`days must be a positive number, got ${days}`);
+  }
+  if (!Number.isFinite(threshold) || threshold < 0) {
+    throw new Error(`threshold must be non-negative, got ${threshold}`);
+  }
+
+  const root = sessionRootOverride || strategy.sessionRoot({ home, env });
+  const now = new Date();
+  const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
+  start.setUTCDate(start.getUTCDate() - (days - 1));
+  const windowStartIso = start.toISOString();
+  const files = strategy.walkSessions({ root, windowStartIso });
+  if (!files || files.length === 0) {
+    return {
+      ok: false,
+      error: "no-local-sessions",
+      source: strategy.id,
+      message: `no local sessions for source=${strategy.id} under ${root}`,
+      rows: [],
+      maxDriftPct: 0,
+    };
+  }
+
+  const local = computeLocalTotals({ files, windowStartIso, strategy });
+
+  let backend;
+  if (dbJson) {
+    try {
+      backend = parseDbJson(dbJson);
+    } catch (err) {
+      return { ok: false, error: "db-json-parse", source: strategy.id, message: err.message, rows: [], maxDriftPct: 0 };
+    }
+  } else if (dbJsonPath) {
+    let blob;
+    try {
+      blob = fs.readFileSync(dbJsonPath, "utf8");
+    } catch (err) {
+      return {
+        ok: false,
+        error: "db-json-read-failed",
+        source: strategy.id,
+        message: `cannot read ${dbJsonPath}: ${err?.message || err}`,
+        rows: [],
+        maxDriftPct: 0,
+      };
+    }
+    backend = parseDbJson(blob);
+  } else {
+    const resolvedUserId = userId || resolveUserIdViaInsforge({ deviceId });
+    if (!resolvedUserId) {
+      return {
+        ok: false,
+        error: "cannot-resolve-user-id",
+        source: strategy.id,
+        message:
+          "cannot resolve user_id; pass userId explicitly, supply dbJson, or make sure `insforge` CLI is linked to the vibeusage workspace and config.deviceId is set",
+        rows: [],
+        maxDriftPct: 0,
+      };
+    }
+    const queryRes = queryDbTotalsViaInsforge({
+      userId: resolvedUserId,
+      source: strategy.id,
+      windowStartIso,
+    });
+    if (!queryRes.ok) {
+      return { ...queryRes, source: strategy.id, rows: [], maxDriftPct: 0 };
+    }
+    backend = queryRes.byDay;
+  }
+
+  const dayKeys = Array.from(new Set([...local.byDay.keys(), ...backend.keys()])).sort();
+  const rows = [];
+  let maxDriftPct = 0;
+  for (const day of dayKeys) {
+    const truth = (local.byDay.get(day) || { total: 0 }).total;
+    const dbTotal = backend.get(day) || 0;
+    const ratio = truth > 0 ? dbTotal / truth : null;
+    const drift = ratio == null ? null : Math.abs(ratio - 1) * 100;
+    if (drift != null && drift > maxDriftPct) maxDriftPct = drift;
+    rows.push({ day, truth, db: dbTotal, ratio, driftPct: drift });
+  }
+
+  return {
+    ok: true,
+    source: strategy.id,
+    displayName: strategy.displayName || strategy.id,
+    windowStartIso,
+    days,
+    thresholdPct: threshold,
+    filesScanned: files.length,
+    usageLines: local.scanned,
+    uniqueMessages: local.uniqueIds,
+    duplicatesSkipped: local.skippedDup,
+    rows,
+    maxDriftPct: Number(maxDriftPct.toFixed(2)),
+    exceedsThreshold: maxDriftPct > threshold,
+  };
+}
+
+function computeLocalTotals({ files, windowStartIso, strategy }) {
+  const byDay = new Map();
+  const seen = new Set();
+  let scanned = 0;
+  let skippedDup = 0;
+
+  const records = typeof strategy.iterateRecords === "function"
+    ? strategy.iterateRecords
+    : defaultIterateRecords;
+
+  for (const filePath of files) {
+    for (const { line, context } of records(filePath)) {
+      const extracted = strategy.extractUsage(line, context);
+      if (!extracted) continue;
+      const { timestamp, dedupeId, channels } = extracted;
+      if (!timestamp || timestamp < windowStartIso) continue;
+      const day = isoDay(timestamp);
+      if (!day) continue;
+
+      scanned += 1;
+      if (dedupeId && seen.has(dedupeId)) {
+        skippedDup += 1;
+        continue;
+      }
+      if (dedupeId) seen.add(dedupeId);
+
+      const input = nonneg(channels.input);
+      const cacheCreation = nonneg(channels.cache_creation);
+      const cacheRead = nonneg(channels.cache_read);
+      const output = nonneg(channels.output);
+      const reasoning = nonneg(channels.reasoning);
+      const total = input + cacheCreation + cacheRead + output + reasoning;
+      if (total === 0) continue;
+
+      let row = byDay.get(day);
+      if (!row) {
+        row = {
+          total: 0,
+          input: 0,
+          cache_creation: 0,
+          cache_read: 0,
+          output: 0,
+          reasoning: 0,
+          messages: 0,
+        };
+        byDay.set(day, row);
+      }
+      row.total += total;
+      row.input += input;
+      row.cache_creation += cacheCreation;
+      row.cache_read += cacheRead;
+      row.output += output;
+      row.reasoning += reasoning;
+      row.messages += 1;
+    }
+  }
+
+  return { byDay, scanned, skippedDup, uniqueIds: seen.size };
+}
+
+function* defaultIterateRecords(filePath) {
+  let text;
+  try {
+    text = fs.readFileSync(filePath, "utf8");
+  } catch (_err) {
+    return;
+  }
+  for (const line of text.split("\n")) {
+    if (!line) continue;
+    yield { line, context: { filePath } };
+  }
+}
+
+function isoDay(ts) {
+  if (typeof ts !== "string") return null;
+  const m = ts.match(/^(\d{4}-\d{2}-\d{2})/);
+  return m ? m[1] : null;
+}
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}
+
+function resolveUserIdViaInsforge({ deviceId }) {
+  if (!deviceId || !UUID_RE.test(String(deviceId))) return null;
+  const r = spawnSync(
+    "insforge",
+    [
+      "db",
+      "query",
+      `SELECT user_id FROM vibeusage_tracker_devices WHERE id='${deviceId}' LIMIT 1`,
+    ],
+    { encoding: "utf8" },
+  );
+  if (r.status !== 0) return null;
+  const m = (r.stdout || "").match(
+    /\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b/i,
+  );
+  return m ? m[1] : null;
+}
+
+function queryDbTotalsViaInsforge({ userId, source, windowStartIso }) {
+  if (!userId || !UUID_RE.test(String(userId))) {
+    return {
+      ok: false,
+      error: "invalid-user-id",
+      message: `refusing to query DB with non-UUID user id '${String(userId).slice(0, 40)}'`,
+    };
+  }
+  if (!source || !SOURCE_ID_RE.test(String(source))) {
+    return {
+      ok: false,
+      error: "invalid-source-id",
+      message: `refusing to query DB with non-identifier source '${String(source).slice(0, 40)}'`,
+    };
+  }
+  if (!windowStartIso || !ISO_TIMESTAMP_RE.test(String(windowStartIso))) {
+    return {
+      ok: false,
+      error: "invalid-window-start",
+      message: `refusing to query DB with non-ISO windowStartIso '${String(windowStartIso).slice(0, 40)}'`,
+    };
+  }
+  const sql =
+    `SELECT DATE(hour_start) AS day, SUM(total_tokens) AS tokens ` +
+    `FROM vibeusage_tracker_hourly ` +
+    `WHERE source='${source}' AND user_id='${userId}' AND hour_start >= '${windowStartIso}' ` +
+    `GROUP BY DATE(hour_start) ORDER BY day`;
+  const r = spawnSync("insforge", ["--json", "db", "query", sql], { encoding: "utf8" });
+  if (r.status !== 0) {
+    return {
+      ok: false,
+      error: "insforge-db-query-failed",
+      message:
+        `\`insforge db query\` failed (${r.status}). Run \`insforge current\` to confirm ` +
+        `the CLI is linked to the vibeusage workspace, or pass dbJson directly.`,
+    };
+  }
+  return { ok: true, byDay: parseDbJson(r.stdout) };
+}
+
+function parseDbJson(blob) {
+  let parsed;
+  if (typeof blob === "object" && blob !== null) {
+    parsed = blob;
+  } else {
+    try {
+      parsed = JSON.parse(blob);
+    } catch (err) {
+      throw new Error(`cannot parse DB JSON: ${err?.message || err}`);
+    }
+  }
+  const rows = Array.isArray(parsed)
+    ? parsed
+    : Array.isArray(parsed?.rows)
+      ? parsed.rows
+      : Array.isArray(parsed?.data)
+        ? parsed.data
+        : null;
+  if (!rows) {
+    throw new Error(
+      `DB JSON shape unexpected (need array of {day, tokens}); got ${
+        Object.keys(parsed || {}).join(",") || "(empty)"
+      }`,
+    );
+  }
+  const byDay = new Map();
+  for (const row of rows) {
+    const rawDay = row?.day ?? row?.date ?? row?.bucket_day;
+    const day = typeof rawDay === "string" ? rawDay.slice(0, 10) : null;
+    if (!day) continue;
+    const total = nonneg(row.tokens ?? row.total_tokens ?? row.total);
+    byDay.set(day, total);
+  }
+  return byDay;
+}
+
+// Registry for doctor --audit-tokens --source routing.
+// Register new strategies as they land in sources/<id>.js.
+function getStrategy(id) {
+  switch (id) {
+    case "claude":
+      // eslint-disable-next-line global-require
+      return require("./sources/claude");
+    case "opencode":
+      // eslint-disable-next-line global-require
+      return require("./sources/opencode");
+    case "codex":
+      // eslint-disable-next-line global-require
+      return require("./sources/codex");
+    case "every-code":
+      // eslint-disable-next-line global-require
+      return require("./sources/every-code");
+    case "gemini":
+      // eslint-disable-next-line global-require
+      return require("./sources/gemini");
+    case "kimi":
+      // eslint-disable-next-line global-require
+      return require("./sources/kimi");
+    case "hermes":
+      // eslint-disable-next-line global-require
+      return require("./sources/hermes");
+    case "openclaw":
+      // eslint-disable-next-line global-require
+      return require("./sources/openclaw");
+    default:
+      return null;
+  }
+}
+
+function listRegisteredSources() {
+  return ["claude", "opencode", "codex", "every-code", "gemini", "kimi", "hermes", "openclaw"];
+}
+
+module.exports = {
+  DEFAULT_DAYS,
+  DEFAULT_THRESHOLD_PCT,
+  runSourceAudit,
+  getStrategy,
+  listRegisteredSources,
+  // Exported for targeted tests that assert the SQL-interpolation inputs are
+  // validated before reaching `insforge db query`.
+  queryDbTotalsViaInsforge,
+  resolveUserIdViaInsforge,
+};

package/src/lib/ops/sources/_rollout-base.js

@@ -0,0 +1,203 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Shared strategy factory for Codex-family rollout audits.
+ *
+ * Codex and Every-Code write identical rollout .jsonl streams into
+ * <home>/<subdir>/sessions/<YYYY>/<MM>/<DD>/rollout-*.jsonl. The directory
+ * layout is the only difference (CODEX_HOME / CODE_HOME). Token accounting
+ * semantics differ from Claude's in two important ways:
+ *
+ *   1. Token events are `payload.type === "token_count"` rows that carry
+ *      `info.total_token_usage` (cumulative) and `info.last_token_usage`
+ *      (delta for the latest API call). Events with `info: null` are
+ *      rate-limit-only pings and must be ignored.
+ *
+ *   2. `input_tokens` already includes cached input, and `output_tokens`
+ *      already includes `reasoning_output_tokens`. Naively summing all five
+ *      channels double-counts. The authoritative per-turn total is simply
+ *      `total_tokens` on the upstream payload, which is what normalizeUsage
+ *      passes through to the DB unchanged.
+ *
+ * Approach:
+ *   - walkSessions prunes by YYYY/MM/DD directories before hitting the jsonl
+ *     files so the auditor does not scan all ~240K Codex rollouts just to
+ *     look at the last 14 days.
+ *   - iterateRecords is stateful per file: it tracks the last seen
+ *     total_token_usage and yields a synthetic delta object whenever the
+ *     total changes (uses last_token_usage when available, otherwise the
+ *     total_prev diff). Duplicate token_count rows with identical totals are
+ *     skipped; that mirrors parseRolloutFile's pickDelta logic.
+ *   - extractUsage routes the authoritative `total_tokens` number into the
+ *     `output` channel and zeroes the rest. The framework sums the five
+ *     channels to compute row.truth, so putting the whole total into one
+ *     channel is a deliberate trick that keeps day totals correct without
+ *     exposing Codex's overlapping channel semantics through the generic
+ *     contract.
+ */
+
+function makeRolloutStrategy({ id, displayName, envKey, defaultSubdir }) {
+  return {
+    id,
+    displayName,
+    sessionRoot({ home, env }) {
+      const base = (env && env[envKey]) || path.join(home, defaultSubdir);
+      return path.join(base, "sessions");
+    },
+    walkSessions({ root, windowStartIso }) {
+      if (!fs.existsSync(root)) return [];
+      // Rollout events written on day N can carry timestamps from day N-1
+      // (sessions straddle midnight). Keep directories starting one day
+      // before the window so we do not drop boundary events.
+      const bufferDay = shiftIsoDay(windowStartIso, -1);
+      const out = [];
+      for (const year of safeReadDirSync(root)) {
+        if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue;
+        const yearDir = path.join(root, year.name);
+        for (const month of safeReadDirSync(yearDir)) {
+          if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue;
+          const monthDir = path.join(yearDir, month.name);
+          for (const day of safeReadDirSync(monthDir)) {
+            if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue;
+            if (bufferDay) {
+              const dayIso = `${year.name}-${month.name}-${day.name}`;
+              if (dayIso < bufferDay) continue;
+            }
+            const dayDir = path.join(monthDir, day.name);
+            for (const f of safeReadDirSync(dayDir)) {
+              if (!f.isFile()) continue;
+              if (!f.name.startsWith("rollout-") || !f.name.endsWith(".jsonl")) continue;
+              out.push(path.join(dayDir, f.name));
+            }
+          }
+        }
+      }
+      return out;
+    },
+    *iterateRecords(filePath) {
+      let text;
+      try {
+        text = fs.readFileSync(filePath, "utf8");
+      } catch (_err) {
+        return;
+      }
+      if (!text) return;
+      let prevTotal = null;
+      for (const line of text.split("\n")) {
+        if (!line || !line.includes("token_count")) continue;
+        let obj;
+        try {
+          obj = JSON.parse(line);
+        } catch (_err) {
+          continue;
+        }
+        const payload = obj?.payload;
+        if (!payload || payload.type !== "token_count") continue;
+        const info = payload.info;
+        if (!info) continue;
+        const total = info.total_token_usage || null;
+        const last = info.last_token_usage || null;
+        if (!total && !last) continue;
+        // Duplicate token_count: same totals, skip.
+        if (prevTotal && total && sameUsage(prevTotal, total)) continue;
+        let delta;
+        if (last && Number(last.total_tokens) > 0) {
+          delta = last;
+        } else if (prevTotal && total) {
+          delta = diffUsage(total, prevTotal);
+        } else if (total) {
+          delta = total;
+        } else {
+          delta = null;
+        }
+        if (total) prevTotal = total;
+        if (!delta || !Number(delta.total_tokens)) continue;
+        yield {
+          line: JSON.stringify({ timestamp: obj.timestamp, delta }),
+          context: { filePath },
+        };
+      }
+    },
+    extractUsage(line) {
+      if (!line) return null;
+      let obj;
+      try {
+        obj = JSON.parse(line);
+      } catch (_err) {
+        return null;
+      }
+      const ts = typeof obj.timestamp === "string" ? obj.timestamp : null;
+      const d = obj.delta;
+      if (!ts || !d) return null;
+      const totalTokens = Number(d.total_tokens);
+      if (!Number.isFinite(totalTokens) || totalTokens <= 0) return null;
+      return {
+        timestamp: ts,
+        dedupeId: null, // per-file dedup already done in iterateRecords
+        channels: {
+          input: 0,
+          cache_creation: 0,
+          cache_read: 0,
+          // Route the authoritative Codex upstream total into a single
+          // channel so the framework's sum-of-channels lands on it. See
+          // module docstring for why we do not split the channels.
+          output: totalTokens,
+          reasoning: 0,
+        },
+      };
+    },
+  };
+}
+
+function safeReadDirSync(p) {
+  try {
+    return fs.readdirSync(p, { withFileTypes: true });
+  } catch (_err) {
+    return [];
+  }
+}
+
+function sameUsage(a, b) {
+  if (!a || !b) return false;
+  for (const k of [
+    "input_tokens",
+    "cached_input_tokens",
+    "output_tokens",
+    "reasoning_output_tokens",
+    "total_tokens",
+  ]) {
+    if (Number(a[k] || 0) !== Number(b[k] || 0)) return false;
+  }
+  return true;
+}
+
+function diffUsage(curr, prev) {
+  if (!curr || !prev) return curr || null;
+  const currTotal = Number(curr.total_tokens || 0);
+  const prevTotal = Number(prev.total_tokens || 0);
+  if (currTotal < prevTotal) return curr; // session reset
+  const out = {};
+  for (const k of [
+    "input_tokens",
+    "cached_input_tokens",
+    "output_tokens",
+    "reasoning_output_tokens",
+    "total_tokens",
+  ]) {
+    out[k] = Math.max(0, Number(curr[k] || 0) - Number(prev[k] || 0));
+  }
+  return out;
+}
+
+function shiftIsoDay(iso, deltaDays) {
+  if (typeof iso !== "string" || !iso) return null;
+  const base = new Date(`${iso.slice(0, 10)}T00:00:00Z`);
+  if (Number.isNaN(base.getTime())) return null;
+  base.setUTCDate(base.getUTCDate() + deltaDays);
+  return base.toISOString().slice(0, 10);
+}
+
+module.exports = { makeRolloutStrategy };

package/src/lib/ops/sources/claude.js

@@ -0,0 +1,52 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+module.exports = {
+  id: "claude",
+  displayName: "Claude Code",
+  sessionRoot({ home }) {
+    return path.join(home, ".claude", "projects");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const dir = path.join(root, entry.name);
+      for (const f of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (!f.isFile()) continue;
+        if (!f.name.endsWith(".jsonl")) continue;
+        out.push(path.join(dir, f.name));
+      }
+    }
+    return out;
+  },
+  extractUsage(line) {
+    if (!line || !line.includes('"usage"')) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    const msg = obj?.message || {};
+    const usage = msg.usage || obj.usage;
+    if (!usage || typeof usage !== "object") return null;
+    const timestamp = typeof obj.timestamp === "string" ? obj.timestamp : null;
+    if (!timestamp) return null;
+
+    return {
+      timestamp,
+      dedupeId: msg.id || obj.requestId || null,
+      channels: {
+        input: usage.input_tokens,
+        cache_creation: usage.cache_creation_input_tokens,
+        cache_read: usage.cache_read_input_tokens,
+        output: usage.output_tokens,
+        reasoning: 0,
+      },
+    };
+  },
+};

package/src/lib/ops/sources/codex.js

@@ -0,0 +1,10 @@
+"use strict";
+
+const { makeRolloutStrategy } = require("./_rollout-base");
+
+module.exports = makeRolloutStrategy({
+  id: "codex",
+  displayName: "Codex CLI",
+  envKey: "CODEX_HOME",
+  defaultSubdir: ".codex",
+});

package/src/lib/ops/sources/every-code.js

@@ -0,0 +1,10 @@
+"use strict";
+
+const { makeRolloutStrategy } = require("./_rollout-base");
+
+module.exports = makeRolloutStrategy({
+  id: "every-code",
+  displayName: "Every Code",
+  envKey: "CODE_HOME",
+  defaultSubdir: ".code",
+});