vibeusage 0.5.0 → 0.6.0

package/src/lib/ops/sources/_rollout-base.js

@@ -0,0 +1,203 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Shared strategy factory for Codex-family rollout audits.
+ *
+ * Codex and Every-Code write identical rollout .jsonl streams into
+ * <home>/<subdir>/sessions/<YYYY>/<MM>/<DD>/rollout-*.jsonl. The directory
+ * layout is the only difference (CODEX_HOME / CODE_HOME). Token accounting
+ * semantics differ from Claude's in two important ways:
+ *
+ *   1. Token events are `payload.type === "token_count"` rows that carry
+ *      `info.total_token_usage` (cumulative) and `info.last_token_usage`
+ *      (delta for the latest API call). Events with `info: null` are
+ *      rate-limit-only pings and must be ignored.
+ *
+ *   2. `input_tokens` already includes cached input, and `output_tokens`
+ *      already includes `reasoning_output_tokens`. Naively summing all five
+ *      channels double-counts. The authoritative per-turn total is simply
+ *      `total_tokens` on the upstream payload, which is what normalizeUsage
+ *      passes through to the DB unchanged.
+ *
+ * Approach:
+ *   - walkSessions prunes by YYYY/MM/DD directories before hitting the jsonl
+ *     files so the auditor does not scan all ~240K Codex rollouts just to
+ *     look at the last 14 days.
+ *   - iterateRecords is stateful per file: it tracks the last seen
+ *     total_token_usage and yields a synthetic delta object whenever the
+ *     total changes (uses last_token_usage when available, otherwise the
+ *     total_prev diff). Duplicate token_count rows with identical totals are
+ *     skipped; that mirrors parseRolloutFile's pickDelta logic.
+ *   - extractUsage routes the authoritative `total_tokens` number into the
+ *     `output` channel and zeroes the rest. The framework sums the five
+ *     channels to compute row.truth, so putting the whole total into one
+ *     channel is a deliberate trick that keeps day totals correct without
+ *     exposing Codex's overlapping channel semantics through the generic
+ *     contract.
+ */
+
+function makeRolloutStrategy({ id, displayName, envKey, defaultSubdir }) {
+  return {
+    id,
+    displayName,
+    sessionRoot({ home, env }) {
+      const base = (env && env[envKey]) || path.join(home, defaultSubdir);
+      return path.join(base, "sessions");
+    },
+    walkSessions({ root, windowStartIso }) {
+      if (!fs.existsSync(root)) return [];
+      // Rollout events written on day N can carry timestamps from day N-1
+      // (sessions straddle midnight). Keep directories starting one day
+      // before the window so we do not drop boundary events.
+      const bufferDay = shiftIsoDay(windowStartIso, -1);
+      const out = [];
+      for (const year of safeReadDirSync(root)) {
+        if (!year.isDirectory() || !/^\d{4}$/.test(year.name)) continue;
+        const yearDir = path.join(root, year.name);
+        for (const month of safeReadDirSync(yearDir)) {
+          if (!month.isDirectory() || !/^\d{2}$/.test(month.name)) continue;
+          const monthDir = path.join(yearDir, month.name);
+          for (const day of safeReadDirSync(monthDir)) {
+            if (!day.isDirectory() || !/^\d{2}$/.test(day.name)) continue;
+            if (bufferDay) {
+              const dayIso = `${year.name}-${month.name}-${day.name}`;
+              if (dayIso < bufferDay) continue;
+            }
+            const dayDir = path.join(monthDir, day.name);
+            for (const f of safeReadDirSync(dayDir)) {
+              if (!f.isFile()) continue;
+              if (!f.name.startsWith("rollout-") || !f.name.endsWith(".jsonl")) continue;
+              out.push(path.join(dayDir, f.name));
+            }
+          }
+        }
+      }
+      return out;
+    },
+    *iterateRecords(filePath) {
+      let text;
+      try {
+        text = fs.readFileSync(filePath, "utf8");
+      } catch (_err) {
+        return;
+      }
+      if (!text) return;
+      let prevTotal = null;
+      for (const line of text.split("\n")) {
+        if (!line || !line.includes("token_count")) continue;
+        let obj;
+        try {
+          obj = JSON.parse(line);
+        } catch (_err) {
+          continue;
+        }
+        const payload = obj?.payload;
+        if (!payload || payload.type !== "token_count") continue;
+        const info = payload.info;
+        if (!info) continue;
+        const total = info.total_token_usage || null;
+        const last = info.last_token_usage || null;
+        if (!total && !last) continue;
+        // Duplicate token_count: same totals, skip.
+        if (prevTotal && total && sameUsage(prevTotal, total)) continue;
+        let delta;
+        if (last && Number(last.total_tokens) > 0) {
+          delta = last;
+        } else if (prevTotal && total) {
+          delta = diffUsage(total, prevTotal);
+        } else if (total) {
+          delta = total;
+        } else {
+          delta = null;
+        }
+        if (total) prevTotal = total;
+        if (!delta || !Number(delta.total_tokens)) continue;
+        yield {
+          line: JSON.stringify({ timestamp: obj.timestamp, delta }),
+          context: { filePath },
+        };
+      }
+    },
+    extractUsage(line) {
+      if (!line) return null;
+      let obj;
+      try {
+        obj = JSON.parse(line);
+      } catch (_err) {
+        return null;
+      }
+      const ts = typeof obj.timestamp === "string" ? obj.timestamp : null;
+      const d = obj.delta;
+      if (!ts || !d) return null;
+      const totalTokens = Number(d.total_tokens);
+      if (!Number.isFinite(totalTokens) || totalTokens <= 0) return null;
+      return {
+        timestamp: ts,
+        dedupeId: null, // per-file dedup already done in iterateRecords
+        channels: {
+          input: 0,
+          cache_creation: 0,
+          cache_read: 0,
+          // Route the authoritative Codex upstream total into a single
+          // channel so the framework's sum-of-channels lands on it. See
+          // module docstring for why we do not split the channels.
+          output: totalTokens,
+          reasoning: 0,
+        },
+      };
+    },
+  };
+}
+
+function safeReadDirSync(p) {
+  try {
+    return fs.readdirSync(p, { withFileTypes: true });
+  } catch (_err) {
+    return [];
+  }
+}
+
+function sameUsage(a, b) {
+  if (!a || !b) return false;
+  for (const k of [
+    "input_tokens",
+    "cached_input_tokens",
+    "output_tokens",
+    "reasoning_output_tokens",
+    "total_tokens",
+  ]) {
+    if (Number(a[k] || 0) !== Number(b[k] || 0)) return false;
+  }
+  return true;
+}
+
+function diffUsage(curr, prev) {
+  if (!curr || !prev) return curr || null;
+  const currTotal = Number(curr.total_tokens || 0);
+  const prevTotal = Number(prev.total_tokens || 0);
+  if (currTotal < prevTotal) return curr; // session reset
+  const out = {};
+  for (const k of [
+    "input_tokens",
+    "cached_input_tokens",
+    "output_tokens",
+    "reasoning_output_tokens",
+    "total_tokens",
+  ]) {
+    out[k] = Math.max(0, Number(curr[k] || 0) - Number(prev[k] || 0));
+  }
+  return out;
+}
+
+function shiftIsoDay(iso, deltaDays) {
+  if (typeof iso !== "string" || !iso) return null;
+  const base = new Date(`${iso.slice(0, 10)}T00:00:00Z`);
+  if (Number.isNaN(base.getTime())) return null;
+  base.setUTCDate(base.getUTCDate() + deltaDays);
+  return base.toISOString().slice(0, 10);
+}
+
+module.exports = { makeRolloutStrategy };

package/src/lib/ops/sources/claude.js

@@ -0,0 +1,52 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+module.exports = {
+  id: "claude",
+  displayName: "Claude Code",
+  sessionRoot({ home }) {
+    return path.join(home, ".claude", "projects");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const dir = path.join(root, entry.name);
+      for (const f of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (!f.isFile()) continue;
+        if (!f.name.endsWith(".jsonl")) continue;
+        out.push(path.join(dir, f.name));
+      }
+    }
+    return out;
+  },
+  extractUsage(line) {
+    if (!line || !line.includes('"usage"')) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    const msg = obj?.message || {};
+    const usage = msg.usage || obj.usage;
+    if (!usage || typeof usage !== "object") return null;
+    const timestamp = typeof obj.timestamp === "string" ? obj.timestamp : null;
+    if (!timestamp) return null;
+
+    return {
+      timestamp,
+      dedupeId: msg.id || obj.requestId || null,
+      channels: {
+        input: usage.input_tokens,
+        cache_creation: usage.cache_creation_input_tokens,
+        cache_read: usage.cache_read_input_tokens,
+        output: usage.output_tokens,
+        reasoning: 0,
+      },
+    };
+  },
+};

package/src/lib/ops/sources/codex.js

@@ -0,0 +1,10 @@
+"use strict";
+
+const { makeRolloutStrategy } = require("./_rollout-base");
+
+module.exports = makeRolloutStrategy({
+  id: "codex",
+  displayName: "Codex CLI",
+  envKey: "CODEX_HOME",
+  defaultSubdir: ".codex",
+});

package/src/lib/ops/sources/every-code.js

@@ -0,0 +1,10 @@
+"use strict";
+
+const { makeRolloutStrategy } = require("./_rollout-base");
+
+module.exports = makeRolloutStrategy({
+  id: "every-code",
+  displayName: "Every Code",
+  envKey: "CODE_HOME",
+  defaultSubdir: ".code",
+});

package/src/lib/ops/sources/gemini.js

@@ -0,0 +1,154 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Gemini audit strategy.
+ *
+ * Gemini writes one JSON per session under
+ *   ~/.gemini/tmp/<hash>/chats/session-YYYY-MM-DDTHH-MM-<id>.json
+ * The file has `{ messages: [ { role, timestamp, model, tokens } ] }` where
+ * `tokens` is the cumulative usage up to that message (not a per-turn delta).
+ *
+ * Channel semantics differ from Claude but match Codex in one important way:
+ *   input + cached + output + tool + thoughts != total
+ * because `tokens.total` is the authoritative upstream count that
+ * src/lib/rollout.js normalizeGeminiTokens passes through as-is to the DB.
+ * Naively summing the five sub-channels double-counts. As with the Codex
+ * strategy, we route `delta.total` into the output channel and zero the rest
+ * so the framework's sum-of-channels row.truth equals the DB total_tokens
+ * without exposing Gemini's internal breakdown through the generic contract.
+ *
+ * Dedupe:
+ *   - Per-file index diff mirrors parseGeminiFile's `lastTotals` state.
+ *   - When `tokens.total` drops (session reset / resume), we treat the current
+ *     cumulative as the delta just like the parser does.
+ */
+
+module.exports = {
+  id: "gemini",
+  displayName: "Gemini CLI",
+  sessionRoot({ home, env }) {
+    const base = (env && env.GEMINI_HOME) || path.join(home, ".gemini");
+    return path.join(base, "tmp");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const hash of safeReadDirSync(root)) {
+      if (!hash.isDirectory()) continue;
+      const chatsDir = path.join(root, hash.name, "chats");
+      for (const f of safeReadDirSync(chatsDir)) {
+        if (!f.isFile()) continue;
+        if (!f.name.startsWith("session-") || !f.name.endsWith(".json")) continue;
+        out.push(path.join(chatsDir, f.name));
+      }
+    }
+    return out;
+  },
+  *iterateRecords(filePath) {
+    let raw;
+    try {
+      raw = fs.readFileSync(filePath, "utf8");
+    } catch (_err) {
+      return;
+    }
+    if (!raw.trim()) return;
+    let session;
+    try {
+      session = JSON.parse(raw);
+    } catch (_err) {
+      return;
+    }
+    const messages = Array.isArray(session?.messages) ? session.messages : [];
+    let prevTotals = null;
+    for (const msg of messages) {
+      if (!msg || typeof msg !== "object") continue;
+      const ts = typeof msg.timestamp === "string" ? msg.timestamp : null;
+      if (!ts) continue;
+      const tokens = msg.tokens;
+      if (!tokens || typeof tokens !== "object") continue;
+
+      const curr = normalizeTokens(tokens);
+      const delta = diffTotals(curr, prevTotals);
+      prevTotals = curr;
+      if (!delta || !delta.total) continue;
+
+      yield {
+        line: JSON.stringify({ timestamp: ts, delta }),
+        context: { filePath },
+      };
+    }
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    const ts = typeof obj.timestamp === "string" ? obj.timestamp : null;
+    const d = obj.delta;
+    if (!ts || !d || !Number(d.total)) return null;
+    return {
+      timestamp: ts,
+      dedupeId: null, // per-file index diff already dedupes
+      channels: {
+        input: 0,
+        cache_creation: 0,
+        cache_read: 0,
+        // Route the authoritative upstream total into a single channel; see
+        // module docstring for why we do not split it.
+        output: Number(d.total),
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function normalizeTokens(tokens) {
+  return {
+    input: nonneg(tokens.input),
+    cached: nonneg(tokens.cached),
+    output: nonneg(tokens.output),
+    tool: nonneg(tokens.tool),
+    thoughts: nonneg(tokens.thoughts),
+    total: nonneg(tokens.total),
+  };
+}
+
+function diffTotals(curr, prev) {
+  if (!curr) return null;
+  if (!prev) {
+    // First message with tokens — the whole cumulative value is the delta.
+    return curr;
+  }
+  // Session reset: upstream total decreased (resume / new session). Trust the
+  // new value as the full delta.
+  if (curr.total < prev.total) return curr;
+  const delta = {
+    input: Math.max(0, curr.input - prev.input),
+    cached: Math.max(0, curr.cached - prev.cached),
+    output: Math.max(0, curr.output - prev.output),
+    tool: Math.max(0, curr.tool - prev.tool),
+    thoughts: Math.max(0, curr.thoughts - prev.thoughts),
+    total: Math.max(0, curr.total - prev.total),
+  };
+  return delta;
+}
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}
+
+function safeReadDirSync(p) {
+  try {
+    return fs.readdirSync(p, { withFileTypes: true });
+  } catch (_err) {
+    return [];
+  }
+}

package/src/lib/ops/sources/hermes.js

@@ -0,0 +1,69 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Hermes audit strategy.
+ *
+ * Hermes does not write raw session logs; it emits one pre-aggregated event
+ * per turn into the vibeusage tracker directory:
+ *   ~/.vibeusage/tracker/hermes.usage.jsonl
+ * Each line is a `{type: "usage", emitted_at, model, input_tokens,
+ * output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens,
+ * total_tokens}` record. src/commands/sync.js parseHermesUsageLedger already
+ * copies `total_tokens` straight into the bucket, so this audit routes the
+ * upstream total into the output channel — same pattern we use for Codex and
+ * Gemini.
+ *
+ * sessionRoot: the tracker directory (NOT `~/.hermes/...` — Hermes usage data
+ * lives under ~/.vibeusage/tracker because Hermes is a plugin that hands
+ * vibeusage ledger rows directly).
+ */
+
+module.exports = {
+  id: "hermes",
+  displayName: "Hermes Plugin",
+  sessionRoot({ home, env }) {
+    const base = (env && env.VIBEUSAGE_HOME) || path.join(home, ".vibeusage");
+    return path.join(base, "tracker");
+  },
+  walkSessions({ root }) {
+    const ledger = path.join(root, "hermes.usage.jsonl");
+    if (!fs.existsSync(ledger)) return [];
+    return [ledger];
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (!event || event.type !== "usage") return null;
+    const timestamp = typeof event.emitted_at === "string" ? event.emitted_at : null;
+    if (!timestamp) return null;
+    const total = nonneg(event.total_tokens);
+    if (total === 0) return null;
+    return {
+      timestamp,
+      // Hermes ledger records do not carry a stable per-event id;
+      // the ledger is append-only and duplicates are prevented at write time.
+      dedupeId: null,
+      channels: {
+        input: 0,
+        cache_creation: 0,
+        cache_read: 0,
+        output: total, // route authoritative upstream total here
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}

package/src/lib/ops/sources/kimi.js

@@ -0,0 +1,105 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Kimi audit strategy.
+ *
+ * Kimi CLI writes one wire log per session:
+ *   ~/.kimi/sessions/<project>/<sessionId>/wire.jsonl
+ * Each StatusUpdate line carries the delta for one Anthropic-compatible
+ * message:
+ *   { timestamp: <unix_seconds float>,
+ *     message: { type: "StatusUpdate",
+ *                payload: { message_id, token_usage: {
+ *                  input_other, input_cache_creation,
+ *                  input_cache_read, output } } } }
+ *
+ * Channel mapping lines up with src/lib/rollout.js normalizeKimiUsage so the
+ * framework's sum-of-channels row.truth equals the DB total_tokens:
+ *     input        = input_other + input_cache_creation
+ *     cache_read   = input_cache_read
+ *     output       = output
+ *     (cache_creation, reasoning) = 0  (already folded into input / n/a)
+ *     total        = input + cache_read + output
+ *
+ * Dedupe key: payload.message_id (chatcmpl-…). Kimi does not currently
+ * duplicate rows the way Claude Code does, but keying on message_id is
+ * free insurance and matches the AGENTS.md intake checklist.
+ */
+
+module.exports = {
+  id: "kimi",
+  displayName: "Kimi CLI",
+  sessionRoot({ home, env }) {
+    const base = (env && env.KIMI_HOME) || path.join(home, ".kimi");
+    return path.join(base, "sessions");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const proj of safeReadDirSync(root)) {
+      if (!proj.isDirectory()) continue;
+      const projDir = path.join(root, proj.name);
+      for (const session of safeReadDirSync(projDir)) {
+        if (!session.isDirectory()) continue;
+        const wire = path.join(projDir, session.name, "wire.jsonl");
+        if (!fs.existsSync(wire)) continue;
+        out.push(wire);
+      }
+    }
+    return out;
+  },
+  extractUsage(line) {
+    if (!line || !line.includes("StatusUpdate")) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (obj?.message?.type !== "StatusUpdate") return null;
+    const payload = obj.message.payload;
+    const tokens = payload?.token_usage;
+    if (!tokens || typeof tokens !== "object") return null;
+    const timestamp = unixSecondsToIso(obj.timestamp);
+    if (!timestamp) return null;
+    return {
+      timestamp,
+      dedupeId: typeof payload.message_id === "string" && payload.message_id
+        ? payload.message_id
+        : null,
+      channels: {
+        input: nonneg(tokens.input_other) + nonneg(tokens.input_cache_creation),
+        cache_creation: 0, // already folded into input per normalizeKimiUsage
+        cache_read: nonneg(tokens.input_cache_read),
+        output: nonneg(tokens.output),
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function unixSecondsToIso(value) {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n <= 0) return null;
+  const ms = n < 1e12 ? Math.floor(n * 1000) : Math.floor(n);
+  const d = new Date(ms);
+  if (Number.isNaN(d.getTime())) return null;
+  return d.toISOString();
+}
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}
+
+function safeReadDirSync(p) {
+  try {
+    return fs.readdirSync(p, { withFileTypes: true });
+  } catch (_err) {
+    return [];
+  }
+}

package/src/lib/ops/sources/openclaw.js

@@ -0,0 +1,64 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * OpenClaw audit strategy.
+ *
+ * Like Hermes, OpenClaw hands vibeusage pre-aggregated ledger rows instead of
+ * raw session logs:
+ *   ~/.vibeusage/tracker/openclaw-usage-ledger.jsonl
+ * Each line is a camelCase event
+ *   { eventId, emittedAt, source, model, inputTokens, cachedInputTokens,
+ *     outputTokens, reasoningOutputTokens, totalTokens }
+ * src/commands/sync.js parseOpenclawSanitizedLedger copies `totalTokens`
+ * straight into the bucket, so this audit routes the upstream total into the
+ * output channel. Dedupe is keyed on eventId, which the ledger writer
+ * already enforces uniqueness of.
+ */
+
+module.exports = {
+  id: "openclaw",
+  displayName: "OpenClaw Plugin",
+  sessionRoot({ home, env }) {
+    const base = (env && env.VIBEUSAGE_HOME) || path.join(home, ".vibeusage");
+    return path.join(base, "tracker");
+  },
+  walkSessions({ root }) {
+    const ledger = path.join(root, "openclaw-usage-ledger.jsonl");
+    if (!fs.existsSync(ledger)) return [];
+    return [ledger];
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (!event || typeof event !== "object") return null;
+    const timestamp = typeof event.emittedAt === "string" ? event.emittedAt : null;
+    if (!timestamp) return null;
+    const total = nonneg(event.totalTokens);
+    if (total === 0) return null;
+    return {
+      timestamp,
+      dedupeId: typeof event.eventId === "string" && event.eventId ? event.eventId : null,
+      channels: {
+        input: 0,
+        cache_creation: 0,
+        cache_read: 0,
+        output: total,
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}