vibeusage 0.5.0 → 0.6.1

package/src/lib/ops/sources/gemini.js

@@ -0,0 +1,154 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Gemini audit strategy.
+ *
+ * Gemini writes one JSON per session under
+ *   ~/.gemini/tmp/<hash>/chats/session-YYYY-MM-DDTHH-MM-<id>.json
+ * The file has `{ messages: [ { role, timestamp, model, tokens } ] }` where
+ * `tokens` is the cumulative usage up to that message (not a per-turn delta).
+ *
+ * Channel semantics differ from Claude but match Codex in one important way:
+ *   input + cached + output + tool + thoughts != total
+ * because `tokens.total` is the authoritative upstream count that
+ * src/lib/rollout.js normalizeGeminiTokens passes through as-is to the DB.
+ * Naively summing the five sub-channels double-counts. As with the Codex
+ * strategy, we route `delta.total` into the output channel and zero the rest
+ * so the framework's sum-of-channels row.truth equals the DB total_tokens
+ * without exposing Gemini's internal breakdown through the generic contract.
+ *
+ * Dedupe:
+ *   - Per-file index diff mirrors parseGeminiFile's `lastTotals` state.
+ *   - When `tokens.total` drops (session reset / resume), we treat the current
+ *     cumulative as the delta just like the parser does.
+ */
+
+module.exports = {
+  id: "gemini",
+  displayName: "Gemini CLI",
+  sessionRoot({ home, env }) {
+    const base = (env && env.GEMINI_HOME) || path.join(home, ".gemini");
+    return path.join(base, "tmp");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const hash of safeReadDirSync(root)) {
+      if (!hash.isDirectory()) continue;
+      const chatsDir = path.join(root, hash.name, "chats");
+      for (const f of safeReadDirSync(chatsDir)) {
+        if (!f.isFile()) continue;
+        if (!f.name.startsWith("session-") || !f.name.endsWith(".json")) continue;
+        out.push(path.join(chatsDir, f.name));
+      }
+    }
+    return out;
+  },
+  *iterateRecords(filePath) {
+    let raw;
+    try {
+      raw = fs.readFileSync(filePath, "utf8");
+    } catch (_err) {
+      return;
+    }
+    if (!raw.trim()) return;
+    let session;
+    try {
+      session = JSON.parse(raw);
+    } catch (_err) {
+      return;
+    }
+    const messages = Array.isArray(session?.messages) ? session.messages : [];
+    let prevTotals = null;
+    for (const msg of messages) {
+      if (!msg || typeof msg !== "object") continue;
+      const ts = typeof msg.timestamp === "string" ? msg.timestamp : null;
+      if (!ts) continue;
+      const tokens = msg.tokens;
+      if (!tokens || typeof tokens !== "object") continue;
+
+      const curr = normalizeTokens(tokens);
+      const delta = diffTotals(curr, prevTotals);
+      prevTotals = curr;
+      if (!delta || !delta.total) continue;
+
+      yield {
+        line: JSON.stringify({ timestamp: ts, delta }),
+        context: { filePath },
+      };
+    }
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    const ts = typeof obj.timestamp === "string" ? obj.timestamp : null;
+    const d = obj.delta;
+    if (!ts || !d || !Number(d.total)) return null;
+    return {
+      timestamp: ts,
+      dedupeId: null, // per-file index diff already dedupes
+      channels: {
+        input: 0,
+        cache_creation: 0,
+        cache_read: 0,
+        // Route the authoritative upstream total into a single channel; see
+        // module docstring for why we do not split it.
+        output: Number(d.total),
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function normalizeTokens(tokens) {
+  return {
+    input: nonneg(tokens.input),
+    cached: nonneg(tokens.cached),
+    output: nonneg(tokens.output),
+    tool: nonneg(tokens.tool),
+    thoughts: nonneg(tokens.thoughts),
+    total: nonneg(tokens.total),
+  };
+}
+
+function diffTotals(curr, prev) {
+  if (!curr) return null;
+  if (!prev) {
+    // First message with tokens — the whole cumulative value is the delta.
+    return curr;
+  }
+  // Session reset: upstream total decreased (resume / new session). Trust the
+  // new value as the full delta.
+  if (curr.total < prev.total) return curr;
+  const delta = {
+    input: Math.max(0, curr.input - prev.input),
+    cached: Math.max(0, curr.cached - prev.cached),
+    output: Math.max(0, curr.output - prev.output),
+    tool: Math.max(0, curr.tool - prev.tool),
+    thoughts: Math.max(0, curr.thoughts - prev.thoughts),
+    total: Math.max(0, curr.total - prev.total),
+  };
+  return delta;
+}
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}
+
+function safeReadDirSync(p) {
+  try {
+    return fs.readdirSync(p, { withFileTypes: true });
+  } catch (_err) {
+    return [];
+  }
+}

package/src/lib/ops/sources/hermes.js

@@ -0,0 +1,69 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Hermes audit strategy.
+ *
+ * Hermes does not write raw session logs; it emits one pre-aggregated event
+ * per turn into the vibeusage tracker directory:
+ *   ~/.vibeusage/tracker/hermes.usage.jsonl
+ * Each line is a `{type: "usage", emitted_at, model, input_tokens,
+ * output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens,
+ * total_tokens}` record. src/commands/sync.js parseHermesUsageLedger already
+ * copies `total_tokens` straight into the bucket, so this audit routes the
+ * upstream total into the output channel — same pattern we use for Codex and
+ * Gemini.
+ *
+ * sessionRoot: the tracker directory (NOT `~/.hermes/...` — Hermes usage data
+ * lives under ~/.vibeusage/tracker because Hermes is a plugin that hands
+ * vibeusage ledger rows directly).
+ */
+
+module.exports = {
+  id: "hermes",
+  displayName: "Hermes Plugin",
+  sessionRoot({ home, env }) {
+    const base = (env && env.VIBEUSAGE_HOME) || path.join(home, ".vibeusage");
+    return path.join(base, "tracker");
+  },
+  walkSessions({ root }) {
+    const ledger = path.join(root, "hermes.usage.jsonl");
+    if (!fs.existsSync(ledger)) return [];
+    return [ledger];
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (!event || event.type !== "usage") return null;
+    const timestamp = typeof event.emitted_at === "string" ? event.emitted_at : null;
+    if (!timestamp) return null;
+    const total = nonneg(event.total_tokens);
+    if (total === 0) return null;
+    return {
+      timestamp,
+      // Hermes ledger records do not carry a stable per-event id;
+      // the ledger is append-only and duplicates are prevented at write time.
+      dedupeId: null,
+      channels: {
+        input: 0,
+        cache_creation: 0,
+        cache_read: 0,
+        output: total, // route authoritative upstream total here
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}

package/src/lib/ops/sources/kimi.js

@@ -0,0 +1,105 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Kimi audit strategy.
+ *
+ * Kimi CLI writes one wire log per session:
+ *   ~/.kimi/sessions/<project>/<sessionId>/wire.jsonl
+ * Each StatusUpdate line carries the delta for one Anthropic-compatible
+ * message:
+ *   { timestamp: <unix_seconds float>,
+ *     message: { type: "StatusUpdate",
+ *                payload: { message_id, token_usage: {
+ *                  input_other, input_cache_creation,
+ *                  input_cache_read, output } } } }
+ *
+ * Channel mapping lines up with src/lib/rollout.js normalizeKimiUsage so the
+ * framework's sum-of-channels row.truth equals the DB total_tokens:
+ *     input        = input_other + input_cache_creation
+ *     cache_read   = input_cache_read
+ *     output       = output
+ *     (cache_creation, reasoning) = 0  (already folded into input / n/a)
+ *     total        = input + cache_read + output
+ *
+ * Dedupe key: payload.message_id (chatcmpl-…). Kimi does not currently
+ * duplicate rows the way Claude Code does, but keying on message_id is
+ * free insurance and matches the AGENTS.md intake checklist.
+ */
+
+module.exports = {
+  id: "kimi",
+  displayName: "Kimi CLI",
+  sessionRoot({ home, env }) {
+    const base = (env && env.KIMI_HOME) || path.join(home, ".kimi");
+    return path.join(base, "sessions");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const proj of safeReadDirSync(root)) {
+      if (!proj.isDirectory()) continue;
+      const projDir = path.join(root, proj.name);
+      for (const session of safeReadDirSync(projDir)) {
+        if (!session.isDirectory()) continue;
+        const wire = path.join(projDir, session.name, "wire.jsonl");
+        if (!fs.existsSync(wire)) continue;
+        out.push(wire);
+      }
+    }
+    return out;
+  },
+  extractUsage(line) {
+    if (!line || !line.includes("StatusUpdate")) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (obj?.message?.type !== "StatusUpdate") return null;
+    const payload = obj.message.payload;
+    const tokens = payload?.token_usage;
+    if (!tokens || typeof tokens !== "object") return null;
+    const timestamp = unixSecondsToIso(obj.timestamp);
+    if (!timestamp) return null;
+    return {
+      timestamp,
+      dedupeId: typeof payload.message_id === "string" && payload.message_id
+        ? payload.message_id
+        : null,
+      channels: {
+        input: nonneg(tokens.input_other) + nonneg(tokens.input_cache_creation),
+        cache_creation: 0, // already folded into input per normalizeKimiUsage
+        cache_read: nonneg(tokens.input_cache_read),
+        output: nonneg(tokens.output),
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function unixSecondsToIso(value) {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n <= 0) return null;
+  const ms = n < 1e12 ? Math.floor(n * 1000) : Math.floor(n);
+  const d = new Date(ms);
+  if (Number.isNaN(d.getTime())) return null;
+  return d.toISOString();
+}
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}
+
+function safeReadDirSync(p) {
+  try {
+    return fs.readdirSync(p, { withFileTypes: true });
+  } catch (_err) {
+    return [];
+  }
+}

package/src/lib/ops/sources/openclaw.js

@@ -0,0 +1,64 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * OpenClaw audit strategy.
+ *
+ * Like Hermes, OpenClaw hands vibeusage pre-aggregated ledger rows instead of
+ * raw session logs:
+ *   ~/.vibeusage/tracker/openclaw-usage-ledger.jsonl
+ * Each line is a camelCase event
+ *   { eventId, emittedAt, source, model, inputTokens, cachedInputTokens,
+ *     outputTokens, reasoningOutputTokens, totalTokens }
+ * src/commands/sync.js parseOpenclawSanitizedLedger copies `totalTokens`
+ * straight into the bucket, so this audit routes the upstream total into the
+ * output channel. Dedupe is keyed on eventId, which the ledger writer
+ * already enforces uniqueness of.
+ */
+
+module.exports = {
+  id: "openclaw",
+  displayName: "OpenClaw Plugin",
+  sessionRoot({ home, env }) {
+    const base = (env && env.VIBEUSAGE_HOME) || path.join(home, ".vibeusage");
+    return path.join(base, "tracker");
+  },
+  walkSessions({ root }) {
+    const ledger = path.join(root, "openclaw-usage-ledger.jsonl");
+    if (!fs.existsSync(ledger)) return [];
+    return [ledger];
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (!event || typeof event !== "object") return null;
+    const timestamp = typeof event.emittedAt === "string" ? event.emittedAt : null;
+    if (!timestamp) return null;
+    const total = nonneg(event.totalTokens);
+    if (total === 0) return null;
+    return {
+      timestamp,
+      dedupeId: typeof event.eventId === "string" && event.eventId ? event.eventId : null,
+      channels: {
+        input: 0,
+        cache_creation: 0,
+        cache_read: 0,
+        output: total,
+        reasoning: 0,
+      },
+    };
+  },
+};
+
+function nonneg(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return Math.floor(n);
+}

package/src/lib/ops/sources/opencode.js

@@ -0,0 +1,100 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * OpenCode audit strategy.
+ *
+ * OpenCode persists one JSON per assistant message under
+ *   ~/.local/share/opencode/storage/message/ses_<session>/msg_<id>.json
+ * Each file looks like:
+ *   {
+ *     role: "assistant",
+ *     id: "msg_...",
+ *     modelID: "...",
+ *     tokens: { input, output, reasoning, cache: { read, write } },
+ *     time: { created, completed }
+ *   }
+ *
+ * Channel mapping matches src/lib/rollout.js normalizeOpencodeTokens so the
+ * audit's truth sum equals what the parser emits into vibeusage_tracker_hourly
+ * (post PR #153, which added cache.read to total):
+ *     total = input + cache.write + cache.read + output + reasoning
+ *
+ * Notes:
+ *   - OPENCODE_HOME / XDG_DATA_HOME env vars override the default root (matches
+ *     the same logic used by src/commands/sync.js).
+ *   - Only assistant messages carry tokens; user messages return null from
+ *     extractUsage so the generic runner skips them.
+ *   - New OpenCode installs may persist into opencode.db (sqlite) instead of
+ *     these JSON files. The audit reports no-local-sessions in that case;
+ *     users can dump the same rows to a JSON file and feed --db-json to
+ *     compare via the backend path.
+ */
+
+module.exports = {
+  id: "opencode",
+  displayName: "OpenCode",
+  sessionRoot({ home, env }) {
+    const xdg = env.XDG_DATA_HOME || path.join(home, ".local", "share");
+    const opencodeHome = env.OPENCODE_HOME || path.join(xdg, "opencode");
+    return path.join(opencodeHome, "storage", "message");
+  },
+  walkSessions({ root }) {
+    if (!fs.existsSync(root)) return [];
+    const out = [];
+    for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const dir = path.join(root, entry.name);
+      for (const f of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (!f.isFile()) continue;
+        if (!f.name.startsWith("msg_") || !f.name.endsWith(".json")) continue;
+        out.push(path.join(dir, f.name));
+      }
+    }
+    return out;
+  },
+  // OpenCode is one JSON per file (not JSONL). Yield the whole file body as a
+  // single "line" so extractUsage can JSON.parse it uniformly with the
+  // line-based contract.
+  *iterateRecords(filePath) {
+    let text;
+    try {
+      text = fs.readFileSync(filePath, "utf8");
+    } catch (_err) {
+      return;
+    }
+    if (!text.trim()) return;
+    yield { line: text, context: { filePath } };
+  },
+  extractUsage(line) {
+    if (!line) return null;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_err) {
+      return null;
+    }
+    if (obj?.role !== "assistant") return null;
+    const tokens = obj.tokens;
+    if (!tokens || typeof tokens !== "object") return null;
+    const completed = obj?.time?.completed;
+    const created = obj?.time?.created;
+    const epochMs = typeof completed === "number" ? completed : typeof created === "number" ? created : null;
+    if (!epochMs || !Number.isFinite(epochMs)) return null;
+
+    const cache = tokens.cache && typeof tokens.cache === "object" ? tokens.cache : {};
+    return {
+      timestamp: new Date(epochMs).toISOString(),
+      dedupeId: typeof obj.id === "string" && obj.id ? obj.id : null,
+      channels: {
+        input: tokens.input,
+        cache_creation: cache.write,
+        cache_read: cache.read,
+        output: tokens.output,
+        reasoning: tokens.reasoning,
+      },
+    };
+  },
+};

package/src/lib/rollout.js

@@ -214,6 +214,7 @@ async function parseClaudeIncremental({
   await ensureDir(path.dirname(queuePath));
   let filesProcessed = 0;
   let eventsAggregated = 0;
+  let dedupSkipped = 0;
 
   const cb = typeof onProgress === "function" ? onProgress : null;
   const files = Array.isArray(projectFiles) ? projectFiles : [];
@@ -283,6 +284,7 @@ async function parseClaudeIncremental({
 
     filesProcessed += 1;
     eventsAggregated += result.eventsAggregated;
+    dedupSkipped += result.dedupSkipped || 0;
 
     if (cb) {
       cb({
@@ -307,7 +309,13 @@ async function parseClaudeIncremental({
     cursors.projectHourly = projectState;
   }
 
-  return { filesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
+  return {
+    filesProcessed,
+    eventsAggregated,
+    bucketsQueued,
+    projectBucketsQueued,
+    dedupSkipped,
+  };
 }
 
 async function parseGeminiIncremental({
@@ -801,16 +809,18 @@ async function parseClaudeFile({
 
   const st = await fs.stat(filePath).catch(() => null);
   if (!st || !st.isFile()) {
-    return { endOffset: startOffset, eventsAggregated: 0, seenIds: seenOrder };
+    return { endOffset: startOffset, eventsAggregated: 0, dedupSkipped: 0, seenIds: seenOrder };
   }
 
   const endOffset = st.size;
-  if (startOffset >= endOffset) return { endOffset, eventsAggregated: 0, seenIds: seenOrder };
+  if (startOffset >= endOffset)
+    return { endOffset, eventsAggregated: 0, dedupSkipped: 0, seenIds: seenOrder };
 
   const stream = fssync.createReadStream(filePath, { encoding: "utf8", start: startOffset });
   const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
 
   let eventsAggregated = 0;
+  let dedupSkipped = 0;
   for await (const line of rl) {
     if (!line || !line.includes('\"usage\"')) continue;
     let obj;
@@ -827,7 +837,10 @@ async function parseClaudeFile({
     // (same `message.id` / `requestId`, different outer `uuid`). Aggregate once per
     // upstream Anthropic response to avoid multi-counting token usage.
     const dedupeId = obj?.message?.id || obj?.requestId || null;
-    if (dedupeId && seenSet.has(dedupeId)) continue;
+    if (dedupeId && seenSet.has(dedupeId)) {
+      dedupSkipped += 1;
+      continue;
+    }
 
     const model = normalizeModelInput(obj?.message?.model || obj?.model) || DEFAULT_MODEL;
     const tokenTimestamp = typeof obj?.timestamp === "string" ? obj.timestamp : null;
@@ -866,7 +879,7 @@ async function parseClaudeFile({
     seenOrder.length > CLAUDE_SEEN_IDS_LIMIT
       ? seenOrder.slice(seenOrder.length - CLAUDE_SEEN_IDS_LIMIT)
       : seenOrder;
-  return { endOffset, eventsAggregated, seenIds: trimmedSeenIds };
+  return { endOffset, eventsAggregated, dedupSkipped, seenIds: trimmedSeenIds };
 }
 
 async function parseKimiFile({
@@ -2457,4 +2470,13 @@ module.exports = {
   bucketKey,
   enqueueTouchedBuckets,
   toUtcHalfHourStart,
+  // Exported for the token-conservation property test (see
+  // test/parser-total-conservation.test.js and AGENTS.md "新 AI CLI Source
+  // 接入 Checklist"). If you add a new normalize<Source>Usage function,
+  // export it here so the conservation test covers it automatically.
+  normalizeUsage,
+  normalizeClaudeUsage,
+  normalizeGeminiTokens,
+  normalizeKimiUsage,
+  normalizeOpencodeTokens,
 };