vibeusage 0.6.1 → 0.6.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vibeusage",
-  "version": "0.6.1",
+  "version": "0.6.3",
   "description": "Codex CLI token usage tracker (macOS-first, notify-driven).",
   "license": "MIT",
   "repository": {
@@ -49,7 +49,8 @@
     "validate:ui-hardcode": "node scripts/ops/validate-ui-hardcode.cjs"
   },
   "dependencies": {
-    "@insforge/sdk": "1.2.2"
+    "@insforge/sdk": "1.2.2",
+    "proper-lockfile": "^4.1.2"
   },
   "devDependencies": {
     "@sourcegraph/scip-typescript": "^0.3.6",
@@ -60,5 +61,8 @@
   ],
   "engines": {
     "node": "20.x"
-  }
+  },
+  "bundleDependencies": [
+    "@insforge/sdk"
+  ]
 }

package/src/commands/doctor.js

@@ -146,8 +146,11 @@ function runAuditTokens({ opts, config }) {
   if (result.exceedsThreshold) {
     process.stderr.write(
       `\nFAIL drift ${result.maxDriftPct.toFixed(2)}% exceeds threshold ${result.thresholdPct}%.\n` +
-        `If vibeusage >= 0.5.0, scrub the Claude/OpenCode cursor block in\n` +
-        `~/.vibeusage/tracker/cursors.json and rerun \`vibeusage sync --drain\`.\n`,
+        `Rebuild this source from its local session files: ` +
+        `\`vibeusage sync --rebuild ${result.source}\`\n` +
+        `(That clears the source's file/bucket/group cursors atomically and ` +
+        `re-parses every session — fixing drift caused by interrupted uploads ` +
+        `or partial cursor edits.)\n`,
     );
     process.exitCode = 1;
   }

package/src/commands/sync.js

@@ -4,6 +4,7 @@ const fs = require("node:fs/promises");
 const cp = require("node:child_process");
 
 const { ensureDir, readJson, writeJson, openLock } = require("../lib/fs");
+const { scrubSourceCursors, listSupportedSources } = require("../lib/cursor-scrub");
 const {
   listRolloutFiles,
   listClaudeProjectFiles,
@@ -65,6 +66,30 @@ async function cmdSync(argv) {
 
     const config = await readJson(configPath);
     const cursors = (await readJson(cursorsPath)) || { version: 1, files: {}, updatedAt: null };
+
+    if (opts.rebuild) {
+      const scrub = scrubSourceCursors({
+        cursors,
+        sourceId: opts.rebuild,
+        home,
+        env: process.env,
+      });
+      // Persist the cleared cursors before parsing begins. If we crash mid-
+      // rebuild, the next sync resumes from a clean state for this source
+      // rather than re-accumulating onto cached totals (the original bug).
+      await writeJson(cursorsPath, cursors);
+      process.stdout.write(
+        `Rebuilding source=${scrub.sourceId}: cleared ${scrub.filesRemoved} file ` +
+          `cursors, ${scrub.bucketsRemoved} hourly buckets, ` +
+          `${scrub.projectBucketsRemoved} project buckets, ` +
+          `${scrub.groupsRemoved} groupQueued entries` +
+          (scrub.extraCursorsCleared.length
+            ? `, ${scrub.extraCursorsCleared.join("+")}`
+            : "") +
+          `.\n`,
+      );
+    }
+
     const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
     let uploadThrottleState = uploadThrottle;
 
@@ -445,6 +470,7 @@ function parseArgs(argv) {
     fromRetry: false,
     fromOpenclaw: false,
     drain: false,
+    rebuild: null,
   };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
@@ -453,7 +479,35 @@ function parseArgs(argv) {
     else if (a === "--from-retry") out.fromRetry = true;
     else if (a === "--from-openclaw") out.fromOpenclaw = true;
     else if (a === "--drain") out.drain = true;
-    else throw new Error(`Unknown option: ${a}`);
+    else if (a === "--rebuild") {
+      const v = argv[++i];
+      if (!v || v.startsWith("--")) {
+        throw new Error("--rebuild requires a source id (e.g. --rebuild claude)");
+      }
+      out.rebuild = v;
+    } else if (a.startsWith("--rebuild=")) {
+      const v = a.slice("--rebuild=".length);
+      if (!v) throw new Error("--rebuild= requires a source id");
+      out.rebuild = v;
+    } else throw new Error(`Unknown option: ${a}`);
+  }
+  if (out.rebuild) {
+    const supported = listSupportedSources();
+    if (!supported.includes(out.rebuild)) {
+      throw new Error(
+        `--rebuild: unknown source '${out.rebuild}'. Supported: ${supported.join(", ")}`,
+      );
+    }
+    // A rebuild always wants a full upload pass, otherwise the freshly-rebuilt
+    // buckets would sit in queue.jsonl behind the default 10-batch cap.
+    out.drain = true;
+    // OpenClaw is the only source whose ledger parsing is gated behind an
+    // explicit flag (see sync flow's `opts.fromOpenclaw ? parseOpenclaw... : noop`).
+    // A `--rebuild=openclaw` that doesn't also turn that flag on would scrub
+    // the OpenClaw cursors and persist the cleared state without ever re-
+    // aggregating from the ledger — leaving totals stale until a later
+    // plugin-triggered sync. Force the flag so rebuild actually rebuilds.
+    if (out.rebuild === "openclaw") out.fromOpenclaw = true;
   }
   return out;
 }

package/src/lib/cursor-scrub.js

@@ -0,0 +1,164 @@
+"use strict";
+
+const path = require("node:path");
+
+// scrubSourceCursors clears every cursor field that, if left in place, would
+// cause a re-parse of a source's session files to *accumulate* into
+// previously-uploaded buckets instead of *rebuilding* them from scratch.
+//
+// This is the helper that fixes the bug behind the recent "DB tokens doubled"
+// incident: clearing only `cursors.files` causes the parser to re-read the
+// jsonl files and add their token totals on top of whatever was still cached
+// in `cursors.hourly.buckets`. The result is buckets at roughly 2x the
+// ground truth.
+//
+// Four cursor surfaces must be cleared in lockstep for a source rebuild to
+// be correct:
+//   1. cursors.files entries whose path lives under that source's session
+//      root (so the parser re-reads each file from offset 0).
+//   2. cursors.hourly.buckets keyed `<source>|<model>|<hour>` (so per-bucket
+//      totals restart at zero before re-aggregation).
+//   3. cursors.hourly.groupQueued keys for that source (so the next sync
+//      re-enqueues each touched bucket to queue.jsonl for upload).
+//   4. cursors.projectHourly.buckets keyed `<project>|<source>|<hour>`
+//      (project-scoped totals are aggregated independently from the global
+//      hourly state and would otherwise stay doubled in the dashboard's
+//      per-project views).
+//
+// `cursors.projectHourly.projects` holds project metadata (git remotes,
+// display names) and is intentionally preserved — it carries no token
+// totals, just identity.
+//
+// Sources that don't use `cursors.files` (sqlite-backed opencode, ledger-
+// backed hermes/openclaw) carry their progress in dedicated cursor fields;
+// those are reset directly.
+
+const SOURCES = {
+  claude: {
+    sessionRoot: ({ home }) => path.join(home, ".claude", "projects"),
+  },
+  codex: {
+    sessionRoot: ({ home, env }) =>
+      path.join(env.CODEX_HOME || path.join(home, ".codex"), "sessions"),
+  },
+  "every-code": {
+    sessionRoot: ({ home, env }) =>
+      path.join(env.CODE_HOME || path.join(home, ".code"), "sessions"),
+  },
+  gemini: {
+    sessionRoot: ({ home, env }) =>
+      path.join(env.GEMINI_HOME || path.join(home, ".gemini"), "tmp"),
+  },
+  kimi: {
+    sessionRoot: ({ home, env }) =>
+      path.join(env.KIMI_HOME || path.join(home, ".kimi"), "sessions"),
+  },
+  opencode: {
+    extraCursorKeys: ["opencode", "opencodeSqlite"],
+  },
+  hermes: {
+    extraCursorKeys: ["hermesLedger"],
+  },
+  openclaw: {
+    extraCursorKeys: ["openclawLedger"],
+  },
+};
+
+function listSupportedSources() {
+  return Object.keys(SOURCES);
+}
+
+function scrubSourceCursors({ cursors, sourceId, home, env = process.env }) {
+  if (!cursors || typeof cursors !== "object") {
+    throw new Error("scrubSourceCursors: cursors must be an object");
+  }
+  const config = SOURCES[sourceId];
+  if (!config) {
+    throw new Error(
+      `scrubSourceCursors: unknown sourceId '${sourceId}'. Supported: ${listSupportedSources().join(", ")}`,
+    );
+  }
+
+  const result = {
+    sourceId,
+    filesRemoved: 0,
+    bucketsRemoved: 0,
+    groupsRemoved: 0,
+    projectBucketsRemoved: 0,
+    extraCursorsCleared: [],
+  };
+
+  // 1) cursors.files — strip every entry whose path lives under this source's
+  // session root, so the parser re-reads them from byte 0.
+  if (config.sessionRoot && cursors.files && typeof cursors.files === "object") {
+    const prefix = config.sessionRoot({ home, env });
+    for (const key of Object.keys(cursors.files)) {
+      if (typeof key !== "string") continue;
+      if (key.startsWith(prefix)) {
+        delete cursors.files[key];
+        result.filesRemoved += 1;
+      }
+    }
+  }
+
+  // 2) cursors.hourly.buckets — strip every bucket keyed for this source so
+  // its totals restart at zero before re-aggregation.
+  if (cursors.hourly && typeof cursors.hourly === "object") {
+    const bucketPrefix = `${sourceId}|`;
+    if (cursors.hourly.buckets && typeof cursors.hourly.buckets === "object") {
+      for (const key of Object.keys(cursors.hourly.buckets)) {
+        if (typeof key === "string" && key.startsWith(bucketPrefix)) {
+          delete cursors.hourly.buckets[key];
+          result.bucketsRemoved += 1;
+        }
+      }
+    }
+    // 3) cursors.hourly.groupQueued — strip per-source enqueue records so
+    // the next sync re-enqueues each touched bucket for upload.
+    if (cursors.hourly.groupQueued && typeof cursors.hourly.groupQueued === "object") {
+      for (const key of Object.keys(cursors.hourly.groupQueued)) {
+        if (typeof key === "string" && key.startsWith(bucketPrefix)) {
+          delete cursors.hourly.groupQueued[key];
+          result.groupsRemoved += 1;
+        }
+      }
+    }
+  }
+
+  // 4) cursors.projectHourly.buckets — keyed `<project_key>|<source>|<hour>`.
+  // Strip the buckets where the middle segment matches this source, leaving
+  // every other source's project-scoped totals (and the projects metadata
+  // map) untouched.
+  if (
+    cursors.projectHourly &&
+    typeof cursors.projectHourly === "object" &&
+    cursors.projectHourly.buckets &&
+    typeof cursors.projectHourly.buckets === "object"
+  ) {
+    for (const key of Object.keys(cursors.projectHourly.buckets)) {
+      if (typeof key !== "string") continue;
+      const parts = key.split("|");
+      if (parts.length >= 2 && parts[1] === sourceId) {
+        delete cursors.projectHourly.buckets[key];
+        result.projectBucketsRemoved += 1;
+      }
+    }
+  }
+
+  // 5) Source-specific top-level cursor fields (opencode sqlite progress,
+  // hermes/openclaw ledger offsets). Resetting these is what makes a
+  // rebuild work for non-file sources.
+  for (const key of config.extraCursorKeys || []) {
+    if (key in cursors) {
+      delete cursors[key];
+      result.extraCursorsCleared.push(key);
+    }
+  }
+
+  return result;
+}
+
+module.exports = {
+  scrubSourceCursors,
+  listSupportedSources,
+};

package/src/lib/fs.js

@@ -1,6 +1,12 @@
 const fs = require("node:fs/promises");
 const path = require("node:path");
 
+// proper-lockfile is required lazily inside openLock(): some callers copy
+// src/lib/fs.js into sandboxes that have no node_modules (e.g. the openclaw
+// session plugin test, which materializes src/ under a tmp dir to test the
+// ledger). Those callers only use ensureDir/readJson/etc and would crash on
+// a top-level require.
+
 async function ensureDir(p) {
   await fs.mkdir(p, { recursive: true });
 }
@@ -47,23 +53,132 @@ async function chmod600IfPossible(filePath) {
   } catch (_e) {}
 }
 
-async function openLock(lockPath, { quietIfLocked }) {
+// proper-lockfile gives us atomic mkdir-based mutual exclusion plus a heart-
+// beat mechanism that auto-recovers from orphan locks without TOCTOU races:
+//
+//   - The holder process refreshes the lock-directory's mtime every `update`
+//     ms. As long as that interval keeps running, the lock is "fresh".
+//   - Any acquirer that finds the existing lock with mtime older than `stale`
+//     ms takes it over via a compare-and-swap that is safe under concurrent
+//     attempts (the library's own contract).
+//   - If the holder dies (crash, SIGKILL, reboot) the heartbeat stops; the
+//     next acquirer sees the stale mtime and recovers automatically.
+//
+// We deliberately set `stale` larger than the default to give a working sync
+// some headroom against transient event-loop pauses (large JSON.parse, GC).
+// We pass `realpath: false` because the lock target may not exist as a file
+// — proper-lockfile creates the lock-directory at `lockPath` directly.
+const LOCK_STALE_MS = 60_000;
+const LOCK_UPDATE_MS = 10_000;
+
+async function openLock(lockPath, { quietIfLocked } = {}) {
+  // Lazy require: see top-of-file note about sandboxed callers.
+  const lockfile = require("proper-lockfile");
+
+  // Migration path: pre-proper-lockfile versions of vibeusage created the
+  // lock as a regular *file* (fs.open with "wx"). proper-lockfile creates
+  // it as a *directory* (mkdir). If we hand a stale legacy file off to
+  // proper-lockfile, its mkdir will EEXIST and its internal rmdir-fallback
+  // will then ENOTDIR, throwing instead of returning ELOCKED. Detect and
+  // resolve that mismatch up front.
+  const migration = await migrateLegacyLockFile(lockPath);
+  if (migration === "yield-to-legacy-holder") {
+    if (!quietIfLocked) process.stdout.write("Another sync is already running.\n");
+    return null;
+  }
+
+  let release;
   try {
-    const handle = await fs.open(lockPath, "wx");
-    return {
-      async release() {
-        await handle.close().catch(() => {});
-      },
-    };
+    release = await lockfile.lock(lockPath, {
+      lockfilePath: lockPath,
+      realpath: false,
+      stale: LOCK_STALE_MS,
+      update: LOCK_UPDATE_MS,
+      retries: 0,
+    });
   } catch (e) {
-    if (e && e.code === "EEXIST") {
-      if (!quietIfLocked) {
-        process.stdout.write("Another sync is already running.\n");
-      }
+    if (e && e.code === "ELOCKED") {
+      if (!quietIfLocked) process.stdout.write("Another sync is already running.\n");
       return null;
     }
     throw e;
   }
+  return {
+    async release() {
+      try {
+        await release();
+      } catch (_e) {
+        // Best-effort cleanup. proper-lockfile throws if the lock was already
+        // compromised (e.g. taken over by another process while we were
+        // running) — there is nothing useful to do at that point.
+      }
+    },
+  };
+}
+
+// Detect a leftover lock file from the previous wx-based scheme. Three cases:
+//   - "orphan"        — proven dead by PID liveness; safe to unlink and migrate.
+//   - "alive"         — recorded PID is still running; yield with the standard
+//                       "another sync running" UX.
+//   - "indeterminate" — empty / corrupt / unreadable file. The original
+//                       production openLock wrote a *zero-byte* file (it never
+//                       called writeFile after fs.open(path, "wx")), so this
+//                       is the **expected** legacy format. We cannot prove
+//                       its holder is dead and we MUST NOT auto-delete: a
+//                       still-running legacy sync would lose its lock and a
+//                       new-format sync would start in parallel. Yield and
+//                       print an actionable manual-cleanup notice.
+async function migrateLegacyLockFile(lockPath) {
+  let stat;
+  try {
+    stat = await fs.lstat(lockPath);
+  } catch (e) {
+    if (e && e.code === "ENOENT") return "no-legacy";
+    throw e;
+  }
+  if (stat.isDirectory()) return "no-legacy"; // already in proper-lockfile format
+
+  const verdict = await classifyLegacyFileLock(lockPath);
+  if (verdict === "orphan") {
+    await fs.unlink(lockPath).catch(() => {});
+    return "migrated";
+  }
+  if (verdict === "indeterminate") {
+    process.stderr.write(
+      `vibeusage: legacy sync.lock at ${lockPath} carries no PID payload, ` +
+        `so we cannot prove its owner is dead. Auto-deletion is unsafe — a ` +
+        `still-running legacy sync would lose its lock. If no legacy ` +
+        `vibeusage sync is actually running, remove it manually: rm ${JSON.stringify(
+          lockPath,
+        )}\n`,
+    );
+  }
+  return "yield-to-legacy-holder";
+}
+
+async function classifyLegacyFileLock(lockPath) {
+  let raw;
+  try {
+    raw = await fs.readFile(lockPath, "utf8");
+  } catch (_e) {
+    return "indeterminate";
+  }
+  if (!raw) return "indeterminate";
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (_e) {
+    return "indeterminate";
+  }
+  const pid = parsed?.pid;
+  if (!Number.isFinite(pid)) return "indeterminate";
+  try {
+    process.kill(pid, 0);
+    return "alive";
+  } catch (e) {
+    if (e && e.code === "ESRCH") return "orphan";
+    return "alive"; // EPERM = pid exists but belongs to another user
+  }
 }
 
 module.exports = {

package/src/lib/ops/sources/claude.js

@@ -11,14 +11,25 @@ module.exports = {
   },
   walkSessions({ root }) {
     if (!fs.existsSync(root)) return [];
+    // Recurse: Claude Code writes the main thread under
+    // `projects/<project>/<session>.jsonl` AND subagent threads under
+    // `projects/<project>/<sessionId>/subagents/agent-*.jsonl`. Subagents
+    // burn real Anthropic tokens, so the audit must include them. Sync's
+    // walkClaudeProjects (rollout.js) already recurses; this mirrors it.
     const out = [];
-    for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
-      if (!entry.isDirectory()) continue;
-      const dir = path.join(root, entry.name);
-      for (const f of fs.readdirSync(dir, { withFileTypes: true })) {
-        if (!f.isFile()) continue;
-        if (!f.name.endsWith(".jsonl")) continue;
-        out.push(path.join(dir, f.name));
+    const stack = [root];
+    while (stack.length) {
+      const dir = stack.pop();
+      let entries;
+      try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+      } catch (_err) {
+        continue;
+      }
+      for (const entry of entries) {
+        const p = path.join(dir, entry.name);
+        if (entry.isDirectory()) stack.push(p);
+        else if (entry.isFile() && entry.name.endsWith(".jsonl")) out.push(p);
       }
     }
     return out;