tokentracker-cli 0.81.2 → 0.82.0

package/src/lib/diagnostics.js

@@ -3,6 +3,7 @@ const path = require("node:path");
 const fs = require("node:fs/promises");
 
 const { readJson } = require("./fs");
+const { readCursorStateSummary } = require("./cursor-store");
 const { readCodexNotify, readEveryCodeNotify } = require("./codex-config");
 const { areClaudeUsageHooksConfigured, buildClaudeHookCommand } = require("./claude-config");
 const {
@@ -59,7 +60,8 @@ async function collectTrackerDiagnostics({
     path.join(home, ".grok");
 
   const config = await readJson(configPath);
-  const cursors = await readJson(cursorsPath);
+  const cursorSummary = await readCursorStateSummary({ trackerDir, cursorsPath });
+  const cursors = cursorSummary.cursors;
   const queueState = (await readJson(queueStatePath)) || { offset: 0 };
   const uploadThrottle = normalizeUploadState(await readJson(uploadThrottlePath));
   const autoRetry = await readJson(autoRetryPath);
@@ -165,10 +167,11 @@ async function collectTrackerDiagnostics({
     },
     parse: {
       updated_at: typeof cursors?.updatedAt === "string" ? cursors.updatedAt : null,
-      file_count:
-        cursors?.files && typeof cursors.files === "object"
-          ? Object.keys(cursors.files).length
-          : null,
+      file_count: cursorSummary.fileCount,
+      cursor_store: cursorSummary.mode,
+      cursor_store_legacy_drift: cursorSummary.legacyDrift === true,
+      codex_file_count: cursorSummary.codexFileCount,
+      codex_event_count: cursorSummary.codexEventCount,
     },
     queue: {
       size_bytes: queueSize,

package/src/lib/fs.js

@@ -1,4 +1,5 @@
 const fs = require("node:fs/promises");
+const crypto = require("node:crypto");
 const path = require("node:path");
 
 async function ensureDir(p) {
@@ -48,28 +49,265 @@ async function chmod600IfPossible(filePath) {
 }
 
 const LOCK_STALE_MS = 5 * 60 * 1000; // 5 minutes
+const LOCK_HEARTBEAT_MS = 30 * 1000;
+const MAX_RECLAIM_DEPTH = 4;
+const TRANSITION_GUARD_RETRY_MS = 5;
+const TRANSITION_GUARD_ATTEMPTS = 1000;
 
-async function openLock(lockPath, { quietIfLocked }) {
+function parseLockOwner(raw) {
+  try {
+    const parsed = JSON.parse(raw);
+    const pid = Number(parsed?.pid);
+    const token = typeof parsed?.token === "string" ? parsed.token : null;
+    if (!Number.isSafeInteger(pid) || pid <= 0 || !token) return null;
+    return { pid, token };
+  } catch (_e) {
+    return null;
+  }
+}
+
+function heartbeatPathFor(lockPath, token) {
+  const tokenDigest = crypto.createHash("sha256").update(token, "utf8").digest("hex");
+  return `${lockPath}.heartbeat.${tokenDigest}`;
+}
+
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    if (e?.code === "ESRCH") return false;
+    // EPERM means the process exists but belongs to another user. Unknown
+    // errors are treated conservatively so a live lock is never reclaimed.
+    return true;
+  }
+}
+
+async function existingLockCanBeReclaimed(lockPath) {
+  let lockHandle = null;
+  try {
+    // Inspect one opened inode instead of stat-ing a path and then reading
+    // that path. The latter is a TOCTOU window and is also unsafe if a
+    // concurrent reclaimer replaces the lock between the two operations.
+    lockHandle = await fs.open(lockPath, "r");
+    const [stat, raw] = await Promise.all([
+      lockHandle.stat(),
+      lockHandle.readFile({ encoding: "utf8" }),
+    ]);
+    const owner = parseLockOwner(raw);
+    if (owner) {
+      if (!isProcessAlive(owner.pid)) return true;
+
+      // New leases keep a token-specific heartbeat separate from the owner
+      // file. This lets a PID that has since been reused be recognized as an
+      // abandoned lease without reclaiming a live, long-running sync whose
+      // heartbeat is still fresh. Locks from before heartbeat support retain
+      // the conservative live-PID behavior.
+      let heartbeatHandle = null;
+      try {
+        heartbeatHandle = await fs.open(heartbeatPathFor(lockPath, owner.token), "r");
+        const heartbeat = await heartbeatHandle.stat();
+        return Date.now() - heartbeat.mtimeMs > LOCK_STALE_MS;
+      } catch (_e) {
+        return false;
+      } finally {
+        await heartbeatHandle?.close().catch(() => {});
+      }
+    }
+    return Date.now() - stat.mtimeMs > LOCK_STALE_MS;
+  } catch (e) {
+    // The lock disappeared between open and inspection, so retry acquisition.
+    if (e?.code === "ENOENT") return true;
+    return false;
+  } finally {
+    await lockHandle?.close().catch(() => {});
+  }
+}
+
+function startLockHeartbeat(handle, heartbeatHandle) {
+  const beat = async () => {
+    try {
+      const now = new Date();
+      await Promise.all([
+        handle.utimes(now, now),
+        heartbeatHandle.utimes(now, now),
+      ]);
+    } catch (_e) {
+      // A failed heartbeat makes the lease eligible for bounded stale
+      // reclamation. The owning sync will still finish or release normally.
+    }
+  };
+  const timer = setInterval(() => {
+    void beat();
+  }, LOCK_HEARTBEAT_MS);
+  timer.unref?.();
+  return timer;
+}
+
+async function releaseOwnedLock(
+  lockPath,
+  handle,
+  heartbeatHandle,
+  heartbeatTimer,
+  token,
+  {
+    beforeReleaseUnlink = null,
+    reclaimDepth = 0,
+    serializeRelease = true,
+  } = {},
+) {
+  let transitionGuard = null;
+  if (serializeRelease) {
+    // Reclaimers use the same guard while moving a stale lease aside. Holding
+    // it across token validation and unlink makes that ownership transition
+    // indivisible: an old owner cannot delete a replacement installed by a
+    // concurrent stale-lock reclaimer.
+    for (let attempt = 0; attempt < TRANSITION_GUARD_ATTEMPTS; attempt += 1) {
+      transitionGuard = await openLock(`${lockPath}.reclaim`, {
+        quietIfLocked: true,
+        reclaimDepth: reclaimDepth + 1,
+        // This short-lived internal guard is the serialization primitive; its
+        // own release must not recursively acquire another transition guard.
+        serializeRelease: false,
+      });
+      if (transitionGuard) break;
+      await new Promise((resolve) => setTimeout(resolve, TRANSITION_GUARD_RETRY_MS));
+    }
+    if (!transitionGuard) {
+      clearInterval(heartbeatTimer);
+      await handle.close().catch(() => {});
+      await heartbeatHandle?.close().catch(() => {});
+      await fs.unlink(heartbeatPathFor(lockPath, token)).catch(() => {});
+      throw new Error(`Timed out serializing lock release: ${lockPath}`);
+    }
+  }
+
+  try {
+    clearInterval(heartbeatTimer);
+    await handle.close().catch(() => {});
+    await heartbeatHandle?.close().catch(() => {});
+    const owner = parseLockOwner(await fs.readFile(lockPath, "utf8"));
+    if (owner?.token === token) {
+      if (typeof beforeReleaseUnlink === "function") {
+        await beforeReleaseUnlink({ lockPath });
+      }
+      await fs.unlink(lockPath).catch(() => {});
+      await fs.unlink(heartbeatPathFor(lockPath, token)).catch(() => {});
+    }
+  } catch (_e) {
+    // Missing or replaced owner paths are already released from this token's
+    // perspective. Other failures retain the conservative no-delete outcome.
+  } finally {
+    await transitionGuard?.release();
+  }
+}
+
+async function openLock(
+  lockPath,
+  {
+    quietIfLocked = false,
+    beforeReclaim = null,
+    beforeReleaseUnlink = null,
+    reclaimDepth = 0,
+    serializeRelease = true,
+  } = {},
+) {
   try {
     const handle = await fs.open(lockPath, "wx");
-    return {
-      async release() {
-        await handle.close().catch(() => {});
-        await fs.unlink(lockPath).catch(() => {});
-      },
-    };
+    const token = crypto.randomUUID();
+    const heartbeatPath = heartbeatPathFor(lockPath, token);
+    let heartbeatHandle = null;
+    try {
+      await handle.writeFile(
+        JSON.stringify({ pid: process.pid, token, createdAt: new Date().toISOString() }) + "\n",
+        "utf8",
+      );
+      heartbeatHandle = await fs.open(heartbeatPath, "wx");
+      await heartbeatHandle.writeFile(`${token}\n`, "utf8");
+      const heartbeatTimer = startLockHeartbeat(handle, heartbeatHandle);
+      return {
+        async release() {
+          await releaseOwnedLock(
+            lockPath,
+            handle,
+            heartbeatHandle,
+            heartbeatTimer,
+            token,
+            {
+              beforeReleaseUnlink,
+              reclaimDepth,
+              serializeRelease,
+            },
+          );
+        },
+      };
+    } catch (e) {
+      await heartbeatHandle?.close().catch(() => {});
+      await fs.unlink(heartbeatPath).catch(() => {});
+      await handle.close().catch(() => {});
+      await fs.unlink(lockPath).catch(() => {});
+      throw e;
+    }
   } catch (e) {
     if (e && e.code === "EEXIST") {
-      // Check if lock is stale
-      try {
-        const stat = await fs.stat(lockPath);
-        if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
-          await fs.unlink(lockPath).catch(() => {});
-          return openLock(lockPath, { quietIfLocked });
+      if (await existingLockCanBeReclaimed(lockPath)) {
+        if (typeof beforeReclaim === "function") {
+          await beforeReclaim({ lockPath });
+        }
+        if (reclaimDepth >= MAX_RECLAIM_DEPTH) return null;
+
+        // Serialize all reclaimers before re-checking the target. A stale
+        // check can be shared by many contenders; only the holder of this
+        // atomic guard may move the old lease out of the way.
+        const reclaimGuard = await openLock(`${lockPath}.reclaim`, {
+          quietIfLocked: true,
+          reclaimDepth: reclaimDepth + 1,
+          serializeRelease: false,
+        });
+        if (!reclaimGuard) return null;
+
+        let quarantinePath = null;
+        try {
+          // Re-check while holding the guard. A competing reclaimer may have
+          // already replaced the target since our initial stale inspection.
+          if (!(await existingLockCanBeReclaimed(lockPath))) return null;
+
+          quarantinePath = `${lockPath}.stale.${process.pid}.${crypto.randomUUID()}`;
+          try {
+            await fs.rename(lockPath, quarantinePath);
+          } catch (renameError) {
+            if (renameError?.code === "ENOENT") {
+              return openLock(lockPath, {
+                quietIfLocked,
+                beforeReleaseUnlink,
+                reclaimDepth,
+                serializeRelease,
+              });
+            }
+            if (!quietIfLocked) {
+              process.stdout.write("Another sync is already running.\n");
+            }
+            return null;
+          }
+
+          const staleOwner = parseLockOwner(
+            await fs.readFile(quarantinePath, "utf8").catch(() => ""),
+          );
+          if (staleOwner) {
+            // The heartbeat name contains the old token, so it cannot belong
+            // to a replacement lease created after the atomic rename.
+            await fs.unlink(heartbeatPathFor(lockPath, staleOwner.token)).catch(() => {});
+          }
+          return await openLock(lockPath, {
+            quietIfLocked,
+            beforeReleaseUnlink,
+            reclaimDepth,
+            serializeRelease,
+          });
+        } finally {
+          if (quarantinePath) await fs.unlink(quarantinePath).catch(() => {});
+          await reclaimGuard.release();
         }
-      } catch (_statErr) {
-        // Lock file disappeared between checks, retry
-        return openLock(lockPath, { quietIfLocked });
       }
       if (!quietIfLocked) {
         process.stdout.write("Another sync is already running.\n");

package/src/lib/git-outcomes.js

@@ -0,0 +1,147 @@
+"use strict";
+
+// Conservative, metadata-only Git attribution. A commit is attributed only
+// when exactly one local AI session overlaps its author timestamp.
+
+const { execFile } = require("node:child_process");
+const fs = require("node:fs");
+const fsp = require("node:fs/promises");
+const os = require("node:os");
+const path = require("node:path");
+const crypto = require("node:crypto");
+
+function resolveAutoOutcomesPath(home = os.homedir()) {
+  return path.join(home, ".tokentracker", "tracker", "auto-outcomes.jsonl");
+}
+
+function readJsonl(filePath) {
+  try { return fs.readFileSync(filePath, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line)); }
+  catch { return []; }
+}
+
+// Async on purpose: this runs inside the local API request path, where a
+// synchronous spawn would freeze every other endpoint for the duration of
+// each git invocation.
+function runGit(cwd, args) {
+  return new Promise((resolve) => {
+    execFile(
+      "git",
+      args,
+      { cwd, encoding: "utf8", timeout: 15_000, maxBuffer: 32 * 1024 * 1024 },
+      (error, stdout) => resolve(error ? "" : String(stdout).trim()),
+    );
+  });
+}
+
+async function repoRoot(cwd) {
+  if (!cwd || !fs.existsSync(cwd)) return null;
+  return (await runGit(cwd, ["rev-parse", "--show-toplevel"])) || null;
+}
+
+async function commitsForWindow(root, from, to) {
+  const format = "%H%x1f%aI%x1f%P%x1f%s%x1e";
+  const raw = await runGit(root, ["log", "--all", `--since=${from}`, `--until=${to}`, `--format=${format}`]);
+  if (!raw) return [];
+  return raw.split("\x1e").map((item) => item.trim()).filter(Boolean).map((item) => {
+    const [sha, timestamp, parents, subject] = item.split("\x1f");
+    return {
+      sha,
+      timestamp,
+      parent_count: String(parents || "").split(/\s+/).filter(Boolean).length,
+      reverted: /^revert\b/i.test(subject || ""),
+    };
+  }).filter((row) => row.sha && Number.isFinite(Date.parse(row.timestamp)));
+}
+
+async function writeAtomic(filePath, content) {
+  await fsp.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
+  const temp = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  await fsp.writeFile(temp, content, { encoding: "utf8", mode: 0o600 });
+  await fsp.rename(temp, filePath);
+}
+
+async function buildGitOutcomesInternal(sessions, { home = os.homedir(), force = false, maxAgeDays = 90 } = {}) {
+  const groups = new Map();
+  const rootCache = new Map();
+  const cutoff = Date.now() - Math.max(1, Number(maxAgeDays) || 90) * 86400_000;
+  for (const session of sessions || []) {
+    if (!session?.project_ref || !session.started_at || !session.ended_at) continue;
+    if (Date.parse(session.ended_at) < cutoff) continue;
+    let root = rootCache.get(session.project_ref);
+    if (root === undefined) {
+      root = await repoRoot(session.project_ref);
+      rootCache.set(session.project_ref, root);
+    }
+    if (!root) continue;
+    if (!groups.has(root)) groups.set(root, []);
+    groups.get(root).push(session);
+  }
+
+  const outputPath = resolveAutoOutcomesPath(home);
+  const metaPath = `${outputPath}.meta.json`;
+  const signatureHash = crypto.createHash("sha256");
+  for (const [root, repoSessions] of [...groups.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+    signatureHash.update(`${root}\0${await runGit(root, ["for-each-ref", "--format=%(objectname)"])}\n`);
+    for (const session of repoSessions) signatureHash.update(`${session.session_hash}\0${session.ended_at}\n`);
+  }
+  const signature = signatureHash.digest("hex");
+  if (!force) {
+    try {
+      const meta = JSON.parse(fs.readFileSync(metaPath, "utf8"));
+      if (meta.version === 1 && meta.signature === signature) return readJsonl(outputPath);
+    } catch { /* first run */ }
+  }
+
+  const outcomes = [];
+  for (const [root, repoSessions] of groups) {
+    const start = repoSessions.reduce((min, row) => !min || row.started_at < min ? row.started_at : min, "");
+    const endMs = Math.max(...repoSessions.map((row) => Date.parse(row.ended_at))) + 60 * 60 * 1000;
+    const commits = await commitsForWindow(root, start, new Date(endMs).toISOString());
+    for (const commit of commits) {
+      const commitMs = Date.parse(commit.timestamp);
+      const candidates = repoSessions.filter((session) => {
+        const sessionStart = Date.parse(session.started_at) - 10 * 60 * 1000;
+        const sessionEnd = Date.parse(session.ended_at) + 60 * 60 * 1000;
+        return commitMs >= sessionStart && commitMs <= sessionEnd;
+      });
+      if (candidates.length !== 1) continue;
+      const session = candidates[0];
+      outcomes.push({
+        timestamp: commit.timestamp,
+        model: session.model || "unknown",
+        tool: session.source || "unknown",
+        accepted: !commit.reverted,
+        task_type: "git_commit",
+        status: commit.reverted ? "reverted" : "committed",
+        session_hash: session.session_hash,
+        commit_hash: commit.sha.slice(0, 12),
+        parent_count: commit.parent_count,
+        confidence: "heuristic",
+        methodology: "single-overlapping-session",
+      });
+    }
+  }
+  outcomes.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+  await writeAtomic(outputPath, outcomes.map((row) => JSON.stringify(row)).join("\n") + (outcomes.length ? "\n" : ""));
+  await writeAtomic(metaPath, `${JSON.stringify({ version: 1, signature, generated_at: new Date().toISOString(), max_age_days: maxAgeDays })}\n`);
+  return outcomes;
+}
+
+// Overlapping outcomes/session-insights requests must not spawn duplicate git
+// scans nor race the atomic sidecar write; share one build per home.
+const gitOutcomesBuilds = new Map();
+
+function buildGitOutcomes(sessions, options = {}) {
+  const home = path.resolve(String(options.home || os.homedir()));
+  const existing = gitOutcomesBuilds.get(home);
+  if (existing) return existing;
+  const promise = buildGitOutcomesInternal(sessions, { ...options, home });
+  gitOutcomesBuilds.set(home, promise);
+  const clear = () => {
+    if (gitOutcomesBuilds.get(home) === promise) gitOutcomesBuilds.delete(home);
+  };
+  promise.then(clear, clear);
+  return promise;
+}
+
+module.exports = { resolveAutoOutcomesPath, buildGitOutcomes, repoRoot, commitsForWindow };