tokentracker-cli 1.1.4 → 1.1.5

package/src/lib/scan-roots.js

@@ -0,0 +1,277 @@
+"use strict";
+
+// Persistent extra scan roots for Codex and Claude Code (#657).
+//
+// Agent harnesses run Codex / Claude Code with a private home
+// (CODEX_HOME=..., CLAUDE_CONFIG_DIR=...). Those sessions were invisible to
+// TokenTracker unless the sync happened to be spawned by a process that
+// inherited the same environment: a hook-fired sync saw them, the native app's
+// background refresh did not. Worse, the two producers then disagreed about
+// where a rollout's cursor lives (the cursor store classifies a rollout as a
+// Codex path — per-day shard — only when it sits under one of ITS codexRoots,
+// everything else goes to core), never saw each other's cursor, and re-parsed
+// the same file from byte 0 on every alternation (#639: one 30-minute bucket
+// re-emitted 138 times, 59B tokens for a 1.2B day).
+//
+// So roots come from ONE persistent list that every producer reads, and the
+// cursor store's codexRoots are derived from that same list.
+//
+// Per provider, in order (first occurrence wins; later duplicates by realpath
+// are dropped):
+//   1. the roots the caller already scans (native home, WSL install)
+//   2. the spawning process's implicit root — CODEX_HOME / CLAUDE_CONFIG_DIR
+//   3. config.scanRoots.<provider>[] from ~/.tokentracker/tracker/config.json
+//
+// Config shape:
+//   { "scanRoots": { "codex": ["~/.local/share/agent/codex"],
+//                    "claude": ["~/.local/share/agent/claude", "~/.claude-native"] } }
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const { readJson } = require("./fs");
+const { resolveTrackerPaths } = require("./tracker-paths");
+
+const PROVIDERS = Object.freeze({
+  codex: Object.freeze({ envVar: "CODEX_HOME" }),
+  claude: Object.freeze({ envVar: "CLAUDE_CONFIG_DIR" }),
+});
+const PROVIDER_NAMES = Object.freeze(Object.keys(PROVIDERS));
+
+// Turn a configured root into a stable absolute path. `~` expands against
+// `home`, and a RELATIVE entry is resolved against `home` too — never against
+// process.cwd(), which differs between a hook-fired sync, the CLI and the
+// desktop app and would make producers scan different directories.
+function expandHome(value, home) {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  if (!trimmed) return null;
+  if (trimmed === "~") return path.resolve(home);
+  if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) {
+    return path.resolve(home, trimmed.slice(2));
+  }
+  return path.isAbsolute(trimmed) ? path.resolve(trimmed) : path.resolve(home, trimmed);
+}
+
+// The spawning process's implicit root — CODEX_HOME / CLAUDE_CONFIG_DIR —
+// normalized ONCE, the same way configured entries are: `~` and relative
+// values are anchored to `home`, never to process.cwd(). Every producer
+// (sync discovery, the cursor store's codexRoots, status, diagnostics, the
+// session browser) must use this value, or a relative CODEX_HOME such as
+// `profiles/codex` is scanned under one directory and cursor-classified
+// under another, and the same rollout replays. Returns null when unset.
+function resolveEnvRoot(provider, { env = process.env, home = os.homedir() } = {}) {
+  const spec = PROVIDERS[provider];
+  if (!spec) return null;
+  const raw = env && typeof env[spec.envVar] === "string" ? env[spec.envVar] : "";
+  return expandHome(raw, home);
+}
+
+// Accepts { codex: string | string[], claude: string | string[] }. Anything
+// else — wrong type, empty strings, unknown providers — is ignored rather than
+// failing the sync: a malformed config must not stop collection.
+function normalizeScanRootsConfig(raw) {
+  const out = {};
+  for (const provider of PROVIDER_NAMES) out[provider] = [];
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return out;
+  for (const provider of PROVIDER_NAMES) {
+    const value = raw[provider];
+    const list = Array.isArray(value) ? value : (typeof value === "string" ? [value] : []);
+    for (const entry of list) {
+      if (typeof entry !== "string" || !entry.trim()) continue;
+      out[provider].push(entry.trim());
+    }
+  }
+  return out;
+}
+
+// { exists, error }: `exists` is true only for a directory that can be
+// LISTED, not merely stat'ed — a stat-able directory whose listing fails
+// (mode 000, ACL) would otherwise be admitted to the scan, yield no files,
+// read as "present" in status/doctor, and slip past the deferred-repair
+// guard in sync. ENOENT and ENOTDIR (or a regular file in a directory's
+// place) mean the root is absent (error null); any other failure (EACCES,
+// ELOOP, EIO, ...) is reported as `error` so callers can say "unreadable"
+// instead of "missing".
+function scanRootDirState(target, deps = {}) {
+  const statSync = deps.statSync || fs.statSync;
+  const opendirSync = deps.opendirSync || fs.opendirSync;
+  try {
+    if (!statSync(target).isDirectory()) return { exists: false, error: null };
+    // Probe listability without reading entries; large roots stay cheap.
+    opendirSync(target).closeSync();
+    return { exists: true, error: null };
+  } catch (e) {
+    const code = e && typeof e.code === "string" ? e.code : "EUNKNOWN";
+    if (code === "ENOENT" || code === "ENOTDIR") return { exists: false, error: null };
+    return { exists: false, error: code };
+  }
+}
+
+function isDirectorySync(target, deps) {
+  return scanRootDirState(target, deps).exists;
+}
+
+function realpathOrSelf(target, deps) {
+  const realpathSync = deps.realpathSync || fs.realpathSync;
+  try {
+    return realpathSync(target);
+  } catch {
+    return target;
+  }
+}
+
+function identityKey(target, deps) {
+  const resolved = path.resolve(target);
+  const real = realpathOrSelf(resolved, deps);
+  return real.replace(/\\/g, "/").replace(/\/+$/, "");
+}
+
+// Drop later entries that resolve (realpath) to a directory already listed.
+// Used at the projects/ level too: two Claude profiles may symlink one
+// projects/ dir, and reading it under both spellings would double-parse every
+// file, leaving correctness to the bounded claudeHashes layer.
+function dedupeDirsByRealpath(dirs, deps = {}) {
+  const seen = new Set();
+  const out = [];
+  for (const dir of Array.isArray(dirs) ? dirs : []) {
+    if (typeof dir !== "string" || !dir) continue;
+    const key = identityKey(dir, deps);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(dir);
+  }
+  return out;
+}
+
+// Resolve the full, ordered, deduplicated root list per provider.
+//
+//   base: { codex: string[], claude: string[] } — roots the caller scans
+//         anyway (native home, WSL install); they keep their position and
+//         are labelled origin "native".
+//   env:  CODEX_HOME / CLAUDE_CONFIG_DIR become origin "env" unless already
+//         covered by base.
+//   config.scanRoots.<provider>[] become origin "config".
+//
+// Every entry: { path, realPath, exists, error, origin }. `exists` is a
+// readable-directory check and `error` carries a stat error code other than
+// absence (see scanRootDirState); non-existent or unreadable configured roots
+// are kept in the list (so cursor-path classification stays stable while a
+// volume is unmounted) and callers that walk directories skip them via
+// `exists`.
+function resolveScanRoots({
+  home = os.homedir(),
+  env = process.env,
+  config = null,
+  base = {},
+  deps = {},
+} = {}) {
+  const configured = normalizeScanRootsConfig(config && typeof config === "object" ? config.scanRoots : null);
+  const out = {};
+  for (const provider of PROVIDER_NAMES) {
+    const candidates = [];
+    for (const root of Array.isArray(base[provider]) ? base[provider] : []) {
+      if (typeof root === "string" && root.trim()) candidates.push({ raw: root, origin: "native" });
+    }
+    const envRoot = resolveEnvRoot(provider, { env, home });
+    if (envRoot) candidates.push({ raw: envRoot, origin: "env" });
+    for (const root of configured[provider]) candidates.push({ raw: root, origin: "config" });
+
+    const seen = new Set();
+    const entries = [];
+    for (const candidate of candidates) {
+      const expanded = expandHome(candidate.raw, home);
+      if (!expanded) continue;
+      const resolved = expanded;
+      const key = identityKey(resolved, deps);
+      if (seen.has(key)) continue;
+      seen.add(key);
+      const state = scanRootDirState(resolved, deps);
+      entries.push({
+        path: resolved,
+        realPath: realpathOrSelf(resolved, deps),
+        exists: state.exists,
+        error: state.error,
+        origin: candidate.origin,
+      });
+    }
+    out[provider] = entries;
+  }
+  return out;
+}
+
+// Append `extraDirs` to `baseDirs`, dropping any extra that resolves (realpath)
+// to a directory already present. Base entries are never collapsed against
+// each other: the native + WSL pair is listed on purpose (see sync.js) and
+// their dedup is handled downstream by file-identity hashes.
+function appendUniqueDirs(baseDirs, extraDirs, deps = {}) {
+  const out = [];
+  const seen = new Set();
+  for (const dir of Array.isArray(baseDirs) ? baseDirs : []) {
+    if (typeof dir !== "string" || !dir) continue;
+    seen.add(identityKey(dir, deps));
+    out.push(dir);
+  }
+  for (const dir of Array.isArray(extraDirs) ? extraDirs : []) {
+    if (typeof dir !== "string" || !dir) continue;
+    const key = identityKey(dir, deps);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(dir);
+  }
+  return out;
+}
+
+// Roots beyond what the caller already scans: the "env" and "config" entries
+// that exist on disk. Paths only, in list order.
+function extraScanRootPaths(entries) {
+  return (Array.isArray(entries) ? entries : [])
+    .filter((entry) => entry && entry.origin !== "native" && entry.exists)
+    .map((entry) => entry.path);
+}
+
+function hasAnyScanChild(root, children, deps = {}) {
+  for (const child of Array.isArray(children) ? children : [children]) {
+    if (isDirectorySync(path.join(root, child), deps)) return true;
+  }
+  return false;
+}
+
+// Human label for status / doctor output.
+function describeScanRootOrigin(entry, provider) {
+  if (!entry || entry.origin === "native") return "native";
+  if (entry.origin === "env") return PROVIDERS[provider]?.envVar || "env";
+  return "scanRoots";
+}
+
+// "" for a readable root, " (missing)" for an absent one, " (unreadable: CODE)"
+// for a stat failure other than absence.
+function describeScanRootState(entry) {
+  if (!entry || entry.exists) return "";
+  return entry.error ? ` (unreadable: ${entry.error})` : " (missing)";
+}
+
+// config.scanRoots as persisted in ~/.tokentracker/tracker/config.json for
+// callers that do not already hold the config (the session browser).
+async function loadScanRootsConfig({ home = os.homedir() } = {}) {
+  const { trackerDir } = await resolveTrackerPaths({ home });
+  const config = await readJson(path.join(trackerDir, "config.json")).catch(() => null);
+  return normalizeScanRootsConfig(config && typeof config === "object" ? config.scanRoots : null);
+}
+
+module.exports = {
+  PROVIDER_NAMES,
+  appendUniqueDirs,
+  dedupeDirsByRealpath,
+  describeScanRootOrigin,
+  describeScanRootState,
+  expandHome,
+  extraScanRootPaths,
+  hasAnyScanChild,
+  loadScanRootsConfig,
+  normalizeScanRootsConfig,
+  resolveEnvRoot,
+  resolveScanRoots,
+  scanRootDirState,
+};

package/src/lib/session-analytics.js

@@ -33,6 +33,14 @@ const { parseCodexRolloutFile } = require("./codex-rollout-parser");
 const { computeRowCost, getModelPricing } = require("./pricing");
 const { USD_TICKS_PER_USD, normalizeGrokUsage } = require("./grok-usage");
 const wsl = require("./wsl-probe");
+const {
+  appendUniqueDirs,
+  dedupeDirsByRealpath,
+  expandHome,
+  loadScanRootsConfig,
+  normalizeScanRootsConfig,
+  scanRootDirState,
+} = require("./scan-roots");
 
 // Bump the sidecar when derived metrics change so cached rows are rebuilt
 // instead of leaving the dashboard on the previous (over-counted) heuristic.
@@ -1074,8 +1082,11 @@ function providerRoots(home, providerDir, env, deps = {}) {
       && path.resolve(home) === path.resolve(homedir())
       && typeof env?.CODEX_HOME === "string"
       && env.CODEX_HOME.trim();
+    // Same normalization as sync / the cursor store (scan-roots.expandHome):
+    // a relative CODEX_HOME is anchored to home, never to process.cwd(), so
+    // the browser walks the exact root the cursors were keyed under.
     const nativeRoot = useProcessCodexHome
-      ? path.resolve(env.CODEX_HOME.trim())
+      ? expandHome(env.CODEX_HOME, home)
       : path.join(home, providerDir);
     roots.push(nativeRoot);
   }
@@ -1095,7 +1106,21 @@ function providerRoots(home, providerDir, env, deps = {}) {
     const wslRoot = discoverWslHome(providerDir, { env });
     if (wslRoot) roots.push(wslRoot);
   }
-  return [...new Set(roots)];
+  // Extra roots (#657), mirroring src/commands/sync.js: CLAUDE_CONFIG_DIR is
+  // the spawning process's implicit Claude root (additive — ~/.claude stays
+  // scanned), and config.scanRoots.<provider> arrives as deps.extraRoots from
+  // discoverSessionFiles. Extras are realpath-deduped against the roots above
+  // so one directory listed under two spellings is walked once.
+  const extras = [];
+  const useProcessClaudeConfigDir = providerDir === ".claude"
+    && path.resolve(home) === path.resolve(homedir())
+    && typeof env?.CLAUDE_CONFIG_DIR === "string"
+    && env.CLAUDE_CONFIG_DIR.trim();
+  if (useProcessClaudeConfigDir) extras.push(expandHome(env.CLAUDE_CONFIG_DIR, home));
+  for (const extra of Array.isArray(deps.extraRoots) ? deps.extraRoots : []) {
+    if (typeof extra === "string" && extra) extras.push(path.resolve(extra));
+  }
+  return appendUniqueDirs([...new Set(roots)], extras);
 }
 
 // Group one logical session discovered under more than one root. A group is
@@ -1221,10 +1246,37 @@ function groupCodexFiles(filePaths) {
 
 async function discoverSessionFiles(home, env = process.env, deps = {}) {
   const grokHome = resolveGrokHome(home);
-  const claudeRoots = providerRoots(home, ".claude", env, deps);
-  const codexRoots = providerRoots(home, ".codex", env, deps);
+  // config.scanRoots (#657): read from the tracker config unless the caller
+  // injects it (tests). A missing or malformed config yields no extras.
+  const scanRootsConfig = deps.scanRootsConfig !== undefined
+    ? normalizeScanRootsConfig(deps.scanRootsConfig)
+    : await loadScanRootsConfig({ home });
+  // expandHome resolves `~` and relative entries against `home` (never cwd);
+  // only readable directories are walked, absence and stat errors alike.
+  const extraRootsFor = (provider) => scanRootsConfig[provider]
+    .map((root) => expandHome(root, home))
+    .filter((root) => root && scanRootDirState(root).exists);
+  const claudeRoots = providerRoots(home, ".claude", env, { ...deps, extraRoots: extraRootsFor("claude") });
+  const codexRoots = providerRoots(home, ".codex", env, { ...deps, extraRoots: extraRootsFor("codex") });
+  // Two distinct roots may share one projects/ dir through a symlink; dedupe
+  // the derived projects dirs by realpath (as sync does) so the browser never
+  // lists the same transcript twice under two spellings.
+  const claudeProjectsDirs = dedupeDirsByRealpath(claudeRoots.map((r) => path.join(r, "projects")));
+  // listClaudeProjectFiles / listRolloutFilesDeep turn a listing error into an
+  // empty result, so a root whose projects/ (or sessions/) exists but cannot be
+  // read would look like "no sessions" and get cached as such. Report those
+  // directories as incomplete discovery instead; the caller decides not to
+  // persist a partial inventory (sync defers its repair on the same state).
+  const incomplete = [];
+  for (const dir of [
+    ...claudeProjectsDirs,
+    ...codexRoots.map((r) => path.join(r, "sessions")),
+  ]) {
+    const state = scanRootDirState(dir);
+    if (state.error !== null) incomplete.push({ path: dir, error: state.error });
+  }
   const [claudeGroups, codexGroups, archivedGroups, grok] = await Promise.all([
-    Promise.all(claudeRoots.map((r) => listClaudeProjectFiles(path.join(r, "projects")))),
+    Promise.all(claudeProjectsDirs.map((dir) => listClaudeProjectFiles(dir))),
     Promise.all(codexRoots.map((r) => listRolloutFilesDeep(path.join(r, "sessions")))),
     Promise.all(codexRoots.map((r) => listRolloutFilesDeep(path.join(r, "archived_sessions")))),
     listGrokSessionFiles(path.join(grokHome, "sessions")),
@@ -1239,7 +1291,20 @@ async function discoverSessionFiles(home, env = process.env, deps = {}) {
   const claude = allClaude.filter((filePaths) => !filePaths.some((filePath) => filePath
     .split(path.sep)
     .some((segment) => segment.endsWith(CLAUDE_MEM_OBSERVER_PROJECT_SUFFIX))));
-  return { claude, codex: groupCodexFiles([...codex, ...archived]), grok };
+  return { claude, codex: groupCodexFiles([...codex, ...archived]), grok, incomplete };
+}
+
+// Warn once per unreadable directory per process: the dashboard polls this
+// path, and a line per poll would flood the serve log.
+const warnedIncompleteDiscovery = new Set();
+function warnIncompleteDiscovery(incomplete) {
+  for (const entry of incomplete) {
+    if (warnedIncompleteDiscovery.has(entry.path)) continue;
+    warnedIncompleteDiscovery.add(entry.path);
+    if (!process.env.NODE_TEST_CONTEXT) {
+      console.warn(`[session-analytics] session directory unreadable (${entry.error}), inventory incomplete: ${entry.path}`);
+    }
+  }
 }
 
 function filesSignature(files) {
@@ -1368,6 +1433,19 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
     } catch { /* first run */ }
   }
   const discovered = await discoverSessionFiles(home);
+  // Incomplete discovery (an unreadable projects/ or sessions/ dir): never let
+  // a partial inventory replace a complete snapshot. Serve the last complete
+  // sidecar when there is one; otherwise build from what is readable but do
+  // not persist it, so the next refresh tries again.
+  const incompleteDirs = Array.isArray(discovered.incomplete) ? discovered.incomplete : [];
+  if (incompleteDirs.length > 0) {
+    warnIncompleteDiscovery(incompleteDirs);
+    if (!force && previousMeta?.version === SIDECAR_VERSION) {
+      const cached = readSidecar(sidecarPath);
+      Object.defineProperty(cached, "incompleteDirs", { value: incompleteDirs, enumerable: false });
+      return cached;
+    }
+  }
   // Codex thread titles are stored separately from rollout files. Include the
   // index in the overall signature so an index-only rename reaches the
   // per-file dependency check below on the next refresh. Grok titles/metadata
@@ -1437,19 +1515,22 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
     nextFiles[cacheKey] = { stat_key: statKey };
   }
   sessions.sort((a, b) => String(b.ended_at || "").localeCompare(String(a.ended_at || "")));
-  const content = sessions.map(serializeSessionRecord).join("\n") + (sessions.length ? "\n" : "");
-  await writeAtomic(sidecarPath, content);
-  const generatedAt = new Date().toISOString();
-  await writeAtomic(metaPath, `${JSON.stringify({
-    version: SIDECAR_VERSION,
-    signature,
-    generated_at: generatedAt,
-    checked_at: generatedAt,
-    files: nextFiles,
-  })}\n`);
+  if (incompleteDirs.length === 0) {
+    const content = sessions.map(serializeSessionRecord).join("\n") + (sessions.length ? "\n" : "");
+    await writeAtomic(sidecarPath, content);
+    const generatedAt = new Date().toISOString();
+    await writeAtomic(metaPath, `${JSON.stringify({
+      version: SIDECAR_VERSION,
+      signature,
+      generated_at: generatedAt,
+      checked_at: generatedAt,
+      files: nextFiles,
+    })}\n`);
+  }
   // Non-enumerable so the array still behaves exactly like a plain row list
   // for every existing caller (map/filter/JSON of the rows is unaffected).
   Object.defineProperty(sessions, "skippedFiles", { value: skippedFiles, enumerable: false });
+  Object.defineProperty(sessions, "incompleteDirs", { value: incompleteDirs, enumerable: false });
   return sessions;
 }
 
@@ -2006,6 +2087,7 @@ module.exports = {
   resumeCommandFor,
   sessionsToCsv,
   providerRoots,
+  discoverSessionFiles,
   dedupeClaudeFilesAcrossRoots,
   analyticsEntryStatKey,
 };

package/dashboard/dist/assets/DialogDescription-Cof52Icj.js

@@ -1 +0,0 @@
-import{r as n,v as a}from"./__vite-browser-external-BFNWaGsk.js";import{u as p}from"./DialogTitle-CVddcRPZ.js";import{u as c}from"./composite-qxBzts0W.js";const x=n.forwardRef(function(e,r){const{render:l,className:u,style:m,id:s,...t}=e,i=p(),o=c(s);return i.useSyncedValueWithCleanup("descriptionElementId",o),a("p",e,{ref:r,props:[{id:o},t]})});export{x as D};

package/dashboard/dist/assets/use-reduced-motion-DrlJ9X6p.js

@@ -1 +0,0 @@
-import{r as o}from"./__vite-browser-external-BFNWaGsk.js";import{h as r,i as t,p as s}from"./react-Djb4VGKb.js";function i(){!r.current&&t();const[e]=o.useState(s.current);return e}export{i as u};