tokentracker-cli 0.84.5 → 0.84.7

package/src/lib/codex-rollout-parser.js

@@ -11,6 +11,12 @@ const path = require("node:path");
 const readline = require("node:readline");
 
 const { listRolloutFiles } = require("./rollout");
+const {
+  consumeUsageDelta,
+  createUsageDeltaState,
+  snapshotUsageBaselines,
+  totalsReset,
+} = require("./codex-token-usage");
 const {
   emptyTotals,
   addInto,
@@ -434,42 +440,10 @@ function normalizeUsage(u) {
   return out;
 }
 
-function totalsReset(curr, prev) {
-  const a = Number(curr?.total_tokens);
-  const b = Number(prev?.total_tokens);
-  if (!Number.isFinite(a) || !Number.isFinite(b)) return false;
-  return a < b;
-}
-
 function pickDelta(lastUsage, totalUsage, prevTotals) {
-  const hasLast = lastUsage && typeof lastUsage === "object";
-  const hasTotal = totalUsage && typeof totalUsage === "object";
-  const hasPrev = prevTotals && typeof prevTotals === "object";
-
-  if (hasTotal && hasPrev) {
-    if (totalsReset(totalUsage, prevTotals)) {
-      const resetUsage = hasLast ? lastUsage : totalUsage;
-      return normalizeUsage(resetUsage);
-    }
-    const delta = {};
-    for (const k of [
-      "input_tokens",
-      "cached_input_tokens",
-      "cache_creation_input_tokens",
-      "output_tokens",
-      "reasoning_output_tokens",
-      "total_tokens",
-    ]) {
-      const a = Number(totalUsage[k]);
-      const b = Number(prevTotals[k]);
-      if (Number.isFinite(a) && Number.isFinite(b)) delta[k] = Math.max(0, a - b);
-    }
-    return normalizeUsage(delta);
-  }
-
-  if (hasLast) return normalizeUsage(lastUsage);
-  if (hasTotal) return normalizeUsage(totalUsage);
-  return null;
+  const state = createUsageDeltaState({ lastTotal: prevTotals });
+  const delta = consumeUsageDelta(state, lastUsage, totalUsage);
+  return delta ? normalizeUsage(delta) : null;
 }
 
 // ---------------------------------------------------------------------------
@@ -596,7 +570,10 @@ async function parseCodexRolloutFile(filePath, {
   let provider = isResuming ? resumeState.provider || null : null;
   let cliVersion = isResuming ? resumeState.cliVersion || null : null;
 
-  let prevTotals = isResuming ? resumeState.prevTotals || null : null;
+  const usageDeltaState = createUsageDeltaState({
+    lastTotal: isResuming ? resumeState.prevTotals || null : null,
+    baselines: isResuming ? resumeState.tokenUsageBaselines || null : null,
+  });
   let pendingToolNames = isResuming
     ? Array.from(resumeState.pendingToolNames || [])
     : [];
@@ -875,8 +852,8 @@ async function parseCodexRolloutFile(filePath, {
       const info = tokenCount.info;
       const lastUsage = info?.last_token_usage;
       const totalUsage = info?.total_token_usage;
-      const delta = pickDelta(lastUsage, totalUsage, prevTotals);
-      if (totalUsage && typeof totalUsage === "object") prevTotals = totalUsage;
+      const rawDelta = consumeUsageDelta(usageDeltaState, lastUsage, totalUsage);
+      const delta = rawDelta ? normalizeUsage(rawDelta) : null;
       const isNewResumeEvent = noteTokenEvent(ts, lastUsage, totalUsage);
       if (inRequestedRange) {
         const eventSessionId = sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath;
@@ -927,7 +904,8 @@ async function parseCodexRolloutFile(filePath, {
       model,
       provider,
       cliVersion,
-      prevTotals,
+      prevTotals: usageDeltaState.lastTotal,
+      tokenUsageBaselines: snapshotUsageBaselines(usageDeltaState),
       pendingToolNames,
       pendingSkills,
       pendingExecDetails,

package/src/lib/codex-token-usage.js

@@ -0,0 +1,195 @@
+"use strict";
+
+// A rollout can contain TokenCount events from more than one Codex SessionState
+// (for example, a parent agent and an embedded reviewer). Each SessionState has
+// its own cumulative total, but the event does not carry a stable stream id.
+// Recover the stream from Codex's invariant: total - last = that stream's prior
+// total. Keeping a small LRU of stream heads also makes repeated rate-limit
+// snapshots idempotent when the streams are interleaved.
+
+const USAGE_FIELDS = [
+  "input_tokens",
+  "cached_input_tokens",
+  "cache_creation_input_tokens",
+  "output_tokens",
+  "reasoning_output_tokens",
+  "total_tokens",
+];
+const MAX_USAGE_BASELINES = 32;
+
+function canonicalUsage(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const out = {};
+  let sawNumber = false;
+  for (const field of USAGE_FIELDS) {
+    const raw = field === "cache_creation_input_tokens"
+      ? value.cache_creation_input_tokens ?? value.cache_write_input_tokens
+      : value[field];
+    const number = Number(raw);
+    if (Number.isFinite(number) && number >= 0) {
+      out[field] = Math.floor(number);
+      sawNumber = true;
+    } else {
+      out[field] = 0;
+    }
+  }
+  return sawNumber ? out : null;
+}
+
+function usageSignature(value) {
+  const usage = canonicalUsage(value);
+  return usage ? USAGE_FIELDS.map((field) => usage[field]).join(":") : null;
+}
+
+function sameUsage(left, right) {
+  const leftSignature = usageSignature(left);
+  return leftSignature !== null && leftSignature === usageSignature(right);
+}
+
+function subtractUsage(totalUsage, lastUsage) {
+  const total = canonicalUsage(totalUsage);
+  const last = canonicalUsage(lastUsage);
+  if (!total || !last) return null;
+  const previous = {};
+  for (const field of USAGE_FIELDS) {
+    if (total[field] < last[field]) return null;
+    previous[field] = total[field] - last[field];
+  }
+  return previous;
+}
+
+function diffUsage(totalUsage, previousUsage) {
+  const total = canonicalUsage(totalUsage);
+  const previous = canonicalUsage(previousUsage);
+  if (!total || !previous) return null;
+  const delta = {};
+  for (const field of USAGE_FIELDS) {
+    delta[field] = Math.max(0, total[field] - previous[field]);
+  }
+  return delta;
+}
+
+function totalsReset(totalUsage, previousUsage) {
+  const total = canonicalUsage(totalUsage);
+  const previous = canonicalUsage(previousUsage);
+  return Boolean(total && previous && total.total_tokens < previous.total_tokens);
+}
+
+function createUsageDeltaState({ lastTotal = null, baselines = null } = {}) {
+  const state = {
+    lastTotal: null,
+    baselines: [],
+    sawDivergentCumulative: false,
+    sawInterleaved: false,
+  };
+  for (const baseline of Array.isArray(baselines) ? baselines : []) {
+    touchBaseline(state, baseline);
+  }
+  if (canonicalUsage(lastTotal)) touchBaseline(state, lastTotal);
+  return state;
+}
+
+function snapshotUsageBaselines(state) {
+  return Array.isArray(state?.baselines)
+    ? state.baselines.map((usage) => ({ ...usage }))
+    : [];
+}
+
+function consumeUsageDelta(state, lastUsage, totalUsage) {
+  if (!state || typeof state !== "object") {
+    throw new TypeError("usage delta state is required");
+  }
+  if (!Array.isArray(state.baselines)) state.baselines = [];
+
+  const last = canonicalUsage(lastUsage);
+  const total = canonicalUsage(totalUsage);
+  if (!total) return last;
+
+  const activeSignature = usageSignature(state.lastTotal);
+  const duplicateIndex = findBaselineIndex(state, total);
+  if (duplicateIndex >= 0) {
+    if (
+      activeSignature !== null &&
+      usageSignature(state.baselines[duplicateIndex]) !== activeSignature
+    ) {
+      state.sawInterleaved = true;
+    }
+    touchBaselineAt(state, duplicateIndex, total);
+    return null;
+  }
+
+  if (last) {
+    const expectedPrevious = subtractUsage(total, last);
+    const lineageIndex = findBaselineIndex(state, expectedPrevious);
+    if (lineageIndex >= 0) {
+      if (
+        activeSignature !== null &&
+        usageSignature(state.baselines[lineageIndex]) !== activeSignature
+      ) {
+        state.sawInterleaved = true;
+      }
+      touchBaselineAt(state, lineageIndex, total);
+      return last;
+    }
+    if (expectedPrevious) {
+      if (canonicalUsage(state.lastTotal)) state.sawDivergentCumulative = true;
+      touchBaseline(state, total);
+      return last;
+    }
+  }
+
+  const active = canonicalUsage(state.lastTotal);
+  if (active && !totalsReset(total, active)) {
+    const delta = diffUsage(total, active);
+    if (!last || Number(delta?.total_tokens || 0) <= Number(last.total_tokens || 0)) {
+      const activeIndex = findBaselineIndex(state, active);
+      touchBaselineAt(state, activeIndex, total);
+      return delta;
+    }
+
+    // A cumulative jump larger than last usage cannot come from the normal
+    // append_last_usage path. Treat it as a new stream instead of charging the
+    // unrelated cumulative gap; a later event will establish its lineage.
+    state.sawDivergentCumulative = true;
+  }
+
+  touchBaseline(state, total);
+  return last || total;
+}
+
+function findBaselineIndex(state, usage) {
+  const signature = usageSignature(usage);
+  if (signature === null) return -1;
+  return state.baselines.findIndex((baseline) => usageSignature(baseline) === signature);
+}
+
+function touchBaselineAt(state, index, usage) {
+  const canonical = canonicalUsage(usage);
+  if (!canonical) return;
+  if (Number.isInteger(index) && index >= 0 && index < state.baselines.length) {
+    state.baselines.splice(index, 1);
+  } else {
+    const duplicateIndex = findBaselineIndex(state, canonical);
+    if (duplicateIndex >= 0) state.baselines.splice(duplicateIndex, 1);
+  }
+  state.baselines.push(canonical);
+  while (state.baselines.length > MAX_USAGE_BASELINES) state.baselines.shift();
+  state.lastTotal = canonical;
+}
+
+function touchBaseline(state, usage) {
+  touchBaselineAt(state, findBaselineIndex(state, usage), usage);
+}
+
+module.exports = {
+  MAX_USAGE_BASELINES,
+  canonicalUsage,
+  consumeUsageDelta,
+  createUsageDeltaState,
+  diffUsage,
+  sameUsage,
+  snapshotUsageBaselines,
+  subtractUsage,
+  totalsReset,
+  usageSignature,
+};

package/src/lib/local-api.js

@@ -2684,7 +2684,10 @@ function createLocalApiHandler({ queuePath }) {
             json(res, { importable, codexDetected: importable.length > 0 });
             return true;
           }
-          json(res, { pets: pets.listInstalledPets() });
+          json(res, {
+            pets: pets.listInstalledPets(),
+            hiddenBuiltinIds: pets.readHiddenBuiltinIds(),
+          });
           return true;
         }
         if (method === "POST") {

package/src/lib/pet-packages.js

@@ -10,6 +10,8 @@ const MAX_PACKAGE_BYTES = 12 * 1024 * 1024;
 const MAX_MANIFEST_BYTES = 16 * 1024;
 const MAX_SPRITESHEET_BYTES = 10 * 1024 * 1024;
 const BUILTIN_IDS = new Set(["clawd", "sprout", "byte", "ember"]);
+const REMOVABLE_BUILTIN_IDS = new Set(["sprout", "byte", "ember"]);
+const HIDDEN_BUILTINS_FILE = ".hidden-builtins.json";
 // codex-pets.net added `kind` to pet.json long after launch, and its enum may keep
 // growing — packages published before that (and future kinds) must stay importable,
 // so kind is optional metadata validated only as a short slug.
@@ -35,6 +37,44 @@ function resolveCodexPetsDir() {
   );
 }
 
+function readHiddenBuiltinIds() {
+  const file = path.join(resolvePetsDir(), HIDDEN_BUILTINS_FILE);
+  try {
+    const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
+    if (!Array.isArray(parsed)) return [];
+    return [...new Set(parsed
+      .map((id) => String(id || "").trim().toLowerCase())
+      .filter((id) => REMOVABLE_BUILTIN_IDS.has(id)))].sort();
+  } catch {
+    return [];
+  }
+}
+
+function hideBuiltinPet(id) {
+  const normalized = String(id || "").trim().toLowerCase();
+  if (normalized === "clawd") throw new Error("Clawd cannot be removed");
+  if (!REMOVABLE_BUILTIN_IDS.has(normalized)) throw new Error("Invalid built-in pet id");
+
+  const root = resolvePetsDir();
+  fs.mkdirSync(root, { recursive: true, mode: 0o700 });
+  const hidden = new Set(readHiddenBuiltinIds());
+  hidden.add(normalized);
+  const destination = path.join(root, HIDDEN_BUILTINS_FILE);
+  const stage = path.join(root, `${HIDDEN_BUILTINS_FILE}.${process.pid}-${crypto.randomBytes(6).toString("hex")}`);
+  const backup = `${stage}-backup`;
+  try {
+    fs.writeFileSync(stage, `${JSON.stringify([...hidden].sort(), null, 2)}\n`, { mode: 0o600 });
+    if (fs.existsSync(destination)) fs.renameSync(destination, backup);
+    fs.renameSync(stage, destination);
+  } catch (error) {
+    try { fs.rmSync(stage, { force: true }); } catch {}
+    if (!fs.existsSync(destination) && fs.existsSync(backup)) fs.renameSync(backup, destination);
+    throw error;
+  }
+  try { fs.rmSync(backup, { force: true }); } catch {}
+  return { id: normalized, hidden: true };
+}
+
 function readUInt24LE(buffer, offset) {
   return buffer[offset] | (buffer[offset + 1] << 8) | (buffer[offset + 2] << 16);
 }
@@ -368,10 +408,11 @@ async function installFromCodexPets(input, fetchImpl = fetch) {
 
 function removeInstalledPet(id) {
   const normalized = String(id || "").trim().toLowerCase();
-  if (!PET_ID_RE.test(normalized) || BUILTIN_IDS.has(normalized)) throw new Error("Invalid custom pet id");
+  if (!PET_ID_RE.test(normalized)) throw new Error("Invalid pet id");
+  if (BUILTIN_IDS.has(normalized)) return hideBuiltinPet(normalized);
   const root = resolvePetsDir();
   const destination = path.join(root, normalized);
-  if (!destination.startsWith(`${root}${path.sep}`)) throw new Error("Invalid custom pet path");
+  if (!destination.startsWith(`${root}${path.sep}`)) throw new Error("Invalid pet path");
   fs.rmSync(destination, { recursive: true, force: true });
   petDirectoryCache.delete(destination);
   return { id: normalized };
@@ -627,6 +668,7 @@ module.exports = {
   petIdFromCodexPetsUrl,
   readCodexBuiltinPets,
   readCodexImportableAsset,
+  readHiddenBuiltinIds,
   readWebpDimensions,
   removeInstalledPet,
   resolveCodexPetsDir,