tokentracker-cli 0.99.0 → 1.0.1

package/src/commands/sync.js

@@ -18,6 +18,7 @@ const {
   updateJsonLocked,
 } = require("../lib/fs");
 const { physicalJsonlRecords } = require("../lib/jsonl-lines");
+const { countRecordOnlyFiles, formatRecordOnlyWarning } = require("../lib/codex-usage-record");
 const {
   listRolloutFiles,
   listRolloutFilesDeep,
@@ -91,6 +92,8 @@ const {
   piAgentDirCollidesWithOmp,
   resolvePrimeAgentSessionFiles,
   parsePrimeAgentIncremental,
+  resolveMinimaxCodeSessionFiles,
+  parseMinimaxCodeIncremental,
   resolveCraftSessionFiles,
   parseCraftIncremental,
   resolveReasonixTelemetryFiles,
@@ -312,6 +315,7 @@ const AUTO_SYNC_SOURCES = new Set([
   "kimi-code",
   "lmstudio",
   "mimo",
+  "minimax-code",
   "omo",
   "omp",
   "opencode",
@@ -2514,6 +2518,34 @@ async function cmdSync(argv, context = {}) {
       }
     }
 
+    // ── MiniMax Code — passive ~/.minimax/v2/sessions/**/messages.jsonl usage reader ──
+    let minimaxCodeResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+    const minimaxCodeFiles = sourceAllowed("minimax-code")
+      ? mergeBothFileSources({ resolveFiles: resolveMinimaxCodeSessionFiles, env: process.env })
+      : [];
+    if (minimaxCodeFiles.length > 0) {
+      if (progress?.enabled) {
+        progress.start(`Parsing MiniMax Code ${renderBar(0)} | buckets 0`);
+      }
+      try {
+        minimaxCodeResult = await parseMinimaxCodeIncremental({
+          sessionFiles: minimaxCodeFiles,
+          cursors,
+          queuePath,
+          env: process.env,
+          onProgress: (p) => {
+            if (!progress?.enabled) return;
+            const pct = p.total > 0 ? p.index / p.total : 1;
+            progress.update(
+              `Parsing MiniMax Code ${renderBar(pct)} ${formatNumber(p.index)}/${formatNumber(p.total)} files | buckets ${formatNumber(p.bucketsQueued)}`,
+            );
+          },
+        });
+      } catch (err) {
+        warnProviderParseFailure("MiniMax Code", err, opts);
+      }
+    }
+
     // ── Craft Agents (passive ~/.craft-agent + workspaces session.jsonl reader) ──
     let craftResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
     const craftFiles = sourceAllowed("craft")
@@ -3014,6 +3046,7 @@ async function cmdSync(argv, context = {}) {
       omoResult.recordsProcessed +
       piResult.recordsProcessed +
       primeAgentResult.recordsProcessed +
+      minimaxCodeResult.recordsProcessed +
       craftResult.recordsProcessed +
       reasonixResult.recordsProcessed +
       grokResult.recordsProcessed +
@@ -3054,6 +3087,7 @@ async function cmdSync(argv, context = {}) {
       omoResult.bucketsQueued +
       piResult.bucketsQueued +
       primeAgentResult.bucketsQueued +
+      minimaxCodeResult.bucketsQueued +
       craftResult.bucketsQueued +
       reasonixResult.bucketsQueued +
       grokResult.bucketsQueued +
@@ -3272,6 +3306,7 @@ async function cmdSync(argv, context = {}) {
     }
 
     if (!opts.auto) {
+      const codexRecordOnlyWarning = formatRecordOnlyWarning(countRecordOnlyFiles(cursors));
       process.stdout.write(
         [
           "Sync finished:",
@@ -3283,6 +3318,7 @@ async function cmdSync(argv, context = {}) {
           runtime.deviceToken && pendingBytes > 0 && !opts.drain
             ? `- Remaining: ${formatBytes(pendingBytes)} pending (run sync again, or use --drain)`
             : null,
+          codexRecordOnlyWarning ? `- Warning: ${codexRecordOnlyWarning}` : null,
           "",
         ]
           .filter(Boolean)
@@ -4779,8 +4815,10 @@ async function migrateRolloutCumulativeDeltaBuckets({ cursors, queuePath, rollou
   // The migration clears Codex buckets and reparses the discovered corpus from
   // byte zero. Persisted event keys belong to the cleared buckets, so retaining
   // them can suppress the rebuild when a moved session still has an old path
-  // cursor. Rebuild the hash inventory together with the buckets.
+  // cursor. Rebuild the hash inventory together with the buckets. Counted
+  // compaction ids (#652) belong to the cleared buckets the same way.
   cursors.codexHashes = [];
+  cursors.codexCompactionResponseIds = [];
 
   const buckets = cursors.hourly?.buckets;
   const retractions = [];
@@ -5332,6 +5370,9 @@ async function repairCodexRescanInflation({
       buckets: tmpCursors.hourly.buckets || {},
       groupQueued: tmpCursors.hourly.groupQueued || {},
       codexHashes: Array.isArray(tmpCursors.codexHashes) ? tmpCursors.codexHashes : [],
+      codexCompactionResponseIds: Array.isArray(tmpCursors.codexCompactionResponseIds)
+        ? tmpCursors.codexCompactionResponseIds
+        : [],
       files: tmpCursors.files || {},
       queueRows: tmpRaw.split("\n").filter((l) => l.trim()),
       projectHourly: tmpCursors.projectHourly || null,
@@ -5486,6 +5527,7 @@ async function repairCodexRescanInflation({
     cursors.files[fp] = v;
   }
   cursors.codexHashes = rebuilt.codexHashes;
+  cursors.codexCompactionResponseIds = rebuilt.codexCompactionResponseIds;
 
   // 3. Project usage mirrors the main Codex repair: drop inflated Codex project
   //    rows, append the rebuilt rows, and swap only Codex project buckets. Project

package/src/lib/codex-rollout-parser.js

@@ -31,6 +31,15 @@ const {
   snapshotCodexModelAttributionState,
 } = require("./codex-model-attribution");
 const { readCodexServiceTier, isPriorityServiceTier } = require("./codex-service-tier");
+const {
+  createUsageRecordState,
+  extractTokenUsageRecord,
+  noteLineAfterUsageRecord,
+  noteTokenCount,
+  noteUsageRecord,
+  snapshotUsageRecordState,
+  takeCompactionOnTokenCount,
+} = require("./codex-usage-record");
 
 const DISCOVERY_FULL_AUDIT_INTERVAL_MS = 60 * 1000;
 const MAX_DISCOVERY_INVENTORIES = 32;
@@ -643,9 +652,37 @@ async function parseCodexRolloutFile(filePath, {
 
   let turnCount = 0;
 
-  function recordModelUsage(delta, rawRequestUsage) {
+  // Compaction records (issue #652, rules in codex-usage-record.js). A resume
+  // state from before #652 with prevTotals has seen a token_count.
+  const usageRecordState = createUsageRecordState(
+    isResuming
+      ? resumeState.usageRecordState || (resumeState.prevTotals ? { sawTokenCount: true } : null)
+      : null,
+  );
+  // Compactions counted in this session, by response_id: the files merged for
+  // a session can repeat a record (an archived copy, a split fragment), and a
+  // resumed parse must not count one from an earlier chunk again.
+  const compactionResponseIds = new Set(
+    isResuming && Array.isArray(resumeState.compactionResponseIds)
+      ? resumeState.compactionResponseIds
+      : [],
+  );
+
+  // Attribution as of now. A compaction record is counted later than it is
+  // read, so it carries the attribution captured when it was read.
+  function currentUsageAttribution() {
+    return {
+      model: currentCodexModel(modelAttributionState) || "unknown",
+      selectedModel: modelAttributionState.selectedModel || null,
+      rerouted: Boolean(modelAttributionState.rerouted),
+      rerouteReason: modelAttributionState.rerouteReason || null,
+      serviceTier,
+    };
+  }
+
+  function recordModelUsage(delta, rawRequestUsage, attribution = currentUsageAttribution()) {
     if (!collectModelUsage || !delta || delta.total_tokens <= 0) return;
-    const effectiveModel = currentCodexModel(modelAttributionState) || "unknown";
+    const effectiveModel = attribution.model;
     let row = byModel.get(effectiveModel);
     if (!row) {
       row = {
@@ -677,13 +714,13 @@ async function parseCodexRolloutFile(filePath, {
     }
     addInto(row, delta);
     row.usage_events += 1;
-    if (modelAttributionState.selectedModel) {
-      row.selected_models.add(modelAttributionState.selectedModel);
+    if (attribution.selectedModel) {
+      row.selected_models.add(attribution.selectedModel);
     }
-    if (modelAttributionState.rerouted) {
+    if (attribution.rerouted) {
       row.rerouted_usage_events += 1;
-      if (modelAttributionState.rerouteReason) {
-        row.reroute_reasons.add(modelAttributionState.rerouteReason);
+      if (attribution.rerouteReason) {
+        row.reroute_reasons.add(attribution.rerouteReason);
       }
     }
     // OpenAI applies the long-context tier to the whole request when its raw
@@ -704,7 +741,7 @@ async function parseCodexRolloutFile(filePath, {
     // instead of replacing it. The two subsets alone cannot say how much usage
     // was on BOTH, so record the intersection explicitly — otherwise a long
     // Fast request prices at 3x Standard input where it should be 4x.
-    if (isPriorityServiceTier(serviceTier)) {
+    if (isPriorityServiceTier(attribution.serviceTier)) {
       row.priority_usage_events += 1;
       row.priority_input_tokens += delta.input_tokens;
       row.priority_cached_input_tokens += delta.cached_input_tokens;
@@ -885,6 +922,14 @@ async function parseCodexRolloutFile(filePath, {
     pendingExecDetails = [];
   }
 
+  function countCompactionRecord(record) {
+    if (compactionResponseIds.has(record.responseId)) return;
+    compactionResponseIds.add(record.responseId);
+    if (!record.inRequestedRange) return;
+    recordModelUsage(record.delta, record.rawUsage, record.attribution);
+    attributeTurn(record.delta);
+  }
+
   function usageEventSignature(lastUsage, totalUsage) {
     return JSON.stringify({ lastUsage, totalUsage });
   }
@@ -952,6 +997,8 @@ async function parseCodexRolloutFile(filePath, {
     // turn_context.payload.model itself would make codex-model-attribution.js
     // one of two authorities on the same question.
     if (typeof onObject === "function") onObject(obj, attributedModel);
+    const usageRecord = extractTokenUsageRecord(obj);
+    if (!usageRecord) noteLineAfterUsageRecord(usageRecordState, obj?.type === "compacted");
     const ts = typeof obj?.timestamp === "string" ? obj.timestamp : null;
     if (!ts) continue;
     const inRequestedRange = isTimestampInRequestedDayRange(ts, { from, to, timeZoneContext });
@@ -1003,13 +1050,42 @@ async function parseCodexRolloutFile(filePath, {
       continue;
     }
 
+    if (usageRecord) {
+      const delta = normalizeUsage(usageRecord.usage);
+      noteUsageRecord(
+        usageRecordState,
+        // Without a response_id a compaction cannot be deduplicated.
+        delta.total_tokens > 0 && usageRecord.responseId
+          ? {
+              responseId: usageRecord.responseId,
+              timestamp: usageRecord.timestamp,
+              inRequestedRange,
+              rawUsage: usageRecord.usage,
+              delta,
+              attribution: currentUsageAttribution(),
+            }
+          : null,
+      );
+      continue;
+    }
+
     const tokenCount = extractTokenCount(obj);
     if (tokenCount) {
       const info = tokenCount.info;
       const lastUsage = info?.last_token_usage;
       const totalUsage = info?.total_token_usage;
+      let compaction = null;
+      if (info && typeof info === "object") {
+        noteTokenCount(usageRecordState);
+        compaction = takeCompactionOnTokenCount(
+          usageRecordState,
+          usageDeltaState.lastTotal,
+          totalUsage,
+        );
+      }
       const rawDelta = consumeUsageDelta(usageDeltaState, lastUsage, totalUsage);
       const delta = rawDelta ? normalizeUsage(rawDelta) : null;
+      if (compaction) countCompactionRecord(compaction);
       const isNewResumeEvent = noteTokenEvent(ts, lastUsage, totalUsage);
       if (inRequestedRange) {
         const eventSessionId = sessionId || rolloutSessionIdFromPath(primaryFilePath) || primaryFilePath;
@@ -1082,6 +1158,8 @@ async function parseCodexRolloutFile(filePath, {
       pendingExecDetails,
       lastTokenTimestamp,
       lastTimestampEventSignatures: Array.from(materializeLastTimestampSignatures()),
+      usageRecordState: snapshotUsageRecordState(usageRecordState),
+      compactionResponseIds: Array.from(compactionResponseIds),
     };
     result.endOffset = readProgress.endOffset;
     result.appendable = Boolean(contentHasher) && (

package/src/lib/codex-usage-record.js

@@ -0,0 +1,195 @@
+"use strict";
+
+// Codex rollouts written since CLI 0.151 carry a top-level
+//   {"type":"token_usage_record","payload":{usage,turn_token_usage,thread_token_usage,...}}
+// next to the event_msg token_count that has always carried usage (issue
+// #652). While both are written they describe the same model response, so
+// token_count stays the only source of normal usage and a record is counted
+// in exactly one case:
+//
+// Compaction. Codex bills the compaction model call in a record that is
+// immediately followed by a top-level `compacted` line, and the next
+// token_count repeats the previous cumulative total (its last_token_usage is
+// the post-compaction context size, not a response). The cumulative delta is
+// zero, so without the record the call is lost. Inside a file that has
+// already shown a token_count, such a record is held as `compaction` and
+// counted when the next token_count arrives with a total identical to the
+// previous one. If that total advanced, the token_count already covers the
+// call and the record is dropped. Each compaction is counted once per
+// payload.response_id: a fork replays its parent's records and a copied
+// rollout repeats them, both with the same id. A record without one is never
+// counted.
+//
+// A file whose records are never matched by a token_count is not counted from
+// its records. It is flagged `recordOnly` instead, and sync surfaces a
+// warning. The proof is the same ordering Codex keeps for every response: its
+// token_count is written after its record and before the next record or the
+// turn end. So a record followed by another record or a turn end with no
+// token_count in between flags the file; a sync that stops between a record
+// and its token_count does not.
+//
+// payload.usage is the per-response delta; turn_token_usage and
+// thread_token_usage are cumulative and are never read. All state below is
+// persisted in the sync cursor and the session parser resume state, so the
+// record, the `compacted` line and the token_count may land in different
+// reads.
+
+const { canonicalUsage, sameUsage } = require("./codex-token-usage");
+
+function extractTokenUsageRecord(obj) {
+  if (obj?.type !== "token_usage_record") return null;
+  const usage = obj.payload?.usage;
+  if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
+  const timestamp = typeof obj.timestamp === "string" ? obj.timestamp : null;
+  if (!timestamp) return null;
+  // canonicalUsage folds cache_write_input_tokens into
+  // cache_creation_input_tokens exactly as the token_count path does.
+  const canonical = canonicalUsage(usage);
+  if (!canonical) return null;
+  const responseId = typeof obj.payload.response_id === "string" && obj.payload.response_id
+    ? obj.payload.response_id
+    : null;
+  return { usage: canonical, timestamp, responseId };
+}
+
+// Compaction calls already counted by sync, by response_id, kept in core
+// cursor state as cursors.codexCompactionResponseIds (oldest first). The id
+// is unique per API response and survives fork replays and file copies, so
+// it is the only dedup key a compaction needs. Compactions are rare (19 on
+// the maintainer's corpus), so the cap is far above any real history.
+const MAX_COMPACTION_RESPONSE_IDS = 20000;
+
+function createCompactionResponseIds(cursors) {
+  const stored = Array.isArray(cursors.codexCompactionResponseIds)
+    ? cursors.codexCompactionResponseIds
+    : [];
+  const ids = new Set(stored);
+  const added = [];
+  return {
+    has: (id) => ids.has(id),
+    add(id) {
+      if (ids.has(id)) return;
+      ids.add(id);
+      added.push(id);
+    },
+    persist() {
+      if (added.length === 0) return;
+      cursors.codexCompactionResponseIds = [...stored, ...added].slice(-MAX_COMPACTION_RESPONSE_IDS);
+    },
+  };
+}
+
+function isCodexTurnEndEvent(obj) {
+  if (obj?.type !== "event_msg") return false;
+  const type = obj.payload?.type;
+  return type === "task_complete" || type === "turn_aborted";
+}
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function createUsageRecordState(snapshot) {
+  const compaction = snapshot?.compaction;
+  return {
+    sawTokenCount: Boolean(snapshot?.sawTokenCount),
+    unmatchedRecord: Boolean(snapshot?.unmatchedRecord),
+    recordOnly: Boolean(snapshot?.recordOnly),
+    compaction: isPlainObject(compaction) && isPlainObject(compaction.candidate)
+      ? { candidate: compaction.candidate, compacted: Boolean(compaction.compacted) }
+      : null,
+  };
+}
+
+// Called for every token_count carrying an info object.
+function noteTokenCount(state) {
+  state.sawTokenCount = true;
+  state.unmatchedRecord = false;
+  state.recordOnly = false;
+}
+
+// Called for every record. `candidate` is what the caller would count if the
+// record turns out to be a compaction call (null when it cannot be counted).
+function noteUsageRecord(state, candidate) {
+  if (state.sawTokenCount) {
+    state.compaction = candidate ? { candidate, compacted: false } : null;
+    return;
+  }
+  if (state.unmatchedRecord) state.recordOnly = true;
+  state.unmatchedRecord = true;
+}
+
+// Turn end: a record still waiting for its token_count will not get one.
+function noteTurnEnd(state) {
+  if (state.sawTokenCount) return;
+  if (state.unmatchedRecord) state.recordOnly = true;
+  state.unmatchedRecord = false;
+}
+
+// True while the line right after a record decides whether it was a
+// compaction call.
+function awaitsCompactedLine(state) {
+  return Boolean(state?.compaction && !state.compaction.compacted);
+}
+
+// Called for each complete line after a record, other than a record. Only a
+// `compacted` line directly after the record keeps it as a candidate.
+function noteLineAfterUsageRecord(state, isCompactedLine) {
+  if (!awaitsCompactedLine(state)) return;
+  if (isCompactedLine) state.compaction.compacted = true;
+  else state.compaction = null;
+}
+
+// Called for every token_count carrying an info object, with the cumulative
+// total that was active before it. Returns the compaction candidate to count,
+// or null.
+function takeCompactionOnTokenCount(state, previousTotal, totalUsage) {
+  const compaction = state.compaction;
+  state.compaction = null;
+  if (!compaction?.compacted) return null;
+  if (!canonicalUsage(previousTotal) || !sameUsage(previousTotal, totalUsage)) return null;
+  return compaction.candidate;
+}
+
+function isRecordOnly(state) {
+  return Boolean(state?.recordOnly && !state.sawTokenCount);
+}
+
+function snapshotUsageRecordState(state) {
+  if (!state || (!state.sawTokenCount && !state.unmatchedRecord && !state.recordOnly)) return null;
+  const out = { sawTokenCount: state.sawTokenCount };
+  if (state.unmatchedRecord) out.unmatchedRecord = true;
+  if (state.recordOnly) out.recordOnly = true;
+  if (state.compaction) out.compaction = state.compaction;
+  return out;
+}
+
+// Sync keeps the flagged rollouts in core cursor state:
+// cursors.codexUsageRecordOnlyFiles = { [rolloutPath]: true }.
+function countRecordOnlyFiles(cursors) {
+  const flagged = cursors?.codexUsageRecordOnlyFiles;
+  return isPlainObject(flagged) ? Object.keys(flagged).length : 0;
+}
+
+function formatRecordOnlyWarning(count) {
+  if (!count) return null;
+  return `${count} Codex session(s) report usage only via token_usage_record; not counted yet (see #652)`;
+}
+
+module.exports = {
+  MAX_COMPACTION_RESPONSE_IDS,
+  createCompactionResponseIds,
+  countRecordOnlyFiles,
+  formatRecordOnlyWarning,
+  awaitsCompactedLine,
+  createUsageRecordState,
+  extractTokenUsageRecord,
+  isCodexTurnEndEvent,
+  isRecordOnly,
+  noteLineAfterUsageRecord,
+  noteTokenCount,
+  noteTurnEnd,
+  noteUsageRecord,
+  snapshotUsageRecordState,
+  takeCompactionOnTokenCount,
+};