tokentracker-cli 1.0.0 → 1.0.2

package/src/lib/rollout.js

@@ -26,6 +26,20 @@ const {
   readCodexServiceTier,
   isPriorityServiceTier,
 } = require("./codex-service-tier");
+const {
+  awaitsCompactedLine,
+  createCompactionResponseIds,
+  createUsageRecordState,
+  extractTokenUsageRecord,
+  isCodexTurnEndEvent,
+  isRecordOnly,
+  noteLineAfterUsageRecord,
+  noteTokenCount,
+  noteTurnEnd,
+  noteUsageRecord,
+  snapshotUsageRecordState,
+  takeCompactionOnTokenCount,
+} = require("./codex-usage-record");
 const {
   DEVIN_TABLE_PROBE_SQL,
   devinUsageSql,
@@ -258,6 +272,10 @@ async function parseRolloutIncremental({
   if (!cursors.files || typeof cursors.files !== "object") {
     cursors.files = {};
   }
+  // Counted Codex compaction calls by response_id (issue #652). Core cursor
+  // state, so a fork replay or a copied rollout under any path or day shard
+  // sees every compaction counted before.
+  const compactionResponseIds = createCompactionResponseIds(cursors);
 
   // Persisted set of seen Codex event keys (sessionUUID:eventTimestamp). Mirrors
   // the claudeHashes pattern: it makes an inode-changing re-scan idempotent so an
@@ -480,6 +498,12 @@ async function parseRolloutIncremental({
     const modelAttributionState = sameInode && !truncated
       ? prev.modelAttributionState || null
       : null;
+    // Only valid for the bytes before the resume offset; a rescan from 0
+    // rebuilds it. A cursor from before #652 with a lastTotal has seen a
+    // token_count already.
+    const usageRecordState = startOffset > 0
+      ? prev?.codexUsageRecord || (prev?.lastTotal ? { sawTokenCount: true } : null)
+      : null;
 
     const codexProjectFastPath =
       projectEnabled && (fileSource === DEFAULT_SOURCE || fileSource === "acode");
@@ -575,6 +599,8 @@ async function parseRolloutIncremental({
           lastModel,
           lastServiceTier,
           modelAttributionState,
+          usageRecordState,
+          compactionResponseIds,
           hourlyState,
           touchedBuckets,
           source: fileSource,
@@ -601,6 +627,20 @@ async function parseRolloutIncremental({
       modelAttributionState: result.modelAttributionState,
       updatedAt: new Date().toISOString(),
     };
+    const nextUsageRecordState = projectContextOnlyScan
+      ? usageRecordState
+      : result.usageRecordState;
+    if (nextUsageRecordState) nextCursor.codexUsageRecord = nextUsageRecordState;
+    // Files whose usage only exists as token_usage_record rows are not
+    // counted (issue #652); keep them in core cursor state so sync and status
+    // can warn without loading per-file cursor shards.
+    if (fileSource === DEFAULT_SOURCE) {
+      if (isRecordOnly(nextUsageRecordState)) {
+        (cursors.codexUsageRecordOnlyFiles ||= {})[key] = true;
+      } else if (cursors.codexUsageRecordOnlyFiles?.[key]) {
+        delete cursors.codexUsageRecordOnlyFiles[key];
+      }
+    }
     if (codexProjectFastPath) {
       nextCursor.projectOffset = result.endOffset;
       nextCursor.projectFileContext = buildProjectFileContext(
@@ -643,6 +683,12 @@ async function parseRolloutIncremental({
     projectState.updatedAt = new Date().toISOString();
     cursors.projectHourly = projectState;
   }
+  compactionResponseIds.persist();
+  // A deleted rollout no longer needs a warning. The map is empty unless a
+  // record-only writer exists, so this is normally free.
+  for (const flaggedPath of Object.keys(cursors.codexUsageRecordOnlyFiles || {})) {
+    if (!fssync.existsSync(flaggedPath)) delete cursors.codexUsageRecordOnlyFiles[flaggedPath];
+  }
 
   return { filesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
 }
@@ -2071,6 +2117,8 @@ async function parseRolloutFile({
   lastModel,
   lastServiceTier,
   modelAttributionState: previousModelAttributionState,
+  usageRecordState: previousUsageRecordState = null,
+  compactionResponseIds = null,
   hourlyState,
   touchedBuckets,
   source,
@@ -2098,6 +2146,7 @@ async function parseRolloutFile({
       lastModel,
       lastServiceTier: typeof lastServiceTier === "string" ? lastServiceTier : null,
       modelAttributionState: previousModelAttributionState,
+      usageRecordState: previousUsageRecordState,
       eventsAggregated: 0,
       projectFileContexts,
     };
@@ -2142,6 +2191,54 @@ async function parseRolloutFile({
   let eventsAggregated = 0;
   let scannedEndOffset = startOffset;
   let committedEndOffset = startOffset;
+  // Compaction records (issue #652, rules in codex-usage-record.js). Codex only.
+  const usageRecordState = source === DEFAULT_SOURCE && compactionResponseIds
+    ? createUsageRecordState(previousUsageRecordState)
+    : null;
+
+  // What a record adds if it turns out to be a compaction call: model, tier
+  // and project are those in effect when the record was written. A record
+  // without a response_id cannot be deduplicated and is never counted.
+  function buildCompactionEvent(record) {
+    if (!record.responseId) return null;
+    const delta = normalizeUsage(record.usage);
+    if (isAllZeroUsage(delta)) return null;
+    delta.conversation_count = 1;
+    const bucketStart = toUtcHalfHourStart(record.timestamp);
+    if (!bucketStart) return null;
+    return {
+      responseId: record.responseId,
+      bucketStart,
+      model,
+      serviceTier,
+      projectKey: currentProjectKey,
+      projectRef: currentProjectRef,
+      delta,
+    };
+  }
+
+  // Counted once per response_id across every rollout: a fork replays its
+  // parent's records and a copied rollout repeats them, both with the same id.
+  function addCompactionEvent(event) {
+    if (compactionResponseIds.has(event.responseId)) return;
+    compactionResponseIds.add(event.responseId);
+    const bucket = getHourlyBucket(hourlyState, source, event.model, event.bucketStart);
+    addTotals(bucket.totals, event.delta);
+    if (isPriorityServiceTier(event.serviceTier)) addPriorityUsage(bucket.totals, event.delta);
+    touchedBuckets.add(bucketKey(source, event.model, event.bucketStart));
+    if (event.projectKey && projectState && projectTouchedBuckets) {
+      const projectBucket = getProjectBucket(
+        projectState,
+        event.projectKey,
+        source,
+        event.bucketStart,
+        event.projectRef,
+      );
+      addTotals(projectBucket.totals, event.delta);
+      projectTouchedBuckets.add(projectBucketKey(event.projectKey, source, event.bucketStart));
+    }
+    eventsAggregated += 1;
+  }
 
   const invalidUtf8 = invalidRecordPolicy === "throw" ? "throw" : "record";
   for await (const record of physicalJsonlRecords(stream, { invalidUtf8 })) {
@@ -2154,6 +2251,26 @@ async function parseRolloutFile({
 
     const { line } = record;
     if (!line) continue;
+    // Only a `compacted` line directly after a record keeps it as a
+    // compaction candidate. A line that is not complete yet decides nothing:
+    // the read stops before it and the next sync sees it whole.
+    if (awaitsCompactedLine(usageRecordState)) {
+      let next;
+      if (!record.terminated || line.includes('"compacted"')) {
+        try {
+          next = JSON.parse(line);
+        } catch {
+          next = undefined;
+        }
+      }
+      if (next !== undefined || record.terminated) {
+        noteLineAfterUsageRecord(usageRecordState, next?.type === "compacted");
+      }
+      if (next?.type === "compacted") {
+        committedEndOffset = scannedEndOffset;
+        continue;
+      }
+    }
     const maybeTokenCount = line.includes('"token_count"');
     const maybeModelReroute =
       !maybeTokenCount &&
@@ -2168,7 +2285,19 @@ async function parseRolloutFile({
     const maybeServiceTier =
       !maybeTokenCount && !maybeModelReroute && !maybeTurnContext &&
       line.includes(CODEX_SERVICE_TIER_MARKER);
-    if (!maybeTokenCount && !maybeTurnContext && !maybeModelReroute && !maybeServiceTier) {
+    const maybeUsageRecord =
+      !maybeTokenCount &&
+      Boolean(usageRecordState) &&
+      (line.includes('"token_usage_record"') ||
+        (usageRecordState.unmatchedRecord &&
+          (line.includes('"task_complete"') || line.includes('"turn_aborted"'))));
+    if (
+      !maybeTokenCount &&
+      !maybeTurnContext &&
+      !maybeModelReroute &&
+      !maybeServiceTier &&
+      !maybeUsageRecord
+    ) {
       if (invalidRecordPolicy === "throw" || !record.terminated) {
         try {
           JSON.parse(line);
@@ -2230,11 +2359,32 @@ async function parseRolloutFile({
       continue;
     }
 
+    if (usageRecordState) {
+      const usageRecord = extractTokenUsageRecord(obj);
+      if (usageRecord) {
+        noteUsageRecord(usageRecordState, buildCompactionEvent(usageRecord));
+        continue;
+      }
+      if (isCodexTurnEndEvent(obj)) {
+        noteTurnEnd(usageRecordState);
+        continue;
+      }
+    }
+
     const token = extractTokenCount(obj);
     if (!token) continue;
 
     const info = token.info;
     if (!info || typeof info !== "object") continue;
+    if (usageRecordState) {
+      noteTokenCount(usageRecordState);
+      const compaction = takeCompactionOnTokenCount(
+        usageRecordState,
+        usageDeltaState.lastTotal,
+        info.total_token_usage,
+      );
+      if (compaction) addCompactionEvent(compaction);
+    }
 
     const tokenTimestamp = typeof token.timestamp === "string" ? token.timestamp : null;
     if (!tokenTimestamp) continue;
@@ -2352,6 +2502,9 @@ async function parseRolloutFile({
     lastModel: model,
     lastServiceTier: serviceTier,
     modelAttributionState: snapshotCodexModelAttributionState(modelAttributionState),
+    usageRecordState: usageRecordState
+      ? snapshotUsageRecordState(usageRecordState)
+      : previousUsageRecordState,
     eventsAggregated,
     projectFileContexts,
   };