tokenusage-cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/LICENSE +22 -0
  3. package/README.md +94 -0
  4. package/THIRD_PARTY_NOTICES.md +140 -0
  5. package/bin/tokenusage.js +69 -0
  6. package/package.json +59 -0
  7. package/src/cli.js +134 -0
  8. package/src/commands/device-login.js +205 -0
  9. package/src/commands/diagnostics.js +38 -0
  10. package/src/commands/doctor.js +110 -0
  11. package/src/commands/init.js +2436 -0
  12. package/src/commands/sessions.js +29 -0
  13. package/src/commands/status.js +1747 -0
  14. package/src/commands/sync.js +7934 -0
  15. package/src/commands/uninstall.js +312 -0
  16. package/src/lib/account-usage-dedup.js +267 -0
  17. package/src/lib/antigravity-paths.js +48 -0
  18. package/src/lib/ark-agent-plan-limits.js +438 -0
  19. package/src/lib/ark-coding-plan-limits.js +537 -0
  20. package/src/lib/async-queue-reader.js +138 -0
  21. package/src/lib/atomic-json.js +27 -0
  22. package/src/lib/bearer-token.js +11 -0
  23. package/src/lib/categorizer-utils.js +232 -0
  24. package/src/lib/claude-categorizer.js +1343 -0
  25. package/src/lib/claude-config.js +279 -0
  26. package/src/lib/claude-user-identity.js +56 -0
  27. package/src/lib/cli-ui.js +179 -0
  28. package/src/lib/cloud-account.js +541 -0
  29. package/src/lib/cloud-device-token.js +141 -0
  30. package/src/lib/cloud-sync-prefs.js +14 -0
  31. package/src/lib/cloud-usage-reset.js +115 -0
  32. package/src/lib/codex-config.js +595 -0
  33. package/src/lib/codex-context-breakdown.js +806 -0
  34. package/src/lib/codex-model-attribution.js +128 -0
  35. package/src/lib/codex-rollout-parser.js +1471 -0
  36. package/src/lib/codex-service-tier.js +49 -0
  37. package/src/lib/codex-token-refresh.js +140 -0
  38. package/src/lib/codex-token-usage.js +195 -0
  39. package/src/lib/codex-usage-record.js +195 -0
  40. package/src/lib/command-runner.js +313 -0
  41. package/src/lib/commandcode-limits.js +317 -0
  42. package/src/lib/context-health.js +124 -0
  43. package/src/lib/cursor-config.js +605 -0
  44. package/src/lib/cursor-store.js +1918 -0
  45. package/src/lib/debug-flags.js +7 -0
  46. package/src/lib/devin-limits.js +249 -0
  47. package/src/lib/devin-usage.js +212 -0
  48. package/src/lib/diagnostics.js +383 -0
  49. package/src/lib/doctor.js +385 -0
  50. package/src/lib/dumate.js +95 -0
  51. package/src/lib/fs.js +455 -0
  52. package/src/lib/function-url.js +14 -0
  53. package/src/lib/gemini-config.js +291 -0
  54. package/src/lib/git-outcomes.js +243 -0
  55. package/src/lib/grok-context-breakdown.js +890 -0
  56. package/src/lib/grok-hook.js +360 -0
  57. package/src/lib/grok-limits.js +865 -0
  58. package/src/lib/grok-usage.js +98 -0
  59. package/src/lib/heatmap-compact.js +114 -0
  60. package/src/lib/ingest-sanitize.js +41 -0
  61. package/src/lib/init-flow.js +42 -0
  62. package/src/lib/install-resolver.js +170 -0
  63. package/src/lib/jsonl-lines.js +134 -0
  64. package/src/lib/kimi-model-repair.js +173 -0
  65. package/src/lib/kimi-profile.js +61 -0
  66. package/src/lib/kimi-session.js +216 -0
  67. package/src/lib/local-api.js +3588 -0
  68. package/src/lib/machine-id.js +238 -0
  69. package/src/lib/multi-install-parser.js +113 -0
  70. package/src/lib/notify-command.js +38 -0
  71. package/src/lib/omp-hook.js +589 -0
  72. package/src/lib/openclaw-cli.js +104 -0
  73. package/src/lib/openclaw-hook.js +375 -0
  74. package/src/lib/openclaw-session-plugin.js +560 -0
  75. package/src/lib/opencode-config.js +121 -0
  76. package/src/lib/opencode-go-limits.js +807 -0
  77. package/src/lib/outcomes-engine.js +253 -0
  78. package/src/lib/passive-mode.js +219 -0
  79. package/src/lib/pet-packages.js +722 -0
  80. package/src/lib/pricing/curated-overrides.json +822 -0
  81. package/src/lib/pricing/index.js +379 -0
  82. package/src/lib/pricing/litellm-fetcher.js +172 -0
  83. package/src/lib/pricing/matcher.js +456 -0
  84. package/src/lib/pricing/seed-snapshot.json +1 -0
  85. package/src/lib/progress.js +76 -0
  86. package/src/lib/project-usage-purge.js +117 -0
  87. package/src/lib/prompt.js +24 -0
  88. package/src/lib/provider-status.js +104 -0
  89. package/src/lib/proxy-env.js +489 -0
  90. package/src/lib/proxy-settings.js +163 -0
  91. package/src/lib/qoder-cn-cli-limits.js +119 -0
  92. package/src/lib/qoder-limits.js +880 -0
  93. package/src/lib/quiet-warnings.js +26 -0
  94. package/src/lib/rollout.js +25966 -0
  95. package/src/lib/runtime-config.js +259 -0
  96. package/src/lib/scan-roots.js +277 -0
  97. package/src/lib/sentry.js +231 -0
  98. package/src/lib/session-analytics.js +2972 -0
  99. package/src/lib/session-performance.js +237 -0
  100. package/src/lib/session-upload.js +379 -0
  101. package/src/lib/skill-usage.js +267 -0
  102. package/src/lib/skills-manager.js +1591 -0
  103. package/src/lib/source-metadata.js +48 -0
  104. package/src/lib/sqlite-reader.js +215 -0
  105. package/src/lib/star-cta.js +73 -0
  106. package/src/lib/subscription-manager.js +293 -0
  107. package/src/lib/subscriptions.js +508 -0
  108. package/src/lib/telemetry.js +193 -0
  109. package/src/lib/tracker-paths.js +16 -0
  110. package/src/lib/trae-cn-config.js +548 -0
  111. package/src/lib/trae-db.js +127 -0
  112. package/src/lib/trae-snapshot.js +230 -0
  113. package/src/lib/trae-sqlite.js +143 -0
  114. package/src/lib/trae-usage.js +121 -0
  115. package/src/lib/upload-queue.js +134 -0
  116. package/src/lib/upload-throttle.js +171 -0
  117. package/src/lib/usage-limits.js +4552 -0
  118. package/src/lib/wrapped-aggregator.js +236 -0
  119. package/src/lib/wsl-probe.js +281 -0
  120. package/src/lib/zcode-limits.js +1172 -0
  121. package/src/lib/zcode-reset-credits.js +66 -0
  122. package/test/cli-smoke.test.js +53 -0
  123. package/test/ingest-sanitize.test.js +162 -0
  124. package/test/quiet-warnings.test.js +53 -0
  125. package/test/session-upload.test.js +427 -0
@@ -0,0 +1,49 @@
1
+ // Codex service-tier attribution.
2
+ //
3
+ // The request tier is NOT on the token_count rows. Codex emits it as its own
4
+ // rollout record — `type: "event_msg"`, `payload.type:
5
+ // "thread_settings_applied"`, `payload.thread_settings.service_tier` — written
6
+ // immediately before the `turn_context` of the turn it applies to. A session's
7
+ // first turn has no such record, and CLI-originated sessions
8
+ // (originator codex_cli_rs / codex-tui) often have none at all: on a 1213-file
9
+ // local corpus only 167 files carried the event, covering 25% of token_count
10
+ // events.
11
+ //
12
+ // Attribution rule, and the only one the log shape permits: a token_count row
13
+ // belongs to the most recent preceding thread_settings_applied. With no
14
+ // preceding record the tier is unknown, and unknown is never guessed — it is
15
+ // billed at Standard, exactly as before this file existed.
16
+
17
+ "use strict";
18
+
19
+ // Cheap line prefilter for parsers that avoid JSON.parse on most records.
20
+ const CODEX_SERVICE_TIER_MARKER = '"thread_settings_applied"';
21
+
22
+ // Observed values on the local corpus are "default" (814) and "priority" (58).
23
+ // "priority" is what the CLI writes for what OpenAI's price table and the Codex
24
+ // UI both call Fast; accept the product-facing spelling too so a future rename
25
+ // does not silently fall back to Standard.
26
+ const PRIORITY_SERVICE_TIERS = new Set(["priority", "fast"]);
27
+
28
+ function readCodexServiceTier(obj) {
29
+ if (!obj || obj.type !== "event_msg") return null;
30
+ const payload = obj.payload;
31
+ if (!payload || typeof payload !== "object") return null;
32
+ if (payload.type !== "thread_settings_applied") return null;
33
+ const settings = payload.thread_settings;
34
+ if (!settings || typeof settings !== "object") return null;
35
+ const tier = settings.service_tier;
36
+ if (typeof tier !== "string" || !tier.trim()) return null;
37
+ return tier.trim().toLowerCase();
38
+ }
39
+
40
+ function isPriorityServiceTier(tier) {
41
+ return typeof tier === "string" && PRIORITY_SERVICE_TIERS.has(tier);
42
+ }
43
+
44
+ module.exports = {
45
+ CODEX_SERVICE_TIER_MARKER,
46
+ PRIORITY_SERVICE_TIERS,
47
+ readCodexServiceTier,
48
+ isPriorityServiceTier,
49
+ };
@@ -0,0 +1,140 @@
1
+ const fs = require("node:fs");
2
+ const path = require("node:path");
3
+
4
+ // Public Codex OAuth client. Same id used by the official `codex` CLI. Aligned with
5
+ // steipete/CodexBar's CodexTokenRefresher.swift — neither is sensitive (it's a public client).
6
+ const REFRESH_ENDPOINT = "https://auth.openai.com/oauth/token";
7
+ const CODEX_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann";
8
+
9
+ // Match the official Codex CLI: when the access token exposes a JWT expiry, refresh only
10
+ // shortly before it expires. Fall back to last_refresh > 8 days for opaque/legacy tokens.
11
+ const REFRESH_THRESHOLD_MS = 8 * 24 * 60 * 60 * 1000;
12
+ const ACCESS_TOKEN_REFRESH_WINDOW_MS = 5 * 60 * 1000;
13
+
14
+ function parseJwtExpirationMs(token) {
15
+ if (typeof token !== "string" || token.length === 0) return null;
16
+ const parts = token.split(".");
17
+ if (parts.length < 2 || !parts[1]) return null;
18
+ try {
19
+ const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
20
+ const exp = Number(payload?.exp);
21
+ return Number.isFinite(exp) ? exp * 1000 : null;
22
+ } catch (_e) {
23
+ return null;
24
+ }
25
+ }
26
+
27
+ function isTokenStale(lastRefreshIso, nowMs = Date.now(), accessToken = null) {
28
+ const expiresAtMs = parseJwtExpirationMs(accessToken);
29
+ if (expiresAtMs !== null) {
30
+ return expiresAtMs <= nowMs + ACCESS_TOKEN_REFRESH_WINDOW_MS;
31
+ }
32
+ if (!lastRefreshIso) return true;
33
+ const ts = Date.parse(lastRefreshIso);
34
+ if (!Number.isFinite(ts)) return true;
35
+ return nowMs - ts > REFRESH_THRESHOLD_MS;
36
+ }
37
+
38
+ async function refreshCodexTokens({ refreshToken, fetchImpl = fetch }) {
39
+ if (typeof refreshToken !== "string" || refreshToken.length === 0) {
40
+ const err = new Error("Codex refresh skipped: no refresh_token in auth.json");
41
+ err.code = "NO_REFRESH_TOKEN";
42
+ throw err;
43
+ }
44
+
45
+ const res = await fetchImpl(REFRESH_ENDPOINT, {
46
+ method: "POST",
47
+ headers: {
48
+ "Content-Type": "application/json",
49
+ Accept: "application/json",
50
+ },
51
+ body: JSON.stringify({
52
+ client_id: CODEX_CLIENT_ID,
53
+ grant_type: "refresh_token",
54
+ refresh_token: refreshToken,
55
+ }),
56
+ });
57
+
58
+ if (res.status === 401) {
59
+ let openaiErrorCode = null;
60
+ try {
61
+ const body = await res.json();
62
+ openaiErrorCode =
63
+ (body && typeof body.error === "object" && body.error.code) ||
64
+ (typeof body?.error === "string" ? body.error : null) ||
65
+ body?.code ||
66
+ null;
67
+ } catch (_e) {
68
+ // Ignore parse failure — surface the generic reason.
69
+ }
70
+ const reason = String(openaiErrorCode || "").toLowerCase();
71
+ let message = "Codex refresh token was rejected. Run `codex login` to re-authenticate.";
72
+ if (reason === "refresh_token_expired") {
73
+ message = "Codex refresh token expired. Run `codex login` to re-authenticate.";
74
+ } else if (reason === "refresh_token_reused") {
75
+ message =
76
+ "Codex refresh token was already used, likely by another Codex process. " +
77
+ "Run `codex login` to re-authenticate.";
78
+ } else if (reason === "refresh_token_invalidated") {
79
+ message = "Codex refresh token was revoked. Run `codex login` to re-authenticate.";
80
+ }
81
+ const err = new Error(message);
82
+ err.code = "REFRESH_TOKEN_EXPIRED";
83
+ err.openaiErrorCode = openaiErrorCode;
84
+ throw err;
85
+ }
86
+
87
+ if (!res.ok) {
88
+ const err = new Error(`Codex token refresh failed: ${res.status}`);
89
+ err.code = "REFRESH_HTTP_ERROR";
90
+ err.status = res.status;
91
+ throw err;
92
+ }
93
+
94
+ const body = await res.json();
95
+ if (!body || typeof body.access_token !== "string" || body.access_token.length === 0) {
96
+ const err = new Error("Codex token refresh response missing access_token");
97
+ err.code = "REFRESH_INVALID_RESPONSE";
98
+ throw err;
99
+ }
100
+
101
+ return {
102
+ access_token: body.access_token,
103
+ refresh_token:
104
+ typeof body.refresh_token === "string" && body.refresh_token.length > 0
105
+ ? body.refresh_token
106
+ : refreshToken,
107
+ id_token: typeof body.id_token === "string" ? body.id_token : null,
108
+ };
109
+ }
110
+
111
+ // Atomic write so a process kill mid-write doesn't corrupt auth.json (which would force the
112
+ // user to re-run `codex` login).
113
+ async function persistRefreshedAuth(authPath, currentAuth, newTokens) {
114
+ const merged = {
115
+ ...currentAuth,
116
+ tokens: {
117
+ ...(currentAuth.tokens || {}),
118
+ access_token: newTokens.access_token,
119
+ refresh_token: newTokens.refresh_token,
120
+ id_token: newTokens.id_token || currentAuth?.tokens?.id_token || null,
121
+ },
122
+ last_refresh: new Date().toISOString(),
123
+ };
124
+
125
+ const tmp = `${authPath}.tmp.${process.pid}.${Date.now()}`;
126
+ await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2), { mode: 0o600 });
127
+ await fs.promises.rename(tmp, authPath);
128
+ return merged;
129
+ }
130
+
131
+ module.exports = {
132
+ REFRESH_ENDPOINT,
133
+ CODEX_CLIENT_ID,
134
+ REFRESH_THRESHOLD_MS,
135
+ ACCESS_TOKEN_REFRESH_WINDOW_MS,
136
+ parseJwtExpirationMs,
137
+ isTokenStale,
138
+ refreshCodexTokens,
139
+ persistRefreshedAuth,
140
+ };
@@ -0,0 +1,195 @@
1
+ "use strict";
2
+
3
+ // A rollout can contain TokenCount events from more than one Codex SessionState
4
+ // (for example, a parent agent and an embedded reviewer). Each SessionState has
5
+ // its own cumulative total, but the event does not carry a stable stream id.
6
+ // Recover the stream from Codex's invariant: total - last = that stream's prior
7
+ // total. Keeping a small LRU of stream heads also makes repeated rate-limit
8
+ // snapshots idempotent when the streams are interleaved.
9
+
10
+ const USAGE_FIELDS = [
11
+ "input_tokens",
12
+ "cached_input_tokens",
13
+ "cache_creation_input_tokens",
14
+ "output_tokens",
15
+ "reasoning_output_tokens",
16
+ "total_tokens",
17
+ ];
18
+ const MAX_USAGE_BASELINES = 32;
19
+
20
+ function canonicalUsage(value) {
21
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
22
+ const out = {};
23
+ let sawNumber = false;
24
+ for (const field of USAGE_FIELDS) {
25
+ const raw = field === "cache_creation_input_tokens"
26
+ ? value.cache_creation_input_tokens ?? value.cache_write_input_tokens
27
+ : value[field];
28
+ const number = Number(raw);
29
+ if (Number.isFinite(number) && number >= 0) {
30
+ out[field] = Math.floor(number);
31
+ sawNumber = true;
32
+ } else {
33
+ out[field] = 0;
34
+ }
35
+ }
36
+ return sawNumber ? out : null;
37
+ }
38
+
39
+ function usageSignature(value) {
40
+ const usage = canonicalUsage(value);
41
+ return usage ? USAGE_FIELDS.map((field) => usage[field]).join(":") : null;
42
+ }
43
+
44
+ function sameUsage(left, right) {
45
+ const leftSignature = usageSignature(left);
46
+ return leftSignature !== null && leftSignature === usageSignature(right);
47
+ }
48
+
49
+ function subtractUsage(totalUsage, lastUsage) {
50
+ const total = canonicalUsage(totalUsage);
51
+ const last = canonicalUsage(lastUsage);
52
+ if (!total || !last) return null;
53
+ const previous = {};
54
+ for (const field of USAGE_FIELDS) {
55
+ if (total[field] < last[field]) return null;
56
+ previous[field] = total[field] - last[field];
57
+ }
58
+ return previous;
59
+ }
60
+
61
+ function diffUsage(totalUsage, previousUsage) {
62
+ const total = canonicalUsage(totalUsage);
63
+ const previous = canonicalUsage(previousUsage);
64
+ if (!total || !previous) return null;
65
+ const delta = {};
66
+ for (const field of USAGE_FIELDS) {
67
+ delta[field] = Math.max(0, total[field] - previous[field]);
68
+ }
69
+ return delta;
70
+ }
71
+
72
+ function totalsReset(totalUsage, previousUsage) {
73
+ const total = canonicalUsage(totalUsage);
74
+ const previous = canonicalUsage(previousUsage);
75
+ return Boolean(total && previous && total.total_tokens < previous.total_tokens);
76
+ }
77
+
78
+ function createUsageDeltaState({ lastTotal = null, baselines = null } = {}) {
79
+ const state = {
80
+ lastTotal: null,
81
+ baselines: [],
82
+ sawDivergentCumulative: false,
83
+ sawInterleaved: false,
84
+ };
85
+ for (const baseline of Array.isArray(baselines) ? baselines : []) {
86
+ touchBaseline(state, baseline);
87
+ }
88
+ if (canonicalUsage(lastTotal)) touchBaseline(state, lastTotal);
89
+ return state;
90
+ }
91
+
92
+ function snapshotUsageBaselines(state) {
93
+ return Array.isArray(state?.baselines)
94
+ ? state.baselines.map((usage) => ({ ...usage }))
95
+ : [];
96
+ }
97
+
98
+ function consumeUsageDelta(state, lastUsage, totalUsage) {
99
+ if (!state || typeof state !== "object") {
100
+ throw new TypeError("usage delta state is required");
101
+ }
102
+ if (!Array.isArray(state.baselines)) state.baselines = [];
103
+
104
+ const last = canonicalUsage(lastUsage);
105
+ const total = canonicalUsage(totalUsage);
106
+ if (!total) return last;
107
+
108
+ const activeSignature = usageSignature(state.lastTotal);
109
+ const duplicateIndex = findBaselineIndex(state, total);
110
+ if (duplicateIndex >= 0) {
111
+ if (
112
+ activeSignature !== null &&
113
+ usageSignature(state.baselines[duplicateIndex]) !== activeSignature
114
+ ) {
115
+ state.sawInterleaved = true;
116
+ }
117
+ touchBaselineAt(state, duplicateIndex, total);
118
+ return null;
119
+ }
120
+
121
+ if (last) {
122
+ const expectedPrevious = subtractUsage(total, last);
123
+ const lineageIndex = findBaselineIndex(state, expectedPrevious);
124
+ if (lineageIndex >= 0) {
125
+ if (
126
+ activeSignature !== null &&
127
+ usageSignature(state.baselines[lineageIndex]) !== activeSignature
128
+ ) {
129
+ state.sawInterleaved = true;
130
+ }
131
+ touchBaselineAt(state, lineageIndex, total);
132
+ return last;
133
+ }
134
+ if (expectedPrevious) {
135
+ if (canonicalUsage(state.lastTotal)) state.sawDivergentCumulative = true;
136
+ touchBaseline(state, total);
137
+ return last;
138
+ }
139
+ }
140
+
141
+ const active = canonicalUsage(state.lastTotal);
142
+ if (active && !totalsReset(total, active)) {
143
+ const delta = diffUsage(total, active);
144
+ if (!last || Number(delta?.total_tokens || 0) <= Number(last.total_tokens || 0)) {
145
+ const activeIndex = findBaselineIndex(state, active);
146
+ touchBaselineAt(state, activeIndex, total);
147
+ return delta;
148
+ }
149
+
150
+ // A cumulative jump larger than last usage cannot come from the normal
151
+ // append_last_usage path. Treat it as a new stream instead of charging the
152
+ // unrelated cumulative gap; a later event will establish its lineage.
153
+ state.sawDivergentCumulative = true;
154
+ }
155
+
156
+ touchBaseline(state, total);
157
+ return last || total;
158
+ }
159
+
160
+ function findBaselineIndex(state, usage) {
161
+ const signature = usageSignature(usage);
162
+ if (signature === null) return -1;
163
+ return state.baselines.findIndex((baseline) => usageSignature(baseline) === signature);
164
+ }
165
+
166
+ function touchBaselineAt(state, index, usage) {
167
+ const canonical = canonicalUsage(usage);
168
+ if (!canonical) return;
169
+ if (Number.isInteger(index) && index >= 0 && index < state.baselines.length) {
170
+ state.baselines.splice(index, 1);
171
+ } else {
172
+ const duplicateIndex = findBaselineIndex(state, canonical);
173
+ if (duplicateIndex >= 0) state.baselines.splice(duplicateIndex, 1);
174
+ }
175
+ state.baselines.push(canonical);
176
+ while (state.baselines.length > MAX_USAGE_BASELINES) state.baselines.shift();
177
+ state.lastTotal = canonical;
178
+ }
179
+
180
+ function touchBaseline(state, usage) {
181
+ touchBaselineAt(state, findBaselineIndex(state, usage), usage);
182
+ }
183
+
184
+ module.exports = {
185
+ MAX_USAGE_BASELINES,
186
+ canonicalUsage,
187
+ consumeUsageDelta,
188
+ createUsageDeltaState,
189
+ diffUsage,
190
+ sameUsage,
191
+ snapshotUsageBaselines,
192
+ subtractUsage,
193
+ totalsReset,
194
+ usageSignature,
195
+ };
@@ -0,0 +1,195 @@
1
+ "use strict";
2
+
3
+ // Codex rollouts written since CLI 0.151 carry a top-level
4
+ // {"type":"token_usage_record","payload":{usage,turn_token_usage,thread_token_usage,...}}
5
+ // next to the event_msg token_count that has always carried usage (issue
6
+ // #652). While both are written they describe the same model response, so
7
+ // token_count stays the only source of normal usage and a record is counted
8
+ // in exactly one case:
9
+ //
10
+ // Compaction. Codex bills the compaction model call in a record that is
11
+ // immediately followed by a top-level `compacted` line, and the next
12
+ // token_count repeats the previous cumulative total (its last_token_usage is
13
+ // the post-compaction context size, not a response). The cumulative delta is
14
+ // zero, so without the record the call is lost. Inside a file that has
15
+ // already shown a token_count, such a record is held as `compaction` and
16
+ // counted when the next token_count arrives with a total identical to the
17
+ // previous one. If that total advanced, the token_count already covers the
18
+ // call and the record is dropped. Each compaction is counted once per
19
+ // payload.response_id: a fork replays its parent's records and a copied
20
+ // rollout repeats them, both with the same id. A record without one is never
21
+ // counted.
22
+ //
23
+ // A file whose records are never matched by a token_count is not counted from
24
+ // its records. It is flagged `recordOnly` instead, and sync surfaces a
25
+ // warning. The proof is the same ordering Codex keeps for every response: its
26
+ // token_count is written after its record and before the next record or the
27
+ // turn end. So a record followed by another record or a turn end with no
28
+ // token_count in between flags the file; a sync that stops between a record
29
+ // and its token_count does not.
30
+ //
31
+ // payload.usage is the per-response delta; turn_token_usage and
32
+ // thread_token_usage are cumulative and are never read. All state below is
33
+ // persisted in the sync cursor and the session parser resume state, so the
34
+ // record, the `compacted` line and the token_count may land in different
35
+ // reads.
36
+
37
+ const { canonicalUsage, sameUsage } = require("./codex-token-usage");
38
+
39
+ function extractTokenUsageRecord(obj) {
40
+ if (obj?.type !== "token_usage_record") return null;
41
+ const usage = obj.payload?.usage;
42
+ if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
43
+ const timestamp = typeof obj.timestamp === "string" ? obj.timestamp : null;
44
+ if (!timestamp) return null;
45
+ // canonicalUsage folds cache_write_input_tokens into
46
+ // cache_creation_input_tokens exactly as the token_count path does.
47
+ const canonical = canonicalUsage(usage);
48
+ if (!canonical) return null;
49
+ const responseId = typeof obj.payload.response_id === "string" && obj.payload.response_id
50
+ ? obj.payload.response_id
51
+ : null;
52
+ return { usage: canonical, timestamp, responseId };
53
+ }
54
+
55
+ // Compaction calls already counted by sync, by response_id, kept in core
56
+ // cursor state as cursors.codexCompactionResponseIds (oldest first). The id
57
+ // is unique per API response and survives fork replays and file copies, so
58
+ // it is the only dedup key a compaction needs. Compactions are rare (19 on
59
+ // the maintainer's corpus), so the cap is far above any real history.
60
+ const MAX_COMPACTION_RESPONSE_IDS = 20000;
61
+
62
+ function createCompactionResponseIds(cursors) {
63
+ const stored = Array.isArray(cursors.codexCompactionResponseIds)
64
+ ? cursors.codexCompactionResponseIds
65
+ : [];
66
+ const ids = new Set(stored);
67
+ const added = [];
68
+ return {
69
+ has: (id) => ids.has(id),
70
+ add(id) {
71
+ if (ids.has(id)) return;
72
+ ids.add(id);
73
+ added.push(id);
74
+ },
75
+ persist() {
76
+ if (added.length === 0) return;
77
+ cursors.codexCompactionResponseIds = [...stored, ...added].slice(-MAX_COMPACTION_RESPONSE_IDS);
78
+ },
79
+ };
80
+ }
81
+
82
+ function isCodexTurnEndEvent(obj) {
83
+ if (obj?.type !== "event_msg") return false;
84
+ const type = obj.payload?.type;
85
+ return type === "task_complete" || type === "turn_aborted";
86
+ }
87
+
88
+ function isPlainObject(value) {
89
+ return Boolean(value) && typeof value === "object" && !Array.isArray(value);
90
+ }
91
+
92
+ function createUsageRecordState(snapshot) {
93
+ const compaction = snapshot?.compaction;
94
+ return {
95
+ sawTokenCount: Boolean(snapshot?.sawTokenCount),
96
+ unmatchedRecord: Boolean(snapshot?.unmatchedRecord),
97
+ recordOnly: Boolean(snapshot?.recordOnly),
98
+ compaction: isPlainObject(compaction) && isPlainObject(compaction.candidate)
99
+ ? { candidate: compaction.candidate, compacted: Boolean(compaction.compacted) }
100
+ : null,
101
+ };
102
+ }
103
+
104
+ // Called for every token_count carrying an info object.
105
+ function noteTokenCount(state) {
106
+ state.sawTokenCount = true;
107
+ state.unmatchedRecord = false;
108
+ state.recordOnly = false;
109
+ }
110
+
111
+ // Called for every record. `candidate` is what the caller would count if the
112
+ // record turns out to be a compaction call (null when it cannot be counted).
113
+ function noteUsageRecord(state, candidate) {
114
+ if (state.sawTokenCount) {
115
+ state.compaction = candidate ? { candidate, compacted: false } : null;
116
+ return;
117
+ }
118
+ if (state.unmatchedRecord) state.recordOnly = true;
119
+ state.unmatchedRecord = true;
120
+ }
121
+
122
+ // Turn end: a record still waiting for its token_count will not get one.
123
+ function noteTurnEnd(state) {
124
+ if (state.sawTokenCount) return;
125
+ if (state.unmatchedRecord) state.recordOnly = true;
126
+ state.unmatchedRecord = false;
127
+ }
128
+
129
+ // True while the line right after a record decides whether it was a
130
+ // compaction call.
131
+ function awaitsCompactedLine(state) {
132
+ return Boolean(state?.compaction && !state.compaction.compacted);
133
+ }
134
+
135
+ // Called for each complete line after a record, other than a record. Only a
136
+ // `compacted` line directly after the record keeps it as a candidate.
137
+ function noteLineAfterUsageRecord(state, isCompactedLine) {
138
+ if (!awaitsCompactedLine(state)) return;
139
+ if (isCompactedLine) state.compaction.compacted = true;
140
+ else state.compaction = null;
141
+ }
142
+
143
+ // Called for every token_count carrying an info object, with the cumulative
144
+ // total that was active before it. Returns the compaction candidate to count,
145
+ // or null.
146
+ function takeCompactionOnTokenCount(state, previousTotal, totalUsage) {
147
+ const compaction = state.compaction;
148
+ state.compaction = null;
149
+ if (!compaction?.compacted) return null;
150
+ if (!canonicalUsage(previousTotal) || !sameUsage(previousTotal, totalUsage)) return null;
151
+ return compaction.candidate;
152
+ }
153
+
154
+ function isRecordOnly(state) {
155
+ return Boolean(state?.recordOnly && !state.sawTokenCount);
156
+ }
157
+
158
+ function snapshotUsageRecordState(state) {
159
+ if (!state || (!state.sawTokenCount && !state.unmatchedRecord && !state.recordOnly)) return null;
160
+ const out = { sawTokenCount: state.sawTokenCount };
161
+ if (state.unmatchedRecord) out.unmatchedRecord = true;
162
+ if (state.recordOnly) out.recordOnly = true;
163
+ if (state.compaction) out.compaction = state.compaction;
164
+ return out;
165
+ }
166
+
167
+ // Sync keeps the flagged rollouts in core cursor state:
168
+ // cursors.codexUsageRecordOnlyFiles = { [rolloutPath]: true }.
169
+ function countRecordOnlyFiles(cursors) {
170
+ const flagged = cursors?.codexUsageRecordOnlyFiles;
171
+ return isPlainObject(flagged) ? Object.keys(flagged).length : 0;
172
+ }
173
+
174
+ function formatRecordOnlyWarning(count) {
175
+ if (!count) return null;
176
+ return `${count} Codex session(s) report usage only via token_usage_record; not counted yet (see #652)`;
177
+ }
178
+
179
+ module.exports = {
180
+ MAX_COMPACTION_RESPONSE_IDS,
181
+ createCompactionResponseIds,
182
+ countRecordOnlyFiles,
183
+ formatRecordOnlyWarning,
184
+ awaitsCompactedLine,
185
+ createUsageRecordState,
186
+ extractTokenUsageRecord,
187
+ isCodexTurnEndEvent,
188
+ isRecordOnly,
189
+ noteLineAfterUsageRecord,
190
+ noteTokenCount,
191
+ noteTurnEnd,
192
+ noteUsageRecord,
193
+ snapshotUsageRecordState,
194
+ takeCompactionOnTokenCount,
195
+ };