tokenusage-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +22 -0
- package/README.md +94 -0
- package/THIRD_PARTY_NOTICES.md +140 -0
- package/bin/tokenusage.js +69 -0
- package/package.json +59 -0
- package/src/cli.js +134 -0
- package/src/commands/device-login.js +205 -0
- package/src/commands/diagnostics.js +38 -0
- package/src/commands/doctor.js +110 -0
- package/src/commands/init.js +2436 -0
- package/src/commands/sessions.js +29 -0
- package/src/commands/status.js +1747 -0
- package/src/commands/sync.js +7934 -0
- package/src/commands/uninstall.js +312 -0
- package/src/lib/account-usage-dedup.js +267 -0
- package/src/lib/antigravity-paths.js +48 -0
- package/src/lib/ark-agent-plan-limits.js +438 -0
- package/src/lib/ark-coding-plan-limits.js +537 -0
- package/src/lib/async-queue-reader.js +138 -0
- package/src/lib/atomic-json.js +27 -0
- package/src/lib/bearer-token.js +11 -0
- package/src/lib/categorizer-utils.js +232 -0
- package/src/lib/claude-categorizer.js +1343 -0
- package/src/lib/claude-config.js +279 -0
- package/src/lib/claude-user-identity.js +56 -0
- package/src/lib/cli-ui.js +179 -0
- package/src/lib/cloud-account.js +541 -0
- package/src/lib/cloud-device-token.js +141 -0
- package/src/lib/cloud-sync-prefs.js +14 -0
- package/src/lib/cloud-usage-reset.js +115 -0
- package/src/lib/codex-config.js +595 -0
- package/src/lib/codex-context-breakdown.js +806 -0
- package/src/lib/codex-model-attribution.js +128 -0
- package/src/lib/codex-rollout-parser.js +1471 -0
- package/src/lib/codex-service-tier.js +49 -0
- package/src/lib/codex-token-refresh.js +140 -0
- package/src/lib/codex-token-usage.js +195 -0
- package/src/lib/codex-usage-record.js +195 -0
- package/src/lib/command-runner.js +313 -0
- package/src/lib/commandcode-limits.js +317 -0
- package/src/lib/context-health.js +124 -0
- package/src/lib/cursor-config.js +605 -0
- package/src/lib/cursor-store.js +1918 -0
- package/src/lib/debug-flags.js +7 -0
- package/src/lib/devin-limits.js +249 -0
- package/src/lib/devin-usage.js +212 -0
- package/src/lib/diagnostics.js +383 -0
- package/src/lib/doctor.js +385 -0
- package/src/lib/dumate.js +95 -0
- package/src/lib/fs.js +455 -0
- package/src/lib/function-url.js +14 -0
- package/src/lib/gemini-config.js +291 -0
- package/src/lib/git-outcomes.js +243 -0
- package/src/lib/grok-context-breakdown.js +890 -0
- package/src/lib/grok-hook.js +360 -0
- package/src/lib/grok-limits.js +865 -0
- package/src/lib/grok-usage.js +98 -0
- package/src/lib/heatmap-compact.js +114 -0
- package/src/lib/ingest-sanitize.js +41 -0
- package/src/lib/init-flow.js +42 -0
- package/src/lib/install-resolver.js +170 -0
- package/src/lib/jsonl-lines.js +134 -0
- package/src/lib/kimi-model-repair.js +173 -0
- package/src/lib/kimi-profile.js +61 -0
- package/src/lib/kimi-session.js +216 -0
- package/src/lib/local-api.js +3588 -0
- package/src/lib/machine-id.js +238 -0
- package/src/lib/multi-install-parser.js +113 -0
- package/src/lib/notify-command.js +38 -0
- package/src/lib/omp-hook.js +589 -0
- package/src/lib/openclaw-cli.js +104 -0
- package/src/lib/openclaw-hook.js +375 -0
- package/src/lib/openclaw-session-plugin.js +560 -0
- package/src/lib/opencode-config.js +121 -0
- package/src/lib/opencode-go-limits.js +807 -0
- package/src/lib/outcomes-engine.js +253 -0
- package/src/lib/passive-mode.js +219 -0
- package/src/lib/pet-packages.js +722 -0
- package/src/lib/pricing/curated-overrides.json +822 -0
- package/src/lib/pricing/index.js +379 -0
- package/src/lib/pricing/litellm-fetcher.js +172 -0
- package/src/lib/pricing/matcher.js +456 -0
- package/src/lib/pricing/seed-snapshot.json +1 -0
- package/src/lib/progress.js +76 -0
- package/src/lib/project-usage-purge.js +117 -0
- package/src/lib/prompt.js +24 -0
- package/src/lib/provider-status.js +104 -0
- package/src/lib/proxy-env.js +489 -0
- package/src/lib/proxy-settings.js +163 -0
- package/src/lib/qoder-cn-cli-limits.js +119 -0
- package/src/lib/qoder-limits.js +880 -0
- package/src/lib/quiet-warnings.js +26 -0
- package/src/lib/rollout.js +25966 -0
- package/src/lib/runtime-config.js +259 -0
- package/src/lib/scan-roots.js +277 -0
- package/src/lib/sentry.js +231 -0
- package/src/lib/session-analytics.js +2972 -0
- package/src/lib/session-performance.js +237 -0
- package/src/lib/session-upload.js +379 -0
- package/src/lib/skill-usage.js +267 -0
- package/src/lib/skills-manager.js +1591 -0
- package/src/lib/source-metadata.js +48 -0
- package/src/lib/sqlite-reader.js +215 -0
- package/src/lib/star-cta.js +73 -0
- package/src/lib/subscription-manager.js +293 -0
- package/src/lib/subscriptions.js +508 -0
- package/src/lib/telemetry.js +193 -0
- package/src/lib/tracker-paths.js +16 -0
- package/src/lib/trae-cn-config.js +548 -0
- package/src/lib/trae-db.js +127 -0
- package/src/lib/trae-snapshot.js +230 -0
- package/src/lib/trae-sqlite.js +143 -0
- package/src/lib/trae-usage.js +121 -0
- package/src/lib/upload-queue.js +134 -0
- package/src/lib/upload-throttle.js +171 -0
- package/src/lib/usage-limits.js +4552 -0
- package/src/lib/wrapped-aggregator.js +236 -0
- package/src/lib/wsl-probe.js +281 -0
- package/src/lib/zcode-limits.js +1172 -0
- package/src/lib/zcode-reset-credits.js +66 -0
- package/test/cli-smoke.test.js +53 -0
- package/test/ingest-sanitize.test.js +162 -0
- package/test/quiet-warnings.test.js +53 -0
- package/test/session-upload.test.js +427 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Codex service-tier attribution.
|
|
2
|
+
//
|
|
3
|
+
// The request tier is NOT on the token_count rows. Codex emits it as its own
|
|
4
|
+
// rollout record — `type: "event_msg"`, `payload.type:
|
|
5
|
+
// "thread_settings_applied"`, `payload.thread_settings.service_tier` — written
|
|
6
|
+
// immediately before the `turn_context` of the turn it applies to. A session's
|
|
7
|
+
// first turn has no such record, and CLI-originated sessions
|
|
8
|
+
// (originator codex_cli_rs / codex-tui) often have none at all: on a 1213-file
|
|
9
|
+
// local corpus only 167 files carried the event, covering 25% of token_count
|
|
10
|
+
// events.
|
|
11
|
+
//
|
|
12
|
+
// Attribution rule, and the only one the log shape permits: a token_count row
|
|
13
|
+
// belongs to the most recent preceding thread_settings_applied. With no
|
|
14
|
+
// preceding record the tier is unknown, and unknown is never guessed — it is
|
|
15
|
+
// billed at Standard, exactly as before this file existed.
|
|
16
|
+
|
|
17
|
+
"use strict";
|
|
18
|
+
|
|
19
|
+
// Cheap line prefilter for parsers that avoid JSON.parse on most records.
|
|
20
|
+
const CODEX_SERVICE_TIER_MARKER = '"thread_settings_applied"';
|
|
21
|
+
|
|
22
|
+
// Observed values on the local corpus are "default" (814) and "priority" (58).
|
|
23
|
+
// "priority" is what the CLI writes for what OpenAI's price table and the Codex
|
|
24
|
+
// UI both call Fast; accept the product-facing spelling too so a future rename
|
|
25
|
+
// does not silently fall back to Standard.
|
|
26
|
+
const PRIORITY_SERVICE_TIERS = new Set(["priority", "fast"]);
|
|
27
|
+
|
|
28
|
+
function readCodexServiceTier(obj) {
|
|
29
|
+
if (!obj || obj.type !== "event_msg") return null;
|
|
30
|
+
const payload = obj.payload;
|
|
31
|
+
if (!payload || typeof payload !== "object") return null;
|
|
32
|
+
if (payload.type !== "thread_settings_applied") return null;
|
|
33
|
+
const settings = payload.thread_settings;
|
|
34
|
+
if (!settings || typeof settings !== "object") return null;
|
|
35
|
+
const tier = settings.service_tier;
|
|
36
|
+
if (typeof tier !== "string" || !tier.trim()) return null;
|
|
37
|
+
return tier.trim().toLowerCase();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function isPriorityServiceTier(tier) {
|
|
41
|
+
return typeof tier === "string" && PRIORITY_SERVICE_TIERS.has(tier);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
module.exports = {
|
|
45
|
+
CODEX_SERVICE_TIER_MARKER,
|
|
46
|
+
PRIORITY_SERVICE_TIERS,
|
|
47
|
+
readCodexServiceTier,
|
|
48
|
+
isPriorityServiceTier,
|
|
49
|
+
};
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
const fs = require("node:fs");
|
|
2
|
+
const path = require("node:path");
|
|
3
|
+
|
|
4
|
+
// Public Codex OAuth client. Same id used by the official `codex` CLI. Aligned with
|
|
5
|
+
// steipete/CodexBar's CodexTokenRefresher.swift — neither is sensitive (it's a public client).
|
|
6
|
+
const REFRESH_ENDPOINT = "https://auth.openai.com/oauth/token";
|
|
7
|
+
const CODEX_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann";
|
|
8
|
+
|
|
9
|
+
// Match the official Codex CLI: when the access token exposes a JWT expiry, refresh only
|
|
10
|
+
// shortly before it expires. Fall back to last_refresh > 8 days for opaque/legacy tokens.
|
|
11
|
+
const REFRESH_THRESHOLD_MS = 8 * 24 * 60 * 60 * 1000;
|
|
12
|
+
const ACCESS_TOKEN_REFRESH_WINDOW_MS = 5 * 60 * 1000;
|
|
13
|
+
|
|
14
|
+
function parseJwtExpirationMs(token) {
|
|
15
|
+
if (typeof token !== "string" || token.length === 0) return null;
|
|
16
|
+
const parts = token.split(".");
|
|
17
|
+
if (parts.length < 2 || !parts[1]) return null;
|
|
18
|
+
try {
|
|
19
|
+
const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
|
|
20
|
+
const exp = Number(payload?.exp);
|
|
21
|
+
return Number.isFinite(exp) ? exp * 1000 : null;
|
|
22
|
+
} catch (_e) {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function isTokenStale(lastRefreshIso, nowMs = Date.now(), accessToken = null) {
|
|
28
|
+
const expiresAtMs = parseJwtExpirationMs(accessToken);
|
|
29
|
+
if (expiresAtMs !== null) {
|
|
30
|
+
return expiresAtMs <= nowMs + ACCESS_TOKEN_REFRESH_WINDOW_MS;
|
|
31
|
+
}
|
|
32
|
+
if (!lastRefreshIso) return true;
|
|
33
|
+
const ts = Date.parse(lastRefreshIso);
|
|
34
|
+
if (!Number.isFinite(ts)) return true;
|
|
35
|
+
return nowMs - ts > REFRESH_THRESHOLD_MS;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function refreshCodexTokens({ refreshToken, fetchImpl = fetch }) {
|
|
39
|
+
if (typeof refreshToken !== "string" || refreshToken.length === 0) {
|
|
40
|
+
const err = new Error("Codex refresh skipped: no refresh_token in auth.json");
|
|
41
|
+
err.code = "NO_REFRESH_TOKEN";
|
|
42
|
+
throw err;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const res = await fetchImpl(REFRESH_ENDPOINT, {
|
|
46
|
+
method: "POST",
|
|
47
|
+
headers: {
|
|
48
|
+
"Content-Type": "application/json",
|
|
49
|
+
Accept: "application/json",
|
|
50
|
+
},
|
|
51
|
+
body: JSON.stringify({
|
|
52
|
+
client_id: CODEX_CLIENT_ID,
|
|
53
|
+
grant_type: "refresh_token",
|
|
54
|
+
refresh_token: refreshToken,
|
|
55
|
+
}),
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
if (res.status === 401) {
|
|
59
|
+
let openaiErrorCode = null;
|
|
60
|
+
try {
|
|
61
|
+
const body = await res.json();
|
|
62
|
+
openaiErrorCode =
|
|
63
|
+
(body && typeof body.error === "object" && body.error.code) ||
|
|
64
|
+
(typeof body?.error === "string" ? body.error : null) ||
|
|
65
|
+
body?.code ||
|
|
66
|
+
null;
|
|
67
|
+
} catch (_e) {
|
|
68
|
+
// Ignore parse failure — surface the generic reason.
|
|
69
|
+
}
|
|
70
|
+
const reason = String(openaiErrorCode || "").toLowerCase();
|
|
71
|
+
let message = "Codex refresh token was rejected. Run `codex login` to re-authenticate.";
|
|
72
|
+
if (reason === "refresh_token_expired") {
|
|
73
|
+
message = "Codex refresh token expired. Run `codex login` to re-authenticate.";
|
|
74
|
+
} else if (reason === "refresh_token_reused") {
|
|
75
|
+
message =
|
|
76
|
+
"Codex refresh token was already used, likely by another Codex process. " +
|
|
77
|
+
"Run `codex login` to re-authenticate.";
|
|
78
|
+
} else if (reason === "refresh_token_invalidated") {
|
|
79
|
+
message = "Codex refresh token was revoked. Run `codex login` to re-authenticate.";
|
|
80
|
+
}
|
|
81
|
+
const err = new Error(message);
|
|
82
|
+
err.code = "REFRESH_TOKEN_EXPIRED";
|
|
83
|
+
err.openaiErrorCode = openaiErrorCode;
|
|
84
|
+
throw err;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (!res.ok) {
|
|
88
|
+
const err = new Error(`Codex token refresh failed: ${res.status}`);
|
|
89
|
+
err.code = "REFRESH_HTTP_ERROR";
|
|
90
|
+
err.status = res.status;
|
|
91
|
+
throw err;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const body = await res.json();
|
|
95
|
+
if (!body || typeof body.access_token !== "string" || body.access_token.length === 0) {
|
|
96
|
+
const err = new Error("Codex token refresh response missing access_token");
|
|
97
|
+
err.code = "REFRESH_INVALID_RESPONSE";
|
|
98
|
+
throw err;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
access_token: body.access_token,
|
|
103
|
+
refresh_token:
|
|
104
|
+
typeof body.refresh_token === "string" && body.refresh_token.length > 0
|
|
105
|
+
? body.refresh_token
|
|
106
|
+
: refreshToken,
|
|
107
|
+
id_token: typeof body.id_token === "string" ? body.id_token : null,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Atomic write so a process kill mid-write doesn't corrupt auth.json (which would force the
|
|
112
|
+
// user to re-run `codex` login).
|
|
113
|
+
async function persistRefreshedAuth(authPath, currentAuth, newTokens) {
|
|
114
|
+
const merged = {
|
|
115
|
+
...currentAuth,
|
|
116
|
+
tokens: {
|
|
117
|
+
...(currentAuth.tokens || {}),
|
|
118
|
+
access_token: newTokens.access_token,
|
|
119
|
+
refresh_token: newTokens.refresh_token,
|
|
120
|
+
id_token: newTokens.id_token || currentAuth?.tokens?.id_token || null,
|
|
121
|
+
},
|
|
122
|
+
last_refresh: new Date().toISOString(),
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
const tmp = `${authPath}.tmp.${process.pid}.${Date.now()}`;
|
|
126
|
+
await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2), { mode: 0o600 });
|
|
127
|
+
await fs.promises.rename(tmp, authPath);
|
|
128
|
+
return merged;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
module.exports = {
|
|
132
|
+
REFRESH_ENDPOINT,
|
|
133
|
+
CODEX_CLIENT_ID,
|
|
134
|
+
REFRESH_THRESHOLD_MS,
|
|
135
|
+
ACCESS_TOKEN_REFRESH_WINDOW_MS,
|
|
136
|
+
parseJwtExpirationMs,
|
|
137
|
+
isTokenStale,
|
|
138
|
+
refreshCodexTokens,
|
|
139
|
+
persistRefreshedAuth,
|
|
140
|
+
};
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// A rollout can contain TokenCount events from more than one Codex SessionState
|
|
4
|
+
// (for example, a parent agent and an embedded reviewer). Each SessionState has
|
|
5
|
+
// its own cumulative total, but the event does not carry a stable stream id.
|
|
6
|
+
// Recover the stream from Codex's invariant: total - last = that stream's prior
|
|
7
|
+
// total. Keeping a small LRU of stream heads also makes repeated rate-limit
|
|
8
|
+
// snapshots idempotent when the streams are interleaved.
|
|
9
|
+
|
|
10
|
+
const USAGE_FIELDS = [
|
|
11
|
+
"input_tokens",
|
|
12
|
+
"cached_input_tokens",
|
|
13
|
+
"cache_creation_input_tokens",
|
|
14
|
+
"output_tokens",
|
|
15
|
+
"reasoning_output_tokens",
|
|
16
|
+
"total_tokens",
|
|
17
|
+
];
|
|
18
|
+
const MAX_USAGE_BASELINES = 32;
|
|
19
|
+
|
|
20
|
+
function canonicalUsage(value) {
|
|
21
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
22
|
+
const out = {};
|
|
23
|
+
let sawNumber = false;
|
|
24
|
+
for (const field of USAGE_FIELDS) {
|
|
25
|
+
const raw = field === "cache_creation_input_tokens"
|
|
26
|
+
? value.cache_creation_input_tokens ?? value.cache_write_input_tokens
|
|
27
|
+
: value[field];
|
|
28
|
+
const number = Number(raw);
|
|
29
|
+
if (Number.isFinite(number) && number >= 0) {
|
|
30
|
+
out[field] = Math.floor(number);
|
|
31
|
+
sawNumber = true;
|
|
32
|
+
} else {
|
|
33
|
+
out[field] = 0;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return sawNumber ? out : null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function usageSignature(value) {
|
|
40
|
+
const usage = canonicalUsage(value);
|
|
41
|
+
return usage ? USAGE_FIELDS.map((field) => usage[field]).join(":") : null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function sameUsage(left, right) {
|
|
45
|
+
const leftSignature = usageSignature(left);
|
|
46
|
+
return leftSignature !== null && leftSignature === usageSignature(right);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function subtractUsage(totalUsage, lastUsage) {
|
|
50
|
+
const total = canonicalUsage(totalUsage);
|
|
51
|
+
const last = canonicalUsage(lastUsage);
|
|
52
|
+
if (!total || !last) return null;
|
|
53
|
+
const previous = {};
|
|
54
|
+
for (const field of USAGE_FIELDS) {
|
|
55
|
+
if (total[field] < last[field]) return null;
|
|
56
|
+
previous[field] = total[field] - last[field];
|
|
57
|
+
}
|
|
58
|
+
return previous;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function diffUsage(totalUsage, previousUsage) {
|
|
62
|
+
const total = canonicalUsage(totalUsage);
|
|
63
|
+
const previous = canonicalUsage(previousUsage);
|
|
64
|
+
if (!total || !previous) return null;
|
|
65
|
+
const delta = {};
|
|
66
|
+
for (const field of USAGE_FIELDS) {
|
|
67
|
+
delta[field] = Math.max(0, total[field] - previous[field]);
|
|
68
|
+
}
|
|
69
|
+
return delta;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function totalsReset(totalUsage, previousUsage) {
|
|
73
|
+
const total = canonicalUsage(totalUsage);
|
|
74
|
+
const previous = canonicalUsage(previousUsage);
|
|
75
|
+
return Boolean(total && previous && total.total_tokens < previous.total_tokens);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function createUsageDeltaState({ lastTotal = null, baselines = null } = {}) {
|
|
79
|
+
const state = {
|
|
80
|
+
lastTotal: null,
|
|
81
|
+
baselines: [],
|
|
82
|
+
sawDivergentCumulative: false,
|
|
83
|
+
sawInterleaved: false,
|
|
84
|
+
};
|
|
85
|
+
for (const baseline of Array.isArray(baselines) ? baselines : []) {
|
|
86
|
+
touchBaseline(state, baseline);
|
|
87
|
+
}
|
|
88
|
+
if (canonicalUsage(lastTotal)) touchBaseline(state, lastTotal);
|
|
89
|
+
return state;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function snapshotUsageBaselines(state) {
|
|
93
|
+
return Array.isArray(state?.baselines)
|
|
94
|
+
? state.baselines.map((usage) => ({ ...usage }))
|
|
95
|
+
: [];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function consumeUsageDelta(state, lastUsage, totalUsage) {
|
|
99
|
+
if (!state || typeof state !== "object") {
|
|
100
|
+
throw new TypeError("usage delta state is required");
|
|
101
|
+
}
|
|
102
|
+
if (!Array.isArray(state.baselines)) state.baselines = [];
|
|
103
|
+
|
|
104
|
+
const last = canonicalUsage(lastUsage);
|
|
105
|
+
const total = canonicalUsage(totalUsage);
|
|
106
|
+
if (!total) return last;
|
|
107
|
+
|
|
108
|
+
const activeSignature = usageSignature(state.lastTotal);
|
|
109
|
+
const duplicateIndex = findBaselineIndex(state, total);
|
|
110
|
+
if (duplicateIndex >= 0) {
|
|
111
|
+
if (
|
|
112
|
+
activeSignature !== null &&
|
|
113
|
+
usageSignature(state.baselines[duplicateIndex]) !== activeSignature
|
|
114
|
+
) {
|
|
115
|
+
state.sawInterleaved = true;
|
|
116
|
+
}
|
|
117
|
+
touchBaselineAt(state, duplicateIndex, total);
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (last) {
|
|
122
|
+
const expectedPrevious = subtractUsage(total, last);
|
|
123
|
+
const lineageIndex = findBaselineIndex(state, expectedPrevious);
|
|
124
|
+
if (lineageIndex >= 0) {
|
|
125
|
+
if (
|
|
126
|
+
activeSignature !== null &&
|
|
127
|
+
usageSignature(state.baselines[lineageIndex]) !== activeSignature
|
|
128
|
+
) {
|
|
129
|
+
state.sawInterleaved = true;
|
|
130
|
+
}
|
|
131
|
+
touchBaselineAt(state, lineageIndex, total);
|
|
132
|
+
return last;
|
|
133
|
+
}
|
|
134
|
+
if (expectedPrevious) {
|
|
135
|
+
if (canonicalUsage(state.lastTotal)) state.sawDivergentCumulative = true;
|
|
136
|
+
touchBaseline(state, total);
|
|
137
|
+
return last;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const active = canonicalUsage(state.lastTotal);
|
|
142
|
+
if (active && !totalsReset(total, active)) {
|
|
143
|
+
const delta = diffUsage(total, active);
|
|
144
|
+
if (!last || Number(delta?.total_tokens || 0) <= Number(last.total_tokens || 0)) {
|
|
145
|
+
const activeIndex = findBaselineIndex(state, active);
|
|
146
|
+
touchBaselineAt(state, activeIndex, total);
|
|
147
|
+
return delta;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// A cumulative jump larger than last usage cannot come from the normal
|
|
151
|
+
// append_last_usage path. Treat it as a new stream instead of charging the
|
|
152
|
+
// unrelated cumulative gap; a later event will establish its lineage.
|
|
153
|
+
state.sawDivergentCumulative = true;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
touchBaseline(state, total);
|
|
157
|
+
return last || total;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function findBaselineIndex(state, usage) {
|
|
161
|
+
const signature = usageSignature(usage);
|
|
162
|
+
if (signature === null) return -1;
|
|
163
|
+
return state.baselines.findIndex((baseline) => usageSignature(baseline) === signature);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function touchBaselineAt(state, index, usage) {
|
|
167
|
+
const canonical = canonicalUsage(usage);
|
|
168
|
+
if (!canonical) return;
|
|
169
|
+
if (Number.isInteger(index) && index >= 0 && index < state.baselines.length) {
|
|
170
|
+
state.baselines.splice(index, 1);
|
|
171
|
+
} else {
|
|
172
|
+
const duplicateIndex = findBaselineIndex(state, canonical);
|
|
173
|
+
if (duplicateIndex >= 0) state.baselines.splice(duplicateIndex, 1);
|
|
174
|
+
}
|
|
175
|
+
state.baselines.push(canonical);
|
|
176
|
+
while (state.baselines.length > MAX_USAGE_BASELINES) state.baselines.shift();
|
|
177
|
+
state.lastTotal = canonical;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function touchBaseline(state, usage) {
|
|
181
|
+
touchBaselineAt(state, findBaselineIndex(state, usage), usage);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
module.exports = {
|
|
185
|
+
MAX_USAGE_BASELINES,
|
|
186
|
+
canonicalUsage,
|
|
187
|
+
consumeUsageDelta,
|
|
188
|
+
createUsageDeltaState,
|
|
189
|
+
diffUsage,
|
|
190
|
+
sameUsage,
|
|
191
|
+
snapshotUsageBaselines,
|
|
192
|
+
subtractUsage,
|
|
193
|
+
totalsReset,
|
|
194
|
+
usageSignature,
|
|
195
|
+
};
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Codex rollouts written since CLI 0.151 carry a top-level
|
|
4
|
+
// {"type":"token_usage_record","payload":{usage,turn_token_usage,thread_token_usage,...}}
|
|
5
|
+
// next to the event_msg token_count that has always carried usage (issue
|
|
6
|
+
// #652). While both are written they describe the same model response, so
|
|
7
|
+
// token_count stays the only source of normal usage and a record is counted
|
|
8
|
+
// in exactly one case:
|
|
9
|
+
//
|
|
10
|
+
// Compaction. Codex bills the compaction model call in a record that is
|
|
11
|
+
// immediately followed by a top-level `compacted` line, and the next
|
|
12
|
+
// token_count repeats the previous cumulative total (its last_token_usage is
|
|
13
|
+
// the post-compaction context size, not a response). The cumulative delta is
|
|
14
|
+
// zero, so without the record the call is lost. Inside a file that has
|
|
15
|
+
// already shown a token_count, such a record is held as `compaction` and
|
|
16
|
+
// counted when the next token_count arrives with a total identical to the
|
|
17
|
+
// previous one. If that total advanced, the token_count already covers the
|
|
18
|
+
// call and the record is dropped. Each compaction is counted once per
|
|
19
|
+
// payload.response_id: a fork replays its parent's records and a copied
|
|
20
|
+
// rollout repeats them, both with the same id. A record without one is never
|
|
21
|
+
// counted.
|
|
22
|
+
//
|
|
23
|
+
// A file whose records are never matched by a token_count is not counted from
|
|
24
|
+
// its records. It is flagged `recordOnly` instead, and sync surfaces a
|
|
25
|
+
// warning. The proof is the same ordering Codex keeps for every response: its
|
|
26
|
+
// token_count is written after its record and before the next record or the
|
|
27
|
+
// turn end. So a record followed by another record or a turn end with no
|
|
28
|
+
// token_count in between flags the file; a sync that stops between a record
|
|
29
|
+
// and its token_count does not.
|
|
30
|
+
//
|
|
31
|
+
// payload.usage is the per-response delta; turn_token_usage and
|
|
32
|
+
// thread_token_usage are cumulative and are never read. All state below is
|
|
33
|
+
// persisted in the sync cursor and the session parser resume state, so the
|
|
34
|
+
// record, the `compacted` line and the token_count may land in different
|
|
35
|
+
// reads.
|
|
36
|
+
|
|
37
|
+
const { canonicalUsage, sameUsage } = require("./codex-token-usage");
|
|
38
|
+
|
|
39
|
+
function extractTokenUsageRecord(obj) {
|
|
40
|
+
if (obj?.type !== "token_usage_record") return null;
|
|
41
|
+
const usage = obj.payload?.usage;
|
|
42
|
+
if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
|
|
43
|
+
const timestamp = typeof obj.timestamp === "string" ? obj.timestamp : null;
|
|
44
|
+
if (!timestamp) return null;
|
|
45
|
+
// canonicalUsage folds cache_write_input_tokens into
|
|
46
|
+
// cache_creation_input_tokens exactly as the token_count path does.
|
|
47
|
+
const canonical = canonicalUsage(usage);
|
|
48
|
+
if (!canonical) return null;
|
|
49
|
+
const responseId = typeof obj.payload.response_id === "string" && obj.payload.response_id
|
|
50
|
+
? obj.payload.response_id
|
|
51
|
+
: null;
|
|
52
|
+
return { usage: canonical, timestamp, responseId };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Compaction calls already counted by sync, by response_id, kept in core
|
|
56
|
+
// cursor state as cursors.codexCompactionResponseIds (oldest first). The id
|
|
57
|
+
// is unique per API response and survives fork replays and file copies, so
|
|
58
|
+
// it is the only dedup key a compaction needs. Compactions are rare (19 on
|
|
59
|
+
// the maintainer's corpus), so the cap is far above any real history.
|
|
60
|
+
const MAX_COMPACTION_RESPONSE_IDS = 20000;
|
|
61
|
+
|
|
62
|
+
function createCompactionResponseIds(cursors) {
|
|
63
|
+
const stored = Array.isArray(cursors.codexCompactionResponseIds)
|
|
64
|
+
? cursors.codexCompactionResponseIds
|
|
65
|
+
: [];
|
|
66
|
+
const ids = new Set(stored);
|
|
67
|
+
const added = [];
|
|
68
|
+
return {
|
|
69
|
+
has: (id) => ids.has(id),
|
|
70
|
+
add(id) {
|
|
71
|
+
if (ids.has(id)) return;
|
|
72
|
+
ids.add(id);
|
|
73
|
+
added.push(id);
|
|
74
|
+
},
|
|
75
|
+
persist() {
|
|
76
|
+
if (added.length === 0) return;
|
|
77
|
+
cursors.codexCompactionResponseIds = [...stored, ...added].slice(-MAX_COMPACTION_RESPONSE_IDS);
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function isCodexTurnEndEvent(obj) {
|
|
83
|
+
if (obj?.type !== "event_msg") return false;
|
|
84
|
+
const type = obj.payload?.type;
|
|
85
|
+
return type === "task_complete" || type === "turn_aborted";
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function isPlainObject(value) {
|
|
89
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function createUsageRecordState(snapshot) {
|
|
93
|
+
const compaction = snapshot?.compaction;
|
|
94
|
+
return {
|
|
95
|
+
sawTokenCount: Boolean(snapshot?.sawTokenCount),
|
|
96
|
+
unmatchedRecord: Boolean(snapshot?.unmatchedRecord),
|
|
97
|
+
recordOnly: Boolean(snapshot?.recordOnly),
|
|
98
|
+
compaction: isPlainObject(compaction) && isPlainObject(compaction.candidate)
|
|
99
|
+
? { candidate: compaction.candidate, compacted: Boolean(compaction.compacted) }
|
|
100
|
+
: null,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Called for every token_count carrying an info object.
|
|
105
|
+
function noteTokenCount(state) {
|
|
106
|
+
state.sawTokenCount = true;
|
|
107
|
+
state.unmatchedRecord = false;
|
|
108
|
+
state.recordOnly = false;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Called for every record. `candidate` is what the caller would count if the
|
|
112
|
+
// record turns out to be a compaction call (null when it cannot be counted).
|
|
113
|
+
function noteUsageRecord(state, candidate) {
|
|
114
|
+
if (state.sawTokenCount) {
|
|
115
|
+
state.compaction = candidate ? { candidate, compacted: false } : null;
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
if (state.unmatchedRecord) state.recordOnly = true;
|
|
119
|
+
state.unmatchedRecord = true;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Turn end: a record still waiting for its token_count will not get one.
|
|
123
|
+
function noteTurnEnd(state) {
|
|
124
|
+
if (state.sawTokenCount) return;
|
|
125
|
+
if (state.unmatchedRecord) state.recordOnly = true;
|
|
126
|
+
state.unmatchedRecord = false;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// True while the line right after a record decides whether it was a
|
|
130
|
+
// compaction call.
|
|
131
|
+
function awaitsCompactedLine(state) {
|
|
132
|
+
return Boolean(state?.compaction && !state.compaction.compacted);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Called for each complete line after a record, other than a record. Only a
|
|
136
|
+
// `compacted` line directly after the record keeps it as a candidate.
|
|
137
|
+
function noteLineAfterUsageRecord(state, isCompactedLine) {
|
|
138
|
+
if (!awaitsCompactedLine(state)) return;
|
|
139
|
+
if (isCompactedLine) state.compaction.compacted = true;
|
|
140
|
+
else state.compaction = null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Called for every token_count carrying an info object, with the cumulative
|
|
144
|
+
// total that was active before it. Returns the compaction candidate to count,
|
|
145
|
+
// or null.
|
|
146
|
+
function takeCompactionOnTokenCount(state, previousTotal, totalUsage) {
|
|
147
|
+
const compaction = state.compaction;
|
|
148
|
+
state.compaction = null;
|
|
149
|
+
if (!compaction?.compacted) return null;
|
|
150
|
+
if (!canonicalUsage(previousTotal) || !sameUsage(previousTotal, totalUsage)) return null;
|
|
151
|
+
return compaction.candidate;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function isRecordOnly(state) {
|
|
155
|
+
return Boolean(state?.recordOnly && !state.sawTokenCount);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function snapshotUsageRecordState(state) {
|
|
159
|
+
if (!state || (!state.sawTokenCount && !state.unmatchedRecord && !state.recordOnly)) return null;
|
|
160
|
+
const out = { sawTokenCount: state.sawTokenCount };
|
|
161
|
+
if (state.unmatchedRecord) out.unmatchedRecord = true;
|
|
162
|
+
if (state.recordOnly) out.recordOnly = true;
|
|
163
|
+
if (state.compaction) out.compaction = state.compaction;
|
|
164
|
+
return out;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Sync keeps the flagged rollouts in core cursor state:
|
|
168
|
+
// cursors.codexUsageRecordOnlyFiles = { [rolloutPath]: true }.
|
|
169
|
+
function countRecordOnlyFiles(cursors) {
|
|
170
|
+
const flagged = cursors?.codexUsageRecordOnlyFiles;
|
|
171
|
+
return isPlainObject(flagged) ? Object.keys(flagged).length : 0;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function formatRecordOnlyWarning(count) {
|
|
175
|
+
if (!count) return null;
|
|
176
|
+
return `${count} Codex session(s) report usage only via token_usage_record; not counted yet (see #652)`;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
module.exports = {
|
|
180
|
+
MAX_COMPACTION_RESPONSE_IDS,
|
|
181
|
+
createCompactionResponseIds,
|
|
182
|
+
countRecordOnlyFiles,
|
|
183
|
+
formatRecordOnlyWarning,
|
|
184
|
+
awaitsCompactedLine,
|
|
185
|
+
createUsageRecordState,
|
|
186
|
+
extractTokenUsageRecord,
|
|
187
|
+
isCodexTurnEndEvent,
|
|
188
|
+
isRecordOnly,
|
|
189
|
+
noteLineAfterUsageRecord,
|
|
190
|
+
noteTokenCount,
|
|
191
|
+
noteTurnEnd,
|
|
192
|
+
noteUsageRecord,
|
|
193
|
+
snapshotUsageRecordState,
|
|
194
|
+
takeCompactionOnTokenCount,
|
|
195
|
+
};
|